network = Network()
network.cuda()
criterion = nn.MSELoss()
optimizer = optim.Adam(network.parameters(), lr=0.0001)
loss_min = np.inf
num_epochs = 1
start_time = time.time()
for epoch in range(1,num_epochs+1):
loss_train = 0
loss_test = 0
running_loss = 0
network.train()
print('size of train loader is: ', len(train_loader))
for step in range(1,len(train_loader)+1):
batch = next(iter(train_loader))
images, landmarks = batch['image'], batch['landmarks']
#RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[64, 600, 800, 3] to have 3 channels, but got 600 channels instead
#using permute below to fix the above error
images = images.permute(0,3,1,2)
images = images.cuda()
landmarks = landmarks.view(landmarks.size(0),-1).cuda()
norm_image = transforms.Normalize([0.3809, 0.3810, 0.3810], [0.1127, 0.1129, 0.1130])
for image in images:
image = image.float()
##image = to_tensor(image) #TypeError: pic should be PIL Image or ndarray. Got <class 'torch.Tensor'>
image = norm_image(image)
###norm_landmarks = transforms.Normalize(0.4949, 0.2165)
###landmarks = norm_landmarks(landmarks)
for landmark in landmarks:
landmark = landmark/743
predictions = network(images)
# clear all the gradients before calculating them
optimizer.zero_grad()
print('predictions are: ', predictions.float())
print('landmarks are: ', landmarks.float())
# find the loss for the current step
loss_train_step = criterion(predictions.float(), landmarks.float())
loss_train_step = loss_train_step.to(torch.float32)
print("loss_train_step before backward: ", loss_train_step)
# calculate the gradients
loss_train_step.backward()
# update the parameters
optimizer.step()
print("loss_train_step after backward: ", loss_train_step)
loss_train += loss_train_step.item()
print("loss_train: ", loss_train)
running_loss = loss_train/step
print('step: ', step)
print('running loss: ', running_loss)
print_overwrite(step, len(train_loader), running_loss, 'train')
network.eval()
with torch.no_grad():
for step in range(1,len(test_loader)+1):
batch = next(iter(train_loader))
images, landmarks = batch['image'], batch['landmarks']
images = images.permute(0,3,1,2)
images = images.cuda()
landmarks = landmarks.view(landmarks.size(0),-1).cuda()
predictions = network(images)
# find the loss for the current step
loss_test_step = criterion(predictions, landmarks)
loss_test += loss_test_step.item()
running_loss = loss_test/step
print_overwrite(step, len(test_loader), running_loss, 'Validation')
loss_train /= len(train_loader)
loss_test /= len(test_loader)
print('\n--------------------------------------------------')
print('Epoch: {} Train Loss: {:.4f} Valid Loss: {:.4f}'.format(epoch, loss_train, loss_test))
print('--------------------------------------------------')
if loss_test < loss_min:
loss_min = loss_test
torch.save(network.state_dict(), '../moth_landmarks.pth')
print("\nMinimum Valid Loss of {:.4f} at epoch {}/{}".format(loss_min, epoch, num_epochs))
print('Model Saved\n')
print('Training Complete')
print("Total Elapsed Time : {} s".format(time.time()-start_time))
--------------------------------------------------------------------
size of train loader is: 90
predictions are: tensor([[-0.2563, -0.3646, 0.3769, 0.1143, 0.0023, 0.2944, -0.1278, 0.4752],
[-0.2647, -0.3612, 0.3365, 0.1329, -0.0065, 0.3049, -0.1599, 0.4826],
[-0.2759, -0.3272, 0.3171, 0.1391, -0.0192, 0.2739, -0.1707, 0.4273],
[-0.2945, -0.3464, 0.3645, 0.1480, -0.0273, 0.2682, -0.1386, 0.4688],
[-0.2539, -0.3436, 0.3657, 0.1210, 0.0040, 0.2758, -0.1780, 0.4699],
[-0.2821, -0.3451, 0.3319, 0.1236, -0.0122, 0.2557, -0.1706, 0.4598],
[-0.2869, -0.2988, 0.3146, 0.1384, -0.0284, 0.2619, -0.1784, 0.4358],
[-0.2417, -0.3456, 0.3381, 0.1549, -0.0175, 0.2926, -0.1432, 0.4399]],
device='cuda:0', grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.0148, 240.8076, 712.0000, 270.0000, 350.0000, 351.0000, 494.0000,
323.0000],
[500.1400, 249.4700, 719.0000, 245.0000, 303.0000, 287.0000, 498.0000,
338.0000],
[486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074,
306.1910],
[494.8400, 247.7400, 712.0000, 274.0000, 315.0000, 325.0000, 458.6980,
306.5814],
[488.8000, 242.8000, 696.0000, 269.0000, 297.0000, 238.0000, 407.0000,
331.0000],
[498.1000, 245.8600, 713.0000, 246.0000, 322.0000, 143.0000, 461.0000,
322.0000],
[497.7600, 246.4500, 715.0000, 254.0000, 288.0000, 229.0000, 466.0000,
317.0000],
[496.3600, 244.8300, 716.0000, 241.0000, 303.0000, 282.0000, 477.0000,
340.0000]], device='cuda:0')
loss_train_step before backward: tensor(170825.2812, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(170825.2812, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: 170825.28125
step: 1
running loss: 170825.28125
Train Steps: 1/90 Loss: 170825.2812 predictions are: tensor([[ 0.0076, -0.2180, 0.6485, 0.2242, 0.1381, 0.3463, 0.0539, 0.6395],
[-0.0352, -0.2353, 0.6883, 0.2799, 0.1462, 0.3891, 0.0105, 0.6426],
[-0.0063, -0.2052, 0.7500, 0.2959, 0.1565, 0.4173, 0.0810, 0.6845],
[-0.0642, -0.2926, 0.7434, 0.2611, 0.1618, 0.4400, 0.0463, 0.6856],
[-0.0411, -0.1966, 0.7119, 0.2435, 0.1632, 0.3932, 0.0168, 0.6313],
[-0.0357, -0.2337, 0.7370, 0.2766, 0.1566, 0.3968, 0.0537, 0.6819],
[ 0.0105, -0.2863, 0.7531, 0.2688, 0.1779, 0.4589, -0.0235, 0.6847],
[-0.0319, -0.2861, 0.7344, 0.2631, 0.1316, 0.4394, 0.0566, 0.6840]],
device='cuda:0', grad_fn=<AddmmBackward>)
landmarks are: tensor([[ nan, nan, 638.5021, 191.6575, 290.0000, 190.0000, 403.1752,
333.7941],
[502.1200, 244.9500, 668.0000, 163.0000, 365.0000, 108.0000, 473.4160,
292.6776],
[492.9800, 236.9300, 707.0000, 271.0000, 340.0000, 311.0000, 467.0000,
330.0000],
[500.0100, 246.3300, 696.0000, 223.0000, 287.0000, 298.0000, 483.0000,
310.0000],
[506.8508, 251.4398, 715.0000, 310.0000, 315.0000, 211.0000, 587.4209,
344.8954],
[498.2935, 243.9347, 681.0000, 343.0000, 360.0000, 303.0000, 482.0000,
321.0000],
[496.1100, 241.6200, 642.3539, 163.6537, 323.3445, 138.6042, 478.4196,
323.4764],
[503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000,
307.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 2
running loss: nan
Train Steps: 2/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[489.9700, 238.4500, 618.0000, 151.0000, 283.0000, 199.0000, 471.0000,
330.0000],
[495.7400, 245.4600, 704.0000, 287.0000, 283.0000, 286.0000, 476.0000,
333.0000],
[501.0700, 243.4400, 704.0000, 230.0000, 292.0000, 223.0000, 509.9838,
288.2302],
[488.7300, 240.1700, 692.0000, 293.0000, 382.0000, 292.0000, 414.0000,
341.0000],
[496.3300, 245.4400, 716.0000, 287.0000, 289.0000, 277.0000, 485.0000,
337.0000],
[484.6700, 238.7000, 663.0000, 216.0000, 272.0000, 243.0000, 442.3150,
327.6658],
[501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471,
292.1411],
[495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262,
295.5407]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 3
running loss: nan
Train Steps: 3/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.5900, 255.9900, 572.0000, 199.0000, 392.0000, 109.0000, 444.8080,
371.0059],
[488.6600, 240.6300, 643.0000, 204.0000, 296.0000, 176.0000, 452.6434,
337.0366],
[502.0600, 247.2100, 699.0000, 188.0000, 338.0000, 133.0000, 496.2461,
293.5293],
[493.1591, 244.5602, 707.0000, 247.0000, 297.0000, 333.0000, 499.0000,
321.0000],
[490.3400, 244.1100, 700.0000, 304.0000, 310.0000, 254.0000, 418.8311,
352.8785],
[502.8574, 242.0584, 655.2804, 144.8417, 340.9529, 143.5759, 509.1774,
321.3521],
[494.7200, 244.7300, 668.0000, 222.0000, 294.0000, 173.0000, 425.0000,
347.0000],
[492.0700, 247.4900, 699.0000, 265.0000, 286.0000, 227.0000, 411.0000,
329.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 4
running loss: nan
Train Steps: 4/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.7500, 245.3800, 626.0000, 150.0000, 336.0000, 149.0000, 479.0000,
340.0000],
[498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000,
311.0000],
[492.3034, 246.7491, 563.0000, 139.0000, 339.0000, 110.0000, 428.0000,
336.0000],
[498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000,
312.0000],
[489.8800, 245.0100, 556.3521, 184.1524, 292.0000, 165.0000, 413.0584,
329.1087],
[484.6600, 239.1800, 665.8854, 277.5496, 307.0000, 299.0000, 411.7228,
327.9374],
[490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792,
324.7162],
[482.0700, 238.7200, 684.0000, 254.0000, 289.0000, 314.0000, 446.5888,
297.9915]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 5
running loss: nan
Train Steps: 5/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[491.6300, 240.4200, 702.0000, 272.0000, 365.0000, 332.0000, 487.0000,
332.0000],
[500.1200, 249.7500, 693.0000, 268.0000, 290.0000, 214.0000, 483.0000,
332.0000],
[488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000,
305.0000],
[489.8800, 245.0100, 556.3521, 184.1524, 292.0000, 165.0000, 413.0584,
329.1087],
[500.8400, 246.3800, 642.0000, 155.0000, 364.0000, 112.0000, 502.5046,
292.1347],
[496.0000, 239.8700, 708.0000, 237.0000, 280.0000, 279.0000, 491.0000,
324.0000],
[495.8700, 247.7900, 701.0000, 247.0000, 292.0000, 294.0000, 456.5611,
306.1910],
[496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000,
356.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 6
running loss: nan
Train Steps: 6/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[489.1000, 241.0600, 577.0000, 118.0000, 301.0000, 162.0000, 470.0000,
332.0000],
[498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000,
321.0000],
[491.4000, 238.9100, 692.0000, 293.0000, 313.0000, 259.0000, 425.2197,
321.0281],
[490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000,
328.0000],
[491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828,
305.8005],
[507.4213, 245.8110, 743.0000, 262.0000, 345.0000, 216.0000, 579.8230,
350.4485],
[ nan, nan, 694.0000, 170.0000, 428.0000, 119.0000, 534.2356,
337.2599],
[499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000,
326.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 7
running loss: nan
Train Steps: 7/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[509.7032, 245.4983, 667.0000, 351.0000, 316.0000, 307.0000, 524.7382,
315.7417],
[495.9600, 243.5600, 696.0000, 294.0000, 333.0000, 300.0000, 493.0000,
328.0000],
[492.4400, 247.4700, 708.0000, 290.0000, 364.0000, 349.0000, 461.1910,
305.0196],
[497.7500, 250.2900, 708.0000, 313.0000, 299.0000, 276.0000, 456.0000,
338.0000],
[495.7000, 245.4200, 676.0000, 234.0000, 286.0000, 236.0000, 478.0000,
335.0000],
[509.1327, 248.6254, 690.0000, 185.0000, 393.0000, 120.0000, 515.8740,
316.4358],
[483.1600, 240.1100, 587.0000, 136.0000, 318.0000, 126.0000, 418.4528,
286.6684],
[499.1492, 246.4364, 652.9544, 165.8065, 290.0000, 216.0000, 479.0000,
342.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 8
running loss: nan
Train Steps: 8/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.1000, 244.9000, 706.1387, 222.8866, 306.0016, 162.7361, 467.5804,
324.7273],
[494.3001, 247.0619, 582.2141, 254.2003, 287.0000, 204.0000, 454.0000,
355.0000],
[486.0700, 237.4800, 696.0000, 280.0000, 363.6628, 302.7423, 418.4528,
297.6010],
[ nan, nan, 715.0000, 171.0000, 373.0000, 187.0000, 592.4862,
331.7068],
[500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154,
295.6488],
[489.1000, 241.0600, 577.0000, 118.0000, 301.0000, 162.0000, 470.0000,
332.0000],
[ nan, nan, 604.2729, 163.3048, 310.0000, 153.0000, 410.0000,
338.0000],
[488.0247, 244.2474, 608.0761, 206.6678, 272.0000, 247.0000, 450.0000,
337.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 9
running loss: nan
Train Steps: 9/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483,
326.7660],
[505.9951, 243.9347, 675.0000, 321.0000, 314.0000, 316.0000, 569.0593,
347.6719],
[494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557,
323.6706],
[492.0181, 236.1169, 695.7136, 309.4855, 371.7718, 319.7672, 483.3020,
309.1750],
[490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470,
329.6944],
[496.2000, 243.9500, 671.8087, 158.8574, 314.0060, 157.6172, 467.5804,
307.4238],
[496.8673, 236.1169, 690.9001, 290.4881, 353.3458, 307.5619, 506.4407,
316.0934],
[500.2902, 239.5567, 719.0000, 286.0000, 319.0000, 331.0000, 556.3961,
317.1299]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 10
running loss: nan
Train Steps: 10/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[490.1000, 242.3100, 659.0000, 238.0000, 290.0000, 216.0000, 471.5194,
368.2727],
[495.9100, 243.6200, 711.0000, 280.0000, 304.0000, 303.0000, 495.0000,
326.0000],
[499.6700, 241.6600, 699.0000, 292.0000, 327.0000, 340.0000, 509.0000,
312.0000],
[493.2500, 240.4900, 685.0000, 340.0000, 351.0000, 296.0000, 446.0000,
334.0000],
[487.2800, 239.8400, 665.1248, 260.0376, 303.0000, 273.0000, 417.0651,
339.3581],
[507.7065, 245.1856, 635.0000, 330.0000, 317.0000, 292.0000, 587.4209,
342.1188],
[497.5500, 246.8200, 654.0000, 169.0000, 314.0000, 167.0000, 472.0000,
321.0000],
[494.5853, 238.3059, 684.1613, 354.8681, 294.6041, 250.8712, 455.0547,
322.6822]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 11
running loss: nan
Train Steps: 11/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[504.7900, 241.0400, 685.0000, 348.0000, 295.0000, 285.0000, 506.0662,
300.3342],
[486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074,
306.1910],
[493.1591, 238.3059, 625.4368, 202.8892, 287.8980, 203.2175, 470.3803,
309.1750],
[482.3900, 238.1600, 639.0000, 235.0000, 276.0000, 220.0000, 421.3020,
281.9830],
[492.9800, 236.9300, 707.0000, 271.0000, 340.0000, 311.0000, 467.0000,
330.0000],
[494.2245, 240.7296, 712.0364, 269.7266, 311.2985, 337.0000, 477.7910,
292.7526],
[490.8772, 241.7457, 661.0000, 201.0000, 290.0000, 184.0000, 454.0000,
310.0000],
[495.9600, 245.1500, 673.1207, 178.4619, 329.3477, 136.4104, 469.4820,
323.4764]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 12
running loss: nan
Train Steps: 12/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471,
292.1411],
[500.0000, 248.7500, 707.0000, 236.0000, 287.0000, 257.0000, 493.0000,
322.0000],
[486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074,
306.1910],
[492.2300, 247.1400, 677.0000, 230.0000, 288.0000, 192.0000, 408.5175,
333.7941],
[500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000,
330.0000],
[497.2900, 250.0000, 687.0000, 335.0000, 318.0000, 310.0000, 462.0000,
340.0000],
[501.8700, 246.5800, 712.0000, 229.0000, 335.0000, 130.0000, 468.6673,
290.0746],
[493.5600, 243.1200, 699.3539, 286.7225, 343.0000, 295.0000, 461.0000,
337.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 13
running loss: nan
Train Steps: 13/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.0900, 241.0100, 703.0000, 306.0000, 326.0000, 315.0000, 473.0000,
302.0000],
[492.8739, 241.4330, 707.0000, 275.0000, 295.0000, 224.0000, 424.8636,
320.6377],
[483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557,
330.5729],
[486.7600, 240.2900, 672.0000, 259.0000, 301.0000, 285.0000, 438.0412,
303.4578],
[503.7131, 240.4949, 732.0000, 259.0000, 341.0000, 183.0000, 580.4562,
324.7654],
[502.4100, 246.0400, 724.0000, 272.0000, 302.0000, 193.0000, 507.0098,
294.9176],
[490.5919, 243.9347, 580.6928, 144.1250, 287.0000, 198.0000, 480.0000,
336.0000],
[494.9600, 246.2100, 570.0437, 124.1114, 316.0000, 151.0000, 473.0000,
341.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 14
running loss: nan
Train Steps: 14/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000,
326.0000],
[495.9000, 240.8800, 697.2715, 313.6593, 306.6687, 286.3209, 470.6230,
308.8831],
[507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504,
326.1537],
[493.1591, 246.1237, 708.0000, 292.0000, 337.0000, 359.0000, 448.7257,
302.2865],
[495.7263, 247.0619, 695.0000, 331.0000, 323.0000, 314.0000, 470.0000,
336.0000],
[495.8200, 244.5800, 635.6002, 147.2001, 358.0301, 112.2785, 471.7639,
320.9747],
[490.3067, 235.8042, 701.4897, 306.3193, 331.7047, 338.5089, 479.9964,
304.8923],
[494.0100, 245.5700, 704.0000, 266.0000, 326.0000, 262.0000, 410.2613,
294.0870]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 15
running loss: nan
Train Steps: 15/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[489.6200, 240.8100, 549.0000, 169.0000, 296.0000, 167.0000, 441.0000,
340.0000],
[494.4200, 243.8400, 576.0000, 148.0000, 342.0000, 142.0000, 477.0000,
373.0000],
[498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000,
321.0000],
[506.2803, 251.7526, 739.0000, 275.0000, 341.0000, 176.0000, 587.4209,
344.8954],
[495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000,
339.0000],
[491.0200, 241.2100, 704.0000, 283.0000, 350.0000, 283.0000, 440.1781,
336.6462],
[498.0700, 246.1700, 686.0000, 207.0000, 351.0000, 124.0000, 463.0000,
323.0000],
[496.2100, 244.5800, 688.8793, 172.7032, 324.0115, 153.2296, 472.5629,
329.7797]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 16
running loss: nan
Train Steps: 16/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396,
367.4918],
[ nan, nan, 669.0000, 199.0000, 285.0000, 202.0000, 426.2881,
308.5337],
[503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324,
332.4009],
[484.8000, 235.4200, 676.0000, 343.0000, 336.0000, 313.0000, 420.2336,
285.1066],
[ nan, nan, 611.8794, 163.3048, 317.0000, 131.0000, 404.7779,
325.3018],
[489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000,
308.0000],
[507.1360, 246.1237, 727.0000, 286.0000, 314.0000, 317.0000, 600.7172,
323.3772],
[497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000,
324.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 17
running loss: nan
Train Steps: 17/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[502.8574, 245.8110, 648.5415, 144.8417, 354.0000, 149.0000, 539.6033,
323.0817],
[492.8300, 245.1400, 702.0000, 239.0000, 300.0000, 227.0000, 410.2613,
292.9156],
[489.1657, 239.8694, 565.0000, 143.0000, 323.0000, 117.0000, 425.5759,
299.5533],
[494.8706, 240.8076, 655.2804, 191.2797, 342.3284, 123.5840, 461.3652,
324.6589],
[487.7600, 239.2700, 692.0000, 315.0000, 337.0000, 312.0000, 454.0000,
303.0000],
[483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557,
330.5729],
[490.2000, 245.0600, 699.0000, 281.0000, 289.0000, 222.0000, 396.7645,
323.8376],
[494.0700, 241.7800, 708.0000, 291.0000, 295.0000, 243.0000, 425.0000,
347.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 18
running loss: nan
Train Steps: 18/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[502.0200, 242.8900, 679.0000, 173.0000, 357.0000, 122.0000, 505.7100,
309.3146],
[501.7700, 247.2200, 723.0000, 247.0000, 298.0000, 192.0000, 494.0000,
315.0000],
[501.1800, 255.0400, 569.0000, 213.0000, 350.0000, 127.0000, 446.9449,
367.1014],
[491.6300, 240.4700, 700.0000, 323.0000, 318.0000, 279.0000, 445.0000,
332.0000],
[495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000,
370.0000],
[506.5656, 247.0619, 739.0000, 256.0000, 321.0000, 284.0000, 601.9836,
326.1537],
[503.4279, 238.9313, 696.0000, 318.0000, 301.0000, 283.0000, 563.9941,
317.8241],
[496.1100, 241.6200, 642.3539, 163.6537, 323.3445, 138.6042, 478.4196,
323.4764]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 19
running loss: nan
Train Steps: 19/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154,
295.6488],
[496.3500, 242.9600, 638.0000, 162.0000, 315.0000, 154.0000, 456.0000,
311.0000],
[489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000,
308.0000],
[497.2200, 247.1100, 615.0000, 138.0000, 336.0000, 137.0000, 474.0000,
319.0000],
[504.5688, 239.8694, 680.0000, 314.0000, 308.0000, 303.0000, 595.6520,
319.2123],
[501.0800, 242.3100, 720.0000, 264.0000, 290.0000, 280.0000, 513.9015,
288.2302],
[501.5100, 244.3800, 712.0000, 296.0000, 290.0000, 251.0000, 503.0000,
309.0000],
[501.8900, 245.2000, 670.0000, 342.0000, 322.0000, 288.0000, 456.0000,
367.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 20
running loss: nan
Train Steps: 20/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[490.3600, 243.8000, 699.0000, 315.0000, 345.0000, 284.0000, 418.4304,
352.4393],
[500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000,
314.0000],
[491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828,
305.8005],
[508.8475, 246.1237, 692.0000, 179.0000, 391.0000, 120.0000, 536.1351,
327.5420],
[499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000,
326.0000],
[494.0900, 241.0100, 703.0000, 306.0000, 326.0000, 315.0000, 473.0000,
302.0000],
[508.5623, 249.5636, 703.0000, 335.0000, 291.0000, 266.0000, 519.0398,
317.8241],
[496.3700, 240.9300, 668.0000, 163.0000, 319.0000, 153.0000, 463.0000,
308.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 21
running loss: nan
Train Steps: 21/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[490.3600, 243.8000, 699.0000, 315.0000, 345.0000, 284.0000, 418.4304,
352.4393],
[491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000,
345.0000],
[ nan, nan, 575.3683, 140.7894, 323.0000, 123.0000, 411.0000,
339.0000],
[501.7164, 244.8729, 726.5198, 293.6544, 296.0000, 257.0000, 532.8420,
316.4105],
[487.3300, 240.0800, 691.0000, 292.0000, 343.0000, 322.0000, 438.7535,
303.8483],
[500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000,
357.0000],
[498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000,
312.0000],
[495.9100, 243.6200, 711.0000, 280.0000, 304.0000, 303.0000, 495.0000,
326.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 22
running loss: nan
Train Steps: 22/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[502.2869, 240.4949, 688.0000, 293.0000, 346.0000, 317.0000, 560.8282,
322.6830],
[490.1900, 247.1700, 692.0000, 318.0000, 361.0000, 315.0000, 420.8047,
327.6445],
[505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925,
341.4247],
[497.0900, 242.4000, 707.0000, 210.0000, 288.0000, 311.0000, 509.0000,
312.0000],
[501.2900, 243.9900, 687.0000, 172.0000, 344.0000, 171.0000, 506.0000,
316.0000],
[488.9500, 241.7100, 691.0000, 288.0000, 390.0000, 305.0000, 461.0000,
334.0000],
[495.1000, 234.6500, 704.0000, 295.0000, 297.0000, 288.0000, 483.0000,
290.0000],
[493.4800, 246.5000, 545.7030, 163.3048, 306.0000, 153.0000, 444.0000,
343.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 23
running loss: nan
Train Steps: 23/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.8300, 248.5600, 700.0000, 342.0000, 319.0000, 283.0000, 481.0000,
328.0000],
[ nan, nan, 567.7618, 140.7894, 340.0000, 111.0000, 414.0000,
335.0000],
[487.7600, 239.2700, 692.0000, 315.0000, 337.0000, 312.0000, 454.0000,
303.0000],
[491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828,
305.8005],
[497.5400, 245.8000, 699.0000, 204.0000, 285.0000, 247.0000, 478.0000,
341.0000],
[497.4900, 236.0200, 695.0000, 316.0000, 345.0000, 298.0000, 479.0000,
299.0000],
[494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070,
323.3411],
[501.1200, 242.0800, 711.0000, 293.0000, 324.0000, 313.0000, 508.9154,
287.4493]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 24
running loss: nan
Train Steps: 24/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[486.9300, 238.5500, 667.0000, 232.0000, 297.0000, 187.0000, 475.0000,
318.0000],
[494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557,
323.6706],
[492.8300, 245.1400, 702.0000, 239.0000, 300.0000, 227.0000, 410.2613,
292.9156],
[507.1360, 244.8729, 674.0000, 325.0000, 308.0000, 290.0000, 586.7878,
345.5895],
[487.6300, 240.1400, 682.6197, 310.0718, 402.5010, 305.6570, 410.0000,
326.0000],
[502.7100, 247.6400, 683.0000, 352.0000, 349.0000, 305.0000, 483.0000,
326.0000],
[500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000,
330.0000],
[494.8706, 240.8076, 655.2804, 191.2797, 342.3284, 123.5840, 461.3652,
324.6589]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 25
running loss: nan
Train Steps: 25/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154,
295.6488],
[489.9400, 241.8100, 692.0000, 292.0000, 399.9338, 306.3606, 410.9215,
346.3862],
[503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000,
307.0000],
[494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557,
323.6706],
[499.7000, 245.3900, 557.1127, 121.6097, 314.0000, 161.0000, 487.0000,
335.0000],
[496.7200, 235.1800, 692.0000, 322.0000, 352.0000, 304.0000, 482.0000,
297.0000],
[496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000,
308.0000],
[486.6900, 238.8800, 687.0000, 314.0000, 368.0000, 322.0000, 454.4241,
300.3342]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 26
running loss: nan
Train Steps: 26/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654],
[490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806,
297.2106],
[502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936,
319.2123],
[501.3200, 243.8900, 665.0000, 148.0000, 383.0000, 104.0000, 505.0000,
308.0000],
[492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000,
348.0000],
[497.7600, 236.2000, 668.0000, 337.0000, 331.0000, 276.0000, 464.0000,
314.0000],
[495.9400, 237.1000, 685.8777, 322.4308, 326.0126, 281.2020, 475.3770,
322.6425],
[487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475,
296.4297]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 27
running loss: nan
Train Steps: 27/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[497.7300, 244.3100, 573.0863, 129.9487, 299.0000, 190.0000, 488.0000,
332.0000],
[491.1800, 244.2000, 696.0000, 300.0000, 369.0000, 294.0000, 420.8344,
351.1215],
[500.8607, 243.9347, 666.0000, 129.0000, 381.0000, 160.0000, 560.1951,
337.9540],
[503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000,
307.0000],
[490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806,
297.2106],
[497.1525, 246.7491, 627.0000, 127.0000, 292.0000, 188.0000, 454.0000,
305.0000],
[491.4477, 242.3712, 659.0000, 200.0000, 326.0000, 127.0000, 410.9736,
298.3820],
[496.4300, 240.2100, 715.0000, 293.0000, 293.0000, 300.0000, 508.5592,
296.8201]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 28
running loss: nan
Train Steps: 28/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.9600, 242.9100, 691.0000, 337.0000, 330.0000, 306.0000, 481.0000,
319.0000],
[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654],
[503.4279, 238.9313, 696.0000, 318.0000, 301.0000, 283.0000, 563.9941,
317.8241],
[501.4312, 241.7457, 680.0000, 161.0000, 315.0000, 210.0000, 548.7982,
317.8241],
[489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000,
308.0000],
[490.9200, 245.0200, 646.1086, 233.3527, 292.0000, 189.0000, 428.4472,
353.7570],
[483.8400, 239.2300, 609.5974, 231.6849, 278.0000, 250.0000, 410.9215,
327.9374],
[496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000,
308.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 29
running loss: nan
Train Steps: 29/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470,
329.6944],
[491.7329, 244.8729, 683.0000, 204.0000, 293.0000, 189.0000, 411.3298,
292.5252],
[ nan, nan, 535.0539, 150.7963, 329.0000, 127.0000, 415.4254,
355.9533],
[501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471,
292.1411],
[ nan, nan, 690.0000, 153.0000, 439.0000, 132.0000, 586.7878,
344.2012],
[490.1900, 247.1700, 692.0000, 318.0000, 361.0000, 315.0000, 420.8047,
327.6445],
[508.5623, 245.8110, 723.0000, 233.0000, 337.0000, 177.0000, 534.8688,
323.3772],
[487.2000, 240.6200, 627.0000, 209.0000, 283.0000, 227.0000, 436.9727,
304.6292]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 30
running loss: nan
Train Steps: 30/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[504.5688, 242.6839, 642.0000, 350.0000, 302.0000, 292.0000, 551.3309,
327.5420],
[490.8800, 245.1000, 535.0000, 139.0000, 309.0000, 142.0000, 441.3725,
346.9719],
[507.1360, 247.0619, 639.0000, 348.0000, 313.0000, 275.0000, 587.4209,
345.5895],
[502.0016, 242.9966, 723.0000, 226.0000, 307.0000, 212.0000, 565.8936,
334.4833],
[486.5300, 242.5300, 558.0000, 115.0000, 328.0000, 119.0000, 440.1781,
334.6939],
[505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925,
341.4247],
[485.1400, 241.1400, 692.0000, 271.0000, 323.0000, 322.0000, 456.2567,
336.5189],
[501.1459, 238.3059, 708.2286, 286.2665, 308.2703, 270.6642, 503.4357,
320.0467]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 31
running loss: nan
Train Steps: 31/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000,
331.0000],
[486.9800, 237.0500, 671.0000, 350.0000, 335.0000, 296.0000, 411.6859,
289.7920],
[500.0800, 246.5000, 711.0000, 282.0000, 346.0000, 349.0000, 486.0000,
309.0000],
[482.6800, 240.6500, 588.0000, 152.0000, 275.0000, 202.0000, 441.2465,
305.0196],
[495.8700, 247.7900, 701.0000, 247.0000, 292.0000, 294.0000, 456.5611,
306.1910],
[489.0300, 246.2400, 692.0000, 255.0000, 314.0000, 358.0000, 462.9718,
306.9719],
[498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000,
311.0000],
[502.0016, 241.4330, 688.0000, 137.0000, 428.0000, 108.0000, 565.8936,
324.7654]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 32
running loss: nan
Train Steps: 32/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.6600, 241.1700, 670.0000, 365.0000, 314.0000, 292.0000, 482.9497,
277.5642],
[502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808,
289.3645],
[494.8706, 243.6220, 700.0000, 303.0000, 283.0000, 262.0000, 465.0000,
365.0000],
[499.2800, 248.5500, 715.0000, 279.0000, 326.0000, 321.0000, 500.0000,
333.0000],
[488.6800, 242.1600, 575.0000, 105.0000, 308.0000, 153.0000, 469.0000,
334.0000],
[494.7600, 242.5300, 616.0000, 135.0000, 325.0000, 127.0000, 461.0000,
309.0000],
[503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324,
332.4009],
[493.4800, 246.5000, 545.7030, 163.3048, 306.0000, 153.0000, 444.0000,
343.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 33
running loss: nan
Train Steps: 33/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.2500, 244.2100, 697.0000, 336.0000, 297.0000, 287.0000, 462.0000,
366.0000],
[483.0400, 236.7800, 673.0000, 293.0000, 285.0000, 273.0000, 421.3020,
281.5925],
[486.0700, 237.4800, 696.0000, 280.0000, 363.6628, 302.7423, 418.4528,
297.6010],
[486.4500, 236.9900, 683.0000, 280.0000, 308.0000, 295.0000, 427.3566,
297.2106],
[503.6500, 239.3200, 720.0000, 249.0000, 289.0000, 232.0000, 512.0000,
306.0000],
[502.8574, 243.3093, 720.0000, 283.0000, 301.0000, 281.0000, 561.4614,
329.6244],
[483.4400, 241.3300, 551.0000, 119.0000, 302.0000, 149.0000, 438.3973,
308.1432],
[495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262,
295.5407]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 34
running loss: nan
Train Steps: 34/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[500.0400, 246.9800, 696.0000, 291.0000, 372.0000, 334.0000, 487.0000,
311.0000],
[ nan, nan, 609.0000, 195.0000, 323.0000, 152.0000, 435.0000,
346.0000],
[500.2902, 239.5567, 719.0000, 286.0000, 319.0000, 331.0000, 556.3961,
317.1299],
[496.3600, 244.8300, 716.0000, 241.0000, 303.0000, 282.0000, 477.0000,
340.0000],
[502.8574, 242.0584, 655.2804, 144.8417, 340.9529, 143.5759, 509.1774,
321.3521],
[495.4411, 242.9966, 585.0000, 146.0000, 326.0000, 127.0000, 451.9311,
339.3793],
[497.9000, 243.6700, 719.0000, 258.0000, 307.0000, 285.0000, 489.0000,
329.0000],
[ nan, nan, 677.0000, 153.0000, 468.0000, 128.0000, 570.3256,
364.3312]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 35
running loss: nan
Train Steps: 35/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070,
323.3411],
[490.5919, 247.0619, 656.0000, 218.0000, 285.0000, 324.0000, 462.9718,
308.1432],
[494.8706, 249.2509, 546.0000, 158.0000, 332.0000, 138.0000, 457.0000,
331.0000],
[501.7900, 244.2400, 699.0000, 336.0000, 294.0000, 227.0000, 474.0855,
284.5056],
[507.7065, 245.4983, 617.0000, 355.0000, 323.0000, 286.0000, 587.4209,
343.5071],
[496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000,
326.0000],
[494.5853, 239.8694, 703.4152, 251.4380, 284.1584, 257.0997, 483.0015,
318.7289],
[500.8607, 239.2440, 723.6318, 252.4934, 288.0082, 277.9555, 525.6299,
309.7392]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 36
running loss: nan
Train Steps: 36/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000,
311.0000],
[495.1558, 237.6805, 607.1456, 135.3430, 300.7195, 196.7750, 494.7211,
320.0467],
[496.8600, 244.1200, 700.0000, 307.0000, 332.0000, 294.0000, 470.0000,
310.0000],
[ nan, nan, 664.0000, 189.0000, 287.0000, 203.0000, 416.6721,
311.6573],
[499.7197, 240.4949, 700.5271, 305.2639, 328.0690, 323.5423, 531.4897,
308.0097],
[486.0200, 240.0200, 681.0000, 311.0000, 360.7842, 319.7291, 414.3940,
316.8095],
[492.8739, 243.3093, 694.0000, 232.0000, 297.0000, 242.0000, 415.6036,
309.7050],
[500.9500, 245.1100, 675.0000, 189.0000, 322.0000, 158.0000, 507.1346,
288.6207]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 37
running loss: nan
Train Steps: 37/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.3500, 242.9600, 638.0000, 162.0000, 315.0000, 154.0000, 456.0000,
311.0000],
[494.8706, 237.9932, 586.9290, 119.5119, 329.5677, 150.5065, 488.4105,
323.6706],
[495.8100, 239.8400, 686.5782, 321.6804, 329.3477, 300.9463, 475.3770,
308.0492],
[501.2500, 246.0400, 668.0000, 140.0000, 316.0000, 177.0000, 501.0800,
295.2583],
[500.7600, 247.1900, 641.0000, 141.0000, 391.0000, 92.0000, 502.5046,
293.6965],
[491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000,
345.0000],
[493.4444, 244.8729, 675.0000, 202.0000, 280.0000, 280.0000, 497.0000,
324.0000],
[495.9900, 244.1900, 715.0000, 251.0000, 283.0000, 274.0000, 494.0000,
324.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 38
running loss: nan
Train Steps: 38/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[493.5600, 243.1200, 699.3539, 286.7225, 343.0000, 295.0000, 461.0000,
337.0000],
[501.0800, 241.7700, 720.0000, 286.0000, 304.0000, 310.0000, 513.1892,
286.2780],
[502.8574, 242.3712, 695.7136, 182.8364, 313.9077, 173.2258, 504.0367,
322.0233],
[501.0900, 244.0100, 724.0000, 251.0000, 302.0000, 276.0000, 504.6415,
291.7443],
[495.4411, 242.9966, 585.0000, 146.0000, 326.0000, 127.0000, 451.9311,
339.3793],
[492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121,
350.7788],
[491.7300, 241.5500, 528.0000, 148.0000, 327.0000, 129.0000, 439.1096,
346.4075],
[501.4500, 243.6300, 668.0000, 146.0000, 366.0000, 137.0000, 508.0000,
318.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 39
running loss: nan
Train Steps: 39/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.1100, 234.6700, 699.0000, 253.0000, 279.0000, 242.0000, 482.0000,
289.0000],
[505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925,
341.4247],
[485.8600, 235.9000, 669.0000, 349.0000, 354.0000, 307.0000, 416.3159,
289.0111],
[491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148,
349.3146],
[498.2500, 240.2200, 700.0000, 315.0000, 306.0000, 314.0000, 509.0000,
300.0000],
[ nan, nan, 711.0000, 186.0000, 421.0000, 169.0000, 571.5920,
360.1664],
[506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061,
303.9413],
[501.1100, 244.2300, 723.0000, 232.0000, 293.0000, 235.0000, 507.0000,
317.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 40
running loss: nan
Train Steps: 40/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[484.7500, 240.1200, 605.0000, 166.0000, 283.0000, 182.0000, 442.3150,
327.2754],
[496.9300, 241.9800, 715.0000, 250.0000, 305.0000, 256.0000, 449.0000,
335.0000],
[495.9000, 240.8800, 697.2715, 313.6593, 306.6687, 286.3209, 470.6230,
308.8831],
[500.0049, 240.8076, 682.0000, 325.0000, 323.0000, 307.0000, 506.0000,
301.0000],
[503.7131, 242.6839, 731.0000, 246.0000, 338.5284, 254.5401, 593.7525,
317.8241],
[496.2700, 244.6800, 704.0000, 305.0000, 312.0000, 300.0000, 488.0000,
335.0000],
[ nan, nan, 548.7455, 131.6165, 332.0000, 112.0000, 412.2570,
343.7506],
[498.6600, 245.4800, 648.0000, 177.0000, 285.0000, 233.0000, 481.0000,
312.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 41
running loss: nan
Train Steps: 41/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.1500, 243.9800, 699.6677, 202.5880, 297.3302, 185.4055, 467.3902,
309.9255],
[494.1500, 245.1300, 699.0000, 237.0000, 302.0000, 336.0000, 498.0000,
342.0000],
[494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221,
317.4111],
[486.8500, 236.3900, 697.0000, 287.0000, 322.0000, 294.0000, 439.8219,
323.3708],
[494.2300, 243.5200, 602.0000, 135.0000, 345.0000, 107.0000, 432.3427,
314.3904],
[506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061,
303.9413],
[490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483,
326.7660],
[493.4200, 246.6600, 521.3622, 172.4778, 295.0000, 169.0000, 418.2301,
350.2430]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 42
running loss: nan
Train Steps: 42/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.9900, 239.9300, 692.2063, 312.4252, 328.0137, 307.5277, 478.0393,
306.1729],
[494.5900, 243.5100, 699.0000, 261.0000, 285.0000, 204.0000, 423.2087,
349.3146],
[501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396,
367.4918],
[502.7100, 247.6400, 683.0000, 352.0000, 349.0000, 305.0000, 483.0000,
326.0000],
[492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000,
322.0000],
[506.5656, 249.8763, 728.0000, 201.0000, 335.0000, 221.0000, 595.0188,
331.7068],
[497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000,
324.0000],
[496.5300, 244.8400, 613.0000, 124.0000, 317.0000, 192.0000, 505.0000,
318.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 43
running loss: nan
Train Steps: 43/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.1000, 242.9900, 620.5920, 134.0372, 356.6960, 107.8909, 478.0393,
325.5612],
[488.0700, 242.5300, 622.0000, 157.0000, 297.0000, 169.0000, 435.1920,
338.9889],
[500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000,
357.0000],
[501.3800, 245.6600, 697.0000, 185.0000, 352.0000, 136.0000, 500.0000,
312.0000],
[500.0049, 237.6805, 693.7882, 295.7652, 335.9166, 298.3375, 504.9382,
318.0700],
[501.2900, 243.9900, 687.0000, 172.0000, 344.0000, 171.0000, 506.0000,
316.0000],
[494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020,
316.7523],
[504.5688, 241.7457, 692.0000, 312.0000, 301.0000, 241.0000, 584.8883,
321.9889]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 44
running loss: nan
Train Steps: 44/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[497.4600, 248.2400, 581.0000, 134.0000, 326.0000, 159.0000, 497.0000,
347.0000],
[493.1591, 238.3059, 625.4368, 202.8892, 287.8980, 203.2175, 470.3803,
309.1750],
[492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000,
322.0000],
[495.4400, 244.7100, 583.0000, 150.0000, 340.0000, 153.0000, 479.1144,
375.9738],
[490.1500, 245.0200, 696.0000, 268.0000, 319.0000, 259.0000, 401.0383,
328.2302],
[494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000,
326.0000],
[487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475,
296.4297],
[506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061,
303.9413]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 45
running loss: nan
Train Steps: 45/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.1800, 245.4400, 716.0000, 212.0000, 288.0000, 238.0000, 503.5731,
294.0870],
[501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396,
367.4918],
[483.8800, 235.7000, 683.0000, 326.0000, 310.0000, 307.0000, 421.3020,
283.1544],
[489.3200, 241.0900, 525.0000, 118.0000, 299.0000, 153.0000, 422.3705,
306.1910],
[501.3600, 244.7900, 675.0000, 158.0000, 381.0000, 108.0000, 500.7239,
314.3904],
[486.3200, 237.8300, 593.6238, 177.4812, 285.0000, 175.0000, 428.0689,
298.7724],
[504.3800, 238.9900, 716.0000, 290.0000, 295.0000, 281.0000, 510.0000,
307.0000],
[490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483,
326.7660]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 46
running loss: nan
Train Steps: 46/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.8800, 245.5700, 633.0000, 152.0000, 343.0000, 148.0000, 478.0000,
342.0000],
[502.9100, 244.7000, 645.0000, 136.0000, 383.0000, 92.0000, 476.2652,
294.7599],
[495.9200, 243.9100, 607.8350, 143.0867, 345.3565, 118.8599, 474.6164,
313.2611],
[500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000,
330.0000],
[501.0300, 253.9500, 633.9382, 277.5496, 303.0000, 173.0000, 445.8765,
362.8064],
[502.1800, 247.0500, 719.0000, 230.0000, 316.0000, 159.0000, 503.2108,
296.3059],
[496.0300, 237.6400, 676.8727, 329.8349, 331.3488, 274.6205, 471.3836,
324.1019],
[495.7263, 235.8042, 703.4152, 284.1557, 332.1768, 327.8600, 504.6377,
314.1167]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 47
running loss: nan
Train Steps: 47/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[ nan, nan, 535.0539, 150.7963, 329.0000, 127.0000, 415.4254,
355.9533],
[486.6400, 237.4500, 691.0000, 297.0000, 349.0000, 305.0000, 427.7128,
298.7724],
[482.6800, 240.6500, 588.0000, 152.0000, 275.0000, 202.0000, 441.2465,
305.0196],
[493.7296, 243.3093, 654.0000, 159.0000, 284.0000, 221.0000, 463.0000,
333.0000],
[495.9900, 244.1900, 715.0000, 251.0000, 283.0000, 274.0000, 494.0000,
324.0000],
[498.0700, 246.1700, 686.0000, 207.0000, 351.0000, 124.0000, 463.0000,
323.0000],
[491.8400, 243.1800, 700.0000, 273.0000, 387.9706, 313.0692, 469.0000,
334.0000],
[494.7600, 244.7600, 707.0000, 277.0000, 387.0000, 339.0000, 494.0000,
351.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 48
running loss: nan
Train Steps: 48/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[492.8739, 242.3712, 602.0000, 128.0000, 330.0000, 124.0000, 463.0000,
307.0000],
[496.0400, 242.3200, 710.4314, 287.9666, 289.9929, 257.0700, 469.2918,
308.8831],
[485.6100, 238.7500, 686.0000, 305.0000, 348.0000, 324.0000, 414.3940,
327.9374],
[496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000,
356.0000],
[502.2869, 240.4949, 688.0000, 293.0000, 346.0000, 317.0000, 560.8282,
322.6830],
[488.3100, 241.7100, 699.0000, 294.0000, 327.0000, 272.0000, 405.5792,
330.5729],
[497.0800, 247.8600, 675.0000, 213.0000, 281.0000, 264.0000, 457.2733,
307.3623],
[494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557,
323.6706]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 49
running loss: nan
Train Steps: 49/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[484.2800, 242.0800, 551.7881, 114.9384, 320.0000, 127.0000, 435.1920,
310.0955],
[497.2900, 246.0200, 588.0000, 124.0000, 346.0000, 123.0000, 476.0000,
320.0000],
[491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000,
345.0000],
[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654],
[495.9800, 239.5800, 691.1306, 221.2412, 292.6610, 188.3306, 480.1311,
322.4341],
[498.2935, 246.4364, 651.0000, 173.0000, 380.0000, 103.0000, 465.0000,
324.0000],
[490.2500, 241.3400, 699.0000, 304.0000, 398.6197, 313.8339, 429.1374,
303.8483],
[495.8800, 245.5700, 633.0000, 152.0000, 343.0000, 148.0000, 478.0000,
342.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 50
running loss: nan
Train Steps: 50/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000,
354.0000],
[507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504,
326.1537],
[490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000,
328.0000],
[487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475,
296.4297],
[500.8607, 239.2440, 723.6318, 252.4934, 288.0082, 277.9555, 525.6299,
309.7392],
[496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000,
308.0000],
[496.2100, 245.7600, 709.0000, 256.0000, 283.0000, 247.0000, 482.0000,
339.0000],
[ nan, nan, 708.0000, 169.0000, 409.0000, 131.0000, 574.7578,
326.1537]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 51
running loss: nan
Train Steps: 51/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[ nan, nan, 581.4534, 139.9555, 330.0000, 116.0000, 409.0000,
323.0000],
[492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121,
350.7788],
[497.7500, 250.2900, 708.0000, 313.0000, 299.0000, 276.0000, 456.0000,
338.0000],
[503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324,
332.4009],
[496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000,
356.0000],
[499.8200, 251.0800, 680.0000, 346.0000, 357.0000, 273.0000, 449.0000,
355.0000],
[501.3200, 243.8900, 665.0000, 148.0000, 383.0000, 104.0000, 505.0000,
308.0000],
[ nan, nan, 517.5590, 116.6062, 322.0000, 120.0000, 410.0000,
332.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 52
running loss: nan
Train Steps: 52/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000,
305.0000],
[491.8400, 239.7800, 679.0000, 232.0000, 279.0000, 244.0000, 469.0000,
300.0000],
[500.2902, 244.5602, 696.0000, 196.0000, 332.0000, 185.0000, 563.9941,
336.5657],
[498.8700, 237.9300, 708.0000, 298.0000, 291.0000, 241.0000, 468.0000,
311.0000],
[490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304,
310.4859],
[ nan, nan, 677.0000, 153.0000, 468.0000, 128.0000, 570.3256,
364.3312],
[494.3001, 247.0619, 582.2141, 254.2003, 287.0000, 204.0000, 454.0000,
355.0000],
[490.2100, 246.9000, 634.6989, 226.6815, 276.0000, 242.0000, 426.9483,
327.3517]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 53
running loss: nan
Train Steps: 53/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361,
317.8241],
[492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000,
348.0000],
[504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961,
321.9889],
[497.7600, 246.4500, 715.0000, 254.0000, 288.0000, 229.0000, 466.0000,
317.0000],
[ nan, nan, 708.0000, 169.0000, 409.0000, 131.0000, 574.7578,
326.1537],
[497.7600, 236.2000, 668.0000, 337.0000, 331.0000, 276.0000, 464.0000,
314.0000],
[507.9900, 242.6000, 699.0000, 339.0000, 308.0000, 285.0000, 512.0750,
297.0000],
[486.3900, 240.6200, 700.0000, 270.0000, 386.0000, 337.0000, 467.0000,
335.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 54
running loss: nan
Train Steps: 54/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[483.1600, 240.1100, 587.0000, 136.0000, 318.0000, 126.0000, 418.4528,
286.6684],
[490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470,
329.6944],
[490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483,
326.7660],
[502.2869, 242.9966, 642.0000, 132.0000, 345.0000, 164.0000, 545.6324,
319.2123],
[486.9100, 241.1400, 622.5284, 256.7020, 290.0000, 261.0000, 411.9899,
317.1024],
[499.6700, 241.6600, 699.0000, 292.0000, 327.0000, 340.0000, 509.0000,
312.0000],
[484.5200, 240.6700, 700.0000, 256.0000, 352.0000, 348.0000, 467.6018,
335.0844],
[488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000,
305.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 55
running loss: nan
Train Steps: 55/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.2400, 244.3600, 655.1108, 143.9094, 352.0268, 123.2475, 474.3377,
330.0576],
[505.1393, 246.4364, 700.0000, 306.0000, 303.0000, 294.0000, 569.6925,
351.8367],
[496.1000, 246.1100, 583.0000, 145.0000, 332.0000, 143.0000, 488.0000,
330.0000],
[495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000,
370.0000],
[ nan, nan, 682.0000, 133.0000, 433.0000, 142.0000, 589.3204,
328.9302],
[507.4213, 245.8110, 743.0000, 262.0000, 345.0000, 216.0000, 579.8230,
350.4485],
[494.5600, 245.9300, 625.0000, 180.0000, 315.0000, 142.0000, 426.0000,
345.0000],
[504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177,
317.8241]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 56
running loss: nan
Train Steps: 56/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654],
[494.5853, 235.8042, 707.2659, 233.4960, 284.4970, 297.9904, 497.7261,
316.7523],
[502.0016, 242.9966, 723.0000, 226.0000, 307.0000, 212.0000, 565.8936,
334.4833],
[ nan, nan, 711.0000, 186.0000, 421.0000, 169.0000, 571.5920,
360.1664],
[503.9300, 240.5100, 676.0000, 321.0000, 337.0000, 300.0000, 508.0000,
306.0000],
[498.7100, 250.7200, 626.0000, 207.0000, 305.0000, 172.0000, 454.0000,
337.0000],
[502.0300, 246.6600, 677.0000, 157.0000, 359.0000, 119.0000, 496.8792,
294.2234],
[491.9500, 243.2500, 537.0000, 140.0000, 322.0000, 121.0000, 417.0651,
341.7008]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 57
running loss: nan
Train Steps: 57/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557,
330.5729],
[486.5985, 241.4330, 699.0000, 241.0000, 295.0000, 237.0000, 424.5074,
308.1432],
[496.2968, 243.3093, 675.0000, 344.0000, 365.9118, 288.0521, 439.0000,
337.0000],
[483.0400, 236.7800, 673.0000, 293.0000, 285.0000, 273.0000, 421.3020,
281.5925],
[497.5500, 246.8200, 654.0000, 169.0000, 314.0000, 167.0000, 472.0000,
321.0000],
[498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000,
346.0000],
[500.9500, 245.1100, 675.0000, 189.0000, 322.0000, 158.0000, 507.1346,
288.6207],
[495.7000, 245.4200, 676.0000, 234.0000, 286.0000, 236.0000, 478.0000,
335.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 58
running loss: nan
Train Steps: 58/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000,
317.0000],
[502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808,
289.3645],
[490.2100, 246.9000, 634.6989, 226.6815, 276.0000, 242.0000, 426.9483,
327.3517],
[488.7400, 242.4700, 558.0000, 190.0000, 281.0000, 203.0000, 412.2570,
319.1522],
[490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000,
328.0000],
[494.5853, 235.8042, 707.2659, 233.4960, 284.4970, 297.9904, 497.7261,
316.7523],
[490.5200, 243.8100, 691.0000, 312.0000, 383.0000, 287.0000, 420.6341,
352.0000],
[487.8500, 239.4600, 691.0000, 283.0000, 341.0000, 298.0000, 417.0000,
339.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 59
running loss: nan
Train Steps: 59/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919,
292.1570],
[486.3900, 240.6200, 700.0000, 270.0000, 386.0000, 337.0000, 467.0000,
335.0000],
[483.8400, 239.2300, 609.5974, 231.6849, 278.0000, 250.0000, 410.9215,
327.9374],
[495.3700, 238.8200, 566.2405, 164.9726, 340.0000, 126.0000, 436.0000,
347.0000],
[502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936,
319.2123],
[496.1000, 244.9000, 706.1387, 222.8866, 306.0016, 162.7361, 467.5804,
324.7273],
[507.1360, 244.8729, 674.0000, 325.0000, 308.0000, 290.0000, 586.7878,
345.5895],
[ nan, nan, 643.0000, 149.0000, 318.0000, 151.0000, 446.0000,
336.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 60
running loss: nan
Train Steps: 60/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000,
324.0000],
[498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000,
346.0000],
[509.1327, 249.8763, 727.0000, 262.0000, 326.0000, 189.0000, 515.8740,
317.8241],
[496.2700, 244.6800, 704.0000, 305.0000, 312.0000, 300.0000, 488.0000,
335.0000],
[495.4411, 249.8763, 707.0000, 282.0000, 332.0000, 292.0000, 434.1604,
315.6382],
[492.8739, 241.4330, 707.0000, 275.0000, 295.0000, 224.0000, 424.8636,
320.6377],
[494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000,
326.0000],
[497.9500, 245.8000, 595.0000, 136.0000, 308.0000, 171.0000, 479.0000,
315.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 61
running loss: nan
Train Steps: 61/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[496.4600, 247.3500, 574.0000, 144.0000, 311.0000, 176.0000, 498.0000,
345.0000],
[498.2400, 247.1600, 635.0000, 134.0000, 373.9316, 106.3581, 495.0000,
326.0000],
[500.9100, 247.8700, 715.0000, 213.0000, 320.0000, 161.0000, 495.0000,
317.0000],
[500.5754, 243.6220, 664.0000, 140.0000, 375.0000, 155.0000, 563.9941,
337.9540],
[505.1393, 246.4364, 700.0000, 306.0000, 303.0000, 294.0000, 569.6925,
351.8367],
[491.7329, 244.8729, 683.0000, 204.0000, 293.0000, 189.0000, 411.3298,
292.5252],
[501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919,
292.1570],
[502.6900, 241.7500, 707.0000, 227.0000, 318.0000, 171.0000, 506.7784,
305.4101]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 62
running loss: nan
Train Steps: 62/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000,
354.0000],
[500.0049, 240.8076, 682.0000, 325.0000, 323.0000, 307.0000, 506.0000,
301.0000],
[496.0300, 237.6400, 676.8727, 329.8349, 331.3488, 274.6205, 471.3836,
324.1019],
[494.7600, 244.7600, 707.0000, 277.0000, 387.0000, 339.0000, 494.0000,
351.0000],
[490.2700, 247.0800, 691.0000, 320.0000, 370.0000, 316.0000, 415.4624,
328.5230],
[494.8400, 247.7400, 712.0000, 274.0000, 315.0000, 325.0000, 458.6980,
306.5814],
[499.1492, 246.4364, 652.9544, 165.8065, 290.0000, 216.0000, 479.0000,
342.0000],
[502.6800, 245.5500, 712.0000, 282.0000, 292.0000, 231.0000, 497.0000,
310.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 63
running loss: nan
Train Steps: 63/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[ nan, nan, 669.0000, 199.0000, 285.0000, 202.0000, 426.2881,
308.5337],
[495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000,
339.0000],
[498.4000, 246.7900, 577.0000, 119.0000, 346.0000, 142.0000, 501.0000,
324.0000],
[ nan, nan, 601.2303, 162.4709, 319.0000, 136.0000, 413.0000,
334.0000],
[508.5623, 247.0619, 672.0000, 150.0000, 433.0000, 98.0000, 538.0345,
335.1775],
[494.4200, 243.8400, 576.0000, 148.0000, 342.0000, 142.0000, 477.0000,
373.0000],
[493.0700, 240.3800, 703.0000, 281.0000, 293.0000, 293.0000, 471.0000,
301.0000],
[496.0500, 246.9500, 698.0000, 284.0000, 296.0000, 193.0000, 430.9181,
346.0170]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 64
running loss: nan
Train Steps: 64/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000,
321.0000],
[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000,
354.0000],
[489.0800, 240.3300, 696.0000, 321.0000, 291.0000, 226.0000, 407.7683,
292.9156],
[507.9918, 248.6254, 740.0000, 246.0000, 330.0000, 225.0000, 570.3256,
356.6957],
[502.2869, 242.9966, 642.0000, 132.0000, 345.0000, 164.0000, 545.6324,
319.2123],
[491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148,
349.3146],
[503.7131, 243.6220, 728.0000, 196.0000, 378.0822, 202.0042, 595.6520,
321.2947],
[501.9300, 247.0000, 648.0244, 348.0684, 320.0000, 275.0000, 446.5888,
367.1014]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 65
running loss: nan
Train Steps: 65/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[491.7329, 246.7491, 639.0000, 192.0000, 302.0000, 166.0000, 412.0000,
333.0000],
[501.0600, 243.8700, 723.0000, 259.0000, 287.0000, 273.0000, 506.0000,
315.0000],
[493.4444, 243.3093, 606.0000, 177.0000, 324.0000, 163.0000, 475.0000,
370.0000],
[503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000,
307.0000],
[494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221,
317.4111],
[499.4344, 247.6873, 621.0000, 163.0000, 306.0000, 188.0000, 497.0000,
325.0000],
[501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919,
292.1570],
[486.3200, 237.8300, 593.6238, 177.4812, 285.0000, 175.0000, 428.0689,
298.7724]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 66
running loss: nan
Train Steps: 66/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[498.6600, 245.4800, 648.0000, 177.0000, 285.0000, 233.0000, 481.0000,
312.0000],
[498.8640, 238.6186, 718.8183, 288.3773, 306.7708, 312.9795, 529.0106,
309.9863],
[502.0016, 241.4330, 617.7352, 124.7889, 351.3434, 134.0229, 514.8118,
317.3988],
[501.2500, 244.2100, 697.0000, 336.0000, 297.0000, 287.0000, 462.0000,
366.0000],
[495.6700, 245.2700, 711.0000, 275.0000, 360.0000, 341.0000, 491.0000,
353.0000],
[495.7300, 239.1300, 704.0000, 277.0000, 335.0000, 287.0000, 455.0000,
333.0000],
[504.5688, 239.8694, 680.0000, 314.0000, 308.0000, 303.0000, 595.6520,
319.2123],
[493.7296, 243.3093, 654.0000, 159.0000, 284.0000, 221.0000, 463.0000,
333.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 67
running loss: nan
Train Steps: 67/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.8400, 245.9700, 571.0000, 128.0000, 320.0000, 159.0000, 486.0000,
338.0000],
[502.1600, 246.1500, 647.0000, 343.0000, 335.0000, 285.0000, 453.0000,
365.0000],
[500.5754, 243.6220, 664.0000, 140.0000, 375.0000, 155.0000, 563.9941,
337.9540],
[496.2100, 244.5800, 688.8793, 172.7032, 324.0115, 153.2296, 472.5629,
329.7797],
[501.1400, 242.9200, 719.0000, 278.0000, 305.0000, 299.0000, 506.0662,
290.5729],
[495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000,
339.0000],
[500.7600, 247.1900, 641.0000, 141.0000, 391.0000, 92.0000, 502.5046,
293.6965],
[ nan, nan, 617.9645, 156.6336, 294.0000, 164.0000, 433.0000,
310.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 68
running loss: nan
Train Steps: 68/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000,
346.0000],
[495.0900, 241.9900, 692.0000, 316.0000, 359.0000, 309.0000, 474.0000,
303.0000],
[488.1300, 241.1800, 691.0000, 223.0000, 290.0000, 301.0000, 483.0000,
330.0000],
[505.6500, 242.8800, 699.0000, 337.0000, 310.0000, 265.0000, 508.9092,
295.6117],
[496.1100, 241.7100, 702.0746, 276.1287, 294.6791, 252.1829, 475.0000,
327.0000],
[502.8574, 256.4433, 680.0000, 270.0000, 362.0000, 155.0000, 435.1920,
372.5677],
[500.0000, 248.7500, 707.0000, 236.0000, 287.0000, 257.0000, 493.0000,
322.0000],
[494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070,
323.3411]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 69
running loss: nan
Train Steps: 69/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[497.7500, 237.4100, 707.0000, 301.0000, 315.0000, 276.0000, 472.0000,
301.0000],
[507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209,
344.8954],
[493.1591, 237.3677, 700.5271, 305.2639, 343.9919, 319.1815, 481.7994,
312.1400],
[498.0600, 251.1200, 708.0000, 330.0000, 304.0000, 255.0000, 450.0000,
337.0000],
[507.9918, 247.0619, 669.0000, 163.0000, 388.0000, 102.0000, 515.2408,
310.1886],
[507.9918, 248.6254, 740.0000, 246.0000, 330.0000, 225.0000, 570.3256,
356.6957],
[495.1000, 234.6500, 704.0000, 295.0000, 297.0000, 288.0000, 483.0000,
290.0000],
[490.5919, 243.9347, 580.6928, 144.1250, 287.0000, 198.0000, 480.0000,
336.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 70
running loss: nan
Train Steps: 70/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209,
344.8954],
[496.4300, 240.2100, 715.0000, 293.0000, 293.0000, 300.0000, 508.5592,
296.8201],
[483.8800, 235.7000, 683.0000, 326.0000, 310.0000, 307.0000, 421.3020,
283.1544],
[490.8700, 247.0400, 696.0000, 310.0000, 335.0000, 305.0000, 411.7228,
329.6944],
[503.1426, 241.4330, 727.0000, 274.0000, 315.0000, 338.0000, 564.6272,
336.5657],
[485.4300, 237.7600, 692.0000, 259.0000, 323.0000, 305.0000, 420.2336,
299.9438],
[496.0400, 242.3200, 710.4314, 287.9666, 289.9929, 257.0700, 469.2918,
308.8831],
[ nan, nan, 682.0000, 133.0000, 433.0000, 142.0000, 589.3204,
328.9302]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 71
running loss: nan
Train Steps: 71/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[507.7065, 249.8763, 731.0000, 239.0000, 310.0000, 259.0000, 597.5515,
328.2361],
[488.1300, 241.1800, 691.0000, 223.0000, 290.0000, 301.0000, 483.0000,
330.0000],
[497.8500, 239.4100, 643.0000, 168.0000, 320.0000, 137.0000, 469.1422,
312.4604],
[ nan, nan, 559.3947, 167.4743, 316.0000, 143.0000, 438.6643,
349.1448],
[501.7164, 242.3712, 720.0000, 195.0000, 395.0000, 138.0000, 575.3909,
324.7654],
[489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000,
331.0000],
[501.8700, 246.5800, 712.0000, 229.0000, 335.0000, 130.0000, 468.6673,
290.0746],
[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 72
running loss: nan
Train Steps: 72/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[484.6700, 238.7000, 663.0000, 216.0000, 272.0000, 243.0000, 442.3150,
327.6658],
[490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304,
310.4859],
[488.9500, 241.7100, 691.0000, 288.0000, 390.0000, 305.0000, 461.0000,
334.0000],
[504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961,
321.9889],
[495.9600, 245.1500, 673.1207, 178.4619, 329.3477, 136.4104, 469.4820,
323.4764],
[486.9300, 238.5500, 667.0000, 232.0000, 297.0000, 187.0000, 475.0000,
318.0000],
[503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324,
332.4009],
[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000,
354.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 73
running loss: nan
Train Steps: 73/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[501.0600, 243.8700, 723.0000, 259.0000, 287.0000, 273.0000, 506.0000,
315.0000],
[501.2000, 245.2900, 642.0000, 111.0000, 333.0000, 150.0000, 503.2169,
296.8201],
[496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000,
326.0000],
[497.9500, 245.8000, 595.0000, 136.0000, 308.0000, 171.0000, 479.0000,
315.0000],
[500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000,
357.0000],
[499.5800, 246.0100, 621.0000, 155.0000, 397.0000, 91.0000, 470.0000,
325.0000],
[486.0900, 237.1500, 650.0000, 235.0000, 282.0000, 245.0000, 427.7128,
297.2106],
[498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000,
312.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 74
running loss: nan
Train Steps: 74/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.8706, 249.2509, 546.0000, 158.0000, 332.0000, 138.0000, 457.0000,
331.0000],
[495.9400, 237.1000, 685.8777, 322.4308, 326.0126, 281.2020, 475.3770,
322.6425],
[502.0016, 241.4330, 617.7352, 124.7889, 351.3434, 134.0229, 514.8118,
317.3988],
[501.7200, 242.6800, 683.0000, 354.0000, 300.0000, 265.0000, 479.1508,
278.9525],
[493.0700, 240.3800, 703.0000, 281.0000, 293.0000, 293.0000, 471.0000,
301.0000],
[485.5300, 238.5200, 690.0000, 305.0000, 351.0000, 329.0000, 452.0000,
298.0000],
[498.2400, 247.1600, 635.0000, 134.0000, 373.9316, 106.3581, 495.0000,
326.0000],
[498.9000, 245.0400, 619.0000, 128.0000, 293.0000, 194.0000, 465.0000,
334.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 75
running loss: nan
Train Steps: 75/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[492.8739, 241.4330, 693.0000, 281.0000, 394.7994, 321.8399, 485.0000,
334.0000],
[490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385,
353.5374],
[492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000,
317.0000],
[492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000,
322.0000],
[ nan, nan, 724.0000, 210.0000, 411.0000, 138.0000, 588.6873,
342.1188],
[501.1400, 242.9200, 719.0000, 278.0000, 305.0000, 299.0000, 506.0662,
290.5729],
[504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177,
317.8241],
[ nan, nan, 638.5021, 191.6575, 290.0000, 190.0000, 403.1752,
333.7941]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 76
running loss: nan
Train Steps: 76/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[504.7900, 241.0400, 685.0000, 348.0000, 295.0000, 285.0000, 506.0662,
300.3342],
[491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000,
345.0000],
[497.4900, 236.0200, 695.0000, 316.0000, 345.0000, 298.0000, 479.0000,
299.0000],
[490.3400, 244.1100, 700.0000, 304.0000, 310.0000, 254.0000, 418.8311,
352.8785],
[ nan, nan, 694.0000, 170.0000, 428.0000, 119.0000, 534.2356,
337.2599],
[492.8739, 242.3712, 602.0000, 128.0000, 330.0000, 124.0000, 463.0000,
307.0000],
[489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000,
331.0000],
[500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000,
314.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 77
running loss: nan
Train Steps: 77/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177,
317.8241],
[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000,
354.0000],
[489.1657, 239.8694, 565.0000, 143.0000, 323.0000, 117.0000, 425.5759,
299.5533],
[497.1525, 246.7491, 627.0000, 127.0000, 292.0000, 188.0000, 454.0000,
305.0000],
[492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000,
317.0000],
[492.2800, 247.0300, 695.0000, 310.0000, 391.5905, 338.8391, 441.9588,
308.1432],
[490.3067, 235.8042, 701.4897, 306.3193, 331.7047, 338.5089, 479.9964,
304.8923],
[485.9900, 240.3900, 662.0000, 295.0000, 324.0000, 306.0000, 413.3255,
316.8095]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 78
running loss: nan
Train Steps: 78/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[485.5300, 238.5200, 690.0000, 305.0000, 351.0000, 329.0000, 452.0000,
298.0000],
[490.0000, 238.9800, 700.0000, 292.0000, 342.0000, 287.0000, 433.0000,
343.0000],
[508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361,
317.8241],
[490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792,
324.7162],
[496.7200, 235.1800, 692.0000, 322.0000, 352.0000, 304.0000, 482.0000,
297.0000],
[504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961,
321.9889],
[501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919,
292.1570],
[490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304,
310.4859]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 79
running loss: nan
Train Steps: 79/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.3001, 240.4949, 688.0120, 240.8839, 314.5486, 150.9156, 456.8577,
326.3061],
[493.1591, 244.5602, 707.0000, 247.0000, 297.0000, 333.0000, 499.0000,
321.0000],
[496.4600, 247.3500, 574.0000, 144.0000, 311.0000, 176.0000, 498.0000,
345.0000],
[494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000,
326.0000],
[497.2500, 245.9600, 578.0000, 122.0000, 335.0000, 133.0000, 478.0000,
317.0000],
[486.4500, 236.9900, 683.0000, 280.0000, 308.0000, 295.0000, 427.3566,
297.2106],
[ nan, nan, 578.0000, 130.0000, 319.0000, 137.0000, 434.1235,
310.8764],
[500.6700, 248.5800, 682.0000, 157.0000, 396.0000, 100.0000, 497.5185,
297.9915]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 80
running loss: nan
Train Steps: 80/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[494.0100, 245.5700, 704.0000, 266.0000, 326.0000, 262.0000, 410.2613,
294.0870],
[500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000,
314.0000],
[ nan, nan, 521.0000, 103.0000, 328.0000, 119.0000, 420.2336,
314.3904],
[507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504,
326.1537],
[491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148,
349.3146],
[492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000,
348.0000],
[501.1200, 242.0800, 711.0000, 293.0000, 324.0000, 313.0000, 508.9154,
287.4493],
[495.6600, 245.6500, 605.0000, 169.0000, 315.0000, 191.0000, 481.0000,
371.0000]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 81
running loss: nan
Train Steps: 81/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[495.4411, 235.8042, 715.9302, 266.2137, 308.1366, 326.1029, 501.0316,
315.7639],
[508.2770, 247.6873, 731.0000, 212.0000, 375.0000, 195.0000, 571.5920,
359.4722],
[497.8600, 241.7000, 719.0000, 252.0000, 301.0000, 344.0000, 510.0000,
310.0000],
[494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221,
317.4111],
[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567,
324.7654],
[502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808,
289.3645],
[495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000,
339.0000],
[491.4477, 242.3712, 659.0000, 200.0000, 326.0000, 127.0000, 410.9736,
298.3820]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 82
running loss: nan
Train Steps: 82/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[486.0200, 240.0200, 681.0000, 311.0000, 360.7842, 319.7291, 414.3940,
316.8095],
[498.1000, 245.8600, 713.0000, 246.0000, 322.0000, 143.0000, 461.0000,
322.0000],
[490.7900, 246.9100, 707.0000, 280.0000, 343.0000, 363.0000, 462.2595,
305.8005],
[502.5721, 242.0584, 626.3995, 124.7889, 362.5892, 124.7976, 512.3327,
319.3755],
[486.8400, 238.8300, 696.0000, 285.0000, 361.0000, 317.0000, 425.9320,
302.2865],
[495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000,
370.0000],
[490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385,
353.5374],
[507.1360, 246.7491, 707.0000, 304.0000, 320.0000, 326.0000, 603.8831,
322.6830]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 83
running loss: nan
Train Steps: 83/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[502.1300, 241.7600, 660.0000, 146.0000, 346.0000, 126.0000, 509.2715,
312.4382],
[486.6900, 238.8800, 687.0000, 314.0000, 368.0000, 322.0000, 454.4241,
300.3342],
[509.1327, 249.2509, 715.0000, 223.0000, 360.0000, 155.0000, 515.8740,
317.1299],
[487.4300, 239.4100, 672.0000, 260.0000, 295.0000, 278.0000, 444.8080,
339.3793],
[507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209,
344.8954],
[490.2000, 245.0600, 699.0000, 281.0000, 289.0000, 222.0000, 396.7645,
323.8376],
[490.8772, 241.7457, 661.0000, 201.0000, 290.0000, 184.0000, 454.0000,
310.0000],
[491.7300, 241.5500, 528.0000, 148.0000, 327.0000, 129.0000, 439.1096,
346.4075]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 84
running loss: nan
Train Steps: 84/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[489.1700, 240.2900, 707.0000, 232.0000, 301.0000, 282.0000, 464.0000,
333.0000],
[495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262,
295.5407],
[ nan, nan, 727.0000, 227.0000, 365.0000, 157.0000, 539.3008,
334.4833],
[489.6200, 240.8100, 549.0000, 169.0000, 296.0000, 167.0000, 441.0000,
340.0000],
[504.1500, 240.4100, 708.0000, 330.0000, 289.0000, 271.0000, 506.7784,
300.7246],
[501.8400, 245.9700, 571.0000, 128.0000, 320.0000, 159.0000, 486.0000,
338.0000],
[501.0300, 253.9500, 633.9382, 277.5496, 303.0000, 173.0000, 445.8765,
362.8064],
[490.6600, 245.6100, 679.5771, 241.6917, 287.0000, 192.0000, 400.2369,
324.4233]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 85
running loss: nan
Train Steps: 85/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[487.6300, 240.1400, 682.6197, 310.0718, 402.5010, 305.6570, 410.0000,
326.0000],
[494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020,
316.7523],
[509.1327, 249.2509, 715.0000, 223.0000, 360.0000, 155.0000, 515.8740,
317.1299],
[500.8607, 241.7457, 697.6390, 201.8338, 293.1961, 225.1964, 522.4747,
310.9746],
[488.0600, 238.6400, 684.0000, 340.0000, 309.0000, 265.0000, 410.2613,
292.1347],
[496.1000, 246.1100, 583.0000, 145.0000, 332.0000, 143.0000, 488.0000,
330.0000],
[502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936,
319.2123],
[488.1900, 240.0300, 619.0000, 215.0000, 277.0000, 228.0000, 441.9588,
341.3316]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 86
running loss: nan
Train Steps: 86/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[ nan, nan, 664.0000, 189.0000, 287.0000, 203.0000, 416.6721,
311.6573],
[490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806,
297.2106],
[494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020,
316.7523],
[487.2000, 240.6200, 627.0000, 209.0000, 283.0000, 227.0000, 436.9727,
304.6292],
[496.8600, 244.1200, 700.0000, 307.0000, 332.0000, 294.0000, 470.0000,
310.0000],
[503.4279, 245.8110, 704.0000, 151.0000, 421.0000, 156.0000, 594.3857,
322.6830],
[490.8700, 247.0400, 696.0000, 310.0000, 335.0000, 305.0000, 411.7228,
329.6944],
[490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792,
324.7162]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 87
running loss: nan
Train Steps: 87/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[492.8600, 245.9100, 699.0000, 263.0000, 303.0000, 329.0000, 448.3696,
301.1151],
[496.2100, 245.7600, 709.0000, 256.0000, 283.0000, 247.0000, 482.0000,
339.0000],
[501.7164, 242.3712, 720.0000, 195.0000, 395.0000, 138.0000, 575.3909,
324.7654],
[495.4600, 246.4600, 595.0000, 162.0000, 292.0000, 221.0000, 499.0000,
343.0000],
[495.9300, 246.6900, 678.0000, 223.0000, 284.0000, 261.0000, 485.0000,
365.0000],
[490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385,
353.5374],
[489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000,
308.0000],
[496.1400, 238.9700, 684.3768, 325.7215, 307.3357, 262.1889, 469.2918,
323.8934]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 88
running loss: nan
Train Steps: 88/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[497.9600, 237.8300, 658.0000, 343.0000, 332.0000, 277.0000, 462.0000,
316.0000],
[503.7131, 242.6839, 731.0000, 246.0000, 338.5284, 254.5401, 593.7525,
317.8241],
[501.7200, 242.6800, 683.0000, 354.0000, 300.0000, 265.0000, 479.1508,
278.9525],
[492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121,
350.7788],
[489.9100, 244.5200, 615.6826, 249.1969, 278.0000, 226.0000, 412.5242,
325.5947],
[496.2400, 244.3600, 655.1108, 143.9094, 352.0268, 123.2475, 474.3377,
330.0576],
[505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925,
341.4247],
[508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361,
317.8241]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 89
running loss: nan
Train Steps: 89/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',
grad_fn=<AddmmBackward>)
landmarks are: tensor([[486.0900, 237.1500, 650.0000, 235.0000, 282.0000, 245.0000, 427.7128,
297.2106],
[496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000,
326.0000],
[489.0500, 245.2700, 548.7455, 132.4504, 349.0000, 102.0000, 415.0000,
332.0000],
[501.4600, 245.6700, 723.0000, 258.0000, 296.0000, 209.0000, 501.0000,
310.0000],
[494.5853, 239.2440, 630.2503, 173.3377, 295.9114, 195.6037, 486.9080,
322.0233],
[501.0800, 241.7700, 720.0000, 286.0000, 304.0000, 310.0000, 513.1892,
286.2780],
[490.0000, 238.9800, 700.0000, 292.0000, 342.0000, 287.0000, 433.0000,
343.0000],
[503.7131, 244.5602, 672.0000, 335.0000, 296.0000, 262.0000, 550.0646,
329.6244]], device='cuda:0')
loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=<MseLossBackward>)
loss_train: nan
step: 90
running loss: nan
Valid Steps: 10/10 Loss: nan
--------------------------------------------------
Epoch: 1 Train Loss: nan Valid Loss: nan
--------------------------------------------------
Training Complete
Total Elapsed Time : 106.72142887115479 s
Comments