network = Network() network.cuda() criterion = nn.MSELoss() optimizer = optim.Adam(network.parameters(), lr=0.0001) loss_min = np.inf num_epochs = 1 start_time = time.time() for epoch in range(1,num_epochs+1): loss_train = 0 loss_test = 0 running_loss = 0 network.train() print('size of train loader is: ', len(train_loader)) for step in range(1,len(train_loader)+1): batch = next(iter(train_loader)) images, landmarks = batch['image'], batch['landmarks'] #RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[64, 600, 800, 3] to have 3 channels, but got 600 channels instead #using permute below to fix the above error images = images.permute(0,3,1,2) images = images.cuda() landmarks = landmarks.view(landmarks.size(0),-1).cuda() norm_image = transforms.Normalize([0.3809, 0.3810, 0.3810], [0.1127, 0.1129, 0.1130]) for image in images: image = image.float() ##image = to_tensor(image) #TypeError: pic should be PIL Image or ndarray. Got image = norm_image(image) ###norm_landmarks = transforms.Normalize(0.4949, 0.2165) ###landmarks = norm_landmarks(landmarks) for landmark in landmarks: landmark = landmark/743 predictions = network(images) # clear all the gradients before calculating them optimizer.zero_grad() print('predictions are: ', predictions.float()) print('landmarks are: ', landmarks.float()) # find the loss for the current step loss_train_step = criterion(predictions.float(), landmarks.float()) loss_train_step = loss_train_step.to(torch.float32) print("loss_train_step before backward: ", loss_train_step) # calculate the gradients loss_train_step.backward() # update the parameters optimizer.step() print("loss_train_step after backward: ", loss_train_step) loss_train += loss_train_step.item() print("loss_train: ", loss_train) running_loss = loss_train/step print('step: ', step) print('running loss: ', running_loss) print_overwrite(step, len(train_loader), running_loss, 'train') network.eval() with torch.no_grad(): for step in range(1,len(test_loader)+1): batch = next(iter(train_loader)) images, landmarks = batch['image'], batch['landmarks'] images = images.permute(0,3,1,2) images = images.cuda() landmarks = landmarks.view(landmarks.size(0),-1).cuda() predictions = network(images) # find the loss for the current step loss_test_step = criterion(predictions, landmarks) loss_test += loss_test_step.item() running_loss = loss_test/step print_overwrite(step, len(test_loader), running_loss, 'Validation') loss_train /= len(train_loader) loss_test /= len(test_loader) print('\n--------------------------------------------------') print('Epoch: {} Train Loss: {:.4f} Valid Loss: {:.4f}'.format(epoch, loss_train, loss_test)) print('--------------------------------------------------') if loss_test < loss_min: loss_min = loss_test torch.save(network.state_dict(), '../moth_landmarks.pth') print("\nMinimum Valid Loss of {:.4f} at epoch {}/{}".format(loss_min, epoch, num_epochs)) print('Model Saved\n') print('Training Complete') print("Total Elapsed Time : {} s".format(time.time()-start_time)) -------------------------------------------------------------------- size of train loader is: 90 predictions are: tensor([[-0.2563, -0.3646, 0.3769, 0.1143, 0.0023, 0.2944, -0.1278, 0.4752], [-0.2647, -0.3612, 0.3365, 0.1329, -0.0065, 0.3049, -0.1599, 0.4826], [-0.2759, -0.3272, 0.3171, 0.1391, -0.0192, 0.2739, -0.1707, 0.4273], [-0.2945, -0.3464, 0.3645, 0.1480, -0.0273, 0.2682, -0.1386, 0.4688], [-0.2539, -0.3436, 0.3657, 0.1210, 0.0040, 0.2758, -0.1780, 0.4699], [-0.2821, -0.3451, 0.3319, 0.1236, -0.0122, 0.2557, -0.1706, 0.4598], [-0.2869, -0.2988, 0.3146, 0.1384, -0.0284, 0.2619, -0.1784, 0.4358], [-0.2417, -0.3456, 0.3381, 0.1549, -0.0175, 0.2926, -0.1432, 0.4399]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.0148, 240.8076, 712.0000, 270.0000, 350.0000, 351.0000, 494.0000, 323.0000], [500.1400, 249.4700, 719.0000, 245.0000, 303.0000, 287.0000, 498.0000, 338.0000], [486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074, 306.1910], [494.8400, 247.7400, 712.0000, 274.0000, 315.0000, 325.0000, 458.6980, 306.5814], [488.8000, 242.8000, 696.0000, 269.0000, 297.0000, 238.0000, 407.0000, 331.0000], [498.1000, 245.8600, 713.0000, 246.0000, 322.0000, 143.0000, 461.0000, 322.0000], [497.7600, 246.4500, 715.0000, 254.0000, 288.0000, 229.0000, 466.0000, 317.0000], [496.3600, 244.8300, 716.0000, 241.0000, 303.0000, 282.0000, 477.0000, 340.0000]], device='cuda:0') loss_train_step before backward: tensor(170825.2812, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(170825.2812, device='cuda:0', grad_fn=) loss_train: 170825.28125 step: 1 running loss: 170825.28125 Train Steps: 1/90 Loss: 170825.2812 predictions are: tensor([[ 0.0076, -0.2180, 0.6485, 0.2242, 0.1381, 0.3463, 0.0539, 0.6395], [-0.0352, -0.2353, 0.6883, 0.2799, 0.1462, 0.3891, 0.0105, 0.6426], [-0.0063, -0.2052, 0.7500, 0.2959, 0.1565, 0.4173, 0.0810, 0.6845], [-0.0642, -0.2926, 0.7434, 0.2611, 0.1618, 0.4400, 0.0463, 0.6856], [-0.0411, -0.1966, 0.7119, 0.2435, 0.1632, 0.3932, 0.0168, 0.6313], [-0.0357, -0.2337, 0.7370, 0.2766, 0.1566, 0.3968, 0.0537, 0.6819], [ 0.0105, -0.2863, 0.7531, 0.2688, 0.1779, 0.4589, -0.0235, 0.6847], [-0.0319, -0.2861, 0.7344, 0.2631, 0.1316, 0.4394, 0.0566, 0.6840]], device='cuda:0', grad_fn=) landmarks are: tensor([[ nan, nan, 638.5021, 191.6575, 290.0000, 190.0000, 403.1752, 333.7941], [502.1200, 244.9500, 668.0000, 163.0000, 365.0000, 108.0000, 473.4160, 292.6776], [492.9800, 236.9300, 707.0000, 271.0000, 340.0000, 311.0000, 467.0000, 330.0000], [500.0100, 246.3300, 696.0000, 223.0000, 287.0000, 298.0000, 483.0000, 310.0000], [506.8508, 251.4398, 715.0000, 310.0000, 315.0000, 211.0000, 587.4209, 344.8954], [498.2935, 243.9347, 681.0000, 343.0000, 360.0000, 303.0000, 482.0000, 321.0000], [496.1100, 241.6200, 642.3539, 163.6537, 323.3445, 138.6042, 478.4196, 323.4764], [503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000, 307.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 2 running loss: nan Train Steps: 2/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[489.9700, 238.4500, 618.0000, 151.0000, 283.0000, 199.0000, 471.0000, 330.0000], [495.7400, 245.4600, 704.0000, 287.0000, 283.0000, 286.0000, 476.0000, 333.0000], [501.0700, 243.4400, 704.0000, 230.0000, 292.0000, 223.0000, 509.9838, 288.2302], [488.7300, 240.1700, 692.0000, 293.0000, 382.0000, 292.0000, 414.0000, 341.0000], [496.3300, 245.4400, 716.0000, 287.0000, 289.0000, 277.0000, 485.0000, 337.0000], [484.6700, 238.7000, 663.0000, 216.0000, 272.0000, 243.0000, 442.3150, 327.6658], [501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471, 292.1411], [495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262, 295.5407]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 3 running loss: nan Train Steps: 3/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.5900, 255.9900, 572.0000, 199.0000, 392.0000, 109.0000, 444.8080, 371.0059], [488.6600, 240.6300, 643.0000, 204.0000, 296.0000, 176.0000, 452.6434, 337.0366], [502.0600, 247.2100, 699.0000, 188.0000, 338.0000, 133.0000, 496.2461, 293.5293], [493.1591, 244.5602, 707.0000, 247.0000, 297.0000, 333.0000, 499.0000, 321.0000], [490.3400, 244.1100, 700.0000, 304.0000, 310.0000, 254.0000, 418.8311, 352.8785], [502.8574, 242.0584, 655.2804, 144.8417, 340.9529, 143.5759, 509.1774, 321.3521], [494.7200, 244.7300, 668.0000, 222.0000, 294.0000, 173.0000, 425.0000, 347.0000], [492.0700, 247.4900, 699.0000, 265.0000, 286.0000, 227.0000, 411.0000, 329.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 4 running loss: nan Train Steps: 4/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.7500, 245.3800, 626.0000, 150.0000, 336.0000, 149.0000, 479.0000, 340.0000], [498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000, 311.0000], [492.3034, 246.7491, 563.0000, 139.0000, 339.0000, 110.0000, 428.0000, 336.0000], [498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000, 312.0000], [489.8800, 245.0100, 556.3521, 184.1524, 292.0000, 165.0000, 413.0584, 329.1087], [484.6600, 239.1800, 665.8854, 277.5496, 307.0000, 299.0000, 411.7228, 327.9374], [490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792, 324.7162], [482.0700, 238.7200, 684.0000, 254.0000, 289.0000, 314.0000, 446.5888, 297.9915]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 5 running loss: nan Train Steps: 5/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[491.6300, 240.4200, 702.0000, 272.0000, 365.0000, 332.0000, 487.0000, 332.0000], [500.1200, 249.7500, 693.0000, 268.0000, 290.0000, 214.0000, 483.0000, 332.0000], [488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000, 305.0000], [489.8800, 245.0100, 556.3521, 184.1524, 292.0000, 165.0000, 413.0584, 329.1087], [500.8400, 246.3800, 642.0000, 155.0000, 364.0000, 112.0000, 502.5046, 292.1347], [496.0000, 239.8700, 708.0000, 237.0000, 280.0000, 279.0000, 491.0000, 324.0000], [495.8700, 247.7900, 701.0000, 247.0000, 292.0000, 294.0000, 456.5611, 306.1910], [496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000, 356.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 6 running loss: nan Train Steps: 6/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[489.1000, 241.0600, 577.0000, 118.0000, 301.0000, 162.0000, 470.0000, 332.0000], [498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000, 321.0000], [491.4000, 238.9100, 692.0000, 293.0000, 313.0000, 259.0000, 425.2197, 321.0281], [490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000, 328.0000], [491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828, 305.8005], [507.4213, 245.8110, 743.0000, 262.0000, 345.0000, 216.0000, 579.8230, 350.4485], [ nan, nan, 694.0000, 170.0000, 428.0000, 119.0000, 534.2356, 337.2599], [499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000, 326.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 7 running loss: nan Train Steps: 7/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[509.7032, 245.4983, 667.0000, 351.0000, 316.0000, 307.0000, 524.7382, 315.7417], [495.9600, 243.5600, 696.0000, 294.0000, 333.0000, 300.0000, 493.0000, 328.0000], [492.4400, 247.4700, 708.0000, 290.0000, 364.0000, 349.0000, 461.1910, 305.0196], [497.7500, 250.2900, 708.0000, 313.0000, 299.0000, 276.0000, 456.0000, 338.0000], [495.7000, 245.4200, 676.0000, 234.0000, 286.0000, 236.0000, 478.0000, 335.0000], [509.1327, 248.6254, 690.0000, 185.0000, 393.0000, 120.0000, 515.8740, 316.4358], [483.1600, 240.1100, 587.0000, 136.0000, 318.0000, 126.0000, 418.4528, 286.6684], [499.1492, 246.4364, 652.9544, 165.8065, 290.0000, 216.0000, 479.0000, 342.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 8 running loss: nan Train Steps: 8/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.1000, 244.9000, 706.1387, 222.8866, 306.0016, 162.7361, 467.5804, 324.7273], [494.3001, 247.0619, 582.2141, 254.2003, 287.0000, 204.0000, 454.0000, 355.0000], [486.0700, 237.4800, 696.0000, 280.0000, 363.6628, 302.7423, 418.4528, 297.6010], [ nan, nan, 715.0000, 171.0000, 373.0000, 187.0000, 592.4862, 331.7068], [500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154, 295.6488], [489.1000, 241.0600, 577.0000, 118.0000, 301.0000, 162.0000, 470.0000, 332.0000], [ nan, nan, 604.2729, 163.3048, 310.0000, 153.0000, 410.0000, 338.0000], [488.0247, 244.2474, 608.0761, 206.6678, 272.0000, 247.0000, 450.0000, 337.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 9 running loss: nan Train Steps: 9/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483, 326.7660], [505.9951, 243.9347, 675.0000, 321.0000, 314.0000, 316.0000, 569.0593, 347.6719], [494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557, 323.6706], [492.0181, 236.1169, 695.7136, 309.4855, 371.7718, 319.7672, 483.3020, 309.1750], [490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470, 329.6944], [496.2000, 243.9500, 671.8087, 158.8574, 314.0060, 157.6172, 467.5804, 307.4238], [496.8673, 236.1169, 690.9001, 290.4881, 353.3458, 307.5619, 506.4407, 316.0934], [500.2902, 239.5567, 719.0000, 286.0000, 319.0000, 331.0000, 556.3961, 317.1299]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 10 running loss: nan Train Steps: 10/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[490.1000, 242.3100, 659.0000, 238.0000, 290.0000, 216.0000, 471.5194, 368.2727], [495.9100, 243.6200, 711.0000, 280.0000, 304.0000, 303.0000, 495.0000, 326.0000], [499.6700, 241.6600, 699.0000, 292.0000, 327.0000, 340.0000, 509.0000, 312.0000], [493.2500, 240.4900, 685.0000, 340.0000, 351.0000, 296.0000, 446.0000, 334.0000], [487.2800, 239.8400, 665.1248, 260.0376, 303.0000, 273.0000, 417.0651, 339.3581], [507.7065, 245.1856, 635.0000, 330.0000, 317.0000, 292.0000, 587.4209, 342.1188], [497.5500, 246.8200, 654.0000, 169.0000, 314.0000, 167.0000, 472.0000, 321.0000], [494.5853, 238.3059, 684.1613, 354.8681, 294.6041, 250.8712, 455.0547, 322.6822]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 11 running loss: nan Train Steps: 11/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[504.7900, 241.0400, 685.0000, 348.0000, 295.0000, 285.0000, 506.0662, 300.3342], [486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074, 306.1910], [493.1591, 238.3059, 625.4368, 202.8892, 287.8980, 203.2175, 470.3803, 309.1750], [482.3900, 238.1600, 639.0000, 235.0000, 276.0000, 220.0000, 421.3020, 281.9830], [492.9800, 236.9300, 707.0000, 271.0000, 340.0000, 311.0000, 467.0000, 330.0000], [494.2245, 240.7296, 712.0364, 269.7266, 311.2985, 337.0000, 477.7910, 292.7526], [490.8772, 241.7457, 661.0000, 201.0000, 290.0000, 184.0000, 454.0000, 310.0000], [495.9600, 245.1500, 673.1207, 178.4619, 329.3477, 136.4104, 469.4820, 323.4764]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 12 running loss: nan Train Steps: 12/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471, 292.1411], [500.0000, 248.7500, 707.0000, 236.0000, 287.0000, 257.0000, 493.0000, 322.0000], [486.9100, 239.8900, 703.0000, 267.0000, 322.0000, 279.0000, 424.5074, 306.1910], [492.2300, 247.1400, 677.0000, 230.0000, 288.0000, 192.0000, 408.5175, 333.7941], [500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000, 330.0000], [497.2900, 250.0000, 687.0000, 335.0000, 318.0000, 310.0000, 462.0000, 340.0000], [501.8700, 246.5800, 712.0000, 229.0000, 335.0000, 130.0000, 468.6673, 290.0746], [493.5600, 243.1200, 699.3539, 286.7225, 343.0000, 295.0000, 461.0000, 337.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 13 running loss: nan Train Steps: 13/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.0900, 241.0100, 703.0000, 306.0000, 326.0000, 315.0000, 473.0000, 302.0000], [492.8739, 241.4330, 707.0000, 275.0000, 295.0000, 224.0000, 424.8636, 320.6377], [483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557, 330.5729], [486.7600, 240.2900, 672.0000, 259.0000, 301.0000, 285.0000, 438.0412, 303.4578], [503.7131, 240.4949, 732.0000, 259.0000, 341.0000, 183.0000, 580.4562, 324.7654], [502.4100, 246.0400, 724.0000, 272.0000, 302.0000, 193.0000, 507.0098, 294.9176], [490.5919, 243.9347, 580.6928, 144.1250, 287.0000, 198.0000, 480.0000, 336.0000], [494.9600, 246.2100, 570.0437, 124.1114, 316.0000, 151.0000, 473.0000, 341.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 14 running loss: nan Train Steps: 14/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000, 326.0000], [495.9000, 240.8800, 697.2715, 313.6593, 306.6687, 286.3209, 470.6230, 308.8831], [507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504, 326.1537], [493.1591, 246.1237, 708.0000, 292.0000, 337.0000, 359.0000, 448.7257, 302.2865], [495.7263, 247.0619, 695.0000, 331.0000, 323.0000, 314.0000, 470.0000, 336.0000], [495.8200, 244.5800, 635.6002, 147.2001, 358.0301, 112.2785, 471.7639, 320.9747], [490.3067, 235.8042, 701.4897, 306.3193, 331.7047, 338.5089, 479.9964, 304.8923], [494.0100, 245.5700, 704.0000, 266.0000, 326.0000, 262.0000, 410.2613, 294.0870]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 15 running loss: nan Train Steps: 15/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[489.6200, 240.8100, 549.0000, 169.0000, 296.0000, 167.0000, 441.0000, 340.0000], [494.4200, 243.8400, 576.0000, 148.0000, 342.0000, 142.0000, 477.0000, 373.0000], [498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000, 321.0000], [506.2803, 251.7526, 739.0000, 275.0000, 341.0000, 176.0000, 587.4209, 344.8954], [495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000, 339.0000], [491.0200, 241.2100, 704.0000, 283.0000, 350.0000, 283.0000, 440.1781, 336.6462], [498.0700, 246.1700, 686.0000, 207.0000, 351.0000, 124.0000, 463.0000, 323.0000], [496.2100, 244.5800, 688.8793, 172.7032, 324.0115, 153.2296, 472.5629, 329.7797]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 16 running loss: nan Train Steps: 16/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396, 367.4918], [ nan, nan, 669.0000, 199.0000, 285.0000, 202.0000, 426.2881, 308.5337], [503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324, 332.4009], [484.8000, 235.4200, 676.0000, 343.0000, 336.0000, 313.0000, 420.2336, 285.1066], [ nan, nan, 611.8794, 163.3048, 317.0000, 131.0000, 404.7779, 325.3018], [489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000, 308.0000], [507.1360, 246.1237, 727.0000, 286.0000, 314.0000, 317.0000, 600.7172, 323.3772], [497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000, 324.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 17 running loss: nan Train Steps: 17/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[502.8574, 245.8110, 648.5415, 144.8417, 354.0000, 149.0000, 539.6033, 323.0817], [492.8300, 245.1400, 702.0000, 239.0000, 300.0000, 227.0000, 410.2613, 292.9156], [489.1657, 239.8694, 565.0000, 143.0000, 323.0000, 117.0000, 425.5759, 299.5533], [494.8706, 240.8076, 655.2804, 191.2797, 342.3284, 123.5840, 461.3652, 324.6589], [487.7600, 239.2700, 692.0000, 315.0000, 337.0000, 312.0000, 454.0000, 303.0000], [483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557, 330.5729], [490.2000, 245.0600, 699.0000, 281.0000, 289.0000, 222.0000, 396.7645, 323.8376], [494.0700, 241.7800, 708.0000, 291.0000, 295.0000, 243.0000, 425.0000, 347.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 18 running loss: nan Train Steps: 18/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[502.0200, 242.8900, 679.0000, 173.0000, 357.0000, 122.0000, 505.7100, 309.3146], [501.7700, 247.2200, 723.0000, 247.0000, 298.0000, 192.0000, 494.0000, 315.0000], [501.1800, 255.0400, 569.0000, 213.0000, 350.0000, 127.0000, 446.9449, 367.1014], [491.6300, 240.4700, 700.0000, 323.0000, 318.0000, 279.0000, 445.0000, 332.0000], [495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000, 370.0000], [506.5656, 247.0619, 739.0000, 256.0000, 321.0000, 284.0000, 601.9836, 326.1537], [503.4279, 238.9313, 696.0000, 318.0000, 301.0000, 283.0000, 563.9941, 317.8241], [496.1100, 241.6200, 642.3539, 163.6537, 323.3445, 138.6042, 478.4196, 323.4764]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 19 running loss: nan Train Steps: 19/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154, 295.6488], [496.3500, 242.9600, 638.0000, 162.0000, 315.0000, 154.0000, 456.0000, 311.0000], [489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000, 308.0000], [497.2200, 247.1100, 615.0000, 138.0000, 336.0000, 137.0000, 474.0000, 319.0000], [504.5688, 239.8694, 680.0000, 314.0000, 308.0000, 303.0000, 595.6520, 319.2123], [501.0800, 242.3100, 720.0000, 264.0000, 290.0000, 280.0000, 513.9015, 288.2302], [501.5100, 244.3800, 712.0000, 296.0000, 290.0000, 251.0000, 503.0000, 309.0000], [501.8900, 245.2000, 670.0000, 342.0000, 322.0000, 288.0000, 456.0000, 367.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 20 running loss: nan Train Steps: 20/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[490.3600, 243.8000, 699.0000, 315.0000, 345.0000, 284.0000, 418.4304, 352.4393], [500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000, 314.0000], [491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828, 305.8005], [508.8475, 246.1237, 692.0000, 179.0000, 391.0000, 120.0000, 536.1351, 327.5420], [499.9900, 243.2300, 701.6359, 283.3869, 373.0000, 322.0000, 493.0000, 326.0000], [494.0900, 241.0100, 703.0000, 306.0000, 326.0000, 315.0000, 473.0000, 302.0000], [508.5623, 249.5636, 703.0000, 335.0000, 291.0000, 266.0000, 519.0398, 317.8241], [496.3700, 240.9300, 668.0000, 163.0000, 319.0000, 153.0000, 463.0000, 308.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 21 running loss: nan Train Steps: 21/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[490.3600, 243.8000, 699.0000, 315.0000, 345.0000, 284.0000, 418.4304, 352.4393], [491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000, 345.0000], [ nan, nan, 575.3683, 140.7894, 323.0000, 123.0000, 411.0000, 339.0000], [501.7164, 244.8729, 726.5198, 293.6544, 296.0000, 257.0000, 532.8420, 316.4105], [487.3300, 240.0800, 691.0000, 292.0000, 343.0000, 322.0000, 438.7535, 303.8483], [500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000, 357.0000], [498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000, 312.0000], [495.9100, 243.6200, 711.0000, 280.0000, 304.0000, 303.0000, 495.0000, 326.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 22 running loss: nan Train Steps: 22/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[502.2869, 240.4949, 688.0000, 293.0000, 346.0000, 317.0000, 560.8282, 322.6830], [490.1900, 247.1700, 692.0000, 318.0000, 361.0000, 315.0000, 420.8047, 327.6445], [505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925, 341.4247], [497.0900, 242.4000, 707.0000, 210.0000, 288.0000, 311.0000, 509.0000, 312.0000], [501.2900, 243.9900, 687.0000, 172.0000, 344.0000, 171.0000, 506.0000, 316.0000], [488.9500, 241.7100, 691.0000, 288.0000, 390.0000, 305.0000, 461.0000, 334.0000], [495.1000, 234.6500, 704.0000, 295.0000, 297.0000, 288.0000, 483.0000, 290.0000], [493.4800, 246.5000, 545.7030, 163.3048, 306.0000, 153.0000, 444.0000, 343.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 23 running loss: nan Train Steps: 23/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.8300, 248.5600, 700.0000, 342.0000, 319.0000, 283.0000, 481.0000, 328.0000], [ nan, nan, 567.7618, 140.7894, 340.0000, 111.0000, 414.0000, 335.0000], [487.7600, 239.2700, 692.0000, 315.0000, 337.0000, 312.0000, 454.0000, 303.0000], [491.4477, 242.0584, 704.0000, 290.0000, 361.0000, 322.0000, 423.0828, 305.8005], [497.5400, 245.8000, 699.0000, 204.0000, 285.0000, 247.0000, 478.0000, 341.0000], [497.4900, 236.0200, 695.0000, 316.0000, 345.0000, 298.0000, 479.0000, 299.0000], [494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070, 323.3411], [501.1200, 242.0800, 711.0000, 293.0000, 324.0000, 313.0000, 508.9154, 287.4493]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 24 running loss: nan Train Steps: 24/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[486.9300, 238.5500, 667.0000, 232.0000, 297.0000, 187.0000, 475.0000, 318.0000], [494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557, 323.6706], [492.8300, 245.1400, 702.0000, 239.0000, 300.0000, 227.0000, 410.2613, 292.9156], [507.1360, 244.8729, 674.0000, 325.0000, 308.0000, 290.0000, 586.7878, 345.5895], [487.6300, 240.1400, 682.6197, 310.0718, 402.5010, 305.6570, 410.0000, 326.0000], [502.7100, 247.6400, 683.0000, 352.0000, 349.0000, 305.0000, 483.0000, 326.0000], [500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000, 330.0000], [494.8706, 240.8076, 655.2804, 191.2797, 342.3284, 123.5840, 461.3652, 324.6589]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 25 running loss: nan Train Steps: 25/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[500.0049, 240.4949, 716.0000, 251.0000, 284.0000, 263.0000, 508.9154, 295.6488], [489.9400, 241.8100, 692.0000, 292.0000, 399.9338, 306.3606, 410.9215, 346.3862], [503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000, 307.0000], [494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557, 323.6706], [499.7000, 245.3900, 557.1127, 121.6097, 314.0000, 161.0000, 487.0000, 335.0000], [496.7200, 235.1800, 692.0000, 322.0000, 352.0000, 304.0000, 482.0000, 297.0000], [496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000, 308.0000], [486.6900, 238.8800, 687.0000, 314.0000, 368.0000, 322.0000, 454.4241, 300.3342]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 26 running loss: nan Train Steps: 26/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654], [490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806, 297.2106], [502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936, 319.2123], [501.3200, 243.8900, 665.0000, 148.0000, 383.0000, 104.0000, 505.0000, 308.0000], [492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000, 348.0000], [497.7600, 236.2000, 668.0000, 337.0000, 331.0000, 276.0000, 464.0000, 314.0000], [495.9400, 237.1000, 685.8777, 322.4308, 326.0126, 281.2020, 475.3770, 322.6425], [487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475, 296.4297]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 27 running loss: nan Train Steps: 27/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[497.7300, 244.3100, 573.0863, 129.9487, 299.0000, 190.0000, 488.0000, 332.0000], [491.1800, 244.2000, 696.0000, 300.0000, 369.0000, 294.0000, 420.8344, 351.1215], [500.8607, 243.9347, 666.0000, 129.0000, 381.0000, 160.0000, 560.1951, 337.9540], [503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000, 307.0000], [490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806, 297.2106], [497.1525, 246.7491, 627.0000, 127.0000, 292.0000, 188.0000, 454.0000, 305.0000], [491.4477, 242.3712, 659.0000, 200.0000, 326.0000, 127.0000, 410.9736, 298.3820], [496.4300, 240.2100, 715.0000, 293.0000, 293.0000, 300.0000, 508.5592, 296.8201]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 28 running loss: nan Train Steps: 28/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.9600, 242.9100, 691.0000, 337.0000, 330.0000, 306.0000, 481.0000, 319.0000], [501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654], [503.4279, 238.9313, 696.0000, 318.0000, 301.0000, 283.0000, 563.9941, 317.8241], [501.4312, 241.7457, 680.0000, 161.0000, 315.0000, 210.0000, 548.7982, 317.8241], [489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000, 308.0000], [490.9200, 245.0200, 646.1086, 233.3527, 292.0000, 189.0000, 428.4472, 353.7570], [483.8400, 239.2300, 609.5974, 231.6849, 278.0000, 250.0000, 410.9215, 327.9374], [496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000, 308.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 29 running loss: nan Train Steps: 29/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470, 329.6944], [491.7329, 244.8729, 683.0000, 204.0000, 293.0000, 189.0000, 411.3298, 292.5252], [ nan, nan, 535.0539, 150.7963, 329.0000, 127.0000, 415.4254, 355.9533], [501.9900, 244.8500, 645.0000, 121.0000, 386.0000, 95.0000, 492.4471, 292.1411], [ nan, nan, 690.0000, 153.0000, 439.0000, 132.0000, 586.7878, 344.2012], [490.1900, 247.1700, 692.0000, 318.0000, 361.0000, 315.0000, 420.8047, 327.6445], [508.5623, 245.8110, 723.0000, 233.0000, 337.0000, 177.0000, 534.8688, 323.3772], [487.2000, 240.6200, 627.0000, 209.0000, 283.0000, 227.0000, 436.9727, 304.6292]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 30 running loss: nan Train Steps: 30/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[504.5688, 242.6839, 642.0000, 350.0000, 302.0000, 292.0000, 551.3309, 327.5420], [490.8800, 245.1000, 535.0000, 139.0000, 309.0000, 142.0000, 441.3725, 346.9719], [507.1360, 247.0619, 639.0000, 348.0000, 313.0000, 275.0000, 587.4209, 345.5895], [502.0016, 242.9966, 723.0000, 226.0000, 307.0000, 212.0000, 565.8936, 334.4833], [486.5300, 242.5300, 558.0000, 115.0000, 328.0000, 119.0000, 440.1781, 334.6939], [505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925, 341.4247], [485.1400, 241.1400, 692.0000, 271.0000, 323.0000, 322.0000, 456.2567, 336.5189], [501.1459, 238.3059, 708.2286, 286.2665, 308.2703, 270.6642, 503.4357, 320.0467]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 31 running loss: nan Train Steps: 31/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000, 331.0000], [486.9800, 237.0500, 671.0000, 350.0000, 335.0000, 296.0000, 411.6859, 289.7920], [500.0800, 246.5000, 711.0000, 282.0000, 346.0000, 349.0000, 486.0000, 309.0000], [482.6800, 240.6500, 588.0000, 152.0000, 275.0000, 202.0000, 441.2465, 305.0196], [495.8700, 247.7900, 701.0000, 247.0000, 292.0000, 294.0000, 456.5611, 306.1910], [489.0300, 246.2400, 692.0000, 255.0000, 314.0000, 358.0000, 462.9718, 306.9719], [498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000, 311.0000], [502.0016, 241.4330, 688.0000, 137.0000, 428.0000, 108.0000, 565.8936, 324.7654]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 32 running loss: nan Train Steps: 32/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.6600, 241.1700, 670.0000, 365.0000, 314.0000, 292.0000, 482.9497, 277.5642], [502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808, 289.3645], [494.8706, 243.6220, 700.0000, 303.0000, 283.0000, 262.0000, 465.0000, 365.0000], [499.2800, 248.5500, 715.0000, 279.0000, 326.0000, 321.0000, 500.0000, 333.0000], [488.6800, 242.1600, 575.0000, 105.0000, 308.0000, 153.0000, 469.0000, 334.0000], [494.7600, 242.5300, 616.0000, 135.0000, 325.0000, 127.0000, 461.0000, 309.0000], [503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324, 332.4009], [493.4800, 246.5000, 545.7030, 163.3048, 306.0000, 153.0000, 444.0000, 343.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 33 running loss: nan Train Steps: 33/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.2500, 244.2100, 697.0000, 336.0000, 297.0000, 287.0000, 462.0000, 366.0000], [483.0400, 236.7800, 673.0000, 293.0000, 285.0000, 273.0000, 421.3020, 281.5925], [486.0700, 237.4800, 696.0000, 280.0000, 363.6628, 302.7423, 418.4528, 297.6010], [486.4500, 236.9900, 683.0000, 280.0000, 308.0000, 295.0000, 427.3566, 297.2106], [503.6500, 239.3200, 720.0000, 249.0000, 289.0000, 232.0000, 512.0000, 306.0000], [502.8574, 243.3093, 720.0000, 283.0000, 301.0000, 281.0000, 561.4614, 329.6244], [483.4400, 241.3300, 551.0000, 119.0000, 302.0000, 149.0000, 438.3973, 308.1432], [495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262, 295.5407]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 34 running loss: nan Train Steps: 34/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[500.0400, 246.9800, 696.0000, 291.0000, 372.0000, 334.0000, 487.0000, 311.0000], [ nan, nan, 609.0000, 195.0000, 323.0000, 152.0000, 435.0000, 346.0000], [500.2902, 239.5567, 719.0000, 286.0000, 319.0000, 331.0000, 556.3961, 317.1299], [496.3600, 244.8300, 716.0000, 241.0000, 303.0000, 282.0000, 477.0000, 340.0000], [502.8574, 242.0584, 655.2804, 144.8417, 340.9529, 143.5759, 509.1774, 321.3521], [495.4411, 242.9966, 585.0000, 146.0000, 326.0000, 127.0000, 451.9311, 339.3793], [497.9000, 243.6700, 719.0000, 258.0000, 307.0000, 285.0000, 489.0000, 329.0000], [ nan, nan, 677.0000, 153.0000, 468.0000, 128.0000, 570.3256, 364.3312]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 35 running loss: nan Train Steps: 35/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070, 323.3411], [490.5919, 247.0619, 656.0000, 218.0000, 285.0000, 324.0000, 462.9718, 308.1432], [494.8706, 249.2509, 546.0000, 158.0000, 332.0000, 138.0000, 457.0000, 331.0000], [501.7900, 244.2400, 699.0000, 336.0000, 294.0000, 227.0000, 474.0855, 284.5056], [507.7065, 245.4983, 617.0000, 355.0000, 323.0000, 286.0000, 587.4209, 343.5071], [496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000, 326.0000], [494.5853, 239.8694, 703.4152, 251.4380, 284.1584, 257.0997, 483.0015, 318.7289], [500.8607, 239.2440, 723.6318, 252.4934, 288.0082, 277.9555, 525.6299, 309.7392]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 36 running loss: nan Train Steps: 36/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[498.6900, 241.4000, 711.0000, 278.0000, 318.0000, 346.0000, 512.0000, 311.0000], [495.1558, 237.6805, 607.1456, 135.3430, 300.7195, 196.7750, 494.7211, 320.0467], [496.8600, 244.1200, 700.0000, 307.0000, 332.0000, 294.0000, 470.0000, 310.0000], [ nan, nan, 664.0000, 189.0000, 287.0000, 203.0000, 416.6721, 311.6573], [499.7197, 240.4949, 700.5271, 305.2639, 328.0690, 323.5423, 531.4897, 308.0097], [486.0200, 240.0200, 681.0000, 311.0000, 360.7842, 319.7291, 414.3940, 316.8095], [492.8739, 243.3093, 694.0000, 232.0000, 297.0000, 242.0000, 415.6036, 309.7050], [500.9500, 245.1100, 675.0000, 189.0000, 322.0000, 158.0000, 507.1346, 288.6207]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 37 running loss: nan Train Steps: 37/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.3500, 242.9600, 638.0000, 162.0000, 315.0000, 154.0000, 456.0000, 311.0000], [494.8706, 237.9932, 586.9290, 119.5119, 329.5677, 150.5065, 488.4105, 323.6706], [495.8100, 239.8400, 686.5782, 321.6804, 329.3477, 300.9463, 475.3770, 308.0492], [501.2500, 246.0400, 668.0000, 140.0000, 316.0000, 177.0000, 501.0800, 295.2583], [500.7600, 247.1900, 641.0000, 141.0000, 391.0000, 92.0000, 502.5046, 293.6965], [491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000, 345.0000], [493.4444, 244.8729, 675.0000, 202.0000, 280.0000, 280.0000, 497.0000, 324.0000], [495.9900, 244.1900, 715.0000, 251.0000, 283.0000, 274.0000, 494.0000, 324.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 38 running loss: nan Train Steps: 38/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[493.5600, 243.1200, 699.3539, 286.7225, 343.0000, 295.0000, 461.0000, 337.0000], [501.0800, 241.7700, 720.0000, 286.0000, 304.0000, 310.0000, 513.1892, 286.2780], [502.8574, 242.3712, 695.7136, 182.8364, 313.9077, 173.2258, 504.0367, 322.0233], [501.0900, 244.0100, 724.0000, 251.0000, 302.0000, 276.0000, 504.6415, 291.7443], [495.4411, 242.9966, 585.0000, 146.0000, 326.0000, 127.0000, 451.9311, 339.3793], [492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121, 350.7788], [491.7300, 241.5500, 528.0000, 148.0000, 327.0000, 129.0000, 439.1096, 346.4075], [501.4500, 243.6300, 668.0000, 146.0000, 366.0000, 137.0000, 508.0000, 318.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 39 running loss: nan Train Steps: 39/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.1100, 234.6700, 699.0000, 253.0000, 279.0000, 242.0000, 482.0000, 289.0000], [505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925, 341.4247], [485.8600, 235.9000, 669.0000, 349.0000, 354.0000, 307.0000, 416.3159, 289.0111], [491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148, 349.3146], [498.2500, 240.2200, 700.0000, 315.0000, 306.0000, 314.0000, 509.0000, 300.0000], [ nan, nan, 711.0000, 186.0000, 421.0000, 169.0000, 571.5920, 360.1664], [506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061, 303.9413], [501.1100, 244.2300, 723.0000, 232.0000, 293.0000, 235.0000, 507.0000, 317.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 40 running loss: nan Train Steps: 40/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[484.7500, 240.1200, 605.0000, 166.0000, 283.0000, 182.0000, 442.3150, 327.2754], [496.9300, 241.9800, 715.0000, 250.0000, 305.0000, 256.0000, 449.0000, 335.0000], [495.9000, 240.8800, 697.2715, 313.6593, 306.6687, 286.3209, 470.6230, 308.8831], [500.0049, 240.8076, 682.0000, 325.0000, 323.0000, 307.0000, 506.0000, 301.0000], [503.7131, 242.6839, 731.0000, 246.0000, 338.5284, 254.5401, 593.7525, 317.8241], [496.2700, 244.6800, 704.0000, 305.0000, 312.0000, 300.0000, 488.0000, 335.0000], [ nan, nan, 548.7455, 131.6165, 332.0000, 112.0000, 412.2570, 343.7506], [498.6600, 245.4800, 648.0000, 177.0000, 285.0000, 233.0000, 481.0000, 312.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 41 running loss: nan Train Steps: 41/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.1500, 243.9800, 699.6677, 202.5880, 297.3302, 185.4055, 467.3902, 309.9255], [494.1500, 245.1300, 699.0000, 237.0000, 302.0000, 336.0000, 498.0000, 342.0000], [494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221, 317.4111], [486.8500, 236.3900, 697.0000, 287.0000, 322.0000, 294.0000, 439.8219, 323.3708], [494.2300, 243.5200, 602.0000, 135.0000, 345.0000, 107.0000, 432.3427, 314.3904], [506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061, 303.9413], [490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483, 326.7660], [493.4200, 246.6600, 521.3622, 172.4778, 295.0000, 169.0000, 418.2301, 350.2430]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 42 running loss: nan Train Steps: 42/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.9900, 239.9300, 692.2063, 312.4252, 328.0137, 307.5277, 478.0393, 306.1729], [494.5900, 243.5100, 699.0000, 261.0000, 285.0000, 204.0000, 423.2087, 349.3146], [501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396, 367.4918], [502.7100, 247.6400, 683.0000, 352.0000, 349.0000, 305.0000, 483.0000, 326.0000], [492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000, 322.0000], [506.5656, 249.8763, 728.0000, 201.0000, 335.0000, 221.0000, 595.0188, 331.7068], [497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000, 324.0000], [496.5300, 244.8400, 613.0000, 124.0000, 317.0000, 192.0000, 505.0000, 318.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 43 running loss: nan Train Steps: 43/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.1000, 242.9900, 620.5920, 134.0372, 356.6960, 107.8909, 478.0393, 325.5612], [488.0700, 242.5300, 622.0000, 157.0000, 297.0000, 169.0000, 435.1920, 338.9889], [500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000, 357.0000], [501.3800, 245.6600, 697.0000, 185.0000, 352.0000, 136.0000, 500.0000, 312.0000], [500.0049, 237.6805, 693.7882, 295.7652, 335.9166, 298.3375, 504.9382, 318.0700], [501.2900, 243.9900, 687.0000, 172.0000, 344.0000, 171.0000, 506.0000, 316.0000], [494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020, 316.7523], [504.5688, 241.7457, 692.0000, 312.0000, 301.0000, 241.0000, 584.8883, 321.9889]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 44 running loss: nan Train Steps: 44/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[497.4600, 248.2400, 581.0000, 134.0000, 326.0000, 159.0000, 497.0000, 347.0000], [493.1591, 238.3059, 625.4368, 202.8892, 287.8980, 203.2175, 470.3803, 309.1750], [492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000, 322.0000], [495.4400, 244.7100, 583.0000, 150.0000, 340.0000, 153.0000, 479.1144, 375.9738], [490.1500, 245.0200, 696.0000, 268.0000, 319.0000, 259.0000, 401.0383, 328.2302], [494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000, 326.0000], [487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475, 296.4297], [506.8508, 245.1856, 712.0000, 237.0000, 312.0000, 177.0000, 520.3061, 303.9413]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 45 running loss: nan Train Steps: 45/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.1800, 245.4400, 716.0000, 212.0000, 288.0000, 238.0000, 503.5731, 294.0870], [501.9800, 249.4200, 667.0000, 348.0000, 301.0000, 252.0000, 443.7396, 367.4918], [483.8800, 235.7000, 683.0000, 326.0000, 310.0000, 307.0000, 421.3020, 283.1544], [489.3200, 241.0900, 525.0000, 118.0000, 299.0000, 153.0000, 422.3705, 306.1910], [501.3600, 244.7900, 675.0000, 158.0000, 381.0000, 108.0000, 500.7239, 314.3904], [486.3200, 237.8300, 593.6238, 177.4812, 285.0000, 175.0000, 428.0689, 298.7724], [504.3800, 238.9900, 716.0000, 290.0000, 295.0000, 281.0000, 510.0000, 307.0000], [490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483, 326.7660]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 46 running loss: nan Train Steps: 46/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.8800, 245.5700, 633.0000, 152.0000, 343.0000, 148.0000, 478.0000, 342.0000], [502.9100, 244.7000, 645.0000, 136.0000, 383.0000, 92.0000, 476.2652, 294.7599], [495.9200, 243.9100, 607.8350, 143.0867, 345.3565, 118.8599, 474.6164, 313.2611], [500.9900, 249.7600, 708.0000, 311.0000, 301.0000, 249.0000, 482.0000, 330.0000], [501.0300, 253.9500, 633.9382, 277.5496, 303.0000, 173.0000, 445.8765, 362.8064], [502.1800, 247.0500, 719.0000, 230.0000, 316.0000, 159.0000, 503.2108, 296.3059], [496.0300, 237.6400, 676.8727, 329.8349, 331.3488, 274.6205, 471.3836, 324.1019], [495.7263, 235.8042, 703.4152, 284.1557, 332.1768, 327.8600, 504.6377, 314.1167]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 47 running loss: nan Train Steps: 47/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[ nan, nan, 535.0539, 150.7963, 329.0000, 127.0000, 415.4254, 355.9533], [486.6400, 237.4500, 691.0000, 297.0000, 349.0000, 305.0000, 427.7128, 298.7724], [482.6800, 240.6500, 588.0000, 152.0000, 275.0000, 202.0000, 441.2465, 305.0196], [493.7296, 243.3093, 654.0000, 159.0000, 284.0000, 221.0000, 463.0000, 333.0000], [495.9900, 244.1900, 715.0000, 251.0000, 283.0000, 274.0000, 494.0000, 324.0000], [498.0700, 246.1700, 686.0000, 207.0000, 351.0000, 124.0000, 463.0000, 323.0000], [491.8400, 243.1800, 700.0000, 273.0000, 387.9706, 313.0692, 469.0000, 334.0000], [494.7600, 244.7600, 707.0000, 277.0000, 387.0000, 339.0000, 494.0000, 351.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 48 running loss: nan Train Steps: 48/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[492.8739, 242.3712, 602.0000, 128.0000, 330.0000, 124.0000, 463.0000, 307.0000], [496.0400, 242.3200, 710.4314, 287.9666, 289.9929, 257.0700, 469.2918, 308.8831], [485.6100, 238.7500, 686.0000, 305.0000, 348.0000, 324.0000, 414.3940, 327.9374], [496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000, 356.0000], [502.2869, 240.4949, 688.0000, 293.0000, 346.0000, 317.0000, 560.8282, 322.6830], [488.3100, 241.7100, 699.0000, 294.0000, 327.0000, 272.0000, 405.5792, 330.5729], [497.0800, 247.8600, 675.0000, 213.0000, 281.0000, 264.0000, 457.2733, 307.3623], [494.5853, 238.3059, 697.6390, 331.6491, 291.0425, 214.9496, 455.6557, 323.6706]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 49 running loss: nan Train Steps: 49/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[484.2800, 242.0800, 551.7881, 114.9384, 320.0000, 127.0000, 435.1920, 310.0955], [497.2900, 246.0200, 588.0000, 124.0000, 346.0000, 123.0000, 476.0000, 320.0000], [491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000, 345.0000], [501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654], [495.9800, 239.5800, 691.1306, 221.2412, 292.6610, 188.3306, 480.1311, 322.4341], [498.2935, 246.4364, 651.0000, 173.0000, 380.0000, 103.0000, 465.0000, 324.0000], [490.2500, 241.3400, 699.0000, 304.0000, 398.6197, 313.8339, 429.1374, 303.8483], [495.8800, 245.5700, 633.0000, 152.0000, 343.0000, 148.0000, 478.0000, 342.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 50 running loss: nan Train Steps: 50/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000, 354.0000], [507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504, 326.1537], [490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000, 328.0000], [487.5800, 238.2200, 695.0000, 286.0000, 388.7642, 292.7355, 415.2475, 296.4297], [500.8607, 239.2440, 723.6318, 252.4934, 288.0082, 277.9555, 525.6299, 309.7392], [496.1400, 243.1500, 691.0000, 317.0000, 363.6628, 306.4949, 472.0000, 308.0000], [496.2100, 245.7600, 709.0000, 256.0000, 283.0000, 247.0000, 482.0000, 339.0000], [ nan, nan, 708.0000, 169.0000, 409.0000, 131.0000, 574.7578, 326.1537]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 51 running loss: nan Train Steps: 51/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[ nan, nan, 581.4534, 139.9555, 330.0000, 116.0000, 409.0000, 323.0000], [492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121, 350.7788], [497.7500, 250.2900, 708.0000, 313.0000, 299.0000, 276.0000, 456.0000, 338.0000], [503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324, 332.4009], [496.1200, 249.0500, 687.0000, 328.0000, 296.0000, 237.0000, 451.0000, 356.0000], [499.8200, 251.0800, 680.0000, 346.0000, 357.0000, 273.0000, 449.0000, 355.0000], [501.3200, 243.8900, 665.0000, 148.0000, 383.0000, 104.0000, 505.0000, 308.0000], [ nan, nan, 517.5590, 116.6062, 322.0000, 120.0000, 410.0000, 332.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 52 running loss: nan Train Steps: 52/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000, 305.0000], [491.8400, 239.7800, 679.0000, 232.0000, 279.0000, 244.0000, 469.0000, 300.0000], [500.2902, 244.5602, 696.0000, 196.0000, 332.0000, 185.0000, 563.9941, 336.5657], [498.8700, 237.9300, 708.0000, 298.0000, 291.0000, 241.0000, 468.0000, 311.0000], [490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304, 310.4859], [ nan, nan, 677.0000, 153.0000, 468.0000, 128.0000, 570.3256, 364.3312], [494.3001, 247.0619, 582.2141, 254.2003, 287.0000, 204.0000, 454.0000, 355.0000], [490.2100, 246.9000, 634.6989, 226.6815, 276.0000, 242.0000, 426.9483, 327.3517]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 53 running loss: nan Train Steps: 53/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361, 317.8241], [492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000, 348.0000], [504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961, 321.9889], [497.7600, 246.4500, 715.0000, 254.0000, 288.0000, 229.0000, 466.0000, 317.0000], [ nan, nan, 708.0000, 169.0000, 409.0000, 131.0000, 574.7578, 326.1537], [497.7600, 236.2000, 668.0000, 337.0000, 331.0000, 276.0000, 464.0000, 314.0000], [507.9900, 242.6000, 699.0000, 339.0000, 308.0000, 285.0000, 512.0750, 297.0000], [486.3900, 240.6200, 700.0000, 270.0000, 386.0000, 337.0000, 467.0000, 335.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 54 running loss: nan Train Steps: 54/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[483.1600, 240.1100, 587.0000, 136.0000, 318.0000, 126.0000, 418.4528, 286.6684], [490.1700, 246.8700, 573.0000, 173.0000, 290.0000, 177.0000, 426.1470, 329.6944], [490.2100, 246.9800, 676.0000, 275.0000, 294.0000, 292.0000, 426.9483, 326.7660], [502.2869, 242.9966, 642.0000, 132.0000, 345.0000, 164.0000, 545.6324, 319.2123], [486.9100, 241.1400, 622.5284, 256.7020, 290.0000, 261.0000, 411.9899, 317.1024], [499.6700, 241.6600, 699.0000, 292.0000, 327.0000, 340.0000, 509.0000, 312.0000], [484.5200, 240.6700, 700.0000, 256.0000, 352.0000, 348.0000, 467.6018, 335.0844], [488.1700, 239.9200, 700.0000, 308.0000, 306.0000, 285.0000, 451.0000, 305.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 55 running loss: nan Train Steps: 55/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.2400, 244.3600, 655.1108, 143.9094, 352.0268, 123.2475, 474.3377, 330.0576], [505.1393, 246.4364, 700.0000, 306.0000, 303.0000, 294.0000, 569.6925, 351.8367], [496.1000, 246.1100, 583.0000, 145.0000, 332.0000, 143.0000, 488.0000, 330.0000], [495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000, 370.0000], [ nan, nan, 682.0000, 133.0000, 433.0000, 142.0000, 589.3204, 328.9302], [507.4213, 245.8110, 743.0000, 262.0000, 345.0000, 216.0000, 579.8230, 350.4485], [494.5600, 245.9300, 625.0000, 180.0000, 315.0000, 142.0000, 426.0000, 345.0000], [504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177, 317.8241]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 56 running loss: nan Train Steps: 56/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654], [494.5853, 235.8042, 707.2659, 233.4960, 284.4970, 297.9904, 497.7261, 316.7523], [502.0016, 242.9966, 723.0000, 226.0000, 307.0000, 212.0000, 565.8936, 334.4833], [ nan, nan, 711.0000, 186.0000, 421.0000, 169.0000, 571.5920, 360.1664], [503.9300, 240.5100, 676.0000, 321.0000, 337.0000, 300.0000, 508.0000, 306.0000], [498.7100, 250.7200, 626.0000, 207.0000, 305.0000, 172.0000, 454.0000, 337.0000], [502.0300, 246.6600, 677.0000, 157.0000, 359.0000, 119.0000, 496.8792, 294.2234], [491.9500, 243.2500, 537.0000, 140.0000, 322.0000, 121.0000, 417.0651, 341.7008]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 57 running loss: nan Train Steps: 57/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[483.3700, 239.4200, 546.4636, 172.4778, 280.0000, 188.0000, 411.4557, 330.5729], [486.5985, 241.4330, 699.0000, 241.0000, 295.0000, 237.0000, 424.5074, 308.1432], [496.2968, 243.3093, 675.0000, 344.0000, 365.9118, 288.0521, 439.0000, 337.0000], [483.0400, 236.7800, 673.0000, 293.0000, 285.0000, 273.0000, 421.3020, 281.5925], [497.5500, 246.8200, 654.0000, 169.0000, 314.0000, 167.0000, 472.0000, 321.0000], [498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000, 346.0000], [500.9500, 245.1100, 675.0000, 189.0000, 322.0000, 158.0000, 507.1346, 288.6207], [495.7000, 245.4200, 676.0000, 234.0000, 286.0000, 236.0000, 478.0000, 335.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 58 running loss: nan Train Steps: 58/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000, 317.0000], [502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808, 289.3645], [490.2100, 246.9000, 634.6989, 226.6815, 276.0000, 242.0000, 426.9483, 327.3517], [488.7400, 242.4700, 558.0000, 190.0000, 281.0000, 203.0000, 412.2570, 319.1522], [490.9100, 237.3000, 672.0000, 196.0000, 280.0000, 252.0000, 469.0000, 328.0000], [494.5853, 235.8042, 707.2659, 233.4960, 284.4970, 297.9904, 497.7261, 316.7523], [490.5200, 243.8100, 691.0000, 312.0000, 383.0000, 287.0000, 420.6341, 352.0000], [487.8500, 239.4600, 691.0000, 283.0000, 341.0000, 298.0000, 417.0000, 339.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 59 running loss: nan Train Steps: 59/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919, 292.1570], [486.3900, 240.6200, 700.0000, 270.0000, 386.0000, 337.0000, 467.0000, 335.0000], [483.8400, 239.2300, 609.5974, 231.6849, 278.0000, 250.0000, 410.9215, 327.9374], [495.3700, 238.8200, 566.2405, 164.9726, 340.0000, 126.0000, 436.0000, 347.0000], [502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936, 319.2123], [496.1000, 244.9000, 706.1387, 222.8866, 306.0016, 162.7361, 467.5804, 324.7273], [507.1360, 244.8729, 674.0000, 325.0000, 308.0000, 290.0000, 586.7878, 345.5895], [ nan, nan, 643.0000, 149.0000, 318.0000, 151.0000, 446.0000, 336.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 60 running loss: nan Train Steps: 60/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[497.1100, 246.9600, 620.0000, 139.0000, 359.0000, 113.0000, 496.0000, 324.0000], [498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000, 346.0000], [509.1327, 249.8763, 727.0000, 262.0000, 326.0000, 189.0000, 515.8740, 317.8241], [496.2700, 244.6800, 704.0000, 305.0000, 312.0000, 300.0000, 488.0000, 335.0000], [495.4411, 249.8763, 707.0000, 282.0000, 332.0000, 292.0000, 434.1604, 315.6382], [492.8739, 241.4330, 707.0000, 275.0000, 295.0000, 224.0000, 424.8636, 320.6377], [494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000, 326.0000], [497.9500, 245.8000, 595.0000, 136.0000, 308.0000, 171.0000, 479.0000, 315.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 61 running loss: nan Train Steps: 61/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[496.4600, 247.3500, 574.0000, 144.0000, 311.0000, 176.0000, 498.0000, 345.0000], [498.2400, 247.1600, 635.0000, 134.0000, 373.9316, 106.3581, 495.0000, 326.0000], [500.9100, 247.8700, 715.0000, 213.0000, 320.0000, 161.0000, 495.0000, 317.0000], [500.5754, 243.6220, 664.0000, 140.0000, 375.0000, 155.0000, 563.9941, 337.9540], [505.1393, 246.4364, 700.0000, 306.0000, 303.0000, 294.0000, 569.6925, 351.8367], [491.7329, 244.8729, 683.0000, 204.0000, 293.0000, 189.0000, 411.3298, 292.5252], [501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919, 292.1570], [502.6900, 241.7500, 707.0000, 227.0000, 318.0000, 171.0000, 506.7784, 305.4101]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 62 running loss: nan Train Steps: 62/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000, 354.0000], [500.0049, 240.8076, 682.0000, 325.0000, 323.0000, 307.0000, 506.0000, 301.0000], [496.0300, 237.6400, 676.8727, 329.8349, 331.3488, 274.6205, 471.3836, 324.1019], [494.7600, 244.7600, 707.0000, 277.0000, 387.0000, 339.0000, 494.0000, 351.0000], [490.2700, 247.0800, 691.0000, 320.0000, 370.0000, 316.0000, 415.4624, 328.5230], [494.8400, 247.7400, 712.0000, 274.0000, 315.0000, 325.0000, 458.6980, 306.5814], [499.1492, 246.4364, 652.9544, 165.8065, 290.0000, 216.0000, 479.0000, 342.0000], [502.6800, 245.5500, 712.0000, 282.0000, 292.0000, 231.0000, 497.0000, 310.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 63 running loss: nan Train Steps: 63/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[ nan, nan, 669.0000, 199.0000, 285.0000, 202.0000, 426.2881, 308.5337], [495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000, 339.0000], [498.4000, 246.7900, 577.0000, 119.0000, 346.0000, 142.0000, 501.0000, 324.0000], [ nan, nan, 601.2303, 162.4709, 319.0000, 136.0000, 413.0000, 334.0000], [508.5623, 247.0619, 672.0000, 150.0000, 433.0000, 98.0000, 538.0345, 335.1775], [494.4200, 243.8400, 576.0000, 148.0000, 342.0000, 142.0000, 477.0000, 373.0000], [493.0700, 240.3800, 703.0000, 281.0000, 293.0000, 293.0000, 471.0000, 301.0000], [496.0500, 246.9500, 698.0000, 284.0000, 296.0000, 193.0000, 430.9181, 346.0170]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 64 running loss: nan Train Steps: 64/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[498.1200, 244.9800, 715.0000, 288.0000, 304.0000, 177.0000, 459.0000, 321.0000], [495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000, 354.0000], [489.0800, 240.3300, 696.0000, 321.0000, 291.0000, 226.0000, 407.7683, 292.9156], [507.9918, 248.6254, 740.0000, 246.0000, 330.0000, 225.0000, 570.3256, 356.6957], [502.2869, 242.9966, 642.0000, 132.0000, 345.0000, 164.0000, 545.6324, 319.2123], [491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148, 349.3146], [503.7131, 243.6220, 728.0000, 196.0000, 378.0822, 202.0042, 595.6520, 321.2947], [501.9300, 247.0000, 648.0244, 348.0684, 320.0000, 275.0000, 446.5888, 367.1014]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 65 running loss: nan Train Steps: 65/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[491.7329, 246.7491, 639.0000, 192.0000, 302.0000, 166.0000, 412.0000, 333.0000], [501.0600, 243.8700, 723.0000, 259.0000, 287.0000, 273.0000, 506.0000, 315.0000], [493.4444, 243.3093, 606.0000, 177.0000, 324.0000, 163.0000, 475.0000, 370.0000], [503.1100, 241.8900, 673.4919, 326.7499, 326.0000, 301.0000, 505.0000, 307.0000], [494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221, 317.4111], [499.4344, 247.6873, 621.0000, 163.0000, 306.0000, 188.0000, 497.0000, 325.0000], [501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919, 292.1570], [486.3200, 237.8300, 593.6238, 177.4812, 285.0000, 175.0000, 428.0689, 298.7724]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 66 running loss: nan Train Steps: 66/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[498.6600, 245.4800, 648.0000, 177.0000, 285.0000, 233.0000, 481.0000, 312.0000], [498.8640, 238.6186, 718.8183, 288.3773, 306.7708, 312.9795, 529.0106, 309.9863], [502.0016, 241.4330, 617.7352, 124.7889, 351.3434, 134.0229, 514.8118, 317.3988], [501.2500, 244.2100, 697.0000, 336.0000, 297.0000, 287.0000, 462.0000, 366.0000], [495.6700, 245.2700, 711.0000, 275.0000, 360.0000, 341.0000, 491.0000, 353.0000], [495.7300, 239.1300, 704.0000, 277.0000, 335.0000, 287.0000, 455.0000, 333.0000], [504.5688, 239.8694, 680.0000, 314.0000, 308.0000, 303.0000, 595.6520, 319.2123], [493.7296, 243.3093, 654.0000, 159.0000, 284.0000, 221.0000, 463.0000, 333.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 67 running loss: nan Train Steps: 67/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.8400, 245.9700, 571.0000, 128.0000, 320.0000, 159.0000, 486.0000, 338.0000], [502.1600, 246.1500, 647.0000, 343.0000, 335.0000, 285.0000, 453.0000, 365.0000], [500.5754, 243.6220, 664.0000, 140.0000, 375.0000, 155.0000, 563.9941, 337.9540], [496.2100, 244.5800, 688.8793, 172.7032, 324.0115, 153.2296, 472.5629, 329.7797], [501.1400, 242.9200, 719.0000, 278.0000, 305.0000, 299.0000, 506.0662, 290.5729], [495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000, 339.0000], [500.7600, 247.1900, 641.0000, 141.0000, 391.0000, 92.0000, 502.5046, 293.6965], [ nan, nan, 617.9645, 156.6336, 294.0000, 164.0000, 433.0000, 310.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 68 running loss: nan Train Steps: 68/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[498.3700, 249.1100, 607.0000, 137.0000, 321.0000, 173.0000, 496.0000, 346.0000], [495.0900, 241.9900, 692.0000, 316.0000, 359.0000, 309.0000, 474.0000, 303.0000], [488.1300, 241.1800, 691.0000, 223.0000, 290.0000, 301.0000, 483.0000, 330.0000], [505.6500, 242.8800, 699.0000, 337.0000, 310.0000, 265.0000, 508.9092, 295.6117], [496.1100, 241.7100, 702.0746, 276.1287, 294.6791, 252.1829, 475.0000, 327.0000], [502.8574, 256.4433, 680.0000, 270.0000, 362.0000, 155.0000, 435.1920, 372.5677], [500.0000, 248.7500, 707.0000, 236.0000, 287.0000, 257.0000, 493.0000, 322.0000], [494.5853, 238.9313, 603.2948, 142.7309, 316.7463, 167.4912, 486.3070, 323.3411]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 69 running loss: nan Train Steps: 69/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[497.7500, 237.4100, 707.0000, 301.0000, 315.0000, 276.0000, 472.0000, 301.0000], [507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209, 344.8954], [493.1591, 237.3677, 700.5271, 305.2639, 343.9919, 319.1815, 481.7994, 312.1400], [498.0600, 251.1200, 708.0000, 330.0000, 304.0000, 255.0000, 450.0000, 337.0000], [507.9918, 247.0619, 669.0000, 163.0000, 388.0000, 102.0000, 515.2408, 310.1886], [507.9918, 248.6254, 740.0000, 246.0000, 330.0000, 225.0000, 570.3256, 356.6957], [495.1000, 234.6500, 704.0000, 295.0000, 297.0000, 288.0000, 483.0000, 290.0000], [490.5919, 243.9347, 580.6928, 144.1250, 287.0000, 198.0000, 480.0000, 336.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 70 running loss: nan Train Steps: 70/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209, 344.8954], [496.4300, 240.2100, 715.0000, 293.0000, 293.0000, 300.0000, 508.5592, 296.8201], [483.8800, 235.7000, 683.0000, 326.0000, 310.0000, 307.0000, 421.3020, 283.1544], [490.8700, 247.0400, 696.0000, 310.0000, 335.0000, 305.0000, 411.7228, 329.6944], [503.1426, 241.4330, 727.0000, 274.0000, 315.0000, 338.0000, 564.6272, 336.5657], [485.4300, 237.7600, 692.0000, 259.0000, 323.0000, 305.0000, 420.2336, 299.9438], [496.0400, 242.3200, 710.4314, 287.9666, 289.9929, 257.0700, 469.2918, 308.8831], [ nan, nan, 682.0000, 133.0000, 433.0000, 142.0000, 589.3204, 328.9302]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 71 running loss: nan Train Steps: 71/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[507.7065, 249.8763, 731.0000, 239.0000, 310.0000, 259.0000, 597.5515, 328.2361], [488.1300, 241.1800, 691.0000, 223.0000, 290.0000, 301.0000, 483.0000, 330.0000], [497.8500, 239.4100, 643.0000, 168.0000, 320.0000, 137.0000, 469.1422, 312.4604], [ nan, nan, 559.3947, 167.4743, 316.0000, 143.0000, 438.6643, 349.1448], [501.7164, 242.3712, 720.0000, 195.0000, 395.0000, 138.0000, 575.3909, 324.7654], [489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000, 331.0000], [501.8700, 246.5800, 712.0000, 229.0000, 335.0000, 130.0000, 468.6673, 290.0746], [501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 72 running loss: nan Train Steps: 72/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[484.6700, 238.7000, 663.0000, 216.0000, 272.0000, 243.0000, 442.3150, 327.6658], [490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304, 310.4859], [488.9500, 241.7100, 691.0000, 288.0000, 390.0000, 305.0000, 461.0000, 334.0000], [504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961, 321.9889], [495.9600, 245.1500, 673.1207, 178.4619, 329.3477, 136.4104, 469.4820, 323.4764], [486.9300, 238.5500, 667.0000, 232.0000, 297.0000, 187.0000, 475.0000, 318.0000], [503.9984, 246.1237, 727.0000, 266.0000, 327.0000, 184.0000, 545.6324, 332.4009], [495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000, 354.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 73 running loss: nan Train Steps: 73/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[501.0600, 243.8700, 723.0000, 259.0000, 287.0000, 273.0000, 506.0000, 315.0000], [501.2000, 245.2900, 642.0000, 111.0000, 333.0000, 150.0000, 503.2169, 296.8201], [496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000, 326.0000], [497.9500, 245.8000, 595.0000, 136.0000, 308.0000, 171.0000, 479.0000, 315.0000], [500.5000, 251.9400, 691.0000, 348.0000, 319.0000, 263.0000, 448.0000, 357.0000], [499.5800, 246.0100, 621.0000, 155.0000, 397.0000, 91.0000, 470.0000, 325.0000], [486.0900, 237.1500, 650.0000, 235.0000, 282.0000, 245.0000, 427.7128, 297.2106], [498.8640, 237.9932, 694.0000, 324.0000, 309.0000, 271.0000, 466.0000, 312.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 74 running loss: nan Train Steps: 74/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.8706, 249.2509, 546.0000, 158.0000, 332.0000, 138.0000, 457.0000, 331.0000], [495.9400, 237.1000, 685.8777, 322.4308, 326.0126, 281.2020, 475.3770, 322.6425], [502.0016, 241.4330, 617.7352, 124.7889, 351.3434, 134.0229, 514.8118, 317.3988], [501.7200, 242.6800, 683.0000, 354.0000, 300.0000, 265.0000, 479.1508, 278.9525], [493.0700, 240.3800, 703.0000, 281.0000, 293.0000, 293.0000, 471.0000, 301.0000], [485.5300, 238.5200, 690.0000, 305.0000, 351.0000, 329.0000, 452.0000, 298.0000], [498.2400, 247.1600, 635.0000, 134.0000, 373.9316, 106.3581, 495.0000, 326.0000], [498.9000, 245.0400, 619.0000, 128.0000, 293.0000, 194.0000, 465.0000, 334.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 75 running loss: nan Train Steps: 75/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[492.8739, 241.4330, 693.0000, 281.0000, 394.7994, 321.8399, 485.0000, 334.0000], [490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385, 353.5374], [492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000, 317.0000], [492.8739, 244.5602, 712.0000, 280.0000, 330.0000, 355.0000, 501.0000, 322.0000], [ nan, nan, 724.0000, 210.0000, 411.0000, 138.0000, 588.6873, 342.1188], [501.1400, 242.9200, 719.0000, 278.0000, 305.0000, 299.0000, 506.0662, 290.5729], [504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177, 317.8241], [ nan, nan, 638.5021, 191.6575, 290.0000, 190.0000, 403.1752, 333.7941]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 76 running loss: nan Train Steps: 76/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[504.7900, 241.0400, 685.0000, 348.0000, 295.0000, 285.0000, 506.0662, 300.3342], [491.6200, 238.9700, 696.0000, 301.0000, 352.0000, 288.0000, 430.0000, 345.0000], [497.4900, 236.0200, 695.0000, 316.0000, 345.0000, 298.0000, 479.0000, 299.0000], [490.3400, 244.1100, 700.0000, 304.0000, 310.0000, 254.0000, 418.8311, 352.8785], [ nan, nan, 694.0000, 170.0000, 428.0000, 119.0000, 534.2356, 337.2599], [492.8739, 242.3712, 602.0000, 128.0000, 330.0000, 124.0000, 463.0000, 307.0000], [489.7400, 240.3700, 708.0000, 253.0000, 327.0000, 331.0000, 485.0000, 331.0000], [500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000, 314.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 77 running loss: nan Train Steps: 77/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[504.2836, 241.4330, 714.0000, 288.0000, 315.0000, 289.0000, 598.8177, 317.8241], [495.2000, 248.0900, 640.0000, 293.0000, 285.2831, 218.8381, 449.0000, 354.0000], [489.1657, 239.8694, 565.0000, 143.0000, 323.0000, 117.0000, 425.5759, 299.5533], [497.1525, 246.7491, 627.0000, 127.0000, 292.0000, 188.0000, 454.0000, 305.0000], [492.9600, 240.8300, 704.0000, 320.0000, 300.0000, 289.0000, 479.0000, 317.0000], [492.2800, 247.0300, 695.0000, 310.0000, 391.5905, 338.8391, 441.9588, 308.1432], [490.3067, 235.8042, 701.4897, 306.3193, 331.7047, 338.5089, 479.9964, 304.8923], [485.9900, 240.3900, 662.0000, 295.0000, 324.0000, 306.0000, 413.3255, 316.8095]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 78 running loss: nan Train Steps: 78/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[485.5300, 238.5200, 690.0000, 305.0000, 351.0000, 329.0000, 452.0000, 298.0000], [490.0000, 238.9800, 700.0000, 292.0000, 342.0000, 287.0000, 433.0000, 343.0000], [508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361, 317.8241], [490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792, 324.7162], [496.7200, 235.1800, 692.0000, 322.0000, 352.0000, 304.0000, 482.0000, 297.0000], [504.8541, 240.4949, 634.8398, 344.7328, 312.0000, 302.0000, 556.3961, 321.9889], [501.9000, 245.9300, 690.0000, 194.0000, 352.0000, 119.0000, 470.0919, 292.1570], [490.5700, 242.2000, 557.0000, 133.0000, 328.0000, 117.0000, 431.6304, 310.4859]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 79 running loss: nan Train Steps: 79/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.3001, 240.4949, 688.0120, 240.8839, 314.5486, 150.9156, 456.8577, 326.3061], [493.1591, 244.5602, 707.0000, 247.0000, 297.0000, 333.0000, 499.0000, 321.0000], [496.4600, 247.3500, 574.0000, 144.0000, 311.0000, 176.0000, 498.0000, 345.0000], [494.0300, 245.4600, 629.0000, 168.0000, 291.0000, 215.0000, 495.0000, 326.0000], [497.2500, 245.9600, 578.0000, 122.0000, 335.0000, 133.0000, 478.0000, 317.0000], [486.4500, 236.9900, 683.0000, 280.0000, 308.0000, 295.0000, 427.3566, 297.2106], [ nan, nan, 578.0000, 130.0000, 319.0000, 137.0000, 434.1235, 310.8764], [500.6700, 248.5800, 682.0000, 157.0000, 396.0000, 100.0000, 497.5185, 297.9915]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 80 running loss: nan Train Steps: 80/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[494.0100, 245.5700, 704.0000, 266.0000, 326.0000, 262.0000, 410.2613, 294.0870], [500.9300, 243.1400, 711.0000, 282.0000, 294.0000, 307.0000, 508.0000, 314.0000], [ nan, nan, 521.0000, 103.0000, 328.0000, 119.0000, 420.2336, 314.3904], [507.1360, 247.3746, 691.0000, 322.0000, 326.0000, 328.0000, 601.3504, 326.1537], [491.0200, 243.2600, 700.0000, 285.0000, 349.0000, 301.0000, 406.9148, 349.3146], [492.0181, 245.8110, 597.4271, 191.6575, 306.0000, 158.0000, 437.0000, 348.0000], [501.1200, 242.0800, 711.0000, 293.0000, 324.0000, 313.0000, 508.9154, 287.4493], [495.6600, 245.6500, 605.0000, 169.0000, 315.0000, 191.0000, 481.0000, 371.0000]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 81 running loss: nan Train Steps: 81/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[495.4411, 235.8042, 715.9302, 266.2137, 308.1366, 326.1029, 501.0316, 315.7639], [508.2770, 247.6873, 731.0000, 212.0000, 375.0000, 195.0000, 571.5920, 359.4722], [497.8600, 241.7000, 719.0000, 252.0000, 301.0000, 344.0000, 510.0000, 310.0000], [494.5853, 237.9932, 661.0566, 183.8918, 282.0930, 249.6721, 495.3221, 317.4111], [501.7164, 242.3712, 731.0000, 225.0000, 370.0000, 157.0000, 578.5567, 324.7654], [502.1200, 243.3900, 664.0000, 159.0000, 349.0000, 111.0000, 491.1808, 289.3645], [495.8600, 244.5100, 692.0000, 337.0000, 332.0000, 262.0000, 436.0000, 339.0000], [491.4477, 242.3712, 659.0000, 200.0000, 326.0000, 127.0000, 410.9736, 298.3820]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 82 running loss: nan Train Steps: 82/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[486.0200, 240.0200, 681.0000, 311.0000, 360.7842, 319.7291, 414.3940, 316.8095], [498.1000, 245.8600, 713.0000, 246.0000, 322.0000, 143.0000, 461.0000, 322.0000], [490.7900, 246.9100, 707.0000, 280.0000, 343.0000, 363.0000, 462.2595, 305.8005], [502.5721, 242.0584, 626.3995, 124.7889, 362.5892, 124.7976, 512.3327, 319.3755], [486.8400, 238.8300, 696.0000, 285.0000, 361.0000, 317.0000, 425.9320, 302.2865], [495.8300, 246.2900, 636.0000, 196.0000, 294.0000, 226.0000, 483.0000, 370.0000], [490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385, 353.5374], [507.1360, 246.7491, 707.0000, 304.0000, 320.0000, 326.0000, 603.8831, 322.6830]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 83 running loss: nan Train Steps: 83/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[502.1300, 241.7600, 660.0000, 146.0000, 346.0000, 126.0000, 509.2715, 312.4382], [486.6900, 238.8800, 687.0000, 314.0000, 368.0000, 322.0000, 454.4241, 300.3342], [509.1327, 249.2509, 715.0000, 223.0000, 360.0000, 155.0000, 515.8740, 317.1299], [487.4300, 239.4100, 672.0000, 260.0000, 295.0000, 278.0000, 444.8080, 339.3793], [507.1360, 249.5636, 672.0000, 337.0000, 306.0000, 249.0000, 587.4209, 344.8954], [490.2000, 245.0600, 699.0000, 281.0000, 289.0000, 222.0000, 396.7645, 323.8376], [490.8772, 241.7457, 661.0000, 201.0000, 290.0000, 184.0000, 454.0000, 310.0000], [491.7300, 241.5500, 528.0000, 148.0000, 327.0000, 129.0000, 439.1096, 346.4075]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 84 running loss: nan Train Steps: 84/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[489.1700, 240.2900, 707.0000, 232.0000, 301.0000, 282.0000, 464.0000, 333.0000], [495.4411, 242.0584, 620.5229, 140.8099, 298.6643, 175.1677, 474.4262, 295.5407], [ nan, nan, 727.0000, 227.0000, 365.0000, 157.0000, 539.3008, 334.4833], [489.6200, 240.8100, 549.0000, 169.0000, 296.0000, 167.0000, 441.0000, 340.0000], [504.1500, 240.4100, 708.0000, 330.0000, 289.0000, 271.0000, 506.7784, 300.7246], [501.8400, 245.9700, 571.0000, 128.0000, 320.0000, 159.0000, 486.0000, 338.0000], [501.0300, 253.9500, 633.9382, 277.5496, 303.0000, 173.0000, 445.8765, 362.8064], [490.6600, 245.6100, 679.5771, 241.6917, 287.0000, 192.0000, 400.2369, 324.4233]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 85 running loss: nan Train Steps: 85/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[487.6300, 240.1400, 682.6197, 310.0718, 402.5010, 305.6570, 410.0000, 326.0000], [494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020, 316.7523], [509.1327, 249.2509, 715.0000, 223.0000, 360.0000, 155.0000, 515.8740, 317.1299], [500.8607, 241.7457, 697.6390, 201.8338, 293.1961, 225.1964, 522.4747, 310.9746], [488.0600, 238.6400, 684.0000, 340.0000, 309.0000, 265.0000, 410.2613, 292.1347], [496.1000, 246.1100, 583.0000, 145.0000, 332.0000, 143.0000, 488.0000, 330.0000], [502.8574, 238.6186, 723.0000, 284.0000, 312.0000, 249.0000, 565.8936, 319.2123], [488.1900, 240.0300, 619.0000, 215.0000, 277.0000, 228.0000, 441.9588, 341.3316]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 86 running loss: nan Train Steps: 86/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[ nan, nan, 664.0000, 189.0000, 287.0000, 203.0000, 416.6721, 311.6573], [490.9200, 242.2600, 685.0000, 243.0000, 305.0000, 153.0000, 408.4806, 297.2106], [494.3001, 239.5567, 714.0048, 287.3219, 310.3356, 293.9974, 483.3020, 316.7523], [487.2000, 240.6200, 627.0000, 209.0000, 283.0000, 227.0000, 436.9727, 304.6292], [496.8600, 244.1200, 700.0000, 307.0000, 332.0000, 294.0000, 470.0000, 310.0000], [503.4279, 245.8110, 704.0000, 151.0000, 421.0000, 156.0000, 594.3857, 322.6830], [490.8700, 247.0400, 696.0000, 310.0000, 335.0000, 305.0000, 411.7228, 329.6944], [490.1900, 243.9500, 684.0000, 334.0000, 372.9783, 308.4714, 405.5792, 324.7162]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 87 running loss: nan Train Steps: 87/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[492.8600, 245.9100, 699.0000, 263.0000, 303.0000, 329.0000, 448.3696, 301.1151], [496.2100, 245.7600, 709.0000, 256.0000, 283.0000, 247.0000, 482.0000, 339.0000], [501.7164, 242.3712, 720.0000, 195.0000, 395.0000, 138.0000, 575.3909, 324.7654], [495.4600, 246.4600, 595.0000, 162.0000, 292.0000, 221.0000, 499.0000, 343.0000], [495.9300, 246.6900, 678.0000, 223.0000, 284.0000, 261.0000, 485.0000, 365.0000], [490.3900, 244.2900, 684.0000, 274.0000, 291.0000, 220.0000, 423.2385, 353.5374], [489.3200, 241.1600, 683.0000, 244.0000, 281.0000, 215.0000, 453.0000, 308.0000], [496.1400, 238.9700, 684.3768, 325.7215, 307.3357, 262.1889, 469.2918, 323.8934]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 88 running loss: nan Train Steps: 88/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[497.9600, 237.8300, 658.0000, 343.0000, 332.0000, 277.0000, 462.0000, 316.0000], [503.7131, 242.6839, 731.0000, 246.0000, 338.5284, 254.5401, 593.7525, 317.8241], [501.7200, 242.6800, 683.0000, 354.0000, 300.0000, 265.0000, 479.1508, 278.9525], [492.0900, 245.0800, 700.0000, 262.0000, 311.0000, 262.0000, 405.3121, 350.7788], [489.9100, 244.5200, 615.6826, 249.1969, 278.0000, 226.0000, 412.5242, 325.5947], [496.2400, 244.3600, 655.1108, 143.9094, 352.0268, 123.2475, 474.3377, 330.0576], [505.1393, 242.9966, 658.1664, 325.8310, 332.0000, 331.0000, 569.6925, 341.4247], [508.8475, 244.5602, 709.0000, 321.0000, 297.0000, 279.0000, 532.3361, 317.8241]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 89 running loss: nan Train Steps: 89/90 Loss: nan predictions are: tensor([[nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan], [nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0', grad_fn=) landmarks are: tensor([[486.0900, 237.1500, 650.0000, 235.0000, 282.0000, 245.0000, 427.7128, 297.2106], [496.5700, 246.5700, 699.0000, 300.0000, 384.0000, 338.0000, 504.0000, 326.0000], [489.0500, 245.2700, 548.7455, 132.4504, 349.0000, 102.0000, 415.0000, 332.0000], [501.4600, 245.6700, 723.0000, 258.0000, 296.0000, 209.0000, 501.0000, 310.0000], [494.5853, 239.2440, 630.2503, 173.3377, 295.9114, 195.6037, 486.9080, 322.0233], [501.0800, 241.7700, 720.0000, 286.0000, 304.0000, 310.0000, 513.1892, 286.2780], [490.0000, 238.9800, 700.0000, 292.0000, 342.0000, 287.0000, 433.0000, 343.0000], [503.7131, 244.5602, 672.0000, 335.0000, 296.0000, 262.0000, 550.0646, 329.6244]], device='cuda:0') loss_train_step before backward: tensor(nan, device='cuda:0', grad_fn=) loss_train_step after backward: tensor(nan, device='cuda:0', grad_fn=) loss_train: nan step: 90 running loss: nan Valid Steps: 10/10 Loss: nan -------------------------------------------------- Epoch: 1 Train Loss: nan Valid Loss: nan -------------------------------------------------- Training Complete Total Elapsed Time : 106.72142887115479 s