exp_all.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# don't use 1.7.2 of scipy !!!\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import DataLoader\n",
    "import copy\n",
    "from matplotlib import pyplot as plt\n",
    "import torch.nn.functional as F\n",
    "import scipy\n",
    "from scipy.sparse.linalg import LinearOperator\n",
    "from scipy.sparse.linalg import eigsh\n",
    "from torch.autograd import Variable, grad\n",
    "from numpy.linalg import eig as eig\n",
    "from torch.distributions.multivariate_normal import MultivariateNormal\n",
    "from utils import *\n",
    "from models.fc import *\n",
    "from models.wide_resnet import wide_resnet_t\n",
    "from models.wide_resnet_1 import WideResNet\n",
    "from models.all_cnn import allcnn_t\n",
    "from models.lenet import lenet\n",
    "import scipy\n",
    "from scipy.linalg import eigh_tridiagonal\n",
    "from functions import *\n",
    "from scipy.optimize import minimize\n",
    "from dataset import *\n",
    "from torch.optim.lr_scheduler import CosineAnnealingLR\n",
    "from backpack import extend, backpack\n",
    "from backpack.extensions import (\n",
    "    KFAC,\n",
    "    KFLR,\n",
    "    KFRA,\n",
    ")\n",
    "\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "# %aimport functions\n",
    "# %aimport analyze\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./lenet_55000_0_mnist/\n",
      "44426\n"
     ]
    }
   ],
   "source": [
    "# prepare dataset\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if device == 'cuda' else {}\n",
    "\n",
    "# dataset\n",
    "num_true = 55000\n",
    "num_prior = 5000\n",
    "num_random = 0\n",
    "num_approx = 10000\n",
    "num_classes = 2\n",
    "dataset = \"mnist\"\n",
    "num_inplanes = 1 if dataset == \"mnist\" else 3\n",
    "\n",
    "# fully connected\n",
    "num_neurons = 600\n",
    "num_layers = 2\n",
    "\n",
    "# wide_resnet\n",
    "depth = 10\n",
    "num_channels = 4\n",
    "widen_factor = 8\n",
    "drop_out = 0\n",
    "\n",
    "# all_cnn\n",
    "c1, c2 = 64, 128\n",
    "\n",
    "\n",
    "\n",
    "model_name = \"lenet\"\n",
    "if model_name == \"fc\":\n",
    "    args = (num_classes, num_layers, num_neurons)\n",
    "if model_name == \"wr\":\n",
    "    args = (depth, num_inplanes, num_classes, num_channels, widen_factor, drop_out)\n",
    "    num_classes = 10\n",
    "if model_name == \"all_cnn\":\n",
    "    args = (num_classes, c1, c2)\n",
    "    num_classes = 10\n",
    "if model_name == \"lenet\":\n",
    "    args = ()\n",
    "    num_classes = 10\n",
    "    \n",
    "path = create_path(model_name, args, num_true, num_random, dataset)\n",
    "print(path)\n",
    "mkdir(path)\n",
    "\n",
    "mc = get_model_class(model_name)\n",
    "model = mc(*args).to(device)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "num_params = sum(param.numel() for param in model.parameters())\n",
    "print(num_params)\n",
    "\n",
    "if dataset == \"mnist\":\n",
    "    train_set, train_set_prior, train_set_approx, test_set = create_mnist(num_classes, num_true, num_prior, num_approx)\n",
    "\n",
    "if dataset == \"cifar10\":\n",
    "    train_set, train_set_prior, train_set_approx, test_set = create_cifar(num_classes, num_true, num_prior, num_approx)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_set,\n",
    "                                          batch_size=500,\n",
    "                                          shuffle = True,\n",
    "                                          **kwargs)\n",
    "\n",
    "train_loader_approx = torch.utils.data.DataLoader(train_set_approx,\n",
    "                                          batch_size=len(train_set_approx.data),\n",
    "                                          shuffle = True,\n",
    "                                          **kwargs)\n",
    "\n",
    "train_loader_FIM = torch.utils.data.DataLoader(train_set_approx,\n",
    "                                          batch_size=1,\n",
    "                                          shuffle = True,\n",
    "                                          **kwargs)\n",
    "\n",
    "train_loader_prior = torch.utils.data.DataLoader(train_set_prior,\n",
    "                                          batch_size=len(train_set_prior.data),\n",
    "                                          shuffle = True,\n",
    "                                          **kwargs)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_set,\n",
    "                                         batch_size=500,\n",
    "                                         shuffle = True,\n",
    "                                         **kwargs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial tr_loss 2.3096226583827626 val_loss 2.309465253353119 tr_err 0.9239272727272727 val_err 0.9274\n",
      "0 tr_loss 0.22986032949252563 val_loss 0.21880549862980841 tr_err 0.06652727272727277 val_err 0.06420000000000003\n",
      "lr 0.0019907662554463535\n",
      "1 tr_loss 0.12806302579966458 val_loss 0.12461539395153523 tr_err 0.037890909090909086 val_err 0.03510000000000002\n",
      "lr 0.0019632923872213654\n",
      "2 tr_loss 0.08269446793604981 val_loss 0.08448399938642978 tr_err 0.024945454545454493 val_err 0.025399999999999978\n",
      "lr 0.001918254893141276\n",
      "3 tr_loss 0.06489361607554284 val_loss 0.06965646538883448 tr_err 0.019290909090909136 val_err 0.02059999999999995\n",
      "lr 0.001856762745781211\n",
      "4 tr_loss 0.048345967229794375 val_loss 0.060224321112036704 tr_err 0.014618181818181863 val_err 0.018100000000000005\n",
      "lr 0.001780330085889911\n",
      "5 tr_loss 0.04417338308624246 val_loss 0.05725962612777948 tr_err 0.013472727272727303 val_err 0.018199999999999994\n",
      "lr 0.0016908389392193551\n",
      "6 tr_loss 0.036107481728223235 val_loss 0.050530843622982505 tr_err 0.011272727272727323 val_err 0.015900000000000025\n",
      "lr 0.0015904928748046603\n",
      "7 tr_loss 0.02993605224076997 val_loss 0.04967172713950276 tr_err 0.008363636363636351 val_err 0.015100000000000002\n",
      "lr 0.0014817627457812108\n",
      "8 tr_loss 0.024362098264761947 val_loss 0.0464993042871356 tr_err 0.006927272727272782 val_err 0.014900000000000024\n",
      "lr 0.0013673258487801733\n",
      "9 tr_loss 0.02169905286282301 val_loss 0.04617901286110282 tr_err 0.006054545454545424 val_err 0.014599999999999946\n",
      "lr 0.00125\n",
      "10 tr_loss 0.018573912944306027 val_loss 0.044488367065787314 tr_err 0.005127272727272758 val_err 0.014399999999999968\n",
      "lr 0.0011326741512198272\n",
      "11 tr_loss 0.016004501719197088 val_loss 0.04046708857640624 tr_err 0.003909090909090862 val_err 0.013499999999999956\n",
      "lr 0.0010182372542187896\n",
      "12 tr_loss 0.013621319889683615 val_loss 0.04074127441272139 tr_err 0.0033454545454545404 val_err 0.013800000000000034\n",
      "lr 0.00090950712519534\n",
      "13 tr_loss 0.012040171590210362 val_loss 0.04215720258653164 tr_err 0.0027454545454544954 val_err 0.013800000000000034\n",
      "lr 0.0008091610607806454\n",
      "14 tr_loss 0.010618969787504862 val_loss 0.03934005494229496 tr_err 0.002454545454545487 val_err 0.012800000000000034\n",
      "lr 0.0007196699141100895\n",
      "15 tr_loss 0.010098245261575689 val_loss 0.04121265020221472 tr_err 0.002036363636363614 val_err 0.013399999999999967\n",
      "lr 0.0006432372542187895\n",
      "16 tr_loss 0.00864567907569422 val_loss 0.0396347348112613 tr_err 0.0014909090909090983 val_err 0.012499999999999956\n",
      "lr 0.0005817451068587242\n",
      "17 tr_loss 0.007415194794620303 val_loss 0.04044843064621091 tr_err 0.0011818181818181728 val_err 0.012800000000000034\n",
      "lr 0.0005367076127786349\n",
      "18 tr_loss 0.006773986565795811 val_loss 0.03800670781638473 tr_err 0.0011818181818181728 val_err 0.012299999999999978\n",
      "lr 0.0005092337445536468\n",
      "19 tr_loss 0.0063445248662240125 val_loss 0.03917480418458581 tr_err 0.0009636363636363887 val_err 0.012700000000000045\n",
      "lr 0.0005\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "###### training ##########\n",
    "##########################\n",
    "epochs = 20\n",
    "torch.save(model.state_dict(), path + \"model_init.pt\")\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr = 2e-3)\n",
    "scheduler = CosineAnnealingLR(optimizer,T_max=epochs, eta_min = 5e-4)\n",
    "\n",
    "tr_err, tr_loss = val(model, device, train_loader, criterion)\n",
    "val_err, val_loss = val(model, device, test_loader, criterion)\n",
    "print(\"initial\", 'tr_loss', tr_loss, 'val_loss', val_loss,  'tr_err', tr_err, 'val_err', val_err)\n",
    "\n",
    "\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    train(model, device, train_loader, criterion, optimizer, epoch)\n",
    "    \n",
    "    tr_err, tr_loss = val(model, device, train_loader, criterion)\n",
    "    val_err, val_loss = val(model, device, test_loader, criterion)\n",
    "\n",
    "    # using cosine annealing scheduler\n",
    "    scheduler.step()\n",
    "\n",
    "    print(epoch, 'tr_loss', tr_loss, 'val_loss', val_loss,  'tr_err', tr_err, 'val_err', val_err)\n",
    "    print(\"lr\", optimizer.param_groups[0]['lr'])\n",
    "\n",
    "torch.save(model.state_dict(), path + \"model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diff_norm_squared tensor(435.3404, device='cuda:0', grad_fn=<PowBackward0>)\n",
      "initial tr_loss 0.006344524988989261 val_loss 0.039174803718924525 tr_err 0.0009636363636363887 val_err 0.012700000000000045\n",
      "207.2321014404297\n",
      "0 tr_loss 0.006344524988989261 val_loss 0.04249177714809775 tr_err 0.0009636363636363887 val_err 0.013599999999999945\n",
      "lr 0.0009939057285945933 decay 0.0012\n",
      "136.79022216796875\n",
      "1 tr_loss 0.006344524988989261 val_loss 0.05051651634275913 tr_err 0.0009636363636363887 val_err 0.013900000000000023\n",
      "lr 0.0009757729755661012 decay 0.0011926868743135119\n",
      "110.39193725585938\n",
      "2 tr_loss 0.006344524988989261 val_loss 0.054887103475630285 tr_err 0.0009636363636363887 val_err 0.016700000000000048\n",
      "lr 0.0009460482294732422 decay 0.0011709275706793213\n",
      "99.6040267944336\n",
      "3 tr_loss 0.006344524988989261 val_loss 0.05519464053213596 tr_err 0.0009636363636363887 val_err 0.01629999999999998\n",
      "lr 0.0009054634122155991 decay 0.0011352578753678906\n",
      "94.70510864257812\n",
      "4 tr_loss 0.006344524988989261 val_loss 0.05578609444200992 tr_err 0.0009636363636363887 val_err 0.01629999999999998\n",
      "lr 0.0008550178566873411 decay 0.0010865560946587188\n",
      "92.58212280273438\n",
      "5 tr_loss 0.006344524988989261 val_loss 0.05466706845909357 tr_err 0.0009636363636363887 val_err 0.015599999999999947\n",
      "lr 0.0007959536998847743 decay 0.0010260214280248092\n",
      "91.74844360351562\n",
      "6 tr_loss 0.006344524988989261 val_loss 0.0509987335652113 tr_err 0.0009636363636363887 val_err 0.015199999999999991\n",
      "lr 0.0007297252973710758 decay 0.0009551444398617292\n",
      "92.269775390625\n",
      "7 tr_loss 0.006344524988989261 val_loss 0.0506030336022377 tr_err 0.0009636363636363887 val_err 0.014100000000000001\n",
      "lr 0.0006579634122155991 decay 0.0008756703568452909\n",
      "93.28910064697266\n",
      "8 tr_loss 0.006344524988989261 val_loss 0.04647498168051243 tr_err 0.0009636363636363887 val_err 0.013499999999999956\n",
      "lr 0.0005824350601949144 decay 0.000789556094658719\n",
      "94.70110321044922\n",
      "9 tr_loss 0.006344524988989261 val_loss 0.04412130787968636 tr_err 0.0009636363636363887 val_err 0.01319999999999999\n",
      "lr 0.0005050000000000001 decay 0.0006989220722338973\n",
      "96.93086242675781\n",
      "10 tr_loss 0.006344524988989261 val_loss 0.043285609036684033 tr_err 0.0009636363636363887 val_err 0.012800000000000034\n",
      "lr 0.000427564939805086 decay 0.0006060000000000001\n",
      "99.17293548583984\n",
      "11 tr_loss 0.006344524988989261 val_loss 0.04146424746140838 tr_err 0.0009636363636363887 val_err 0.013299999999999979\n",
      "lr 0.0003520365877844012 decay 0.0005130779277661031\n",
      "101.40798950195312\n",
      "12 tr_loss 0.006344524988989261 val_loss 0.039168785139918325 tr_err 0.0009636363636363887 val_err 0.012499999999999956\n",
      "lr 0.00028027470262892445 decay 0.00042244390534128143\n",
      "103.82876586914062\n",
      "13 tr_loss 0.006344524988989261 val_loss 0.03758167633786798 tr_err 0.0009636363636363887 val_err 0.011499999999999955\n",
      "lr 0.0002140463001152259 decay 0.00033632964315470935\n",
      "106.00387573242188\n",
      "14 tr_loss 0.006344524988989261 val_loss 0.03599430052563548 tr_err 0.0009636363636363887 val_err 0.011099999999999999\n",
      "lr 0.00015498214331265904 decay 0.00025685556013827106\n",
      "107.88545227050781\n",
      "15 tr_loss 0.006344524988989261 val_loss 0.03474368816241622 tr_err 0.0009636363636363887 val_err 0.011099999999999999\n",
      "lr 0.00010453658778440108 decay 0.00018597857197519084\n",
      "109.36643981933594\n",
      "16 tr_loss 0.006344524988989261 val_loss 0.03382088243961334 tr_err 0.0009636363636363887 val_err 0.010399999999999965\n",
      "lr 6.395177052675796e-05 decay 0.00012544390534128128\n",
      "110.37709045410156\n",
      "17 tr_loss 0.006344524988989261 val_loss 0.033331524766981604 tr_err 0.0009636363636363887 val_err 0.010399999999999965\n",
      "lr 3.422702443389901e-05 decay 7.674212463210954e-05\n",
      "110.9874496459961\n",
      "18 tr_loss 0.006344524988989261 val_loss 0.03327103340998292 tr_err 0.0009636363636363887 val_err 0.010499999999999954\n",
      "lr 1.6094271405406862e-05 decay 4.1072429320678816e-05\n",
      "111.2977523803711\n",
      "19 tr_loss 0.006344524988989261 val_loss 0.03306981166824698 tr_err 0.0009636363636363887 val_err 0.010600000000000054\n",
      "lr 1e-05 decay 1.9313125686488235e-05\n"
     ]
    }
   ],
   "source": [
    "\n",
    "##################################\n",
    "##### decay model ################\n",
    "##################################\n",
    "\n",
    "if model_name == \"fc\" or \"lenet\":\n",
    "\n",
    "    epochs = 20\n",
    "\n",
    "    model_init = mc(*args)\n",
    "    model_init.load_state_dict(torch.load(path + \"model_init.pt\", map_location='cpu'))\n",
    "    model_ana = mc(*args)\n",
    "    model_ana.load_state_dict(torch.load(path + \"model.pt\", map_location='cpu'))\n",
    "    model_init, model_ana = model_init.to(device), model_ana.to(device)\n",
    "\n",
    "    optimizer = optim.Adam(model_ana.parameters(), lr = 1e-3)\n",
    "    scheduler = CosineAnnealingLR(optimizer,T_max=epochs, eta_min = 1e-5)\n",
    "\n",
    "    diff = diff_norm(list(model_ana.parameters()), list(model_init.parameters()))\n",
    "    print(\"diff_norm_squared\", diff**2)\n",
    "\n",
    "    tr_err, tr_loss = val(model_ana, device, train_loader, criterion)\n",
    "    val_err, val_loss = val(model_ana, device, test_loader, criterion)\n",
    "    print(\"initial\", 'tr_loss', tr_loss, 'val_loss', val_loss,  'tr_err', tr_err, 'val_err', val_err)\n",
    "\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "\n",
    "        decay = 1.2*optimizer.param_groups[0]['lr']\n",
    "\n",
    "        train_decay(model_ana, model_init, decay, device, train_loader, criterion, optimizer, epoch) \n",
    "        tr_err_true, tr_loss_true = val(model_ana, device, train_loader, criterion)\n",
    "        val_err, val_loss = val(model_ana, device, test_loader, criterion)\n",
    "\n",
    "        # using cosine annealing scheduler\n",
    "        scheduler.step()\n",
    "\n",
    "        print(epoch, 'tr_loss', tr_loss, 'val_loss', val_loss,  'tr_err', tr_err, 'val_err', val_err)\n",
    "        print(\"lr\", optimizer.param_groups[0]['lr'], \"decay\", decay)\n",
    "\n",
    "    torch.save(model_ana.state_dict(), path + \"model_ana.pt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_params': 44426, 'te_tl_ve_vl': (0.0009636363636363887, 0.006344524861990728, 0.012700000000000045, 0.03917480600066483), 'tea, tla, vea, vla': (0.007490909090909104, 0.028154844096438453, 0.010600000000000054, 0.033069811388850213), 'norm_init, norm_trained, norm_ana': (8.887020111083984, 23.712791442871094, 14.88017749786377), 'diff_trained, diff_ana': (20.86481285095215, 10.549897193908691)}\n"
     ]
    }
   ],
   "source": [
    "##########################################\n",
    "##### recap of training statistics #######\n",
    "##########################################\n",
    "\n",
    "model = mc(*args)\n",
    "model.load_state_dict(torch.load(path + \"model.pt\", map_location='cpu'))\n",
    "model_init = mc(*args)\n",
    "model_init.load_state_dict(torch.load(path + \"model_init.pt\", map_location='cpu'))\n",
    "model, model_init = model.to(device), model_init.to(device)\n",
    "\n",
    "num_params = sum(param.numel() for param in model.parameters())\n",
    "norm_init = norm_2(model_init)\n",
    "norm_trained = norm_2(model)\n",
    "diff = diff_norm(list(model.parameters()), list(model_init.parameters()))\n",
    "\n",
    "tr_err, tr_loss = val(model, device, train_loader, criterion)\n",
    "val_err, val_loss = val(model, device, test_loader, criterion)\n",
    "stat_models = dict({\"num_params\":num_params, \"te_tl_ve_vl\":(tr_err, tr_loss, val_err, val_loss), \"norm_init, norm_trained\": (norm_init.item(), norm_trained.item()), \"diff_trained\": (diff.item())})\n",
    "\n",
    "if model_name == \"fc\" or \"lenet\":\n",
    "    model_ana = mc(*args)\n",
    "    model_ana.load_state_dict(torch.load(path + \"model_ana.pt\", map_location='cpu'))\n",
    "    model_ana = model_ana.to(device)\n",
    "    norm_ana = norm_2(model_ana)\n",
    "    diff_ana = diff_norm(list(model_ana.parameters()), list(model_init.parameters()))\n",
    "    tr_err_ana, tr_loss_ana = val(model_ana, device, train_loader, criterion)\n",
    "    val_err_ana, val_loss_ana = val(model_ana, device, test_loader, criterion)\n",
    "    stat_models = dict({\"num_params\":num_params, \"te_tl_ve_vl\":(tr_err, tr_loss, val_err, val_loss), \"tea, tla, vea, vla\":(tr_err_ana, tr_loss_ana, val_err_ana, val_loss_ana), \"norm_init, norm_trained, norm_ana\": (norm_init.item(), norm_trained.item(), norm_ana.item()), \"diff_trained, diff_ana\": (diff.item(), diff_ana.item())})\n",
    "\n",
    "torch.save(stat_models, path + \"stat_models.pt\")\n",
    "print(stat_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#################################################\n",
    "###### kfac of FIM at init and Hess at end ######\n",
    "###### and their top eigen values and vecs ######\n",
    "#################################################\n",
    "\n",
    "if model_name == \"fc\":\n",
    "    kfac_list_init = FIM_kfac(model_init, train_loader_prior, mc=500, device=\"cpu\", mode = \"kfra\", empirical = True)\n",
    "    eigspace_list_init, eigval_list_init = eigspace_FIM_kron(kfac_list_init)\n",
    "\n",
    "    kfac_list_end = FIM_kfac(model, train_loader_approx, mc=500, device=\"cpu\", mode = \"kfra\", empirical = True)\n",
    "    eigspace_list_end, eigval_list_end = eigspace_FIM_kron(kfac_list_end)\n",
    "\n",
    "    torch.save((kfac_list_init, eigspace_list_init, eigval_list_init), path + \"kfac_all_init.pt\")\n",
    "    torch.save((kfac_list_end, eigspace_list_end, eigval_list_end), path + \"kfac_all_end.pt\")\n",
    "\n",
    "\n",
    "    kfac_val_init, kfac_vec_init = kfac_top_eigvec(kfac_list_init, model, 2000)\n",
    "    kfac_val_end, kfac_vec_end = kfac_top_eigvec(kfac_list_end, model, 2000)\n",
    "\n",
    "    torch.save((kfac_val_init, kfac_vec_init), path + \"kfac_init.pt\")\n",
    "    torch.save((kfac_val_end, kfac_vec_end), path + \"kfac_end.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################\n",
    "###### calculating eigen values of ###########################\n",
    "######  logit Jacobian #######################################\n",
    "##############################################################\n",
    "\n",
    "model = mc(*args)\n",
    "model.load_state_dict(torch.load(path + \"model.pt\", map_location='cpu'))\n",
    "model = model.to(device)\n",
    "\n",
    "\n",
    "k = 1000\n",
    "FIM_logit, L_logit, u = logit_jacobian(model, 0, criterion, train_loader_FIM, 'cpu', k)\n",
    "torch.save((FIM_logit, L_logit, u), path + \"FIM_logit_end.pt\")\n",
    "torch.save(L_logit, path + \"stat_logit.pt\")\n",
    "\n",
    "L_logit_list = []\n",
    "for i in range(num_classes):\n",
    "    FIM_logit, L_logit, u = logit_jacobian(model, i, criterion, train_loader_FIM, 'cpu', k)\n",
    "    L_logit_list.append(L_logit)\n",
    "\n",
    "torch.save(L_logit_list, path + \"stat_logit_all.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_31053/1389885423.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFIM_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mL_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mu_i\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"FIM_true_init.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFIM_tt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mL_tt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mu_tt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"FIM_true_end.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFIM_te\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mL_te\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mu_te\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"FIM_em_end.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(obj, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization)\u001b[0m\n\u001b[1;32m    377\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_use_new_zipfile_serialization\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0m_open_zipfile_writer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_zipfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 379\u001b[0;31m                 \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopened_zipfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    380\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m         \u001b[0m_legacy_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_save\u001b[0;34m(obj, zip_file, pickle_module, pickle_protocol)\u001b[0m\n\u001b[1;32m    482\u001b[0m     \u001b[0mpickler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    483\u001b[0m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersistent_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 484\u001b[0;31m     \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    485\u001b[0m     \u001b[0mdata_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_buf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m     \u001b[0mzip_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_record\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mpersistent_id\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    459\u001b[0m     \u001b[0mid_map\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 461\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    462\u001b[0m         \u001b[0;31m# FIXME: the docs say that persistent_id should only return a string\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m         \u001b[0;31m# but torch store returns tuples. This works only in the binary protocol\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "##############################################################\n",
    "###### calculating eigen values of ###########################\n",
    "######  true FIM at init, true FIM at the end of training ####\n",
    "######   empirical FIM end ###################################\n",
    "##############################################################\n",
    "\n",
    "\n",
    "\n",
    "model = mc(*args)\n",
    "model.load_state_dict(torch.load(path + \"model.pt\", map_location='cpu'))\n",
    "model_init = mc(*args)\n",
    "model_init.load_state_dict(torch.load(path + \"model_init.pt\", map_location='cpu'))\n",
    "model, model_init= model.to(device), model_init.to(device)\n",
    "\n",
    "FIM_i, L_i, u_i = FIM_true(model_init, criterion, train_loader_FIM, \"cpu\", 800)\n",
    "FIM_tt, L_tt, u_tt = FIM_true(model, criterion, train_loader_FIM, \"cpu\", 800)\n",
    "FIM_te, L_te, u_te = FIM2(model, criterion, train_loader_FIM, \"cpu\", 800)\n",
    "\n",
    "torch.save((FIM_i, L_i, u_i), path + \"FIM_true_init.pt\")\n",
    "torch.save((FIM_tt, L_tt, u_tt), path + \"FIM_true_end.pt\")\n",
    "torch.save((FIM_te, L_te, u_te), path + \"FIM_em_end.pt\")\n",
    "\n",
    "\n",
    "if model_name == \"fc\" or \"lenet\":\n",
    "    model_ana = mc(*args)\n",
    "    model_ana.load_state_dict(torch.load(path + \"model_ana.pt\", map_location='cpu'))\n",
    "    model_ana = model_ana.to(device)\n",
    "    FIM_tt_ana, L_tt_ana, u_tt_ana = FIM_true(model_ana, criterion, train_loader_FIM, \"cpu\", 800)\n",
    "    FIM_te_ana, L_te_ana, u_te_ana = FIM2(model_ana, criterion, train_loader_FIM, \"cpu\", 800)\n",
    "    torch.save((FIM_tt_ana, L_tt_ana, u_tt_ana), path + \"FIM_true_end_ana.pt\")\n",
    "    torch.save((FIM_te_ana, L_te_ana, u_te_ana), path + \"FIM_em_end_ana.pt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([800])\n"
     ]
    }
   ],
   "source": [
    "L, eig, u = torch.load(path + \"FIM_true_init.pt\")\n",
    "print(eig.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0., device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.3171, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(1.3722, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(1.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.6625, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.7013, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.6781, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.4989, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.3307, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.5343, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2887, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2682, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2774, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.3945, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2921, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2127, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.2363, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1796, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1772, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1317, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1274, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1290, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1175, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1190, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1093, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1323, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1003, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1015, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0892, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0954, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0841, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0791, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1013, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.1054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0745, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0738, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0805, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0864, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0697, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0685, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0753, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0675, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0614, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0622, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0638, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0569, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0563, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0597, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0591, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0617, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0612, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0653, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0648, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0553, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0513, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0559, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0478, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0508, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0577, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0479, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0578, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0520, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0455, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0408, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0441, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0445, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0436, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0456, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0423, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0447, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0423, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0392, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0390, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0408, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0401, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0460, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0445, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0473, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0427, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0434, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0416, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0369, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0390, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0374, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0366, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0391, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0374, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0334, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0324, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0305, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0316, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0388, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0360, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0335, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0372, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0365, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0342, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0279, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0266, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0321, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0419, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0346, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0289, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0269, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0313, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0288, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0290, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0266, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0282, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0282, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0282, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0278, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0247, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0246, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0307, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0278, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0267, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0261, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0261, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0247, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0257, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0251, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0267, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0285, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0266, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0261, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0280, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0247, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0238, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0244, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0259, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0233, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0255, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0249, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0240, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0219, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0235, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0226, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0217, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0228, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0209, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0206, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0222, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0197, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0198, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0222, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0225, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0214, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0245, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0242, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0200, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0190, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0201, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0195, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0214, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0197, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0189, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0191, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0180, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0170, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0182, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0184, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0180, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0184, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0195, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0210, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0191, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0189, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0168, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0179, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0191, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0197, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0185, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0179, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0179, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0187, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0203, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0188, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0181, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0174, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0175, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0172, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0173, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0173, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0169, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0178, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0187, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0177, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0167, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0164, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0169, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0171, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0163, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0147, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0138, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0172, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0176, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0162, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0152, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0160, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0158, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0159, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0168, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0165, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0167, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0159, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0148, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0159, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0163, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0151, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0164, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0159, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0145, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0161, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0159, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0147, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0152, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0151, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0136, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0137, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0150, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0150, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0142, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0141, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0134, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0137, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0142, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0138, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0140, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0141, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0153, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0141, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0136, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0138, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0132, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0136, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0138, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0126, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0132, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0142, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0139, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0142, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0134, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0131, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0135, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0140, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0134, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0136, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0133, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0133, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0140, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0135, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0128, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0118, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0117, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0116, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0119, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0124, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0120, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0124, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0129, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0135, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0133, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0126, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0126, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0126, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0124, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0122, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0126, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0128, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0117, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0122, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0122, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0116, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0117, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0130, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0124, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0125, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0124, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0115, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0112, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0116, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0118, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0118, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0114, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0111, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0110, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0104, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0101, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0099, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0107, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0113, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0107, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0114, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0108, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0108, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0110, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0113, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0119, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0105, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0108, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0112, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0106, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0104, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0108, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0106, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0106, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0104, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0107, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0113, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0116, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0101, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0104, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0108, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0100, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0098, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0098, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0102, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0105, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0103, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0101, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0104, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0109, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0105, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0102, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0098, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0096, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0100, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0096, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0096, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0098, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0094, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0092, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0088, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0092, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0094, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0096, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0086, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0092, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0096, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0094, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0093, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0094, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0091, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0087, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0098, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0097, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0089, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0089, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0084, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0089, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0087, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0090, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0091, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0092, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0095, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0089, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0081, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0084, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0091, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0091, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0091, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0084, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0085, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0080, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0080, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0080, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0081, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0080, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0083, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0082, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0078, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0078, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0079, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0077, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0071, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0075, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0071, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0075, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0081, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0078, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0070, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0074, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0072, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0073, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0076, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0070, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0070, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0070, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0071, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0069, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0067, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0065, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0068, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0066, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0059, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0063, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0059, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0059, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0064, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0061, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0059, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0059, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0062, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0060, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0057, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0058, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0055, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0056, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0054, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0051, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0053, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0052, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0050, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0049, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0048, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0046, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0047, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0045, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0044, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0043, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0042, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0041, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0039, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0040, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0037, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0036, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0038, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0035, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0034, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0033, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0032, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0031, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0030, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0029, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0027, device='cuda:0')\n",
      "vhp iteration tensor(0.0063, device='cuda:0') tensor(0.0028, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "\n",
    "###############################################\n",
    "###### calculating eigen values ###############\n",
    "###### of hess using scipy.sparse #############\n",
    "###############################################\n",
    "\n",
    "\n",
    "k = 500\n",
    "eigenvalues, eigenvectors = hess_scipy(model, k, train_loader_approx, criterion, device)\n",
    "torch.save((eigenvalues, eigenvectors), path + \"eig_hess_scipy.pt\")\n",
    "\n",
    "if model_name == \"fc\" or \"lenet\":\n",
    "    eigenvalues, eigenvectors = hess_scipy(model_ana, k, train_loader_approx, criterion, device)\n",
    "    torch.save((eigenvalues, eigenvectors), path + \"eig_hess_ana_scipy.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################################\n",
    "###### Overlap of FIM at init and end of training #####\n",
    "#######################################################\n",
    "\n",
    "\n",
    "FIM_i, L_i, u_i = torch.load(path + \"FIM_true_init.pt\")\n",
    "FIM_tt, L_tt, u_tt = torch.load(path + \"FIM_true_end.pt\")\n",
    "FIM_te, L_te, u_te = torch.load(path + \"FIM_em_end.pt\")\n",
    "eig_hess, u_hess = torch.load(path + \"eig_hess_scipy.pt\")\n",
    "\n",
    "over_i_tt = overlap(u_i, u_tt, 500, \"cpu\")\n",
    "over_te_hess = overlap(u_te, u_hess, 500, \"cpu\")\n",
    "\n",
    "diff = list_to_vec(diff_list(list(model.parameters()), list(model_init.parameters())))\n",
    "frac_all = proj(diff, u_i, 1000, \"cpu\")\n",
    "\n",
    "stat1 = dict({\"FIM_init\":L_i, \"FIM_tt\":L_tt, \"FIM_te\":L_te, \"eig_hess\":eig_hess})\n",
    "stat2 = dict({\"over_i_tt\":over_i_tt, \"over_te_hess\":over_te_hess})\n",
    "stat3 = dict({\"frac_all\":frac_all})\n",
    "\n",
    "\n",
    "if model_name == \"fc\":\n",
    "    FIM_tt_ana, L_tt_ana, u_tt_ana = torch.load(path + \"FIM_true_end_ana.pt\")\n",
    "    eig_hess_ana, u_hess_ana = torch.load(path + \"eig_hess_ana_scipy.pt\")\n",
    "    kfac_eig_val_init, kfac_eig_vec_init = torch.load(path + \"kfac_init.pt\")\n",
    "    kfac_eig_val_end, kfac_eig_vec_end = torch.load(path + \"kfac_end.pt\")\n",
    "    over_i_tt_ana = overlap(u_i, u_tt_ana, 500, \"cpu\")\n",
    "    over_kfi_i = overlap(kfac_eig_vec_init, u_i, 500, \"cpu\")\n",
    "    over_kfe_hess = overlap(kfac_eig_vec_end, u_hess, 500, \"cpu\")\n",
    "    stat1 = dict({\"FIM_init\":L_i, \"FIM_tt\":L_tt, \"FIM_te\":L_te, \"FIM_tt_ana\":L_tt_ana, \"eig_hess\":eig_hess, \"eig_hess_ana\":eig_hess_ana , \"kfac_eig_val_init\": kfac_eig_val_init, \"kfac_eig_val_end\": kfac_eig_val_end})\n",
    "    stat2 = dict({\"over_i_tt\":over_i_tt,\"over_i_tt_ana\":over_i_tt_ana, \"over_te_hess\":over_te_hess, \"over_kfi_i\":over_kfi_i, \"over_kfe_hess\":over_kfe_hess})\n",
    "    diff_ana = list_to_vec(diff_list(list(model_ana.parameters()), list(model_init.parameters())))\n",
    "    frac_all_ana = proj(diff_ana, u_i, 1000, \"cpu\")\n",
    "    stat3 = dict({\"frac_all\":frac_all, \"frac_all_ana\":frac_all_ana})\n",
    "\n",
    "torch.save((stat1, stat2, stat3), path + \"stat_overlap.pt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "832802\n",
      "norm_diff tensor(8.0422, device='cuda:1', grad_fn=<SqrtBackward>)\n",
      "300\n",
      "37\n",
      "tr_loss_true_avg 0.25375252024693923 val_loss_avg 0.29349191740155217\n",
      "tr_err_true_avg 0.08788363636363636 val_err_avg 0.09372\n",
      "37 epsilon 808.946087201348 bd 0.5461233924913831 sec 0.4878810942173004 KL 26160.27734375\n",
      "KL1, KL2, third 0.0 52320.5546875 22.323504046087738\n",
      "{'lbd_bar': tensor([808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461,\n",
      "        808.9461, 808.9461, 808.9461, 808.9461, 808.9461, 808.9461],\n",
      "       device='cuda:1'), 'te_tl_ve_vl': (0.08788363636363636, 0.25375252024693923, 0.09372, 0.29349191740155217), 'epsilon_bd_kl_kl1_kl2': (808.946087201348, 0.5461233924913831, 26160.27734375, 0.0, 52320.5546875)}\n",
      "300\n",
      "38\n",
      "tr_loss_true_avg 0.22369188430634415 val_loss_avg 0.2589626793563366\n",
      "tr_err_true_avg 0.0774909090909091 val_err_avg 0.08312\n",
      "38 epsilon 894.0236898660163 bd 0.5581505454549518 sec 0.512874960899353 KL 28911.576171875\n",
      "KL1, KL2, third -6.103515625e-05 57823.15234375 22.37684054025206\n"
     ]
    }
   ],
   "source": [
    "##############################################\n",
    "####### analytical methods for posterior #####\n",
    "####### calculation ##########################\n",
    "##############################################\n",
    "\n",
    "\n",
    "if model_name == \"fc\" or \"lenet\":\n",
    "\n",
    "    def PB(lbd_bar,eig_hess, epsilon, norm_diff, b, c, delta):\n",
    "\n",
    "        # eig_hess, epsilon, norm_diff, b, c, delta = args\n",
    "        gap = np.sum(eig_hess / lbd_bar)\n",
    "\n",
    "        KL1 = np.sum(np.log(lbd_bar / epsilon) + epsilon / lbd_bar) - len(lbd_bar)\n",
    "        KL2 = epsilon*(norm_diff**2)\n",
    "        KL = (KL1 + KL2) / 2\n",
    "        sec = np.sqrt((KL + 2*np.log(b*np.log(c*epsilon)) + np.log((num_true*np.pi**2) / (6*delta))) / (2*(num_true-1)))\n",
    "        PB = gap + sec\n",
    "\n",
    "        return PB\n",
    "\n",
    "\n",
    "    def cov(eig_hess, epsilon, norm_diff, b, c, delta):\n",
    "        norm_diff = norm_diff.cpu().detach().numpy()\n",
    "        eig_hess = eig_hess.cpu().detach().numpy()\n",
    "        args = (eig_hess, epsilon, norm_diff, b, c, delta)\n",
    "        lbd_bar = np.ones(len(eig_hess))\n",
    "        res = minimize(PB, lbd_bar, args = args, method = 'BFGS', tol=1e-6)\n",
    "\n",
    "        sol = torch.tensor(res.x)\n",
    "        sol = sol.float()\n",
    "\n",
    "        return sol\n",
    "\n",
    "\n",
    "    k = 300\n",
    "\n",
    "    if model_name == \"fc\":\n",
    "        eig_hess, u = torch.load(path + \"eig_hess_ana_scipy.pt\")\n",
    "        model_use = model_ana\n",
    "    if model_name == \"lenet\":\n",
    "        eig_hess, u = torch.load(path + \"eig_hess_scipy.pt\")\n",
    "        model_use = model\n",
    "\n",
    "    eig_hess, u = eig_hess[:k], u[:, :k]\n",
    "    eig_hess, u = ((1 / np.log(2))*eig_hess).to(device), u.to(device)\n",
    "\n",
    "\n",
    "\n",
    "    norm_diff = diff_norm(list(model_use.parameters()), list(model_init.parameters()))\n",
    "    mean = list_to_vec(list(model_use.parameters()))\n",
    "\n",
    "    print(\"norm_diff\" ,norm_diff)\n",
    "\n",
    "\n",
    "    n_total = 5\n",
    "    b = 10\n",
    "    c = 0.05\n",
    "    delta = 0.025\n",
    "    mode = \"analytic\"\n",
    "\n",
    "\n",
    "    for j in range(60):\n",
    "        j += 1\n",
    "        epsilon = (1/c)*np.exp(j/b)\n",
    "\n",
    "        if mode == \"analytic\":\n",
    "            lbd_bar = 4*(num_true-1)*eig_hess + epsilon\n",
    "\n",
    "        if mode == \"numeric\":\n",
    "            lbd_bar = cov(eig_hess, epsilon, norm_diff, b, c, delta)\n",
    "\n",
    "        if mode == \"isotropic\":\n",
    "            lbd_bar = torch.ones(eig_hess.shape) * epsilon\n",
    "            \n",
    "        lbd_bar = lbd_bar.to(device)\n",
    "        \n",
    "        KL_1 = (torch.sum(torch.log(lbd_bar/epsilon)) + torch.sum(epsilon/lbd_bar) - len(lbd_bar))\n",
    "        KL_2 = epsilon*(norm_diff**2)\n",
    "        KL = (KL_1 + KL_2) / 2\n",
    "        penalty = (np.log((np.pi)**2*(j**2)*num_true/(delta*6)))\n",
    "        sec = torch.sqrt((KL + penalty) / (2*(num_true-1)))\n",
    "\n",
    "\n",
    "        tr_err_true_sum, tr_loss_true_sum, val_err_sum, val_loss_sum = 0, 0, 0, 0\n",
    "\n",
    "        for k in range(n_total):\n",
    "            # print(j, k)\n",
    "\n",
    "            r = torch.randn(num_params).to(device)\n",
    "            r1 = u.T@r\n",
    "            r1 = (u * (1/torch.sqrt(lbd_bar))) @ r1\n",
    "            r2_com = u @ (u.T@r)\n",
    "            r2 = (r - r2_com) / np.sqrt(epsilon)\n",
    "\n",
    "            param_tuple = tuple(vec_to_list((r1+r2+mean), model_use))\n",
    "            for (p1, p2) in zip(model_use.parameters(), param_tuple):\n",
    "                p1.data = p2.data\n",
    "\n",
    "            tr_err_true, tr_loss_true = val(model_use, device, train_loader, criterion)\n",
    "            val_err, val_loss = val(model_use, device, test_loader, criterion)\n",
    "\n",
    "            tr_err_true_sum += tr_err_true\n",
    "            tr_loss_true_sum += tr_loss_true\n",
    "            val_err_sum += val_err\n",
    "            val_loss_sum += val_loss\n",
    "\n",
    "\n",
    "\n",
    "        tr_err_true_avg = tr_err_true_sum / n_total\n",
    "        tr_loss_true_avg = (tr_loss_true_sum / n_total)\n",
    "        val_err_avg = val_err_sum / n_total\n",
    "        val_loss_avg = val_loss_sum / n_total\n",
    "\n",
    "        bd = approximate_BPAC_bound(1-tr_err_true_avg, sec.item())\n",
    "\n",
    "\n",
    "        print(j)\n",
    "        print('tr_loss_true_avg', tr_loss_true_avg, 'val_loss_avg', val_loss_avg)\n",
    "        print('tr_err_true_avg', tr_err_true_avg, 'val_err_avg', val_err_avg)\n",
    "\n",
    "        print(j, \"epsilon\", epsilon, \"bd\", bd, \"sec\", sec.item(), \"KL\", KL.item())\n",
    "        print(\"KL1, KL2, third\", KL_1.item(), KL_2.item(), penalty)\n",
    "\n",
    "\n",
    "        # if j == 37:\n",
    "        #     stat = dict({\"lbd_bar\":lbd_bar, \"te_tl_ve_vl\":(tr_err_true_avg, tr_loss_true_avg, val_err_avg, val_loss_avg), \"epsilon_bd_kl_kl1_kl2\":(epsilon, bd, KL.item(), KL_1.item(), KL_2.item())})\n",
    "        #     torch.save(stat, path + \"stat_\" + mode + \".pt\")\n",
    "        #     print(stat)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f9dc663c12342db218673069099c5f4d734584a05de3b02b725edaf289dd9612"
  },
  "kernelspec": {
   "display_name": "Python 3.7.12 64-bit ('pytorch_latest_p37': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}