pytorch优化对比.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from benchmark_funs import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "def evaluate(opti, p_ind=1000):\n",
    "    s = 0\n",
    "    for fun in benchmark_functions:\n",
    "        assert fun.larger_better == False # 最小化\n",
    "        interval_len = fun.ub - fun.lb\n",
    "        # X = nn.init.kaiming_uniform_(torch.empty(1000, fun.dimension))\n",
    "        # X = nn.init.trunc_normal_(torch.empty(1000, fun.dimension), fun.lb+interval_len/2, interval_len/6, a=fun.lb, b=fun.ub)\n",
    "        X = nn.init.kaiming_uniform_(torch.empty(p_ind, fun.dimension))\n",
    "        X = X.to(device)\n",
    "        X.requires_grad_(True)\n",
    "        optimizer = opti([X])\n",
    "        # box = tqdm.tqdm(range(1000), desc=f\"正在优化{fun.name}\")\n",
    "        box = tqdm.tqdm(range(fun.get_budget()//p_ind), desc=f\"正在优化{fun.name}\")\n",
    "        best_loss = 1e10\n",
    "        patience = 0\n",
    "        for i in box:\n",
    "            # X[X<fun.lb] = fun.lb\n",
    "            # X[X>fun.ub] = fun.ub\n",
    "            loss = fun(X).min() - fun.optival\n",
    "            # loss = fun(X).sum()\n",
    "            loss_item = loss.detach().numpy().item()\n",
    "            box.set_postfix(loss=loss_item)\n",
    "            if loss_item<best_loss:\n",
    "                patience = 0\n",
    "                best_loss = loss_item\n",
    "            else:\n",
    "                patience +=1\n",
    "                if patience>100:\n",
    "                    break\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            if X.grad.norm()<1e-8:\n",
    "                break\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "        s+=loss.detach().numpy().item()\n",
    "    return s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "正在优化Sphere:   1%|          | 361/30000 [00:01<01:33, 317.77it/s, loss=2.18e-17]\n",
      "正在优化Rosenbrock:  15%|█▌        | 4622/30000 [00:15<01:24, 300.83it/s, loss=0.0948]  \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_3562268/2379154280.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mopti\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopti\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_ind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/tmp/ipykernel_3562268/3977840881.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(opti, p_ind)\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0mbox\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_postfix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0;36m1e-8\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/team/package_managers/anaconda3/envs/torch2.0/lib/python3.9/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    261\u001b[0m                                                f\"but got {result}.\")\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m                 \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    264\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/team/package_managers/anaconda3/envs/torch2.0/lib/python3.9/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36m_use_grad\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'differentiable'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprev_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/team/package_managers/anaconda3/envs/torch2.0/lib/python3.9/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    141\u001b[0m                 state_steps)\n\u001b[1;32m    142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m             adam(params_with_grad,\n\u001b[0m\u001b[1;32m    144\u001b[0m                  \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m                  \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/team/package_managers/anaconda3/envs/torch2.0/lib/python3.9/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, differentiable, fused, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_single_tensor_adam\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m     func(params,\n\u001b[0m\u001b[1;32m    294\u001b[0m          \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m          \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/team/package_managers/anaconda3/envs/torch2.0/lib/python3.9/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36m_single_tensor_adam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable, differentiable)\u001b[0m\n\u001b[1;32m    401\u001b[0m                 \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sqs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction2_sqrt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 403\u001b[0;31m                 \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction2_sqrt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    404\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m             \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcdiv_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp_avg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdenom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstep_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "opti = lambda p: optim.Adam(p, lr=0.1)\n",
    "evaluate(opti, p_ind=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "正在优化Sphere:  30%|███       | 91/300 [00:00<00:00, 461.87it/s, loss=2.16e-17]\n",
      "正在优化Rosenbrock: 100%|██████████| 300/300 [00:00<00:00, 354.72it/s, loss=23.6]\n",
      "正在优化Step:   0%|          | 0/300 [00:00<?, ?it/s, loss=0]\n",
      "正在优化NoisyQuartic:  78%|███████▊  | 235/300 [00:00<00:00, 586.32it/s, loss=23.7] \n",
      "正在优化Schwefel226: 100%|██████████| 300/300 [00:00<00:00, 552.30it/s, loss=1.25e+4]\n",
      "正在优化Ackley: 100%|██████████| 300/300 [00:00<00:00, 700.40it/s, loss=0.0211]\n",
      "正在优化Schwefel222:  31%|███▏      | 94/300 [00:00<00:00, 737.86it/s, loss=1.64e-17]\n",
      "正在优化Holder: 100%|██████████| 20/20 [00:00<00:00, 823.19it/s, loss=17.5]\n",
      "正在优化Camel3: 100%|██████████| 20/20 [00:00<00:00, 714.58it/s, loss=1.78e-6]\n",
      "正在优化Michal: 100%|██████████| 20/20 [00:00<00:00, 758.59it/s, loss=0.843]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "12525.051821757954"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# opti = lambda p: optim.AdamW(p, lr=3e-1)\n",
    "opti = lambda p: optim.RMSprop(p, lr=1e-2)\n",
    "# evaluate(opti, p_ind=10)\n",
    "evaluate(opti, p_ind=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "正在优化Sphere: 100%|██████████| 1000/1000 [00:02<00:00, 379.85it/s, loss=0]      \n",
      "正在优化Rosenbrock: 100%|██████████| 1000/1000 [00:02<00:00, 382.24it/s, loss=2.54e+8]\n",
      "正在优化Step: 100%|██████████| 1000/1000 [00:01<00:00, 743.10it/s, loss=13872.0]\n",
      "正在优化NoisyQuartic: 100%|██████████| 1000/1000 [00:03<00:00, 319.81it/s, loss=3.11] \n",
      "正在优化Schwefel226: 100%|██████████| 1000/1000 [00:01<00:00, 582.55it/s, loss=8.45e+3]\n",
      "正在优化Ackley: 100%|██████████| 1000/1000 [00:01<00:00, 760.50it/s, loss=14.8]\n",
      "正在优化Schwefel222: 100%|██████████| 1000/1000 [00:01<00:00, 523.88it/s, loss=0]      \n",
      "正在优化Holder: 100%|██████████| 1000/1000 [00:02<00:00, 376.04it/s, loss=0.396]\n",
      "正在优化Camel3: 100%|██████████| 1000/1000 [00:03<00:00, 328.15it/s, loss=0]      \n",
      "正在优化Michal: 100%|██████████| 1000/1000 [00:03<00:00, 294.12it/s, loss=1.37] \n"
     ]
    }
   ],
   "source": [
    "opti = lambda p: optim.SGD(p, lr=0.1)\n",
    "evaluate(opti)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "正在优化Sphere: 100%|██████████| 1000/1000 [00:01<00:00, 530.79it/s, loss=9.05e+3]\n",
      "正在优化Rosenbrock: 100%|██████████| 1000/1000 [00:01<00:00, 611.18it/s, loss=7.33e+6]\n",
      "正在优化Step: 100%|██████████| 1000/1000 [00:01<00:00, 835.39it/s, loss=13376.0]\n",
      "正在优化NoisyQuartic: 100%|██████████| 1000/1000 [00:01<00:00, 592.47it/s, loss=22.9]\n",
      "正在优化Schwefel226: 100%|██████████| 1000/1000 [00:01<00:00, 774.13it/s, loss=-2.06e+3]\n",
      "正在优化Ackley: 100%|██████████| 1000/1000 [00:01<00:00, 697.71it/s, loss=14.5]\n",
      "正在优化Schwefel222: 100%|██████████| 1000/1000 [00:01<00:00, 664.02it/s, loss=93.6]\n",
      "正在优化Holder: 100%|██████████| 1000/1000 [00:02<00:00, 451.23it/s, loss=-9.99]\n",
      "正在优化Camel3: 100%|██████████| 1000/1000 [00:03<00:00, 327.15it/s, loss=1.15e-31]\n",
      "正在优化Michal: 100%|██████████| 1000/1000 [00:03<00:00, 290.30it/s, loss=-1.8]\n"
     ]
    }
   ],
   "source": [
    "opti = lambda p: optim.AdamW(p, lr=3e-4)\n",
    "evaluate(opti)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch2.0",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}