animatediff_lora_with_celeb.ipynb

{
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.13",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "accelerator": "GPU",
    "kaggle": {
      "accelerator": "gpu",
      "dataSources": [
        {
          "sourceId": 8495004,
          "sourceType": "datasetVersion",
          "datasetId": 5068797
        }
      ],
      "dockerImageVersionId": 30699,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abhijitpal1247/animatediff_exp/blob/main/animatediff_lora_with_celeb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash\n",
        "pip install -q peft\n",
        "pip install -q --upgrade transformers\n",
        "pip install -q diffusers\n",
        "pip install -q python-dotenv\n",
        "pip install -q openai==0.28\n",
        "mkdir hennessy_weights\n",
        "mkdir celebrity_weights\n",
        "wget -q https://civitai.com/api/download/models/319628 -O hennessy_weights/pytorch_lora.safetensors\n",
        "wget -q https://civitai.com/api/download/models/13560 -O celebrity_weights/pytorch_lora.safetensors"
      ],
      "metadata": {
        "id": "KHXBoM_9ifK4",
        "execution": {
          "iopub.status.busy": "2024-05-30T18:40:33.990261Z",
          "iopub.execute_input": "2024-05-30T18:40:33.990649Z",
          "iopub.status.idle": "2024-05-30T18:42:00.870190Z",
          "shell.execute_reply.started": "2024-05-30T18:40:33.990619Z",
          "shell.execute_reply": "2024-05-30T18:42:00.869028Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Enter product details { display-mode: \"form\" }\n",
        "\n",
        "from dotenv import load_dotenv\n",
        "\n",
        "load_dotenv(\"/kaggle/input/env-sample/.env\")\n",
        "\n",
        "celebrity = 'The Rock - Dwayne Johnson' # @param {type:\"string\"}\n",
        "company_name = 'hennessy' # @param {type:\"string\"}\n",
        "product = 'bottle' # @param {type:\"string\"}"
      ],
      "metadata": {
        "id": "HFCPnaWolBlY",
        "execution": {
          "iopub.status.busy": "2024-05-30T18:42:00.872469Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell gives a function for querying openai endpoint with backoff\n",
        "import os\n",
        "import time\n",
        "import random\n",
        "import openai\n",
        "\n",
        "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
        "\n",
        "# define a retry decorator\n",
        "def retry_with_exponential_backoff(\n",
        "    func,\n",
        "    initial_delay: float = 2,\n",
        "    exponential_base: float = 2,\n",
        "    jitter: bool = True,\n",
        "    max_retries: int = 30,\n",
        "    errors: tuple = (openai.error.RateLimitError, openai.error.APIError, openai.error.Timeout, openai.error.ServiceUnavailableError),\n",
        "):\n",
        "    \"\"\"Retry a function with exponential backoff.\"\"\"\n",
        "\n",
        "    def wrapper(*args, **kwargs):\n",
        "        # Initialize variables\n",
        "        num_retries = 0\n",
        "        delay = initial_delay\n",
        "\n",
        "        # Loop until a successful response or max_retries is hit or an exception is raised\n",
        "        while True:\n",
        "            try:\n",
        "                return func(*args, **kwargs)\n",
        "\n",
        "            # Retry on specific errors\n",
        "            except errors as e:\n",
        "                print(str(e))\n",
        "                # Increment retries\n",
        "                num_retries += 1\n",
        "\n",
        "                # Check if max retries has been reached\n",
        "                if num_retries > max_retries:\n",
        "                    raise Exception(\n",
        "                        f\"Maximum number of retries ({max_retries}) exceeded.\"\n",
        "                    )\n",
        "\n",
        "                # Increment the delay\n",
        "                delay *= exponential_base * (1 + jitter * random.random())\n",
        "\n",
        "                # Sleep for the delay\n",
        "                time.sleep(delay)\n",
        "\n",
        "            # Raise exceptions for any errors not specified\n",
        "            except Exception as e:\n",
        "                raise e\n",
        "\n",
        "    return wrapper\n",
        "\n",
        "@retry_with_exponential_backoff\n",
        "def completions_with_backoff(**kwargs):\n",
        "    return openai.ChatCompletion.create(**kwargs)"
      ],
      "metadata": {
        "id": "h_xTDlvPlBiC",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell gives a function for querying openai endpoint with multiple completions\n",
        "def get_openai_completion(model, max_tokens, temperature, n, messages,\n",
        "                          return_completions_only=False):\n",
        "    response= completions_with_backoff(model=model, max_tokens=max_tokens,\n",
        "                                       temperature=temperature, n=n,\n",
        "                                       messages=messages)\n",
        "\n",
        "    if return_completions_only:\n",
        "        continuations=[]\n",
        "        for res in response[\"choices\"]:\n",
        "            if \"message\" in res:\n",
        "                continuations.append(res[\"message\"][\"content\"])\n",
        "            else:\n",
        "                continuations.append(res[\"text\"])\n",
        "        return continuations\n",
        "    return response"
      ],
      "metadata": {
        "id": "nfgKMQX2lsis",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell creates script\n",
        "system_message = \"You are a storyboard writer. You write the story board descriptions for ads \"\n",
        "prompt = f\"\"\"provide a storyboard for ad for the product {product} of this company: {company_name}\n",
        "The ad will be a gif based video so our timeframe is very small.\n",
        "Instructions:\n",
        "1. Be as descriptive as possible\n",
        "2. Describe the background, the relative-product placement and every other minute detail\n",
        "3. Describe the entire set of actions that takes place in the ad video\n",
        "4. Keep the storyboard concise, only one line and is of max 40 words describing the entire ad\n",
        "5. Do not provide frame wise or scene wise description, provide overall idea or concept of ad\n",
        "6. It should just have small set of actions, so that it fits within the timeframe\n",
        "7. The video involves an actor/actress: {celebrity}\n",
        "9. You do not have any type of overlay available so you can not display logo\n",
        "10. You cannot display any tagline or any text\n",
        "11. There is no audio associated with the video\n",
        "12. You have a camera which can take a fixed aspect ratio video\n",
        "\n",
        "Storyboard:\"\"\"\n",
        "\n",
        "messages = [{\"role\": \"system\", \"content\": system_message}]\n",
        "messages.append({\"role\": \"user\", \"content\": prompt})\n",
        "completions = get_openai_completion(model=\"gpt-3.5-turbo\",\n",
        "                                    max_tokens=1000, n=1,\n",
        "                                    temperature=1.0, messages=messages,\n",
        "                                    return_completions_only=True)\n",
        "script = completions[0]"
      ],
      "metadata": {
        "id": "VZpZasYNlsfV",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell prints the script\n",
        "script"
      ],
      "metadata": {
        "id": "Tuw2do3ilBf3",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This cell creates prompt from script\n",
        "system_message = \"You are a prompt writter for text to video model. Given description you will write a prompt that will generate great video\"\n",
        "prompt = f\"\"\"provide the prompt for the following video description: {script}\n",
        "Instructions:\n",
        "    The focus of the prompt should be on : {product} of {company_name} company\n",
        "    Like shown in the example, describe the {product} and the relative action that is taking place in the video\n",
        "    keep the background description simple\n",
        "    keep the prompt size small like 30-40 word length, do not go beyond it\n",
        "    mention the actor/actress, their attire and the action they will be performing\n",
        "    keep it concise\n",
        "    Mention the company name along with the product\n",
        "    Follow the below examples and be creative\n",
        "\n",
        "Example:\n",
        "    1. (a person drinking from a bottle of coca cola, sitting on top of ice covered ground and water splashing with a red background and a red background), red background, simple background, english text,Best quality,masterpiece,ultra high res,\n",
        "    2. (a bottle of coca cola is being thrown into the air with ice cubes on the bottom of it), no humans, simple background, white background, still life, english text, coca-cola, bottle, soda, food focus, realistic,Best quality,masterpiece,ultra high res\n",
        "    3. (a coca cola bottle with a red circle around it and daisies in the background with a blue sky and white daisies), no humans, flower, white flower, still life, bottle, water, english text,\n",
        "    4. (a person holding coca cola can) intricate playground atmosphere, atmospheric lighting, 4k UHD, hyper detailed, vibrant colors playground background, epic composition, octane render, sharp focus, high resolution isometric, in the center shot,\n",
        "    5. (a big bottle in the playground, inside the bottle there's a coca cola can) intricate playground atmosphere, atmospheric oliva lighting, on the table, 4k UHD, hyper detailed, vibrant colors playground background, epic composition, octane render, sharp focus, high resolution isometric, in the center shot\n",
        "    6. (a poster of a coca cola advertisement with six bottles of coke on it and a bag of soda on the side),no humans,english text,still life,artist name,border,Best quality,masterpiece,ultra high res,\n",
        "    7. (a rabbit sitting next to a coca cola bottle and jumping in a field of grass and flowers with a sun shining behind it), no humans, rabbit, coca-cola, animal focus, animal, plant, outdoors, signature, bottle, nature, grass, tree,Best quality,masterpiece,ultra high res\n",
        "    8. (A person taking a Coca-Cola bottle from a fridge), beautiful interior, high quality, 8k, warm light\n",
        "    9. (A rabbit holding a coca cola bottle in its paws and sipping it), in a field of grass and weeds with the sun shining behind it,no humans,bottle,blurry,animal,animal focus,rabbit,outdoors,grass,blurry background,coca-cola,plant,realistic,mushroom,whiskers, Best quality,masterpiece,ultra high res\n",
        "    10. extremely detailed photo a man wearing iron man armor, lorarobertdj, stylish short beard, model photoshoot, 8k UHD, ultra detailed, RAW photography\n",
        "    11. extremely detailed photo a man wearing batman armor, lorarobertdj, stylish short beard, model photoshoot, 8k UHD, ultra detailed, RAW photography\n",
        "\n",
        "prompt:\"\"\"\n",
        "messages = [{\"role\": \"system\", \"content\": system_message}]\n",
        "messages.append({\"role\": \"user\", \"content\": prompt})\n",
        "completions = get_openai_completion(model=\"gpt-3.5-turbo\",\n",
        "                                  max_tokens=1000, n=1,\n",
        "                                  temperature=1.0, messages=messages,\n",
        "                                  return_completions_only=True)\n",
        "prompt = completions[0]"
      ],
      "metadata": {
        "id": "GylQQEBtlBdy",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell prints the prompt\n",
        "prompt"
      ],
      "metadata": {
        "id": "nhxX4JUV4Kx7",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#This cell provides a ipywidget for modifying prompt\n",
        "import ipywidgets as widgets\n",
        "from ipywidgets import interact, interactive, fixed\n",
        "from IPython.display import display\n",
        "\n",
        "text_input = widgets.Text(\n",
        "    value=prompt,\n",
        "    description='Edit prompt and add LoRA tagwords',\n",
        "    disabled=False,\n",
        "    layout = widgets.Layout(width='2000px')\n",
        ")\n",
        "\n",
        "display(text_input)"
      ],
      "metadata": {
        "id": "pypB3ab13xck",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(text_input.value)"
      ],
      "metadata": {
        "id": "bNhlB3Vrl0Ar",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from diffusers import AnimateDiffPipeline, LCMScheduler, MotionAdapter\n",
        "from diffusers.utils import export_to_gif\n",
        "\n",
        "adapter = MotionAdapter.from_pretrained(\"wangfuyun/AnimateLCM\")\n",
        "pipe = AnimateDiffPipeline.from_pretrained(\"runwayml/stable-diffusion-v1-5\", motion_adapter=adapter, variant=\"fp16\")\n",
        "pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule=\"linear\")\n",
        "\n",
        "pipe.load_lora_weights(\"hennessy_weights/\", adapter_name=\"hennessy-lora\")\n",
        "pipe.load_lora_weights(\"celebrity_weights/\", adapter_name=\"celebrity-lora\")\n",
        "pipe.load_lora_weights(\"wangfuyun/AnimateLCM\", weight_name=\"AnimateLCM_sd15_t2v_lora.safetensors\", adapter_name=\"lcm-lora\")\n",
        "pipe.load_lora_weights(\"guoyww/animatediff-motion-lora-tilt-up\", adapter_name=\"tilt-up\")\n",
        "# pipe.load_lora_weights(\n",
        "#     \"guoyww/animatediff-motion-lora-zoom-out\", adapter_name=\"zoom-out\",\n",
        "# )\n",
        "# pipe.load_lora_weights(\n",
        "#     \"guoyww/animatediff-motion-lora-pan-left\", adapter_name=\"pan-left\",\n",
        "# )\n",
        "\n",
        "pipe.set_adapters([\"lcm-lora\", \"tilt-up\", \"hennessy-lora\", \"celebrity-lora\"],\n",
        " [1.0, 0.8, 0.9, 0.9])\n",
        "pipe.enable_vae_slicing()\n",
        "pipe.enable_model_cpu_offload()"
      ],
      "metadata": {
        "id": "NV-Ra6Cqlz9U",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = pipe(\n",
        "    prompt=text_input.value,\n",
        "    negative_prompt=\"(artist name),(worst quality:2),(low quality:2),(normal quality:2),lowres,bad anatomy,bad hands,monochrome, ugly, boring, cartoon, anime, disfigured, multiple heads, multiple people, extra fingers, fused limbs\",\n",
        "    num_frames=16,\n",
        "    guidance_scale=1.5,\n",
        "    num_inference_steps=25,\n",
        "    generator=torch.Generator(\"cpu\").manual_seed(0),\n",
        ")\n",
        "frames = output.frames[0]\n",
        "export_to_gif(frames, \"animatelcm-motion-lora.gif\")"
      ],
      "metadata": {
        "id": "qfXA9tNPlz6Z",
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "T_TggIpRrfYc"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}