LTX_Video_with_Start_&_End_frames.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **LTX-VIDEO WITH START & END FRAMES**"
      ],
      "metadata": {
        "id": "f4p1ysFKMbs_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- You can use the free T4 GPU to run this. For faster video generation, use higher GPUs.\n",
        "- This notebook is mainly for generating animations from simple transitions between two images. It doesn't do well for fast motions like walking or running.\n",
        "- Use detailed prompts to generate good videos."
      ],
      "metadata": {
        "id": "EBB00lC6q-DA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Prepare Environment\n",
        "%cd /content\n",
        "Always_Load_Models_for_Inference = False\n",
        "Use_t5xxl_fp16 = False\n",
        "\n",
        "!pip install -q torchsde einops diffusers accelerate xformers\n",
        "!pip install av\n",
        "!git clone https://github.com/Isi-dev/ComfyUI\n",
        "%cd /content/ComfyUI\n",
        "!apt -y install -qq aria2 ffmpeg\n",
        "\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Isi99999/LTX-Video/resolve/main/ltx-video-2b-v0.9.5.safetensors -d /content/ComfyUI/models/checkpoints -o ltx-video-2b-v0.9.5.safetensors\n",
        "if Use_t5xxl_fp16:\n",
        "    !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Isi99999/LTX-Video/resolve/main/t5xxl_fp16.safetensors -d /content/ComfyUI/models/text_encoders -o t5xxl_fp16.safetensors\n",
        "else:\n",
        "    !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Isi99999/LTX-Video/resolve/main/t5xxl_fp8_e4m3fn_scaled.safetensors -d /content/ComfyUI/models/text_encoders -o t5xxl_fp8_e4m3fn_scaled.safetensors\n",
        "\n",
        "import torch\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "import gc\n",
        "import sys\n",
        "import random\n",
        "import os\n",
        "import imageio\n",
        "from google.colab import files\n",
        "from IPython.display import display, HTML\n",
        "sys.path.insert(0, '/content/ComfyUI')\n",
        "\n",
        "from comfy import model_management\n",
        "\n",
        "from nodes import (\n",
        "    CheckpointLoaderSimple,\n",
        "    CLIPLoader,\n",
        "    CLIPTextEncode,\n",
        "    VAEDecode,\n",
        "    LoadImage\n",
        ")\n",
        "\n",
        "from comfy_extras.nodes_custom_sampler import (\n",
        "    KSamplerSelect,\n",
        "    SamplerCustom\n",
        ")\n",
        "\n",
        "from comfy_extras.nodes_lt import (\n",
        "    EmptyLTXVLatentVideo,\n",
        "    LTXVPreprocess,\n",
        "    LTXVAddGuide,\n",
        "    LTXVScheduler,\n",
        "    LTXVConditioning,\n",
        "    LTXVCropGuides\n",
        ")\n",
        "\n",
        "checkpoint_loader = CheckpointLoaderSimple()\n",
        "clip_loader = CLIPLoader()\n",
        "clip_encode_positive = CLIPTextEncode()\n",
        "clip_encode_negative = CLIPTextEncode()\n",
        "load_image = LoadImage()\n",
        "empty_latent = EmptyLTXVLatentVideo()\n",
        "preprocess = LTXVPreprocess()\n",
        "add_guide = LTXVAddGuide()\n",
        "scheduler = LTXVScheduler()\n",
        "sampler_select = KSamplerSelect()\n",
        "conditioning = LTXVConditioning()\n",
        "sampler = SamplerCustom()\n",
        "vae_decode = VAEDecode()\n",
        "crop_guides = LTXVCropGuides()\n",
        "\n",
        "def clear_gpu_memory():\n",
        "    gc.collect()\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.empty_cache()\n",
        "        torch.cuda.ipc_collect()\n",
        "    for obj in list(globals().values()):\n",
        "        if torch.is_tensor(obj) or (hasattr(obj, \"data\") and torch.is_tensor(obj.data)):\n",
        "            del obj\n",
        "    gc.collect()\n",
        "\n",
        "def upload_image():\n",
        "    \"\"\"Handle image upload in Colab and store in /content/ComfyUI/input/\"\"\"\n",
        "    from google.colab import files\n",
        "    import os\n",
        "    import shutil\n",
        "\n",
        "    os.makedirs('/content/ComfyUI/input', exist_ok=True)\n",
        "\n",
        "    uploaded = files.upload()\n",
        "\n",
        "    # Move each uploaded file to ComfyUI input directory\n",
        "    for filename in uploaded.keys():\n",
        "        src_path = f'/content/ComfyUI/{filename}'\n",
        "        dest_path = f'/content/ComfyUI/input/{filename}'\n",
        "\n",
        "        shutil.move(src_path, dest_path)\n",
        "        print(f\"Image saved to: {dest_path}\")\n",
        "        return dest_path\n",
        "\n",
        "    return None\n",
        "\n",
        "def generate_video(\n",
        "    image_path: str = None,\n",
        "    guide_image_path: str = None,\n",
        "    positive_prompt: str = \"A red fox moving gracefully, its russet coat vibrant against the white landscape, leaving perfect star-shaped prints behind as steam rises from its breath in the crisp winter air. The scene is wrapped in snow-muffled silence, broken only by the gentle murmur of water still flowing beneath the ice.\",\n",
        "    negative_prompt: str = \"low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly\",\n",
        "    width: int = 768,\n",
        "    height: int = 512,\n",
        "    seed: int = 397166166231987,\n",
        "    steps: int = 30,\n",
        "    cfg_scale: float = 2.05,\n",
        "    sampler_name: str = \"euler\",\n",
        "    length: int = 97,\n",
        "    fps: int = 24,\n",
        "    guide_strength: float = 0.1,\n",
        "    guide_frame: int = -1\n",
        "):\n",
        "    with torch.inference_mode():\n",
        "        print(\"Loading Text_Encoder...\")\n",
        "        clip = clip_loader.load_clip(\"t5xxl_fp8_e4m3fn_scaled.safetensors\", \"ltxv\", \"default\")[0]\n",
        "        print(\"Loaded Text_Encoder!\")\n",
        "\n",
        "    try:\n",
        "        assert width % 32 == 0, \"Width must be divisible by 32\"\n",
        "        assert height % 32 == 0, \"Height must be divisible by 32\"\n",
        "\n",
        "        positive = clip_encode_positive.encode(clip, positive_prompt)[0]\n",
        "        negative = clip_encode_negative.encode(clip, negative_prompt)[0]\n",
        "\n",
        "        del clip\n",
        "        torch.cuda.empty_cache()\n",
        "        gc.collect()\n",
        "        print(\"Text_Encoder removed from memory\")\n",
        "\n",
        "        if image_path is None:\n",
        "            print(\"Please upload the main image file:\")\n",
        "            image_path = upload_image()\n",
        "        if image_path is None:\n",
        "            print(\"No main image uploaded!\")\n",
        "\n",
        "        if guide_image_path is None:\n",
        "            print(\"Please upload the guide image file:\")\n",
        "            guide_image_path = upload_image()\n",
        "        if guide_image_path is None:\n",
        "            print(\"No guide image uploaded!\")\n",
        "\n",
        "        loaded_image = load_image.load_image(image_path)[0]\n",
        "        processed_image = preprocess.preprocess(loaded_image, 35)[0]\n",
        "\n",
        "        loaded_guide_image = load_image.load_image(guide_image_path)[0]\n",
        "        processed_guide_image = preprocess.preprocess(loaded_guide_image, 40)[0]\n",
        "\n",
        "        print(\"Loading model & VAE...\")\n",
        "        model, _, vae = checkpoint_loader.load_checkpoint(\"ltx-video-2b-v0.9.5.safetensors\")\n",
        "        print(\"Loaded model & VAE!\")\n",
        "\n",
        "        # Create empty latent video\n",
        "        latent_video = empty_latent.generate(width, height, length)[0]\n",
        "\n",
        "        # First guide pass\n",
        "        guided_positive, guided_negative, guided_latent_1 = add_guide.generate(\n",
        "            positive=positive,\n",
        "            negative=negative,\n",
        "            vae=vae,\n",
        "            latent=latent_video,\n",
        "            image=processed_image,\n",
        "            frame_idx=0,\n",
        "            strength=1\n",
        "        )\n",
        "\n",
        "        # Second guide pass (from the other image)\n",
        "        guided_positive, guided_negative, guided_latent = add_guide.generate(\n",
        "            positive=guided_positive,\n",
        "            negative=guided_negative,\n",
        "            vae=vae,\n",
        "            latent=guided_latent_1,\n",
        "            image=processed_guide_image,\n",
        "            frame_idx=guide_frame,\n",
        "            strength=guide_strength\n",
        "        )\n",
        "\n",
        "        # Get sigmas for sampling\n",
        "        sigmas = scheduler.get_sigmas(steps, cfg_scale, 0.95, True, 0.1, guided_latent_1)[0]\n",
        "        selected_sampler = sampler_select.get_sampler(sampler_name)[0]\n",
        "\n",
        "        # Apply conditioning\n",
        "        conditioned_positive, conditioned_negative = conditioning.append(\n",
        "            guided_positive,\n",
        "            guided_negative,\n",
        "            25.0\n",
        "        )\n",
        "\n",
        "        print(\"Generating video...\")\n",
        "\n",
        "        # Sample the video\n",
        "        sampled = sampler.sample(\n",
        "            model=model,\n",
        "            add_noise=True,\n",
        "            noise_seed=seed if seed != 0 else random.randint(0, 2**32),\n",
        "            cfg=cfg_scale,\n",
        "            positive=conditioned_positive,\n",
        "            negative=conditioned_negative,\n",
        "            sampler=selected_sampler,\n",
        "            sigmas=sigmas,\n",
        "            latent_image=guided_latent\n",
        "        )[0]\n",
        "\n",
        "        # Crop guides if needed\n",
        "        cropped_latent = crop_guides.crop(\n",
        "            conditioned_positive,\n",
        "            conditioned_negative,\n",
        "            sampled\n",
        "        )[2]\n",
        "\n",
        "        del model\n",
        "        torch.cuda.empty_cache()\n",
        "        gc.collect()\n",
        "        print(\"Model removed from memory\")\n",
        "\n",
        "        with torch.no_grad():\n",
        "            try:\n",
        "                print(\"Decoding Latents...\")\n",
        "                decoded = vae_decode.decode(vae, cropped_latent)[0].detach()\n",
        "                print(\"Latents Decoded!\")\n",
        "                del vae\n",
        "                torch.cuda.empty_cache()\n",
        "                gc.collect()\n",
        "                print(\"VAE removed from memory\")\n",
        "            except Exception as e:\n",
        "                print(f\"Error during decoding: {str(e)}\")\n",
        "                raise\n",
        "\n",
        "        # Save as MP4\n",
        "        output_path = \"/content/output.mp4\"\n",
        "        frames_np = (decoded.cpu().numpy() * 255).astype(np.uint8)\n",
        "        with imageio.get_writer(output_path, fps=fps) as writer:\n",
        "            for frame in frames_np:\n",
        "                writer.append_data(frame)\n",
        "\n",
        "        print(f\"\\nVideo generation complete!\")\n",
        "        print(f\"Saved {len(decoded)} frames to {output_path}\")\n",
        "        display_video(output_path)\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"Error during video generation: {str(e)}\")\n",
        "        raise\n",
        "    finally:\n",
        "        clear_gpu_memory()\n",
        "\n",
        "def display_video(video_path):\n",
        "    \"\"\"Display video in Colab notebook with proper HTML5 player\"\"\"\n",
        "    from IPython.display import HTML\n",
        "    from base64 import b64encode\n",
        "\n",
        "    mp4 = open(video_path,'rb').read()\n",
        "    data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "\n",
        "    display(HTML(f\"\"\"\n",
        "    <video width=512 controls autoplay loop>\n",
        "        <source src=\"{data_url}\" type=\"video/mp4\">\n",
        "    </video>\n",
        "    \"\"\"))\n",
        "\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "rrXFIT4fMfyJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Run Video Generation\n",
        "positive_prompt = \"Flowers growing from the sides of a vase\" # @param {\"type\":\"string\"}\n",
        "negative_prompt = \"low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly\" # @param {\"type\":\"string\"}\n",
        "width = 512 # @param {\"type\":\"number\"}\n",
        "height = 768 # @param {\"type\":\"number\"}\n",
        "seed = 397166166231987 # @param {\"type\":\"integer\"}\n",
        "steps = 25 # @param {\"type\":\"integer\", \"min\":1, \"max\":100}\n",
        "cfg_scale = 2.05 # @param {\"type\":\"number\", \"min\":1, \"max\":20}\n",
        "sampler_name = \"euler\" # @param [\"euler\", \"dpmpp_2m\", \"ddim\", \"lms\"]\n",
        "frames = 49 # @param {\"type\":\"integer\", \"min\":1, \"max\":120}\n",
        "guide_strength = 1 # @param {\"type\":\"number\", \"min\":0, \"max\":1}\n",
        "guide_frame = -1 # @param {\"type\":\"integer\"}\n",
        "\n",
        "# @title Run Video Generation\n",
        "print(\"Starting video generation workflow...\")\n",
        "with torch.inference_mode():\n",
        "    generate_video(\n",
        "        image_path=None,\n",
        "        guide_image_path=None,\n",
        "        positive_prompt=positive_prompt,\n",
        "        negative_prompt=negative_prompt,\n",
        "        width=width,\n",
        "        height=height,\n",
        "        seed=seed,\n",
        "        steps=steps,\n",
        "        cfg_scale=cfg_scale,\n",
        "        sampler_name=sampler_name,\n",
        "        length=frames,\n",
        "        guide_strength=guide_strength,\n",
        "        guide_frame=guide_frame\n",
        "    )\n",
        "clear_gpu_memory()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "roC59_oNNflb"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}