diff --git a/pynq_composable/notebooks/04_OpenCV_vs_Composable_Overlay.ipynb b/pynq_composable/notebooks/04_OpenCV_vs_Composable_Overlay.ipynb new file mode 100644 index 0000000..949c02d --- /dev/null +++ b/pynq_composable/notebooks/04_OpenCV_vs_Composable_Overlay.ipynb @@ -0,0 +1,397 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0a2dcafd-aed4-4fb9-b825-2a33545636f7", + "metadata": {}, + "source": [ + "# OpenCV vs Composable Overlay \n", + "\n", + "
\n", + "Please use Jupyter labs http://<board_ip_address>/lab for this notebook.\n", + "
\n", + "\n", + "\n", + "## Aims\n", + "* Compare a pure OpenCV pipeline with a composable pipeline\n", + "\n", + "## Table of Contents\n", + "* [Software Video Pipeline](#sw)\n", + "* [Hardware Video Pipeline](#hw)\n", + "* [Conclusion](#conclusion)\n", + "\n", + "----\n", + "\n", + "## Revision History\n", + "\n", + "* v1.0 | 04 March 2024 | First notebook revision." + ] + }, + { + "cell_type": "markdown", + "id": "a224ec5a", + "metadata": {}, + "source": [ + "## Software Video Pipeline " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca9a68c6-7985-4e3b-95eb-d68a39fed157", + "metadata": {}, + "outputs": [], + "source": [ + "import cv2\n", + "import numpy as np\n", + "import PIL.Image\n", + "from pynq.lib.video import VideoMode\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "id": "d33d4aef", + "metadata": {}, + "source": [ + "Setup video mode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f89f0d88-dac2-4a32-baf9-e33303c10ead", + "metadata": {}, + "outputs": [], + "source": [ + "mode = VideoMode(1280, 720, 24, 60)" + ] + }, + { + "cell_type": "markdown", + "id": "6b337b92", + "metadata": {}, + "source": [ + "Open and configure the webcam" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0700b4e-d56d-4637-bf73-183d3bcd71df", + "metadata": {}, + "outputs": [], + "source": [ + "webcam = cv2.VideoCapture(0 + cv2.CAP_V4L2)\n", + "webcam.set(cv2.CAP_PROP_FRAME_WIDTH, mode.width)\n", + "webcam.set(cv2.CAP_PROP_FRAME_HEIGHT, mode.height)\n", + "webcam.set(cv2.CAP_PROP_FPS, mode.fps)" + ] + }, + { + "cell_type": "markdown", + "id": "98a92183", + "metadata": {}, + "source": [ + "Check webcamera status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0524ca5b-1c4f-49c2-ae88-83e8b1397f38", + "metadata": {}, + "outputs": [], + "source": [ + "ret, frame = webcam.read()\n", + "if not ret:\n", + " print('Camera cannot be open')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6a01e53-3ad1-4f4e-8295-c5e613c22cdc", + "metadata": {}, + "outputs": [], + "source": [ + "def show_image(frame):\n", + " return PIL.Image.fromarray(frame[:,:,[2,1,0]])" + ] + }, + { + "cell_type": "markdown", + "id": "6aceb8e2", + "metadata": {}, + "source": [ + "Define function to implement color detect pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b5ce21e-cbda-45a8-9b87-650ab01ccf2f", + "metadata": {}, + "outputs": [], + "source": [ + "kernel = np.ones((3,3), np.uint8)\n", + "def run_sw_pipeline():\n", + " ret, frame = webcam.read()\n", + " frame_hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)\n", + " #frame_threshold0 = cv2.inRange(frame_hsv, (22, 38, 160), (38, 75, 179))\n", + " #frame_threshold1 = cv2.inRange(frame_hsv, (150, 150, 150), (255, 255, 255))\n", + " frame_threshold2 = cv2.inRange(frame_hsv, (60, 60, 60), (255, 255, 255))\n", + " #frame_threshold = frame_threshold0 | frame_threshold1 | frame_threshold2\n", + " erode_0 = cv2.erode(frame_threshold2, kernel, iterations=1)\n", + " dilate_0 = cv2.erode(erode_0, kernel, iterations=1)\n", + " dilate_1 = cv2.erode(dilate_0, kernel, iterations=1)\n", + " erode_1 = cv2.erode(dilate_1, kernel, iterations=1)\n", + " frame_rgb = cv2.cvtColor(erode_1, cv2.COLOR_GRAY2RGB)\n", + " return frame_rgb & frame" + ] + }, + { + "cell_type": "markdown", + "id": "77731b04", + "metadata": {}, + "source": [ + "Check software pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02749cd7-d434-459e-bc21-bdca3a209db4", + "metadata": {}, + "outputs": [], + "source": [ + "res = run_sw_pipeline()\n", + "show_image(res)" + ] + }, + { + "cell_type": "markdown", + "id": "15d3c1ec", + "metadata": {}, + "source": [ + "Run software pipeline for several frames to get frames per second" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1bca68d4-6de8-447d-ad71-5e9d8e1c0372", + "metadata": {}, + "outputs": [], + "source": [ + "frames = 120\n", + "start_time = time.time()\n", + "for _ in range(frames):\n", + " res = run_sw_pipeline()\n", + "end_time = time.time()\n", + "\n", + "print(\"Took {:.3f} seconds to process {} frames. Software achieved {:.2f} FPS\".format(ex_time := end_time-start_time, frames, frames/ex_time))" + ] + }, + { + "cell_type": "markdown", + "id": "c22a2bd1-d410-4fa4-9f46-a3c26fe7f2b3", + "metadata": {}, + "source": [ + "## Hardware Video Pipeline " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5288fe66-3fa4-4420-8add-a0b81434c55a", + "metadata": {}, + "outputs": [], + "source": [ + "from pynq import Overlay\n", + "import pynq_composable" + ] + }, + { + "cell_type": "markdown", + "id": "9abe04d3", + "metadata": {}, + "source": [ + "Download overlay and create objects" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d39deb-97c4-4021-b001-6f8d38efd02b", + "metadata": {}, + "outputs": [], + "source": [ + "ol = Overlay(\"cv_dfx_3_pr.bit\")\n", + "\n", + "cpipe = ol.composable" + ] + }, + { + "cell_type": "markdown", + "id": "1fac27c7", + "metadata": {}, + "source": [ + "Define objects to control the DMA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1989b140-6201-40d9-ac04-55861a2563f6", + "metadata": {}, + "outputs": [], + "source": [ + "writechannel = ol.video.axi_vdma.writechannel\n", + "readchannel = ol.video.axi_vdma.readchannel\n", + "writechannel.mode = mode\n", + "readchannel.mode = mode\n", + "writechannel.start()\n", + "readchannel.start()" + ] + }, + { + "cell_type": "markdown", + "id": "0ca141a9", + "metadata": {}, + "source": [ + "Download partial bitstreams and compose pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a599aff-d0d3-4e21-9de8-a22fd4c6cadf", + "metadata": {}, + "outputs": [], + "source": [ + "cpipe.load('pr_0/dilate_accel')\n", + "cpipe.load('pr_1/dilate_accel')\n", + "cpipe.load('pr_2/bitwise_and_accel')\n", + "\n", + "pipeline = [cpipe.ps_video_in, cpipe.duplicate_accel,\n", + " [[cpipe.rgb2hsv_accel, cpipe.colorthresholding_accel, cpipe.pr_0.erode_accel,\n", + " cpipe.pr_0.dilate_accel, cpipe.pr_1.dilate_accel, cpipe.pr_1.erode_accel,\n", + " cpipe.gray2rgb_accel], [1]],\n", + " cpipe.pr_2.bitwise_and_accel, cpipe.ps_video_out]\n", + "\n", + "cpipe.compose(pipeline)\n", + "cpipe.graph" + ] + }, + { + "cell_type": "markdown", + "id": "0702314f", + "metadata": {}, + "source": [ + "Run hardware pipeline via the VDMA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "033c3b67-e3d9-4b95-bc9f-41f3cc9daaf2", + "metadata": {}, + "outputs": [], + "source": [ + "def run_hw_pipeline():\n", + " fpgaframe = writechannel.newframe()\n", + " _, fpgaframe[:] = webcam.read()\n", + " writechannel.writeframe(fpgaframe)\n", + " res = readchannel.readframe()\n", + " return res" + ] + }, + { + "cell_type": "markdown", + "id": "7c49bd5d", + "metadata": {}, + "source": [ + "Run software pipeline for several frames to get frames per second" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64c21582-f822-4318-94c6-b70a8260b731", + "metadata": {}, + "outputs": [], + "source": [ + "frames = 120\n", + "start_time = time.time()\n", + "for _ in range(frames):\n", + " res = run_hw_pipeline()\n", + "end_time = time.time()\n", + "\n", + "print(\"Took {:.3f} seconds to process {} frames. Hardware achieved {:.2f} FPS\".format(ex_time := end_time-start_time, frames, frames/ex_time))" + ] + }, + { + "cell_type": "markdown", + "id": "ca70214a", + "metadata": {}, + "source": [ + "Release the camera and overlay" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6cdf002-bd9d-4649-bd42-7bc0732a899f", + "metadata": {}, + "outputs": [], + "source": [ + "webcam.release()\n", + "ol.free()" + ] + }, + { + "cell_type": "markdown", + "id": "52a0ec19", + "metadata": {}, + "source": [ + "## Conclusion \n", + "\n", + "This notebooks showed the performance of a software video pipeline and a hardware video pipeline" + ] + }, + { + "cell_type": "markdown", + "id": "95d68b05", + "metadata": {}, + "source": [ + "Copyright © 2024 AMD, Inc\n", + "\n", + "SPDX-License-Identifier: BSD-3-Clause\n", + "\n", + "----" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}