NodeHaveTextVideo(eachFrame).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a3298af6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 48ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 1ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 5ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 1ms/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 1ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 1ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 9ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 2ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 17ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 5ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 2ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 5ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 6ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 11ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 0s/step\n",
      "1/1 [==============================] - 0s 11ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 11ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "1/1 [==============================] - 0s 11ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "1/1 [==============================] - 0s 0s/step\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "# 載入模型\n",
    "model = tf.keras.models.load_model(r\"C:\\Users\\B00\\Desktop\\LINproject\\numpyModel\", compile=False)\n",
    "interpreter = tf.lite.Interpreter(model_path=r\"C:\\Users\\B00\\Desktop\\for project\\movenet file\\lite-model_movenet_singlepose_lightning_3.tflite\")\n",
    "interpreter.allocate_tensors()\n",
    "\n",
    "def draw_box_with_text_on_frame(frame, x, y, width, height, text, color=(0,0,0), thickness=2):\n",
    "    \"\"\"在视频帧上绘制带有文本的方框\"\"\"\n",
    "    text_size, _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)\n",
    "    text_x = x + (width - text_size[0]) // 2\n",
    "    text_y = y + height + 30\n",
    "    cv2.putText(frame, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2)\n",
    "\n",
    "# Function to pad image to make it square\n",
    "def pad_image(image):\n",
    "    height, width, _ = image.shape\n",
    "    size = max(height, width)\n",
    "    pad_y = (size - height) // 2\n",
    "    pad_x = (size - width) // 2\n",
    "    padded_image = cv2.copyMakeBorder(image, pad_y, pad_y, pad_x, pad_x, cv2.BORDER_CONSTANT, value=[0, 0, 0])\n",
    "    return padded_image\n",
    "\n",
    "def main():\n",
    "    video_path = r\"C:\\Users\\B00\\Desktop\\for project\\tmp\\GetTimeflow\\videos\\normal\\normal8.mp4\"\n",
    "    cap = cv2.VideoCapture(video_path)\n",
    "\n",
    "    if not cap.isOpened():\n",
    "        print(\"无法打开视频文件。请检查文件路径。\")\n",
    "        exit()\n",
    "\n",
    "    frame_count = 1\n",
    "\n",
    "    while True:\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        if not ret:\n",
    "            break\n",
    "\n",
    "        original_frame = frame.copy()\n",
    "\n",
    "        # for cnn model\n",
    "        image_height = 192\n",
    "        image_width = 192\n",
    "        if ret:\n",
    "            # 调整帧的大小并正规化\n",
    "            frame = tf.image.resize_with_pad(frame, image_height, image_width)\n",
    "            frame = cv2.cvtColor(np.array(frame), cv2.COLOR_BGR2RGB)  # 将图像转换为RGB格式\n",
    "            cv2.imwrite(fr'C:\\Users\\B00\\Desktop\\bodyImage\\fornobody\\frame\\_{frame_count}.jpg', frame)\n",
    "            frame_count += 1\n",
    "            frame = frame.astype(np.uint8)  # 确保图像数据类型是uint8\n",
    "\n",
    "            # 添加一个维度以匹配模型的输入形状\n",
    "            frame = np.expand_dims(frame, axis=0)\n",
    "\n",
    "            input_image = tf.cast(frame, dtype=tf.float32)\n",
    "\n",
    "            # Setup input and output\n",
    "            input_details = interpreter.get_input_details()\n",
    "            output_details = interpreter.get_output_details()\n",
    "\n",
    "            interpreter.set_tensor(input_details[0][\"index\"], input_image)\n",
    "            interpreter.invoke()\n",
    "            keypoints_with_scores = interpreter.get_tensor(output_details[0][\"index\"])\n",
    "\n",
    "            now_input = keypoints_with_scores\n",
    "\n",
    "            with tf.device('/CPU:0'):  # 强制使用 CPU\n",
    "                # 使用模型进行预测\n",
    "                prediction = model.predict(now_input)\n",
    "\n",
    "            # 找出最可能的类别\n",
    "            class_index = np.argmax(prediction)\n",
    "\n",
    "            if class_index == 0:\n",
    "                output_text = \"standing\"\n",
    "            elif class_index == 1:\n",
    "                output_text = \"sitting\"\n",
    "            elif class_index == 2:\n",
    "                output_text = \"lying\"\n",
    "        else:\n",
    "            output_text = \"Hello, OpenCV!\"\n",
    "\n",
    "        # 在视频帧上绘制方框并显示输出信息\n",
    "        frame_height, frame_width, _ = original_frame.shape  # 使用原始的frame\n",
    "        box_width, box_height = 0, 0\n",
    "        x = 50\n",
    "        y = 50\n",
    "       \n",
    "        draw_box_with_text_on_frame(original_frame, x, y, box_width, box_height, output_text)\n",
    "\n",
    "        # 显示输出\n",
    "        cv2.imshow(\"Video\", original_frame)\n",
    "\n",
    "        # 按下 'q' 键退出循环\n",
    "        if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "    # 释放资源\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d1ff132",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Import matplotlib libraries\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.collections import LineCollection\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "# Some modules to display an animation using imageio.\n",
    "#import imageio\n",
    "#from IPython.display import HTML, display\n",
    "\n",
    "\n",
    "def movenet(input_image):\n",
    "\n",
    "    # TF Lite format expects tensor type of uint8.\n",
    "    input_image = tf.cast(input_image, dtype=tf.uint8)\n",
    "    input_details = interpreter.get_input_details()\n",
    "    output_details = interpreter.get_output_details()\n",
    "    interpreter.set_tensor(input_details[0]['index'], input_image.numpy())\n",
    "    # Invoke inference.\n",
    "    interpreter.invoke()\n",
    "    # Get the model prediction.\n",
    "    keypoints_with_scores = interpreter.get_tensor(output_details[0]['index'])\n",
    "    return keypoints_with_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af9b18b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confidence score to determine whether a keypoint prediction is reliable.\n",
    "MIN_CROP_KEYPOINT_SCORE = 0.2\n",
    "\n",
    "def init_crop_region(image_height, image_width):\n",
    "  \"\"\"Defines the default crop region.\n",
    "\n",
    "  The function provides the initial crop region (pads the full image from both\n",
    "  sides to make it a square image) when the algorithm cannot reliably determine\n",
    "  the crop region from the previous frame.\n",
    "  \"\"\"\n",
    "  if image_width > image_height:\n",
    "    box_height = image_width / image_height\n",
    "    box_width = 1.0\n",
    "    y_min = (image_height / 2 - image_width / 2) / image_height\n",
    "    x_min = 0.0\n",
    "  else:\n",
    "    box_height = 1.0\n",
    "    box_width = image_height / image_width\n",
    "    y_min = 0.0\n",
    "    x_min = (image_width / 2 - image_height / 2) / image_width\n",
    "\n",
    "  return {\n",
    "    'y_min': y_min,\n",
    "    'x_min': x_min,\n",
    "    'y_max': y_min + box_height,\n",
    "    'x_max': x_min + box_width,\n",
    "    'height': box_height,\n",
    "    'width': box_width\n",
    "  }\n",
    "\n",
    "def torso_visible(keypoints):\n",
    "  \"\"\"Checks whether there are enough torso keypoints.\n",
    "\n",
    "  This function checks whether the model is confident at predicting one of the\n",
    "  shoulders/hips which is required to determine a good crop region.\n",
    "  \"\"\"\n",
    "  return ((keypoints[0, 0, KEYPOINT_DICT['left_hip'], 2] >\n",
    "           MIN_CROP_KEYPOINT_SCORE or\n",
    "          keypoints[0, 0, KEYPOINT_DICT['right_hip'], 2] >\n",
    "           MIN_CROP_KEYPOINT_SCORE) and\n",
    "          (keypoints[0, 0, KEYPOINT_DICT['left_shoulder'], 2] >\n",
    "           MIN_CROP_KEYPOINT_SCORE or\n",
    "          keypoints[0, 0, KEYPOINT_DICT['right_shoulder'], 2] >\n",
    "           MIN_CROP_KEYPOINT_SCORE))\n",
    "\n",
    "def determine_torso_and_body_range(\n",
    "    keypoints, target_keypoints, center_y, center_x):\n",
    "  \"\"\"Calculates the maximum distance from each keypoints to the center location.\n",
    "\n",
    "  The function returns the maximum distances from the two sets of keypoints:\n",
    "  full 17 keypoints and 4 torso keypoints. The returned information will be\n",
    "  used to determine the crop size. See determineCropRegion for more detail.\n",
    "  \"\"\"\n",
    "  torso_joints = ['left_shoulder', 'right_shoulder', 'left_hip', 'right_hip']\n",
    "  max_torso_yrange = 0.0\n",
    "  max_torso_xrange = 0.0\n",
    "  for joint in torso_joints:\n",
    "    dist_y = abs(center_y - target_keypoints[joint][0])\n",
    "    dist_x = abs(center_x - target_keypoints[joint][1])\n",
    "    if dist_y > max_torso_yrange:\n",
    "      max_torso_yrange = dist_y\n",
    "    if dist_x > max_torso_xrange:\n",
    "      max_torso_xrange = dist_x\n",
    "\n",
    "  max_body_yrange = 0.0\n",
    "  max_body_xrange = 0.0\n",
    "  for joint in KEYPOINT_DICT.keys():\n",
    "    if keypoints[0, 0, KEYPOINT_DICT[joint], 2] < MIN_CROP_KEYPOINT_SCORE:\n",
    "      continue\n",
    "    dist_y = abs(center_y - target_keypoints[joint][0]);\n",
    "    dist_x = abs(center_x - target_keypoints[joint][1]);\n",
    "    if dist_y > max_body_yrange:\n",
    "      max_body_yrange = dist_y\n",
    "\n",
    "    if dist_x > max_body_xrange:\n",
    "      max_body_xrange = dist_x\n",
    "\n",
    "  return [max_torso_yrange, max_torso_xrange, max_body_yrange, max_body_xrange]\n",
    "\n",
    "def determine_crop_region(\n",
    "      keypoints, image_height,\n",
    "      image_width):\n",
    "  \"\"\"Determines the region to crop the image for the model to run inference on.\n",
    "\n",
    "  The algorithm uses the detected joints from the previous frame to estimate\n",
    "  the square region that encloses the full body of the target person and\n",
    "  centers at the midpoint of two hip joints. The crop size is determined by\n",
    "  the distances between each joints and the center point.\n",
    "  When the model is not confident with the four torso joint predictions, the\n",
    "  function returns a default crop which is the full image padded to square.\n",
    "  \"\"\"\n",
    "  target_keypoints = {}\n",
    "  for joint in KEYPOINT_DICT.keys():\n",
    "    target_keypoints[joint] = [\n",
    "      keypoints[0, 0, KEYPOINT_DICT[joint], 0] * image_height,\n",
    "      keypoints[0, 0, KEYPOINT_DICT[joint], 1] * image_width\n",
    "    ]\n",
    "\n",
    "  if torso_visible(keypoints):\n",
    "    center_y = (target_keypoints['left_hip'][0] +\n",
    "                target_keypoints['right_hip'][0]) / 2;\n",
    "    center_x = (target_keypoints['left_hip'][1] +\n",
    "                target_keypoints['right_hip'][1]) / 2;\n",
    "\n",
    "    (max_torso_yrange, max_torso_xrange,\n",
    "      max_body_yrange, max_body_xrange) = determine_torso_and_body_range(\n",
    "          keypoints, target_keypoints, center_y, center_x)\n",
    "\n",
    "    crop_length_half = np.amax(\n",
    "        [max_torso_xrange * 1.9, max_torso_yrange * 1.9,\n",
    "          max_body_yrange * 1.2, max_body_xrange * 1.2])\n",
    "\n",
    "    tmp = np.array(\n",
    "        [center_x, image_width - center_x, center_y, image_height - center_y])\n",
    "    crop_length_half = np.amin(\n",
    "        [crop_length_half, np.amax(tmp)]);\n",
    "\n",
    "    crop_corner = [center_y - crop_length_half, center_x - crop_length_half];\n",
    "\n",
    "    if crop_length_half > max(image_width, image_height) / 2:\n",
    "      return init_crop_region(image_height, image_width)\n",
    "    else:\n",
    "      crop_length = crop_length_half * 2;\n",
    "      return {\n",
    "        'y_min': crop_corner[0] / image_height,\n",
    "        'x_min': crop_corner[1] / image_width,\n",
    "        'y_max': (crop_corner[0] + crop_length) / image_height,\n",
    "        'x_max': (crop_corner[1] + crop_length) / image_width,\n",
    "        'height': (crop_corner[0] + crop_length) / image_height -\n",
    "            crop_corner[0] / image_height,\n",
    "        'width': (crop_corner[1] + crop_length) / image_width -\n",
    "            crop_corner[1] / image_width\n",
    "      }\n",
    "  else:\n",
    "    return init_crop_region(image_height, image_width)\n",
    "\n",
    "def crop_and_resize(image, crop_region, crop_size):\n",
    "  \"\"\"Crops and resize the image to prepare for the model input.\"\"\"\n",
    "  boxes=[[crop_region['y_min'], crop_region['x_min'],\n",
    "          crop_region['y_max'], crop_region['x_max']]]\n",
    "  output_image = tf.image.crop_and_resize(\n",
    "      image, box_indices=[0], boxes=boxes, crop_size=crop_size)\n",
    "  return output_image\n",
    "\n",
    "def run_inference(movenet, image, crop_region, crop_size):\n",
    "  \"\"\"Runs model inferece on the cropped region.\n",
    "\n",
    "  The function runs the model inference on the cropped region and updates the\n",
    "  model output to the original image coordinate system.\n",
    "  \"\"\"\n",
    "  image_height, image_width, _ = image.shape\n",
    "  input_image = crop_and_resize(\n",
    "    tf.expand_dims(image, axis=0), crop_region, crop_size=crop_size)\n",
    "  # Run model inference.\n",
    "  keypoints_with_scores = movenet(input_image)\n",
    "  # Update the coordinates.\n",
    "  for idx in range(17):\n",
    "    keypoints_with_scores[0, 0, idx, 0] = (\n",
    "        crop_region['y_min'] * image_height +\n",
    "        crop_region['height'] * image_height *\n",
    "        keypoints_with_scores[0, 0, idx, 0]) / image_height\n",
    "    keypoints_with_scores[0, 0, idx, 1] = (\n",
    "        crop_region['x_min'] * image_width +\n",
    "        crop_region['width'] * image_width *\n",
    "        keypoints_with_scores[0, 0, idx, 1]) / image_width\n",
    "  return keypoints_with_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0e6b9894",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dictionary that maps from joint names to keypoint indices.\n",
    "KEYPOINT_DICT = {\n",
    "    'nose': 0,\n",
    "    'left_eye': 1,\n",
    "    'right_eye': 2,\n",
    "    'left_ear': 3,\n",
    "    'right_ear': 4,\n",
    "    'left_shoulder': 5,\n",
    "    'right_shoulder': 6,\n",
    "    'left_elbow': 7,\n",
    "    'right_elbow': 8,\n",
    "    'left_wrist': 9,\n",
    "    'right_wrist': 10,\n",
    "    'left_hip': 11,\n",
    "    'right_hip': 12,\n",
    "    'left_knee': 13,\n",
    "    'right_knee': 14,\n",
    "    'left_ankle': 15,\n",
    "    'right_ankle': 16\n",
    "}\n",
    "\n",
    "# Maps bones to a matplotlib color name.\n",
    "KEYPOINT_EDGE_INDS_TO_COLOR = {\n",
    "    (0, 1): 'm',\n",
    "    (0, 2): 'c',\n",
    "    (1, 3): 'm',\n",
    "    (2, 4): 'c',\n",
    "    (0, 5): 'm',\n",
    "    (0, 6): 'c',\n",
    "    (5, 7): 'm',\n",
    "    (7, 9): 'm',\n",
    "    (6, 8): 'c',\n",
    "    (8, 10): 'c',\n",
    "    (5, 6): 'y',\n",
    "    (5, 11): 'm',\n",
    "    (6, 12): 'c',\n",
    "    (11, 12): 'y',\n",
    "    (11, 13): 'm',\n",
    "    (13, 15): 'm',\n",
    "    (12, 14): 'c',\n",
    "    (14, 16): 'c'\n",
    "}\n",
    "\n",
    "def _keypoints_and_edges_for_display(keypoints_with_scores,\n",
    "                                     height,\n",
    "                                     width,\n",
    "                                     keypoint_threshold=0.11):\n",
    "  \"\"\"Returns high confidence keypoints and edges for visualization.\n",
    "\n",
    "  Args:\n",
    "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
    "      the keypoint coordinates and scores returned from the MoveNet model.\n",
    "    height: height of the image in pixels.\n",
    "    width: width of the image in pixels.\n",
    "    keypoint_threshold: minimum confidence score for a keypoint to be\n",
    "      visualized.\n",
    "\n",
    "  Returns:\n",
    "    A (keypoints_xy, edges_xy, edge_colors) containing:\n",
    "      * the coordinates of all keypoints of all detected entities;\n",
    "      * the coordinates of all skeleton edges of all detected entities;\n",
    "      * the colors in which the edges should be plotted.\n",
    "  \"\"\"\n",
    "  keypoints_all = []\n",
    "  keypoint_edges_all = []\n",
    "  edge_colors = []\n",
    "  num_instances, _, _, _ = keypoints_with_scores.shape\n",
    "  for idx in range(num_instances):\n",
    "    kpts_x = keypoints_with_scores[0, idx, :, 1]\n",
    "    kpts_y = keypoints_with_scores[0, idx, :, 0]\n",
    "    kpts_scores = keypoints_with_scores[0, idx, :, 2]\n",
    "    kpts_absolute_xy = np.stack(\n",
    "        [width * np.array(kpts_x), height * np.array(kpts_y)], axis=-1)\n",
    "    kpts_above_thresh_absolute = kpts_absolute_xy[\n",
    "        kpts_scores > keypoint_threshold, :]\n",
    "    keypoints_all.append(kpts_above_thresh_absolute)\n",
    "\n",
    "    for edge_pair, color in KEYPOINT_EDGE_INDS_TO_COLOR.items():\n",
    "      if (kpts_scores[edge_pair[0]] > keypoint_threshold and\n",
    "          kpts_scores[edge_pair[1]] > keypoint_threshold):\n",
    "        x_start = kpts_absolute_xy[edge_pair[0], 0]\n",
    "        y_start = kpts_absolute_xy[edge_pair[0], 1]\n",
    "        x_end = kpts_absolute_xy[edge_pair[1], 0]\n",
    "        y_end = kpts_absolute_xy[edge_pair[1], 1]\n",
    "        line_seg = np.array([[x_start, y_start], [x_end, y_end]])\n",
    "        keypoint_edges_all.append(line_seg)\n",
    "        edge_colors.append(color)\n",
    "  if keypoints_all:\n",
    "    keypoints_xy = np.concatenate(keypoints_all, axis=0)\n",
    "  else:\n",
    "    keypoints_xy = np.zeros((0, 17, 2))\n",
    "\n",
    "  if keypoint_edges_all:\n",
    "    edges_xy = np.stack(keypoint_edges_all, axis=0)\n",
    "  else:\n",
    "    edges_xy = np.zeros((0, 2, 2))\n",
    "  return keypoints_xy, edges_xy, edge_colors\n",
    "\n",
    "\n",
    "def draw_prediction_on_image(\n",
    "    image, keypoints_with_scores, crop_region=None, close_figure=False,\n",
    "    output_image_height=None):\n",
    "  \"\"\"Draws the keypoint predictions on image.\n",
    "\n",
    "  Args:\n",
    "    image: A numpy array with shape [height, width, channel] representing the\n",
    "      pixel values of the input image.\n",
    "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
    "      the keypoint coordinates and scores returned from the MoveNet model.\n",
    "    crop_region: A dictionary that defines the coordinates of the bounding box\n",
    "      of the crop region in normalized coordinates (see the init_crop_region\n",
    "      function below for more detail). If provided, this function will also\n",
    "      draw the bounding box on the image.\n",
    "    output_image_height: An integer indicating the height of the output image.\n",
    "      Note that the image aspect ratio will be the same as the input image.\n",
    "\n",
    "  Returns:\n",
    "    A numpy array with shape [out_height, out_width, channel] representing the\n",
    "    image overlaid with keypoint predictions.\n",
    "  \"\"\"\n",
    "  height, width, channel = image.shape\n",
    "  aspect_ratio = float(width) / height\n",
    "  fig, ax = plt.subplots(figsize=(12 * aspect_ratio, 12))\n",
    "  # To remove the huge white borders\n",
    "  fig.tight_layout(pad=0)\n",
    "  ax.margins(0)\n",
    "  ax.set_yticklabels([])\n",
    "  ax.set_xticklabels([])\n",
    "  plt.axis('off')\n",
    "\n",
    "  im = ax.imshow(image)\n",
    "  line_segments = LineCollection([], linewidths=(4), linestyle='solid')\n",
    "  ax.add_collection(line_segments)\n",
    "  # Turn off tick labels\n",
    "  scat = ax.scatter([], [], s=60, color='#FF1493', zorder=3)\n",
    "\n",
    "  (keypoint_locs, keypoint_edges,\n",
    "   edge_colors) = _keypoints_and_edges_for_display(\n",
    "       keypoints_with_scores, height, width)\n",
    "\n",
    "  line_segments.set_segments(keypoint_edges)\n",
    "  line_segments.set_color(edge_colors)\n",
    "  if keypoint_edges.shape[0]:\n",
    "    line_segments.set_segments(keypoint_edges)\n",
    "    line_segments.set_color(edge_colors)\n",
    "  if keypoint_locs.shape[0]:\n",
    "    scat.set_offsets(keypoint_locs)\n",
    "\n",
    "  if crop_region is not None:\n",
    "    xmin = max(crop_region['x_min'] * width, 0.0)\n",
    "    ymin = max(crop_region['y_min'] * height, 0.0)\n",
    "    rec_width = min(crop_region['x_max'], 0.99) * width - xmin\n",
    "    rec_height = min(crop_region['y_max'], 0.99) * height - ymin\n",
    "    rect = patches.Rectangle(\n",
    "        (xmin,ymin),rec_width,rec_height,\n",
    "        linewidth=1,edgecolor='b',facecolor='none')\n",
    "    ax.add_patch(rect)\n",
    "\n",
    "  fig.canvas.draw()\n",
    "  image_from_plot = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
    "  image_from_plot = image_from_plot.reshape(\n",
    "      fig.canvas.get_width_height()[::-1] + (3,))\n",
    "  plt.close(fig)\n",
    "  if output_image_height is not None:\n",
    "    output_image_width = int(output_image_height / height * width)\n",
    "    image_from_plot = cv2.resize(\n",
    "        image_from_plot, dsize=(output_image_width, output_image_height),\n",
    "         interpolation=cv2.INTER_CUBIC)\n",
    "  return image_from_plot\n",
    "\n",
    "def to_gif(images, fps):\n",
    "  \"\"\"Converts image sequence (4D numpy array) to gif.\"\"\"\n",
    "  imageio.mimsave('./animation.gif', images, fps=fps)\n",
    "  return embed.embed_file('./animation.gif')\n",
    "\n",
    "def progress(value, max=100):\n",
    "  return HTML(\"\"\"\n",
    "      <progress\n",
    "          value='{value}'\n",
    "          max='{max}',\n",
    "          style='width: 100%'\n",
    "      >\n",
    "          {value}\n",
    "      </progress>\n",
    "  \"\"\".format(value=value, max=max))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c26f1b82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4eb038be",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "in user code:\n\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2341, in predict_function  *\n        return step_function(self, iterator)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2327, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2315, in run_step  **\n        outputs = model.predict_step(data)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2283, in predict_step\n        return self(x, training=False)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\utils\\traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\input_spec.py\", line 298, in assert_input_compatibility\n        raise ValueError(\n\n    ValueError: Input 0 of layer \"sequential\" is incompatible with the layer: expected shape=(None, 150, 51), found shape=(None, 1, 17, 3)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 141\u001b[0m\n\u001b[0;32m    138\u001b[0m     cv2\u001b[38;5;241m.\u001b[39mdestroyAllWindows()\n\u001b[0;32m    140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 141\u001b[0m     \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[4], line 87\u001b[0m, in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m     83\u001b[0m now_input \u001b[38;5;241m=\u001b[39m keypoints_with_scores\n\u001b[0;32m     85\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/CPU:0\u001b[39m\u001b[38;5;124m'\u001b[39m):  \u001b[38;5;66;03m# 强制使用 CPU\u001b[39;00m\n\u001b[0;32m     86\u001b[0m     \u001b[38;5;66;03m# 使用模型进行预测\u001b[39;00m\n\u001b[1;32m---> 87\u001b[0m     prediction \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnow_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     89\u001b[0m \u001b[38;5;66;03m# 找出最可能的类别\u001b[39;00m\n\u001b[0;32m     90\u001b[0m class_index \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margmax(prediction)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[0;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[1;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[0;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[1;32m~\\AppData\\Local\\Temp\\__autograph_generated_file_26ifcsj.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__predict_function\u001b[1;34m(iterator)\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     14\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[0;32m     17\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "\u001b[1;31mValueError\u001b[0m: in user code:\n\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2341, in predict_function  *\n        return step_function(self, iterator)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2327, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2315, in run_step  **\n        outputs = model.predict_step(data)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\training.py\", line 2283, in predict_step\n        return self(x, training=False)\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\utils\\traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\keras\\src\\engine\\input_spec.py\", line 298, in assert_input_compatibility\n        raise ValueError(\n\n    ValueError: Input 0 of layer \"sequential\" is incompatible with the layer: expected shape=(None, 150, 51), found shape=(None, 1, 17, 3)\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "# 載入模型\n",
    "model = tf.keras.models.load_model(r\"C:\\Users\\ASUS\\Desktop\\school\\project\\model\\GRUModel\", compile=False)\n",
    "interpreter = tf.lite.Interpreter(model_path=r\"C:\\Users\\ASUS\\Desktop\\school\\project\\TFLiteMovenet\\lite-model_movenet_singlepose_lightning_3.tflite\")\n",
    "interpreter.allocate_tensors()\n",
    "\n",
    "def draw_box_with_text_on_frame(frame, x, y, width, height, text, color=(0,0,0), thickness=2):\n",
    "    \"\"\"在视频帧上绘制带有文本的方框\"\"\"\n",
    "    text_size, _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 1)\n",
    "    text_x = x + (width - text_size[0]) // 2\n",
    "    text_y = y + height + 130\n",
    "    cv2.putText(frame, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, 2, color, 2)\n",
    "\n",
    "def main():\n",
    "    video_path = r\"C:\\Users\\ASUS\\Desktop\\school\\project\\poseture\\videos\\production_id-4423925 (1080p).mp4\"\n",
    "    cap = cv2.VideoCapture(video_path)\n",
    "\n",
    "    if not cap.isOpened():\n",
    "        print(\"无法打开视频文件。请检查文件路径。\")\n",
    "        exit()\n",
    "\n",
    "    frame_count = 1\n",
    "\n",
    "    # Get video properties\n",
    "    num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "    frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "    frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "    \n",
    "    # 定义输出视频的相关参数\n",
    "    output_path = r\"C:\\Users\\ASUS\\Desktop\\school\\project\\poseture\\videos\\gru.mp4\"  # 输出视频路径\n",
    "    fourcc = cv2.VideoWriter_fourcc('X','V','I','D')  # 编解码器\n",
    "    frame_rate = 30  # 帧速率（可以根据需要调整）\n",
    "#     frame_width, frame_height = 1920, 1080  # 输出视频的帧宽和帧高（可以根据需要调整）\n",
    "    out = cv2.VideoWriter(output_path, fourcc, frame_rate, (frame_width, frame_height))\n",
    "\n",
    "    output_images = []\n",
    "    while True:\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        if not ret:\n",
    "            break\n",
    "\n",
    "        original_frame = frame.copy()\n",
    "\n",
    "        # for cnn model\n",
    "        image_height = 192\n",
    "        image_width = 192\n",
    "        if ret:\n",
    "            # 调整帧的大小并正规化\n",
    "            frame = tf.image.resize_with_pad(frame, image_height, image_width)\n",
    "            frame = cv2.cvtColor(np.array(frame), cv2.COLOR_BGR2RGB)  # 将图像转换为RGB格式\n",
    "            cv2.imwrite(fr'C:\\Users\\B00\\Desktop\\bodyImage\\fornobody\\frame\\_{frame_count}.jpg', frame)\n",
    "            frame_count += 1\n",
    "            frame = frame.astype(np.uint8)  # 确保图像数据类型是uint8\n",
    "\n",
    "            # 添加一个维度以匹配模型的输入形状\n",
    "            frame = np.expand_dims(frame, axis=0)\n",
    "\n",
    "            input_image = tf.cast(frame, dtype=tf.float32)\n",
    "\n",
    "            # Setup input and output\n",
    "            input_details = interpreter.get_input_details()\n",
    "            output_details = interpreter.get_output_details()\n",
    "\n",
    "            interpreter.set_tensor(input_details[0][\"index\"], input_image)\n",
    "            interpreter.invoke()\n",
    "            keypoints_with_scores = interpreter.get_tensor(output_details[0][\"index\"])\n",
    "\n",
    "            now_input = keypoints_with_scores\n",
    "\n",
    "            with tf.device('/CPU:0'):  # 强制使用 CPU\n",
    "                # 使用模型进行预测\n",
    "                prediction = model.predict(now_input)\n",
    "\n",
    "            # 找出最可能的类别\n",
    "            class_index = np.argmax(prediction)\n",
    "\n",
    "            if class_index == 0:\n",
    "                output_text = \"standing\"\n",
    "            elif class_index == 1:\n",
    "                output_text = \"sitting\"\n",
    "            elif class_index == 2:\n",
    "                output_text = \"falldown\"\n",
    "        else:\n",
    "            output_text = \"Hello, OpenCV!\"\n",
    "\n",
    "        # 在视频帧上绘制方框并显示输出信息\n",
    "        box_width, box_height = 0, 0\n",
    "        x = 50\n",
    "        y = 50\n",
    "        \n",
    "   \n",
    "\n",
    "         # Draw keypoints on the frame\n",
    "        original_frame = draw_prediction_on_image(\n",
    "            original_frame, keypoints_with_scores, crop_region=None,\n",
    "            close_figure=True, output_image_height=frame_height)\n",
    "        \n",
    "        draw_box_with_text_on_frame(original_frame, x, y, box_width, box_height, output_text)\n",
    "        \n",
    "         # Check and adjust the size of the processed frame\n",
    "        height, width, _ = original_frame.shape\n",
    "        if height != frame_height or width != frame_width:\n",
    "            original_frame = cv2.resize(original_frame, (frame_width, frame_height))\n",
    "    \n",
    "        output_images.append(original_frame)\n",
    "        \n",
    "#         # 将当前帧写入输出视频\n",
    "#         out.write(original_frame)\n",
    "\n",
    "        # 显示输出\n",
    "        cv2.imshow(\"Video\", original_frame)\n",
    "        \n",
    "        # 按下 'q' 键退出循环\n",
    "        if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "    # 释放资源\n",
    "    for processed_frame in output_images:\n",
    "        out.write(processed_frame)\n",
    "        \n",
    "    cap.release()\n",
    "    out.release()\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d110a976",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8f1a28b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}