version 0.1.0

README.md

@@ -1,5 +1,5 @@
 # VTS FULLBODY TRACKING
-_Version ALPHA 0.0.2_
+_Version ALPHA 0.1.0_
 
 This plugin integrates full body tracking functionality using Mediapipe. 
 It allows users to use tracked body parameters as inputs to control your Live2D model in [VTube Studio](https://denchisoft.com/). 
@@ -49,9 +49,9 @@ If you prefer not to execute the .exe file, you can run the Python code from Git
 - **Peview Camera**: Displays the image captured by the camera
 - **Annotated Values**: Shows the X, Y, Z values for each body part
 
-| default                                                  | Preview Camera                                                       | Annotated Values                                                         |
-|----------------------------------------------------------|----------------------------------------------------------------------|--------------------------------------------------------------------------|
-| ![exemple_preview.png](readme_img/exemple_preview.png)   | ![exemple_camera_preview.png](readme_img/exemple_camera_preview.png) | ![exemple_annotated_values.png](readme_img/exemple_annotated_values.png) |
+| default                                                  | Preview Camera                                                       | 
+|----------------------------------------------------------|----------------------------------------------------------------------|
+| ![exemple_preview.png](readme_img/exemple_preview.png)   | ![exemple_camera_preview.png](readme_img/exemple_camera_preview.png) | 
 
 **note**: You need to reload the plugin to switch settings.
 
@@ -60,15 +60,16 @@ If you prefer not to execute the .exe file, you can run the Python code from Git
 Press `q` or `ESC` in the Tracking Preview window to stop the plugin.
 
 ## Custom Parameters
-![List Bodyparts MediaPipe](readme_img/list_bodyparts.png)
-*BlazePose 33 keypoint topology as COCO (colored with green) superset*
+![List Bodyparts MediaPipe](readme_img/list_bodyparts.png)<br/>
+*BlazePose 33 keypoint topology as COCO (colored with green) superset*<br/>
 https://blog.research.google/2020/08/on-device-real-time-body-pose-tracking.html
 
 This plugin will create new parameters in Vtube Studio, for each body part a parameter `_X`, `_Y`, `_Z` and `_VISIBILITY` will be available in your Vtube Studio.
 
-- x, y: Real-world 3-dimensional coordinates in meters, with the midpoint of the hips as the origin.
-- z: depth, with the depth at the midpoint of the hips as the origin. The smaller the value, the closer the landmark is to the camera. The magnitude of z uses roughly the same scale as x.
-- visibility: The likelihood of the landmark being visible within the image.
+- **X**: Controls the left-to-right position.
+- **Y**: Controls the top-to-bottom position.
+- **Z**: Controls the backward-to-forward position.
+- **VISIBILITY**: Controls the visibility, from visible to hidden.
 
 ### Body
 
@@ -130,7 +131,7 @@ python app.py
 ### Build executable
 
 ```shell
- pyinstaller --name VTS_Fullbody_Tracking-0.0.2 --add-data='models/*:models' -F -w .\app.py
+ pyinstaller --name VTS_Fullbody_Tracking-0.1.0 --add-data='models/*:models' -F -w .\app.py
 ```
 
 ## Documentations

app.py

@@ -6,7 +6,7 @@
 import cv2
 import os
 
-from utils_mediapipe import draw_landmarks_on_image, BodyParts
+from utils_mediapipe import draw_landmarks_on_image, BodyParts, BodyCenters, get_part_from_name
 from ui import window_tracking_configuration
 
 RESULT = None
@@ -20,7 +20,7 @@
 }
 
 
-async def main(camera_id, preview_enabled, annotate_enabled):
+async def main(camera_id, preview_enabled):
     # ----- MEDIAPIPE: LANDMARKER CONFIGURATION -----------
 
     # Create a PoseLandmarker object
@@ -61,7 +61,7 @@ def set_result(result: PoseLandmarkerResult, output_image: mp.Image, timestamp_m
 
     for parameter_name in parameter_names:
         # Add custom parameters in VTube Studio
-        send_request_new_parameter = vts.vts_request.requestCustomParameter(parameter_name, min=-1, max=1)
+        send_request_new_parameter = vts.vts_request.requestCustomParameter(parameter_name, min=-10, max=10)
         await vts.request(send_request_new_parameter)
 
     # ---- LIVE TRACKING ----------------
@@ -94,32 +94,80 @@ def set_result(result: PoseLandmarkerResult, output_image: mp.Image, timestamp_m
                             parameters = RESULT.pose_world_landmarks[0]
 
                             # Display pose result in additional window
-                            annotated_image = draw_landmarks_on_image(image.numpy_view(), RESULT, preview_enabled, annotate_enabled)
+                            annotated_image = draw_landmarks_on_image(image.numpy_view(), RESULT, preview_enabled)
                             cv2.imshow('Body Tracking', annotated_image)
                             if cv2.waitKey(1) & 0xFF in [ord('q'), 27]:
                                 cv2.destroyAllWindows()
                                 running = False
                                 break
 
                     if parameters:
+                        # Prepare values to send
+                        data = get_bodyparts_values(parameters)
+                        values = list(data.values())
+                        names = list(data.keys())
+
                         print('Update parameters in Vtube Studio')
                         # -- Update parameters in VTube Studio
-                        values = [parameters[body_part.value].__getattribute__(angle) for angle in ['x', 'y', 'z', 'visibility'] for body_part in BodyParts]
-                        send_request_parameter = vts.vts_request.requestSetMultiParameterValue(parameter_names, values, weight=1, face_found=True, mode='set')
+                        send_request_parameter = vts.vts_request.requestSetMultiParameterValue(names, values, weight=1, face_found=True, mode='set')
                         await vts.request(send_request_parameter)
 
 
+def get_bodyparts_values(parameters):
+    i = 0
+    values = {}
+    # Go through each tracked body part
+    for part in parameters:
+        # Find center for this part of the body
+        part_name = get_part_from_name(i)
+        part_center = BodyCenters[part_name.name]
+        center = parameters[part_center.value.value]
+        # Calculate values from new center
+        data = calcul_data(part, center, part_name.name)
+        values.update(data)
+        i += 1
+    return values
+
+
+def calcul_data(part, center, name):
+    """
+    Calculate body part values
+    :param part: Landmarks, body part to recalculate
+    :param center: Landmarks, body part used as new center
+    :param name: String, name of body part to calculate
+    :return: Dict with new values for each axis
+    """
+
+    x_name = name + '_X'
+    y_name = name + '_Y'
+    z_name = name + '_Z'
+    v_name = name + '_VISIBILITY'
+
+    x = (part.x - center.x) * 10
+    y = (part.y - center.y) * 10
+    z = (part.z - center.z) * 10
+
+    data = {
+        x_name: x,
+        y_name: y,
+        z_name: z,
+        v_name: part.visibility,
+    }
+
+    return data
+
+
 if __name__ == '__main__':
 
     print('=== VTS FULLBODY TRACKING ===')
 
     # --- OPEN USER WINDOW : CONFIGURATION TRACKING
 
     root, settings = window_tracking_configuration()
-    camera_id, preview_enabled, annotate_enabled = settings
+    camera_id, preview_enabled = settings
     # ========= START TRACKING ==========
 
-    asyncio.run(main(camera_id, preview_enabled, annotate_enabled))
+    asyncio.run(main(camera_id, preview_enabled))
 
     # ========= STOP PLUGIN ==========
 

ui.py

@@ -30,8 +30,7 @@ def get_configuration():
         camera_id = available_cameras[camera_index]['id']
 
         preview_enabled = preview_checkbox_var.get()
-        annotate_enabled = annotate_checkbox_var.get()
-        return camera_id, preview_enabled, annotate_enabled
+        return camera_id, preview_enabled
 
     root = Tk()
     root.title("Camera Selection")
@@ -53,11 +52,6 @@ def get_configuration():
     preview_checkbox = Checkbutton(root, text="Show Camera View", variable=preview_checkbox_var)
     preview_checkbox.pack()
 
-    # -- Option for showing annotated values when displaying tracking pose
-    annotate_checkbox_var = BooleanVar()
-    annotate_checkbox = Checkbutton(root, text="Annotated Values", variable=annotate_checkbox_var)
-    annotate_checkbox.pack()
-
     # -- Submit Configuration
     start_button = Button(root, text="Start Tracking", command=root.quit)
     start_button.pack()

utils_mediapipe.py

@@ -46,7 +46,56 @@ class BodyParts(Enum):
     RIGHT_FOOT_INDEX = 32
 
 
-def draw_landmarks_on_image(rgb_image, detection_result, preview=False, annotated=False):
+class BodyCenters(Enum):
+    """
+    Enum class defining the center of body parts.
+    """
+
+    NOSE = BodyParts.RIGHT_HIP
+    LEFT_EYE_INNER = BodyParts.LEFT_EYE
+    LEFT_EYE = BodyParts.RIGHT_HIP
+    LEFT_EYE_OUTER = BodyParts.RIGHT_HIP
+    RIGHT_EYE_INNER = BodyParts.RIGHT_HIP
+    RIGHT_EYE = BodyParts.RIGHT_HIP
+    RIGHT_EYE_OUTER = BodyParts.RIGHT_HIP
+    LEFT_EAR = BodyParts.RIGHT_HIP
+    RIGHT_EAR = BodyParts.RIGHT_HIP
+    MOUTH_LEFT = BodyParts.RIGHT_HIP
+    MOUTH_RIGHT = BodyParts.RIGHT_HIP
+
+    LEFT_SHOULDER = BodyParts.RIGHT_HIP
+    RIGHT_SHOULDER = BodyParts.RIGHT_HIP
+    LEFT_ELBOW = BodyParts.LEFT_SHOULDER
+    RIGHT_ELBOW = BodyParts.RIGHT_SHOULDER
+    LEFT_WRIST = BodyParts.LEFT_ELBOW
+    RIGHT_WRIST = BodyParts.RIGHT_ELBOW
+    LEFT_PINKY = BodyParts.LEFT_WRIST
+    RIGHT_PINKY = BodyParts.RIGHT_WRIST
+    LEFT_INDEX = BodyParts.LEFT_WRIST
+    RIGHT_INDEX = BodyParts.RIGHT_WRIST
+    LEFT_THUMB = BodyParts.LEFT_WRIST
+    RIGHT_THUMB = BodyParts.RIGHT_WRIST
+
+    LEFT_HIP = BodyParts.RIGHT_HIP
+    RIGHT_HIP = BodyParts.LEFT_HIP
+    LEFT_KNEE = BodyParts.LEFT_HIP
+    RIGHT_KNEE = BodyParts.RIGHT_HIP
+    LEFT_ANKLE = BodyParts.LEFT_KNEE
+    RIGHT_ANKLE = BodyParts.RIGHT_KNEE
+    LEFT_HEEL = BodyParts.LEFT_ANKLE
+    RIGHT_HEEL = BodyParts.RIGHT_ANKLE
+    LEFT_FOOT_INDEX = BodyParts.LEFT_ANKLE
+    RIGHT_FOOT_INDEX = BodyParts.RIGHT_ANKLE
+
+
+def get_part_from_name(i):
+    for part in BodyParts:
+        if part.value == i:
+            return part
+    raise None
+
+
+def draw_landmarks_on_image(rgb_image, detection_result, preview=False):
     """
     Draw landmarks on the input image.
 
@@ -84,25 +133,4 @@ def draw_landmarks_on_image(rgb_image, detection_result, preview=False, annotate
           solutions.pose.POSE_CONNECTIONS,
           solutions.drawing_styles.get_default_pose_landmarks_style())
 
-        # -- Draw annotated values on the image
-        if annotated:
-
-            i = 0
-            pworld = detection_result.pose_world_landmarks[0]
-
-            for landmark in pose_landmarks:
-                # Convert normalized coordinates to image coordinates
-                image_height, image_width, _ = annotated_image.shape
-                landmark_px = (int(landmark.x * image_width), int(landmark.y * image_height))
-
-                # Add x, y, z values next to each landmark
-                text = f"x: {pworld[i].x:.2f}"
-                cv2.putText(annotated_image, text, (landmark_px[0], landmark_px[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 212, 0), 1, cv2.LINE_AA)
-                text = f"y: {pworld[i].y:.2f}"
-                cv2.putText(annotated_image, text, (landmark_px[0], landmark_px[1] + 15), cv2.FONT_HERSHEY_SIMPLEX, 0.4,(112, 0, 255), 1, cv2.LINE_AA)
-                text = f"z: {pworld[i].z:.2f}"
-                cv2.putText(annotated_image, text, (landmark_px[0], landmark_px[1] + 30), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (80, 255, 90), 1, cv2.LINE_AA)
-
-                i += 1
-
     return annotated_image