Merge pull request #433 from mprib/430-create-autorig_configjson-from…

…-calibration-footage

430 create autorig configjson from calibration footage

dev/dev_low_pass_filter.py

@@ -23,7 +23,7 @@
 
 # specify a source directory (with recordings)
 from pyxy3d.helper import copy_contents
-from pyxy3d.post_processor import PostProcessor
+from pyxy3d.post_processing.post_processor import PostProcessor
 from pyxy3d.trackers.tracker_enum import TrackerEnum
 import toml
 from PyQt6.QtWidgets import QApplication

dev/dev_post_processer.py

@@ -1,4 +1,4 @@
-from pyxy3d.post_processor import PostProcessor
+from pyxy3d.post_processing.post_processor import PostProcessor
 import toml
 from pathlib import Path
 from pyxy3d.configurator import Configurator

pyproject.toml

@@ -17,6 +17,7 @@ PyOpenGL = "^3.1.6"
 toml = "^0.10.2"
 numba = "^0.56.4"
 mediapipe = "0.10.1"
+siuba = "^0.4.2"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.3.0"

pyxy3d/cameras/synchronizer.py

@@ -281,7 +281,7 @@ def synch_frames_worker(self):
                 q.put(self.current_sync_packet)
                 if self.current_sync_packet is not None:
                     logger.debug(f"Placing new synched frames packet on queue with {self.current_sync_packet.frame_packet_count} frames")
-                    logger.info(f"Placing new synched frames with index {self.current_sync_packet.sync_index}")
+                    logger.debug(f"Placing new synched frames with index {self.current_sync_packet.sync_index}")
 
                     # provide infrequent notice of synchronizer activity
                     if self.current_sync_packet.sync_index % 100 == 0:

pyxy3d/gui/post_processing_widget.py

@@ -36,7 +36,7 @@
     QVBoxLayout,
 )
 
-from pyxy3d.post_processor import PostProcessor
+from pyxy3d.post_processing.post_processor import PostProcessor
 from pyxy3d.session.session import Session
 from pyxy3d.cameras.synchronizer import Synchronizer
 from pyxy3d import __root__

pyxy3d/gui/single_main_widget.py

@@ -122,6 +122,8 @@ def connect_menu_actions(self):
 
     def mode_change_action(self):
         action = self.sender()
+
+        # create a reverse lookup dictionary to pull the mode enum that should be activated
         SessionModeLookup = {mode.value: mode for mode in SessionMode}
         mode = SessionModeLookup[action.text()]
         logger.info(f"Attempting to set session mode to {mode.value}")
@@ -131,6 +133,10 @@ def mode_change_action(self):
 
 
     def update_central_widget_mode(self):
+        """
+        This will be triggered whenever the session successfully completes a mode change and emits
+        a signal to that effect.
+        """
         logger.info(f"Begin process of updating central widget")
         # Delete the current central widget
         old_widget = self.centralWidget()

pyxy3d/interface.py

@@ -53,7 +53,25 @@ def name(self)->str:
         Used for file naming creation
         """
         pass
-
+
+    @property
+    def metarig_symmetrical_measures(self):
+        """
+        a dictionary of key: value in the form Measure_Name:[pointA, pointB]
+        when processed, the mean distances (excluding outliers) of both 
+        left_pointA,left_pointB and right_pointA, right_pointB will be calculated. 
+        The mean of the two sides will be taken
+        """
+        raise NotImplementedError(f"Tracker {self.name} has not provided its measures for configuring a metarig")
+
+    @property
+    def metarig_bilateral_measures(self):
+        """
+        a dictionary of key: value in the form Measure_Name:[side_pointA, side_pointB]
+        when processed, mean distance (excluding outliers) between the two points will be calculated and stored as the measure
+        """
+        raise NotImplementedError(f"Tracker {self.name} has not provided its measures for configuring a metarig")
+
     @abstractmethod
     def get_point_name(self, point_id:int) -> str:
         """
@@ -118,7 +136,6 @@ class FramePacket:
     port: int
     frame_time: float
     frame: np.ndarray
-    # frame_index: int = None
     points: PointPacket = None
     draw_instructions: callable = None
 

pyxy3d/post_processing/blender_tools.py

@@ -0,0 +1,80 @@
+
+import pyxy3d.logger
+import json
+
+logger = pyxy3d.logger.get(__name__)
+from pyxy3d import __root__
+import pytest
+import shutil
+import cv2
+from pathlib import Path
+import time
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+from pyxy3d.trackers.tracker_enum import TrackerEnum
+
+def calculate_distance(xyz_trajectory_data:pd.DataFrame, point1:str, point2:str):
+    """
+    Given a set of xyz trajectories from tracked and triangulated landmarks, calculate the 
+    mean distance between two named points, excluding outlier points for data cleanliness
+    
+    """
+    # calculate the distance
+    distances = np.sqrt((xyz_trajectory_data[point1 + '_x'] - xyz_trajectory_data[point2 + '_x']) ** 2 +
+                     (xyz_trajectory_data[point1 + '_y'] - xyz_trajectory_data[point2 + '_y']) ** 2 +
+                     (xyz_trajectory_data[point1 + '_z'] - xyz_trajectory_data[point2 + '_z']) ** 2)
+
+
+        # calculate Q1, Q3 and IQR for outlier detection
+    Q1 = distances.quantile(0.25)
+    Q3 = distances.quantile(0.75)
+    IQR = Q3 - Q1
+
+    # filter out the outliers
+    filtered_distances = distances[(distances >= Q1 - 1.5 * IQR) & (distances <= Q3 + 1.5 * IQR)]
+
+    # calculate the average length
+    average_length = filtered_distances.mean()
+    return average_length
+
+
+def generate_metarig_config(tracker_enum: TrackerEnum, xyz_csv_path:Path):
+    """
+    Stores metarig config json file within the tracker sub-directory within a recording folder 
+    """        
+    tracker = tracker_enum.value()
+
+    xyz_trajectories = pd.read_csv(xyz_csv_path)
+    json_path = Path(xyz_csv_path.parent, f"metarig_config_{tracker.name}.json")
+
+    # for testing purposes, need to make sure that this file is not there before proceeding
+    json_path.unlink(missing_ok=True)
+    assert not json_path.exists()
+
+    autorig_config = {} # Dictionary that will become json
+
+    # average distances across the bilateral measures
+    for measure, points in tracker.metarig_bilateral_measures.items():
+        logger.info(f"Calculating mean distance (IQR) for {measure}")
+
+        mean_distance = 0
+        for side in ["left", "right"]:
+
+            point1 = f"{side}_{points[0]}"
+            point2 = f"{side}_{points[1]}"
+
+            distance = calculate_distance(xyz_trajectories, point1,point2)
+            logger.info(f"Between {point1} and {point2} the mean distance is {distance}")     
+            mean_distance += distance/2
+        autorig_config[measure] = round(mean_distance,4)
+
+    # calculate distance across symmetrical measures       
+    for measure, points in tracker.metarig_symmetrical_measures.items():
+        distance = calculate_distance(xyz_trajectories, points[0],points[1])
+        autorig_config[measure] = round(distance,4)
+
+    # save output to file that will be in same folder as the associated xy_csv data
+    with open(json_path,"w") as f:
+        json.dump(autorig_config, f, indent=4)    

pyxy3d/post_processor.py → pyxy3d/post_processing/post_processor.py

@@ -35,7 +35,6 @@ class PostProcessor:
     # progress_update = pyqtSignal(dict)  # {"stage": str, "percent":int}
 
     def __init__(self,config:Configurator ):
-        # super().__init__()
         self.config = config
 
     def create_xyz(self, recording_path: Path, tracker_enum: TrackerEnum,) -> None:
@@ -105,12 +104,6 @@ def create_xy(self, recording_path: Path, tracker_enum: TrackerEnum,):
             sleep(1)
             percent_complete = int((video_recorder.sync_index / sync_index_count) * 100)
             logger.info(f"(Stage 1 of 2): {percent_complete}% of frames processed for (x,y) landmark detection")
-            # self.progress_update.emit(
-            #     {
-            #         "stage": "Estimating (x,y) landmark positions (stage 1 of 2)",
-            #         "percent": percent_complete,
-            #     }
-            # )
 
 
     def triangulate_xy_data(self, xy_data: pd.DataFrame) -> Dict[str, List]:
@@ -137,15 +130,15 @@ def triangulate_xy_data(self, xy_data: pd.DataFrame) -> Dict[str, List]:
 
         for index in xy_data["sync_index"].unique():
             active_index = xy_data["sync_index"] == index
-            cameras = xy_data["port"][active_index].to_numpy()
+            port = xy_data["port"][active_index].to_numpy()
             point_ids = xy_data["point_id"][active_index].to_numpy()
             img_loc_x = xy_data["img_loc_x"][active_index].to_numpy()
             img_loc_y = xy_data["img_loc_y"][active_index].to_numpy()
             imgs_xy = np.vstack([img_loc_x, img_loc_y]).T
 
             # the fancy part
             point_id_xyz, points_xyz = triangulate_sync_index(
-                projection_matrices, cameras, point_ids, imgs_xy
+                projection_matrices, port, point_ids, imgs_xy
             )
 
             if len(point_id_xyz) > 0:
@@ -165,20 +158,5 @@ def triangulate_xy_data(self, xy_data: pd.DataFrame) -> Dict[str, List]:
                     f"(Stage 2 of 2): Triangulation of (x,y) point estimates is {percent_complete}% complete"
                 )
                 last_log_update = int(time())
-                # self.progress_update.emit(
-                #     {
-                #         "stage": "Triangulating (x,y,z) estimates (stage 2 of 2)",
-                #         "percent": percent_complete,
-                #     }
-                # )
-
-        # signalling the progress bar can now close
-        # self.progress_update.emit(
-        #     {
-        #         "stage": "Triangulating (x,y,z) estimates (stage 2 of 2)",
-        #         "percent": 100,
-        #         "close": True
-        #     }
-        # )
 
         return xyz_history

pyxy3d/trackers/holistic_opensim_tracker.py

@@ -125,6 +125,44 @@
 }
 
 
+METARIG_BILATERAL_MEAUSURES = {
+    "Hip_Shoulder_Distance":["hip", "shoulder"],
+    "Shoulder_Inner_Eye_Distance":["inner_eye", "shoulder"],
+    "Palm": ["index_finger_MCP", "pinky_MCP"],
+    "Foot":["heel", "foot_index"],  
+    "Upper_Arm":["shoulder","elbow"],
+    "Forearm":["elbow", "wrist"],
+    "Wrist_to_MCP1":["wrist", "thumb_MCP"],
+    "Wrist_to_MCP2":["wrist", "index_finger_MCP"],
+    "Wrist_to_MCP3":["wrist", "middle_finger_MCP"],
+    "Wrist_to_MCP4":["wrist", "ring_finger_MCP"],
+    "Wrist_to_MCP5":["wrist", "pinky_MCP"],
+    "Prox_Phalanx_1":["thumb_MCP", "thumb_IP"],
+    "Prox_Phalanx_2":["index_finger_MCP", "index_finger_PIP"],
+    "Prox_Phalanx_3":["middle_finger_MCP", "middle_finger_PIP"],
+    "Prox_Phalanx_4":["ring_finger_MCP", "ring_finger_PIP"],
+    "Prox_Phalanx_5":["pinky_MCP", "pinky_PIP"],
+    "Mid_Phalanx_2":["index_finger_PIP", "index_finger_DIP"],
+    "Mid_Phalanx_3":["middle_finger_PIP","middle_finger_DIP"],
+    "Mid_Phalanx_4":["ring_finger_PIP", "ring_finger_DIP"],
+    "Mid_Phalanx_5":["pinky_PIP", "pinky_DIP"],
+    "Dist_Phalanx_1":["thumb_IP", "thumb_tip"],
+    "Dist_Phalanx_2":["index_finger_DIP", "index_finger_tip"],
+    "Dist_Phalanx_3":["middle_finger_DIP","middle_finger_tip"],
+    "Dist_Phalanx_4":["ring_finger_DIP", "middle_finger_tip"],
+    "Dist_Phalanx_5":["pinky_DIP", "pinky_tip"],
+    "Thigh_Length":["hip","knee"],
+    "Shin_Length": ["knee", "ankle"]
+}
+
+
+METARIG_SYMMETRICAL_MEASURES = {
+    "Shoulder_Width":["left_shoulder", "right_shoulder"],
+    "Hip_Width":["left_hip", "right_hip"],
+    "Inner_Eye_Distance":["left_inner_eye", "right_inner_eye"]
+}
+
+
 # keep ids in distinct ranges to avoid clashes
 POSE_OFFSET = 0
 RIGHT_HAND_OFFSET = 100
@@ -144,6 +182,14 @@ def __init__(self) -> None:
     def name(self):
         return "HOLISTIC_OPENSIM"
 
+    @property
+    def metarig_symmetrical_measures(self):
+        return METARIG_SYMMETRICAL_MEASURES
+
+    @property
+    def metarig_bilateral_measures(self):
+        return METARIG_BILATERAL_MEAUSURES
+
     def run_frame_processor(self, port: int, rotation_count: int):
         # Create a MediaPipe pose instance
         with mp.solutions.holistic.Holistic(

pyxy3d/trackers/holistic_tracker.py

@@ -112,6 +112,42 @@
     220: "left_pinky_tip",
 }
 
+METARIG_BILATERAL_MEAUSURES = {
+    "Hip_Shoulder_Distance":["hip", "shoulder"],
+    "Shoulder_Inner_Eye_Distance":["inner_eye", "shoulder"],
+    "Palm": ["index_finger_MCP", "pinky_MCP"],
+    "Foot":["heel", "foot_index"],  
+    "Upper_Arm":["shoulder","elbow"],
+    "Forearm":["elbow", "wrist"],
+    "Wrist_to_MCP1":["wrist", "thumb_MCP"],
+    "Wrist_to_MCP2":["wrist", "index_finger_MCP"],
+    "Wrist_to_MCP3":["wrist", "middle_finger_MCP"],
+    "Wrist_to_MCP4":["wrist", "ring_finger_MCP"],
+    "Wrist_to_MCP5":["wrist", "pinky_MCP"],
+    "Prox_Phalanx_1":["thumb_MCP", "thumb_IP"],
+    "Prox_Phalanx_2":["index_finger_MCP", "index_finger_PIP"],
+    "Prox_Phalanx_3":["middle_finger_MCP", "middle_finger_PIP"],
+    "Prox_Phalanx_4":["ring_finger_MCP", "ring_finger_PIP"],
+    "Prox_Phalanx_5":["pinky_MCP", "pinky_PIP"],
+    "Mid_Phalanx_2":["index_finger_PIP", "index_finger_DIP"],
+    "Mid_Phalanx_3":["middle_finger_PIP","middle_finger_DIP"],
+    "Mid_Phalanx_4":["ring_finger_PIP", "ring_finger_DIP"],
+    "Mid_Phalanx_5":["pinky_PIP", "pinky_DIP"],
+    "Dist_Phalanx_1":["thumb_IP", "thumb_tip"],
+    "Dist_Phalanx_2":["index_finger_DIP", "index_finger_tip"],
+    "Dist_Phalanx_3":["middle_finger_DIP","middle_finger_tip"],
+    "Dist_Phalanx_4":["ring_finger_DIP", "middle_finger_tip"],
+    "Dist_Phalanx_5":["pinky_DIP", "pinky_tip"],
+    "Thigh_Length":["hip","knee"],
+    "Shin_Length": ["knee", "ankle"]
+}
+
+
+METARIG_SYMMETRICAL_MEASURES = {
+    "Shoulder_Width":["left_shoulder", "right_shoulder"],
+    "Hip_Width":["left_hip", "right_hip"],
+    "Inner_Eye_Distance":["left_inner_eye", "right_inner_eye"]
+}
 
 # keep ids in distinct ranges to avoid clashes
 POSE_OFFSET = 0
@@ -131,7 +167,16 @@ def __init__(self) -> None:
     @property
     def name(self):
         return "HOLISTIC"
-
+
+
+    @property
+    def metarig_symmetrical_measures(self):
+        return METARIG_SYMMETRICAL_MEASURES
+
+    @property
+    def metarig_bilateral_measures(self):
+        return METARIG_BILATERAL_MEAUSURES
+
     def run_frame_processor(self, port: int, rotation_count: int):
         # Create a MediaPipe pose instance
         with mp.solutions.holistic.Holistic(