Label balancing and symmetric behaviors for training classifiers

classify.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 
 import numpy as np
+import pandas as pd
 
 from src import APP_NAME
 from src.classifier import Classifier, ClassifierType
@@ -109,11 +110,10 @@ def classify_pose(classifier: Classifier, input_pose_file: Path, out_dir: Path,
             distance_scale_factor=distance_scale_factor,
             extended_features=classifier.extended_features
         ).get_features(window_size, use_social)
+        per_frame_features = pd.DataFrame(IdentityFeatures.merge_per_frame_features(features['per_frame'], use_social))
+        window_features = pd.DataFrame(IdentityFeatures.merge_window_features(features['window'], use_social))
 
-        data = Classifier.combine_data(
-            features['per_frame'],
-            features['window']
-        )
+        data = Classifier.combine_data(per_frame_features, window_features)
 
         if data.shape[0] > 0:
             pred = classifier.predict(data)
@@ -125,8 +125,9 @@ def classify_pose(classifier: Classifier, input_pose_file: Path, out_dir: Path,
             # for each row of the pred_prob array we just computed.
             pred_prob = pred_prob[np.arange(len(pred_prob)), pred]
 
-            prediction_labels[curr_id, features['frame_indexes']] = pred
-            prediction_prob[curr_id, features['frame_indexes']] = pred_prob
+            # Only copy out predictions where there was a valid pose
+            prediction_labels[curr_id, features['frame_indexes']] = pred[features['frame_indexes']]
+            prediction_prob[curr_id, features['frame_indexes']] = pred_prob[features['frame_indexes']]
     cli_progress_bar(len(pose_est.identities), len(pose_est.identities),
                      complete_as_percent=False, suffix='identities')
 
@@ -178,6 +179,8 @@ def train(
           f"{__CLASSIFIER_CHOICES[classifier.classifier_type]}")
     print(f"  Window Size: {training_file['window_size']}")
     print(f"  Social: {training_file['has_social_features']}")
+    print(f"  Balanced Labels: {training_file['balance_labels']}")
+    print(f"  Symmetric Behavior: {training_file['symmetric']}")
     print(f"  Distance Unit: {training_file['distance_unit'].name}")
 
     training_features = classifier.combine_data(training_file['per_frame'],
@@ -190,6 +193,8 @@ def train(
         behavior,
         training_file['window_size'],
         training_file['has_social_features'],
+        training_file['balance_labels'],
+        training_file['symmetric'],
         training_file['extended_features'],
         training_file['distance_unit'],
         random_seed=training_file['training_seed']

docs/features/features.md

@@ -0,0 +1,89 @@
+# Per-Frame Features
+
+## Egocentric Animal Features
+
+116 features from 12 animal keypoints
+
+* point mask (boolean, 12 values)
+* pairwise distances between all points (66 values)
+* angles between a subset of connected points (11 values)
+    * nose, base neck, right front paw
+    * nose, base neck, left front paw
+    * right front paw, base neck, center spine
+    * left front paw, base neck, center spine
+    * base neck, center spine, base tail
+    * right rear paw, base tail, center spine
+    * left rear paw, base tail, center spine
+    * right rear paw, base tail, mid tail
+    * left rear paw, base tail, mid tail
+    * center spine, base tail, mid tail
+    * base tail, mid tail, tip tail
+* angular velocity (using base tail → base neck bearing)
+* velocities (each velocity listed below consists of two features – the direction and magnitude components)
+    * centroid
+    * 12 keypoints
+
+## Social features (v3 pose files)
+
+21 features from social context
+
+* distance to closest mouse
+* distance to closest in field of view (fov)
+* fov angle
+* pairwise distances (9 values)
+    * pairwise distances between (nose, base neck, tail) points of subject and closest mouse
+* pairwise distances fov (9 values)
+    * pairwise distances between (nose, base neck, tail) points of subject and closest fov mouse
+
+## Static object features (v5 pose files)
+
+### Arena Corners
+
+2 features from arena corners
+
+* distance to corner using the convex hull center
+* bearing to corner using angle of the base neck - nose vector
+
+### Water Spout (Lixit)
+
+1 feature from lixit
+
+* distance from nose to nearest lixit
+
+### Food Hopper
+
+10 features from food hopper
+
+* signed distance from keypoint to food hopper border (positive = inside, negative = outside)
+    * nose
+    * left ear
+    * right ear
+    * base neck
+    * left front paw
+    * right front paw
+    * center spine
+    * left rear paw
+    * right rear paw
+    * base tail
+
+# Window Features
+
+Window features describe how a specific per-frame feature may be changing over time. We use a centered window around the frame of interest. These calculated features are added to the total feature vector. Window features calculated are different whether or not the per-frame feature is non-circular or circular.
+
+## Non circular measurements
+
+* mean
+* median
+* standard deviation
+* max
+* min
+
+## Circular measurements
+
+* circstd
+* cirmean
+
+### Circular Feature List
+
+* Angles
+* Bearings