Added rotation transform to the tracker bounding-box.

BrennoCaldato · BrennoCaldato · commit 4fdd325ea3eb · 2020-12-14T18:08:44.000-03:00
Changed the bounding-box struct point convention to (cx, cy, width, height, angle).
The GetFrame member function from Tracker class now uses the cv::RotatedRect object to represent the bounding-box and draw it on screen.
The JSON and Protobuf communication between the backend and frontend still uses the (x1,y1)(x2,y2) point convention, the backend performs the adequate transformations.
diff --git a/src/KeyFrameBBox.cpp b/src/KeyFrameBBox.cpp
@@ -142,14 +142,14 @@ KeyFrameBBox::KeyFrameBBox() : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y
 }
 
 // Add a BBox to the BoxVec map
-void KeyFrameBBox::AddBox(int64_t _frame_num, float _x1, float _y1, float _width, float _height)
+void KeyFrameBBox::AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle)
 {
     // Check if the given frame number is valid
     if (_frame_num < 0)
         return;
 
     // Instantiate a new bounding-box
-    BBox newBBox = BBox(_x1, _y1, _width, _height);
+    BBox newBBox = BBox(_cx, _cy, _width, _height, _angle);
 
     // Get the time of given frame
     double time = this->FrameNToTime(_frame_num, 1.0);
@@ -231,25 +231,12 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number)
         BBox currentBBox = currentBBoxIterator->second;
 
         // Adjust the BBox properties by the Keyframes values
-        currentBBox.x1 += this->delta_x.GetValue(frame_number);
-        currentBBox.y1 += this->delta_y.GetValue(frame_number);
+        currentBBox.cx += this->delta_x.GetValue(frame_number);
+        currentBBox.cy += this->delta_y.GetValue(frame_number);
         currentBBox.width *= this->scale_x.GetValue(frame_number);
         currentBBox.height *= this->scale_y.GetValue(frame_number);
+        currentBBox.angle += this->rotation.GetValue(frame_number);
     
-        /* TODO - Add rotation
-            (x1,y1) -> current point
-            (x1',y1') -> transformed point
-            (xc, yc) -> center of the BBox
-            (xc, yc) = (x1 + w/2, y1 + h/2)
-            rot -> rotation angle [radians]
-            x1' = xc + (x1 - xc)*cos(rot) - (y1-yc)*sin(rot)
-            y1' = yc + (x1 - xc)*sin(rot) + (y1-yc)*cos(rot)
-            ***
-            x1' = (x1 + w/2) + (-w/2)*cos(rot) - (-h/2)*sin(rot)
-            y1' = (y1 + h/2) + (-w/2)*sin(rot) + (-h/2)*cos(rot) 
-        currentBBox.x1 += currentBBox.width/2 - (currentBBox.width/2)*cos(rot*PI/180) + (currentBBox.height)*sin(rot*PI/180);
-        currentBBox.y1 += currentBBox.height/2 - (currentBBox.width/2)*sin(rot*PI/180) - (currentBBox.height)*cos(rot*PI/180);
-        */
         return currentBBox;
     }
 
@@ -263,37 +250,47 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number)
                                              previousBBox, currentBBox, time);
 
     // Adjust the BBox properties by the Keyframes values
-    interpolatedBBox.x1 += this->delta_x.GetValue(frame_number);
-    interpolatedBBox.y1 += this->delta_y.GetValue(frame_number);
+    interpolatedBBox.cx += this->delta_x.GetValue(frame_number);
+    interpolatedBBox.cy += this->delta_y.GetValue(frame_number);
     interpolatedBBox.width *= this->scale_x.GetValue(frame_number);
     interpolatedBBox.height *= this->scale_y.GetValue(frame_number);
+    interpolatedBBox.angle += this->rotation.GetValue(frame_number);
     
     return interpolatedBBox;
 }
 
 // Interpolate the bouding-boxes properties
 BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target)
 {
-
-    Point p1_left(t1, left.x1, openshot::InterpolationType::LINEAR);
-    Point p1_right(t2, right.x1, openshot::InterpolationType::LINEAR);
-    Point p1 = InterpolateBetween(p1_left, p1_right, target, 0.01);
-
-    Point p2_left(t1, left.y1, openshot::InterpolationType::LINEAR);
-    Point p2_right(t2, right.y1, openshot::InterpolationType::LINEAR);
-    Point p2 = InterpolateBetween(p2_left, p2_right, target, 0.01);
-
-    Point p3_left(t1, left.height, openshot::InterpolationType::LINEAR);
-    Point p3_right(t2, right.height, openshot::InterpolationType::LINEAR);
-    Point p3 = InterpolateBetween(p3_left, p3_right, target, 0.01);
-
-    Point p4_left(t1, left.width, openshot::InterpolationType::LINEAR);
-    Point p4_right(t2, right.width, openshot::InterpolationType::LINEAR);
-    Point p4 = InterpolateBetween(p4_left, p4_right, target, 0.01);
-
-    BBox ans(p1.co.Y, p2.co.Y, p4.co.Y, p3.co.Y);
-
-    return ans;
+    // Interpolate the x-coordinate of the center point
+    Point cx_left(t1, left.cx, openshot::InterpolationType::LINEAR);
+    Point cx_right(t2, right.cx, openshot::InterpolationType::LINEAR);
+    Point cx = InterpolateBetween(cx_left, cx_right, target, 0.01);
+
+    // Interpolate de y-coordinate of the center point
+    Point cy_left(t1, left.cy, openshot::InterpolationType::LINEAR);
+    Point cy_right(t2, right.cy, openshot::InterpolationType::LINEAR);
+    Point cy = InterpolateBetween(cy_left, cy_right, target, 0.01);
+
+    // Interpolate the width
+    Point width_left(t1, left.width, openshot::InterpolationType::LINEAR);
+    Point width_right(t2, right.width, openshot::InterpolationType::LINEAR);
+    Point width = InterpolateBetween(width_left, width_right, target, 0.01);
+
+    // Interpolate the height
+    Point height_left(t1, left.height, openshot::InterpolationType::LINEAR);
+    Point height_right(t2, right.height, openshot::InterpolationType::LINEAR);
+    Point height = InterpolateBetween(height_left, height_right, target, 0.01);
+
+    // Interpolate the rotation angle
+    Point angle_left(t1, left.angle, openshot::InterpolationType::LINEAR);
+    Point angle_right(t1, right.angle, openshot::InterpolationType::LINEAR);
+    Point angle = InterpolateBetween(angle_left, angle_right, target, 0.01);
+
+    // Create a bounding box with the interpolated points
+    BBox interpolatedBox(cx.co.Y, cy.co.Y, width.co.Y, height.co.Y, angle.co.Y);
+
+    return interpolatedBox;
 }
 
 // Update object's BaseFps
@@ -348,16 +345,18 @@ bool KeyFrameBBox::LoadBoxData(std::string inputFilePath)
 
         // Get bounding box data from current frame
         const libopenshottracker::Frame::Box &box = pbFrameData.bounding_box();
-        
-        float x1 = box.x1();
-        float y1 = box.y1();
-        float x2 = box.x2();
-        float y2 = box.y2();
 
-        if ( (x1 >= 0.0) && (y1 >= 0.0) && (x2 >= 0.0) && (y2 >= 0.0) )
+        float width = box.x2() - box.x1();
+        float height = box.y2() - box.y1();
+        float cx = box.x1() + width/2;
+        float cy = box.y1() + height/2;
+        float angle = 0.0;
+
+
+        if ( (cx >= 0.0) && (cy >= 0.0) && (width >= 0.0) && (height >= 0.0) )
         {
             // The bounding-box properties are valid, so add it to the BoxVec map
-            this->AddBox(frame_number, x1, y1, (x2 - x1), (y2 - y1));
+            this->AddBox(frame_number, cx, cy, width, height, angle);
         }
     }
     
@@ -464,7 +463,6 @@ void KeyFrameBBox::SetJsonValue(const Json::Value root)
             // Insert BBox into the BoxVec map 
             BBox box;
             box.SetJsonValue(existing_point["data"]);
-            //BoxVec.insert({existing_point["time"].asDouble(), box});
             BoxVec[existing_point["time"].asDouble()] = box;
         }
     }
@@ -482,4 +480,4 @@ void KeyFrameBBox::SetJsonValue(const Json::Value root)
         rotation.SetJsonValue(root["rotation"]);
 
     return;
-}
+}
diff --git a/src/KeyFrameBBox.h b/src/KeyFrameBBox.h
@@ -60,10 +60,11 @@ namespace openshot
 
     struct BBox
     {
-        float x1 = -1; ///< x-coordinate of the top left corner
-        float y1 = -1; ///< y-coordinate of the top left corner
+        float cx = -1; ///< x-coordinate of the bounding box center
+        float cy = -1; ///< y-coordinate of the bounding box center
         float width = -1; ///< bounding box width
         float height = -1; ///< bounding box height
+        float angle = -1; ///< bounding box rotation angle [degrees]
 
         /// Blank constructor
         BBox()
@@ -72,18 +73,21 @@ namespace openshot
         }
 
         /// Default constructor, which takes the bounding box top-left corner coordinates, width and height.
-        /// @param _x1 X-coordinate of the top left corner
-        /// @param _y1 Y-coordinate of the top left corner
+        /// @param _cx X-coordinate of the bounding box center
+        /// @param _cy Y-coordinate of the bounding box center
         /// @param _width Bounding box width
-        /// @param _height Bouding box height
-        BBox(float _x1, float _y1, float _width, float _height)
+        /// @param _height Bounding box height
+        /// @param _angle Bounding box rotation angle [degrees]
+        BBox(float _cx, float _cy, float _width, float _height, float _angle)
         {
-            x1 = _x1;
-            y1 = _y1;
+            cx = _cx;
+            cy = _cy;
             width = _width;
             height = _height;
+            angle = _angle;
         }
 
+        
         /// Generate JSON string of this object
         std::string Json() const
         {
@@ -94,10 +98,11 @@ namespace openshot
         Json::Value JsonValue() const
         {
             Json::Value root;
-            root["x1"] = x1;
-            root["y1"] = y1;
-            root["height"] = height;
+            root["cx"] = cx;
+            root["cy"] = cy;
             root["width"] = width;
+            root["height"] = height;
+            root["angle"] = angle;
 
             return root;
         }
@@ -124,15 +129,17 @@ namespace openshot
         {
 
             // Set data from Json (if key is found)
-            if (!root["x1"].isNull())
-                x1 = root["x1"].asDouble();
-            if (!root["y1"].isNull())
-                y1 = root["y1"].asDouble();
-            if (!root["height"].isNull())
-                height = root["height"].asDouble();
+            if (!root["cx"].isNull())
+                cx = root["cx"].asDouble();
+            if (!root["cy"].isNull())
+                cy = root["cy"].asDouble();
             if (!root["width"].isNull())
                 width = root["width"].asDouble();
-        }
+            if (!root["height"].isNull())
+                height = root["height"].asDouble();
+            if (!root["angle"].isNull())
+                angle = root["angle"].asDouble();
+        } 
     };
 
     /**
@@ -166,7 +173,7 @@ namespace openshot
         KeyFrameBBox();
 
         /// Add a BBox to the BoxVec map
-        void AddBox(int64_t _frame_num, float _x1, float _y1, float _width, float _height);
+        void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle);
         
         /// Update object's BaseFps
         void SetBaseFPS(Fraction fps);
@@ -214,4 +221,4 @@ namespace openshot
 
 } // namespace openshot
 
-#endif
+#endif
diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp
@@ -73,24 +73,30 @@ std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t f
 	cv::Mat frame_image = frame->GetImageCV();
 
     // Check if frame isn't NULL
-    if(!frame_image.empty()){
-
+    if(!frame_image.empty())
+	{
         // Check if track data exists for the requested frame
-        if (trackedData.Contains(frame_number)) {
-
+        if (trackedData.Contains(frame_number)) 
+		{
 			// Get the width and height of the image
 			float fw = frame_image.size().width;
         	float fh = frame_image.size().height;
 
 			// Get the bounding-box of given frame
 			BBox fd = this->trackedData.GetValue(frame_number);
 
-            // Draw the bounding-box on the image
-            cv::Rect2d box((int)( (fd.x1 ) * fw ),
-						   (int)( (fd.y1 ) * fh ),
-						   (int)( (fd.width) * fw),
-						   (int)( (fd.height) * fh) );
-            cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 );
+			// Create a rotated rectangle object that holds the bounding box
+			cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ), 
+								  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ), 
+								  (int) (fd.angle) );
+			// Get the bouding box vertices
+			cv::Point2f vertices[4];
+			box.points(vertices);
+			// Draw the bounding-box on the image
+			for (int i = 0; i < 4; i++)
+			{
+				cv::line(frame_image, vertices[i], vertices[(i+1)%4], cv::Scalar(255,0,0), 2);
+			}
         }
     }
 
@@ -194,10 +200,10 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
 	// Get the bounding-box for the given-frame
 	BBox fd = trackedData.GetValue(requested_frame);
 	// Add the data of given frame bounding-box to the JSON object
-	root["x1"] = add_property_json("X1", fd.x1, "float", "", NULL, 0.0, 1.0, false, requested_frame);
-	root["y1"] = add_property_json("Y1", fd.y1, "float", "", NULL, 0.0, 1.0, false, requested_frame);
-	root["x2"] = add_property_json("X2", fd.x1+fd.width, "float", "", NULL, 0.0, 1.0, false, requested_frame);
-	root["y2"] = add_property_json("Y2", fd.y1+fd.height, "float", "", NULL, 0.0, 1.0, false, requested_frame);
+	root["x1"] = add_property_json("X1", fd.cx-(fd.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
+	root["y1"] = add_property_json("Y1", fd.cy-(fd.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
+	root["x2"] = add_property_json("X2", fd.cx+(fd.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
+	root["y2"] = add_property_json("Y2", fd.cy+(fd.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
 
 	// Add the bounding-box Keyframes to the JSON object
 	root["delta_x"] = add_property_json("Displacement X-axis", trackedData.delta_x.GetValue(requested_frame), "float", "", &trackedData.delta_x, -1.0, 1.0, false, requested_frame);
@@ -260,4 +266,4 @@ void Tracker::SetJson(int64_t requested_frame, const std::string value)
 		throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
 	}
 	return;
-}
+}
diff --git a/tests/KeyFrame_Tests.cpp b/tests/KeyFrame_Tests.cpp
