included CVStabilization into Clip

Author: BrennoCaldato

include/CVStabilization.h

@@ -1,27 +1,59 @@
+/**
+ * @file
+ * @brief Header file for CVStabilization class
+ * @author Jonathan Thomas <[email protected]>
+ *
+ * @ref License
+ */
+
+/* LICENSE
+ *
+ * Copyright (c) 2008-2019 OpenShot Studios, LLC
+ * <http://www.openshotstudios.com/>. This file is part of
+ * OpenShot Library (libopenshot), an open-source project dedicated to
+ * delivering high quality video editing and animation solutions to the
+ * world. For more information visit <http://www.openshot.org/>.
+ *
+ * OpenShot Library (libopenshot) is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * OpenShot Library (libopenshot) is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef OPENSHOT_STABILIZATION_H
+#define OPENSHOT_STABILIZATION_H
+
 #define int64 opencv_broken_int
 #define uint64 opencv_broken_uint
 #include <opencv2/opencv.hpp>
 #include <opencv2/core.hpp>
 #undef uint64
 #undef int64
 #include <cmath>
-#include "Clip.h"
 
 using namespace std;
 
-// struct TransformParam
-// {
-//     TransformParam() {}
-//     TransformParam(double _dx, double _dy, double _da) {
-//         dx = _dx;
-//         dy = _dy;
-//         da = _da;
-//     }
+struct TransformParam
+{
+    TransformParam() {}
+    TransformParam(double _dx, double _dy, double _da) {
+        dx = _dx;
+        dy = _dy;
+        da = _da;
+    }
 
-//     double dx;
-//     double dy;
-//     double da; // angle
-// };
+    double dx;
+    double dy;
+    double da; // angle
+};
 
 struct CamTrajectory
 {
@@ -44,13 +76,14 @@ class CVStabilization {
     cv::Mat prev, prev_grey;
 
     public:
-    const int SMOOTHING_RADIUS = 30; // In frames. The larger the more stable the video, but less reactive to sudden panning
-    const int HORIZONTAL_BORDER_CROP = 20; // In pixels. Crops the border to reduce the black borders from stabilisation being too noticeable.
+    const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning
     std::vector <TransformParam> prev_to_cur_transform; // previous to current
 
     CVStabilization();
 
-    void ProcessVideo(openshot::Clip &video);
+    CVStabilization(int _smoothingWindow);
+
+    // void ProcessVideo(openshot::Clip &video);
 
     // Track current frame features and find the relative transformation
     void TrackFrameFeatures(cv::Mat frame, int frameNum);
@@ -62,6 +95,8 @@ class CVStabilization {
     std::vector<TransformParam> GenNewCamPosition(std::vector <CamTrajectory> &smoothed_trajectory);
 
     // Send smoothed camera transformation to be applyed on clip
-    void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector <TransformParam> &new_prev_to_cur_transform);
+    // void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector <TransformParam> &new_prev_to_cur_transform);
+
+};
 
-};
+#endif

include/Clip.h

@@ -38,6 +38,8 @@
 	#include <opencv2/core.hpp>
 	#undef uint64
 	#undef int64	
+
+	#include "CVStabilization.h"
 #endif
 
 #include <memory>
@@ -56,26 +58,9 @@
 #include "JuceHeader.h"
 
 
-
-
-// TODO: move to stabilization effect
-struct TransformParam
-{
-	TransformParam() {}
-	TransformParam(double _dx, double _dy, double _da) {
-		dx = _dx;
-		dy = _dy;
-		da = _da;
-	}
-
-	double dx;
-	double dy;
-	double da; // angle
-};
-
-
 namespace openshot {
 
+
 	/// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted
 	/// from lowest layer to top layer (since that is sequence clips are combined), and then by
 	/// position, and then by effect order.
@@ -140,6 +125,14 @@ namespace openshot {
 		/// (reader member variable itself may have been replaced)
 		openshot::ReaderBase* allocated_reader;
 
+		#ifdef USE_OPENCV
+		/// Smoothed transformation for all the clip frames
+		std::vector <TransformParam> new_prev_to_cur_transform;
+		/// apply the smoothed transformation warp when retrieving a frame
+		bool hasStabilization = false;
+		void apply_stabilization(std::shared_ptr<openshot::Frame> f, int64_t frame_number);
+		#endif
+
 		/// Adjust frame number minimum value
 		int64_t adjust_frame_number_minimum(int64_t frame_number);
 
@@ -167,21 +160,15 @@ namespace openshot {
 		/// Reverse an audio buffer
 		void reverse_buffer(juce::AudioSampleBuffer* buffer);
 
+
+
 	public:
 		openshot::GravityType gravity;   ///< The gravity of a clip determines where it snaps to its parent
 		openshot::ScaleType scale;		 ///< The scale determines how a clip should be resized to fit its parent
 		openshot::AnchorType anchor;     ///< The anchor determines what parent a clip should snap to
 		openshot::FrameDisplayType display; ///< The format to display the frame number (if any)
 		openshot::VolumeMixType mixing;  ///< What strategy should be followed when mixing audio with other clips
 
-
-		#ifdef USE_OPENCV
-		std::vector <TransformParam> new_prev_to_cur_transform;
-		bool hasStabilization = false;
-		void apply_stabilization(std::shared_ptr<openshot::Frame> f, int64_t frame_number);
-		#endif
-
-
 		/// Default Constructor
 		Clip();
 
@@ -244,6 +231,9 @@ namespace openshot {
 		bool Waveform() { return waveform; } ///< Get the waveform property of this clip
 		void Waveform(bool value) { waveform = value; } ///< Set the waveform property of this clip
 
+		/// Stabilize the clip using opencv and opticalflow
+		void stabilize_video();
+
 		// Scale, Location, and Alpha curves
 		openshot::Keyframe scale_x; ///< Curve representing the horizontal scaling in percent (0 to 1)
 		openshot::Keyframe scale_y; ///< Curve representing the vertical scaling in percent (0 to 1)

src/CVStabilization.cpp

@@ -1,43 +1,48 @@
 #include "../include/CVStabilization.h"
 
-CVStabilization::CVStabilization(){
 
-}
+CVStabilization::CVStabilization():smoothingWindow(30) {}
+
+CVStabilization::CVStabilization(int _smoothingWindow): smoothingWindow(_smoothingWindow){}
+
+// void CVStabilization::ProcessVideo(openshot::Clip &video){
+//     // Make sure Clip is opened
+//     video.Open();
+//     // Get total number of frames
+//     int videoLenght = video.Reader()->info.video_length;
+
+//     // Get first Opencv image
+//     std::shared_ptr<openshot::Frame> f = video.GetFrame(0);
+//     cv::Mat prev = f->GetImageCV();
+//     // OpticalFlow works with grayscale images
+//     cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY);
 
-void CVStabilization::ProcessVideo(openshot::Clip &video){
-    // Make sure Clip is opened
-    video.Open();
-    // Get total number of frames
-    int videoLenght = video.Reader()->info.video_length;
-
-    // Get first Opencv image
-    std::shared_ptr<openshot::Frame> f = video.GetFrame(0);
-    cv::Mat prev = f->GetImageCV();
-    // OpticalFlow works with grayscale images
-    cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY);
-
-    // Extract and track opticalflow features for each frame
-    for (long int frame_number = 1; frame_number <= videoLenght; frame_number++)
-    {
-        std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
+//     // Extract and track opticalflow features for each frame
+//     for (long int frame_number = 1; frame_number <= videoLenght; frame_number++)
+//     {
+//         std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
 
-        // Grab Mat image
-        cv::Mat cvimage = f->GetImageCV();
-        cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
-        TrackFrameFeatures(cvimage, frame_number);
-    }
+//         // Grab Mat image
+//         cv::Mat cvimage = f->GetImageCV();
+//         cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
+//         TrackFrameFeatures(cvimage, frame_number);
+//     }
 
-    vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
+//     vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
 
-    vector <CamTrajectory> smoothed_trajectory = SmoothTrajectory(trajectory);
+//     vector <CamTrajectory> smoothed_trajectory = SmoothTrajectory(trajectory);
 
-    vector <TransformParam> new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory);
+//     vector <TransformParam> new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory);
 
-    ApplyNewTrajectoryToClip(video, new_prev_to_cur_transform);
-}
+//     ApplyNewTrajectoryToClip(video, new_prev_to_cur_transform);
+// }
 
 // Track current frame features and find the relative transformation
 void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){
+    if(prev_grey.empty()){
+        prev_grey = frame;
+        return;
+    }
 
     // OpticalFlow features vector
     vector <cv::Point2f> prev_corner, cur_corner;
@@ -113,7 +118,7 @@ vector <CamTrajectory> CVStabilization::SmoothTrajectory(vector <CamTrajectory>
         double sum_a = 0;
         int count = 0;
 
-        for(int j=-SMOOTHING_RADIUS; j <= SMOOTHING_RADIUS; j++) {
+        for(int j=-smoothingWindow; j <= smoothingWindow; j++) {
             if(i+j >= 0 && i+j < trajectory.size()) {
                 sum_x += trajectory[i+j].x;
                 sum_y += trajectory[i+j].y;
@@ -164,9 +169,9 @@ vector <TransformParam> CVStabilization::GenNewCamPosition(vector <CamTrajectory
     return new_prev_to_cur_transform;
 }
 
-// Send smoothed camera transformation to be applyed on clip 
-void CVStabilization::ApplyNewTrajectoryToClip(openshot::Clip &video, vector <TransformParam> &new_prev_to_cur_transform){
+// // Send smoothed camera transformation to be applyed on clip 
+// void CVStabilization::ApplyNewTrajectoryToClip(openshot::Clip &video, vector <TransformParam> &new_prev_to_cur_transform){
 
-    video.new_prev_to_cur_transform = new_prev_to_cur_transform;
-    video.hasStabilization = true;
-}
+//     video.new_prev_to_cur_transform = new_prev_to_cur_transform;
+//     video.hasStabilization = true;
+// }

src/Clip.cpp

@@ -1075,3 +1075,45 @@ std::shared_ptr<Frame> Clip::apply_effects(std::shared_ptr<Frame> frame)
 	// Return modified frame
 	return frame;
 }
+
+#ifdef USE_OPENCV
+void Clip::stabilize_video(){
+	// create CVStabilization object
+	CVStabilization stabilizer; 
+
+    // Make sure Clip is opened
+    Open();
+    // Get total number of frames
+    int videoLenght = Reader()->info.video_length;
+
+    // Get first Opencv image
+    // std::shared_ptr<openshot::Frame> f = GetFrame(0);
+    // cv::Mat prev = f->GetImageCV();
+    // // OpticalFlow works with grayscale images
+    // cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY);
+
+    // Extract and track opticalflow features for each frame
+    for (long int frame_number = 0; frame_number <= videoLenght; frame_number++)
+    {
+        std::shared_ptr<openshot::Frame> f = GetFrame(frame_number);
+        
+        // Grab Mat image
+        cv::Mat cvimage = f->GetImageCV();
+        cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
+        stabilizer.TrackFrameFeatures(cvimage, frame_number);
+    }
+
+    vector <CamTrajectory> trajectory = stabilizer.ComputeFramesTrajectory();
+
+    vector <CamTrajectory> smoothed_trajectory = stabilizer.SmoothTrajectory(trajectory);
+
+	// Get the smoothed trajectory
+    new_prev_to_cur_transform = stabilizer.GenNewCamPosition(smoothed_trajectory);
+	// Will apply the smoothed transformation warp when retrieving a frame
+	hasStabilization = true;
+}
+#else
+void Clip::stabilize_video(){
+	throw "Please compile libopenshot with OpenCV to use this feature";
+}
+#endif

src/examples/Boneyard Memories.mp4

@@ -164,7 +164,7 @@ int main(int argc, char* argv[]) {
         displayTrackedData(r9);
     if(SMOOTH_VIDEO){
         CVStabilization stabilization;
-        stabilization.ProcessVideo(r9);
+        r9.stabilize_video();
         displayStabilization(r9);
     }