Merge pull request #932 from OpenShot/effect-sequencing

Allow Effects to be applied BEFORE or AFTER a clip's keyframes

Author: jonoomph

src/Clip.cpp

@@ -440,21 +440,18 @@ std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> backgroun
 		// Apply waveform image (if any)
 		apply_waveform(frame, background_frame);
 
-		// Apply local effects to the frame (if any)
-		apply_effects(frame);
+		// Apply effects BEFORE applying keyframes (if any local or global effects are used)
+		apply_effects(frame, background_frame, options, true);
 
-		// Apply global timeline effects (i.e. transitions & masks... if any)
-		if (timeline != NULL && options != NULL) {
-			if (options->is_top_clip) {
-				// Apply global timeline effects (only to top clip... if overlapping, pass in timeline frame number)
-				Timeline* timeline_instance = static_cast<Timeline*>(timeline);
-				frame = timeline_instance->apply_effects(frame, background_frame->number, Layer());
-			}
-		}
-
-		// Apply keyframe / transforms
+		// Apply keyframe / transforms to current clip image
 		apply_keyframes(frame, background_frame);
 
+		// Apply effects AFTER applying keyframes (if any local or global effects are used)
+		apply_effects(frame, background_frame, options, false);
+
+		// Apply background canvas (i.e. flatten this image onto previous layer image)
+		apply_background(frame, background_frame);
+
 		// Add final frame to cache
 		final_cache.Add(frame);
 
@@ -1202,16 +1199,41 @@ void Clip::RemoveEffect(EffectBase* effect)
 	final_cache.Clear();
 }
 
+// Apply background image to the current clip image (i.e. flatten this image onto previous layer)
+void Clip::apply_background(std::shared_ptr<openshot::Frame> frame, std::shared_ptr<openshot::Frame> background_frame) {
+	// Add background canvas
+	std::shared_ptr<QImage> background_canvas = background_frame->GetImage();
+	QPainter painter(background_canvas.get());
+	painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
+
+	// Composite a new layer onto the image
+	painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
+	painter.drawImage(0, 0, *frame->GetImage());
+	painter.end();
+
+	// Add new QImage to frame
+	frame->AddImage(background_canvas);
+}
+
 // Apply effects to the source frame (if any)
-void Clip::apply_effects(std::shared_ptr<Frame> frame)
+void Clip::apply_effects(std::shared_ptr<Frame> frame, std::shared_ptr<Frame> background_frame, TimelineInfoStruct* options, bool before_keyframes)
 {
-	// Find Effects at this position and layer
 	for (auto effect : effects)
 	{
 		// Apply the effect to this frame
-		frame = effect->GetFrame(frame, frame->number);
+		if (effect->info.apply_before_clip && before_keyframes) {
+			effect->GetFrame(frame, frame->number);
+		} else if (!effect->info.apply_before_clip && !before_keyframes) {
+			effect->GetFrame(frame, frame->number);
+		}
+	}
 
-	} // end effect loop
+	if (timeline != NULL && options != NULL) {
+		// Apply global timeline effects (i.e. transitions & masks... if any)
+		Timeline* timeline_instance = static_cast<Timeline*>(timeline);
+		options->is_before_clip_keyframes = before_keyframes;
+		timeline_instance->apply_effects(frame, background_frame->number, Layer(), options);
+	}
 }
 
 // Compare 2 floating point numbers for equality
@@ -1228,20 +1250,16 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<Frame>
 		return;
 	}
 
-	// Get image from clip
+	// Get image from clip, and create transparent background image
 	std::shared_ptr<QImage> source_image = frame->GetImage();
-	std::shared_ptr<QImage> background_canvas = background_frame->GetImage();
+	std::shared_ptr<QImage> background_canvas = std::make_shared<QImage>(background_frame->GetImage()->width(),
+																		 background_frame->GetImage()->height(),
+																		 QImage::Format_RGBA8888_Premultiplied);
+	background_canvas->fill(QColor(Qt::transparent));
 
 	// Get transform from clip's keyframes
 	QTransform transform = get_transform(frame, background_canvas->width(), background_canvas->height());
 
-	// Debug output
-	ZmqLogger::Instance()->AppendDebugMethod(
-		"Clip::ApplyKeyframes (Transform: Composite Image Layer: Prepare)",
-		"frame->number", frame->number,
-		"background_canvas->width()", background_canvas->width(),
-		"background_canvas->height()", background_canvas->height());
-
 	// Load timeline's new frame image into a QPainter
 	QPainter painter(background_canvas.get());
 	painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);

src/Clip.h

@@ -127,8 +127,11 @@ namespace openshot {
 		/// Adjust frame number minimum value
 		int64_t adjust_frame_number_minimum(int64_t frame_number);
 
+		/// Apply background image to the current clip image (i.e. flatten this image onto previous layer)
+		void apply_background(std::shared_ptr<openshot::Frame> frame, std::shared_ptr<openshot::Frame> background_frame);
+
 		/// Apply effects to the source frame (if any)
-		void apply_effects(std::shared_ptr<openshot::Frame> frame);
+		void apply_effects(std::shared_ptr<openshot::Frame> frame, std::shared_ptr<openshot::Frame> background_frame, TimelineInfoStruct* options, bool before_keyframes);
 
 		/// Apply keyframes to an openshot::Frame and use an existing background frame (if any)
 		void apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<Frame> background_frame);

src/EffectBase.cpp

@@ -30,7 +30,6 @@ void EffectBase::InitEffectInfo()
 	End(0.0);
 	Order(0);
 	ParentClip(NULL);
-
 	parentEffect = NULL;
 
 	info.has_video = false;
@@ -39,6 +38,7 @@ void EffectBase::InitEffectInfo()
 	info.name = "";
 	info.description = "";
 	info.parent_effect_id = "";
+	info.apply_before_clip = true;
 }
 
 // Display file information
@@ -51,6 +51,8 @@ void EffectBase::DisplayInfo(std::ostream* out) {
 	*out << "--> Description: " << info.description << std::endl;
 	*out << "--> Has Video: " << info.has_video << std::endl;
 	*out << "--> Has Audio: " << info.has_audio << std::endl;
+	*out << "--> Apply Before Clip Keyframes: " << info.apply_before_clip << std::endl;
+	*out << "--> Order: " << order << std::endl;
 	*out << "----------------------------" << std::endl;
 }
 
@@ -85,6 +87,7 @@ Json::Value EffectBase::JsonValue() const {
 	root["has_video"] = info.has_video;
 	root["has_audio"] = info.has_audio;
 	root["has_tracked_object"] = info.has_tracked_object;
+	root["apply_before_clip"] = info.apply_before_clip;
 	root["order"] = Order();
 
 	// return JsonValue
@@ -145,6 +148,9 @@ void EffectBase::SetJsonValue(const Json::Value root) {
 	if (!my_root["order"].isNull())
 		Order(my_root["order"].asInt());
 
+	if (!my_root["apply_before_clip"].isNull())
+		info.apply_before_clip = my_root["apply_before_clip"].asBool();
+
 	if (!my_root["parent_effect_id"].isNull()){
 		info.parent_effect_id = my_root["parent_effect_id"].asString();
 		if (info.parent_effect_id.size() > 0 && info.parent_effect_id != "" && parentEffect == NULL)
@@ -169,6 +175,28 @@ Json::Value EffectBase::JsonInfo() const {
 	return root;
 }
 
+// Get all properties for a specific frame
+Json::Value EffectBase::BasePropertiesJSON(int64_t requested_frame) const {
+	// Generate JSON properties list
+	Json::Value root;
+	root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
+	root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 30 * 60 * 60 * 48, false, requested_frame);
+	root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
+	root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 30 * 60 * 60 * 48, false, requested_frame);
+	root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 30 * 60 * 60 * 48, false, requested_frame);
+	root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 30 * 60 * 60 * 48, true, requested_frame);
+
+	// Add replace_image choices (dropdown style)
+	root["apply_before_clip"] = add_property_json("Apply Before Clip Keyframes", info.apply_before_clip, "int", "", NULL, 0, 1, false, requested_frame);
+	root["apply_before_clip"]["choices"].append(add_property_choice_json("Yes", true, info.apply_before_clip));
+	root["apply_before_clip"]["choices"].append(add_property_choice_json("No", false, info.apply_before_clip));
+
+	// Set the parent effect which properties this effect will inherit
+	root["parent_effect_id"] = add_property_json("Parent", 0.0, "string", info.parent_effect_id, NULL, -1, -1, false, requested_frame);
+
+	return root;
+}
+
 /// Parent clip object of this reader (which can be unparented and NULL)
 openshot::ClipBase* EffectBase::ParentClip() {
 	return clip;

src/EffectBase.h

@@ -40,6 +40,7 @@ namespace openshot
 		bool has_video;	///< Determines if this effect manipulates the image of a frame
 		bool has_audio;	///< Determines if this effect manipulates the audio of a frame
 		bool has_tracked_object; ///< Determines if this effect track objects through the clip
+		bool apply_before_clip; ///< Apply effect before we evaluate the clip's keyframes
 	};
 
 	/**
@@ -58,7 +59,6 @@ namespace openshot
 		openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any)
 
 	public:
-
 		/// Parent effect (which properties will set this effect properties)
 		EffectBase* parentEffect;
 
@@ -106,7 +106,11 @@ namespace openshot
 			return;
 		};
 
-		Json::Value JsonInfo() const; ///< Generate JSON object of meta data / info
+		/// Generate JSON object of meta data / info
+		Json::Value JsonInfo() const; 
+
+		/// Generate JSON object of base properties (recommended to be used by all effects)
+		Json::Value BasePropertiesJSON(int64_t requested_frame) const;
 
 		/// Get the order that this effect should be executed.
 		int Order() const { return order; }

src/Timeline.cpp

@@ -523,7 +523,7 @@ double Timeline::calculate_time(int64_t number, Fraction rate)
 }
 
 // Apply effects to the source frame (if any)
-std::shared_ptr<Frame> Timeline::apply_effects(std::shared_ptr<Frame> frame, int64_t timeline_frame_number, int layer)
+std::shared_ptr<Frame> Timeline::apply_effects(std::shared_ptr<Frame> frame, int64_t timeline_frame_number, int layer, TimelineInfoStruct* options)
 {
 	// Debug output
 	ZmqLogger::Instance()->AppendDebugMethod(
@@ -541,21 +541,19 @@ std::shared_ptr<Frame> Timeline::apply_effects(std::shared_ptr<Frame> frame, int
 
 		bool does_effect_intersect = (effect_start_position <= timeline_frame_number && effect_end_position >= timeline_frame_number && effect->Layer() == layer);
 
-		// Debug output
-		ZmqLogger::Instance()->AppendDebugMethod(
-			"Timeline::apply_effects (Does effect intersect)",
-			"effect->Position()", effect->Position(),
-			"does_effect_intersect", does_effect_intersect,
-			"timeline_frame_number", timeline_frame_number,
-			"layer", layer);
-
 		// Clip is visible
 		if (does_effect_intersect)
 		{
 			// Determine the frame needed for this clip (based on the position on the timeline)
 			long effect_start_frame = (effect->Start() * info.fps.ToDouble()) + 1;
 			long effect_frame_number = timeline_frame_number - effect_start_position + effect_start_frame;
 
+			if (!options->is_top_clip)
+				continue; // skip effect, if overlapped/covered by another clip on same layer
+
+			if (options->is_before_clip_keyframes != effect->info.apply_before_clip)
+				continue; // skip effect, if this filter does not match
+
 			// Debug output
 			ZmqLogger::Instance()->AppendDebugMethod(
 				"Timeline::apply_effects (Process Effect)",
@@ -615,6 +613,7 @@ void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, in
 	// Create timeline options (with details about this current frame request)
 	TimelineInfoStruct* options = new TimelineInfoStruct();
 	options->is_top_clip = is_top_clip;
+	options->is_before_clip_keyframes = true;
 
 	// Get the clip's frame, composited on top of the current timeline frame
 	std::shared_ptr<Frame> source_frame;

src/Timeline.h

@@ -68,15 +68,13 @@ namespace openshot {
 	/// the Clip with the highest end-frame number using std::max_element
 	struct CompareClipEndFrames {
 		bool operator()(const openshot::Clip* lhs, const openshot::Clip* rhs) {
-			return (lhs->Position() + lhs->Duration())
-			       <= (rhs->Position() + rhs->Duration());
+			return (lhs->Position() + lhs->Duration()) <= (rhs->Position() + rhs->Duration());
 	}};
 
 	/// Like CompareClipEndFrames, but for effects
 	struct CompareEffectEndFrames {
 		bool operator()(const openshot::EffectBase* lhs, const openshot::EffectBase* rhs) {
-			return (lhs->Position() + lhs->Duration())
-				<= (rhs->Position() + rhs->Duration());
+			return (lhs->Position() + lhs->Duration()) <= (rhs->Position() + rhs->Duration());
 	}};
 
 	/**
@@ -231,7 +229,7 @@ namespace openshot {
 		/// @param convert_absolute_paths Should all paths be converted to absolute paths (relative to the location of projectPath)
 		Timeline(const std::string& projectPath, bool convert_absolute_paths);
 
-        virtual ~Timeline();
+		virtual ~Timeline();
 
 		/// Add to the tracked_objects map a pointer to a tracked object (TrackedObjectBBox)
 		void AddTrackedObject(std::shared_ptr<openshot::TrackedObjectBase> trackedObject);
@@ -240,9 +238,9 @@ namespace openshot {
 		/// Return the ID's of the tracked objects as a list of strings
 		std::list<std::string> GetTrackedObjectsIds() const;
 		/// Return the trackedObject's properties as a JSON string
-        #ifdef USE_OPENCV
+		#ifdef USE_OPENCV
 		std::string GetTrackedObjectValues(std::string id, int64_t frame_number) const;
-        #endif
+		#endif
 
 		/// @brief Add an openshot::Clip to the timeline
 		/// @param clip Add an openshot::Clip to the timeline. A clip can contain any type of Reader.
@@ -252,8 +250,8 @@ namespace openshot {
 		/// @param effect Add an effect to the timeline. An effect can modify the audio or video of an openshot::Frame.
 		void AddEffect(openshot::EffectBase* effect);
 
-        /// Apply global/timeline effects to the source frame (if any)
-        std::shared_ptr<openshot::Frame> apply_effects(std::shared_ptr<openshot::Frame> frame, int64_t timeline_frame_number, int layer);
+		/// Apply global/timeline effects to the source frame (if any)
+		std::shared_ptr<openshot::Frame> apply_effects(std::shared_ptr<openshot::Frame> frame, int64_t timeline_frame_number, int layer, TimelineInfoStruct* options);
 
 		/// Apply the timeline's framerate and samplerate to all clips
 		void ApplyMapperToClips();
@@ -266,7 +264,7 @@ namespace openshot {
 
 		/// Clear all clips, effects, and frame mappers from timeline (and free memory)
 		void Clear();
-        
+
 		/// Clear all cache for this timeline instance, including all clips' cache
 		/// @param deep If True, clear all FrameMappers and nested Readers (QtImageReader, FFmpegReader, etc...)
 		void ClearAllCache(bool deep=false);

src/TimelineBase.h

@@ -18,21 +18,22 @@
 
 
 namespace openshot {
-    // Forward decl
-    class Clip;
-
-    /**
-     * @brief This struct contains info about the current Timeline clip instance
-     *
-     * When the Timeline requests an openshot::Frame instance from a Clip, it passes
-     * this struct along, with some additional details from the Timeline, such as if this clip is
-     * above or below overlapping clips, etc... This info can help determine if a Clip should apply
-     * global effects from the Timeline, such as a global Transition/Mask effect.
-     */
-    struct TimelineInfoStruct
-    {
-        bool is_top_clip;                 ///< Is clip on top (if overlapping another clip)
-    };
+	// Forward decl
+	class Clip;
+
+	/**
+	 * @brief This struct contains info about the current Timeline clip instance
+	 *
+	 * When the Timeline requests an openshot::Frame instance from a Clip, it passes
+	 * this struct along, with some additional details from the Timeline, such as if this clip is
+	 * above or below overlapping clips, etc... This info can help determine if a Clip should apply
+	 * global effects from the Timeline, such as a global Transition/Mask effect.
+	 */
+	struct TimelineInfoStruct
+	{
+		bool is_top_clip;				 ///< Is clip on top (if overlapping another clip)
+		bool is_before_clip_keyframes;	///< Is this before clip keyframes are applied
+	};
 
 	/**
 	 * @brief This class represents a timeline (used for building generic timeline implementations)