Skip to content

Commit 3daa5bd

Browse files
jonoomphjonoomph
committed
Large refactor of OpenMP integration (or rather the removal of OpenMP on the Timeline and FFmpeg-related classes). The logic behind this decision, was based on profiling libopenshot and the amount of wasted CPU idle time on all the various threads. The slow code is still synchronous, and all the threads must wait on each other, adding additional overhead. So, removing lots of unneeded threads, and simplifying the underlying Timeline->Clip->FFmpegReader flow. Also, removed 2 calls to QPainter::drawImage, by improving the flexibility of Clip->GetFrame.
1 parent 15695e3 commit 3daa5bd

File tree

15 files changed

+1037
-1242
lines changed

15 files changed

+1037
-1242
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@
66
.project
77
.cproject
88
/.metadata/
9+
cmake-build-debug/*
910
tags
1011
*~
11-

‎examples/Example.cpp‎

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -39,51 +39,52 @@ using namespace openshot;
3939

4040
int main(int argc, char* argv[]) {
4141

42-
Settings *s = Settings::Instance();
43-
s->HARDWARE_DECODER = 2; // 1 VA-API, 2 NVDEC, 6 VDPAU
44-
s->HW_DE_DEVICE_SET = 0;
42+
// Types for storing time durations in whole and fractional milliseconds
43+
using ms = std::chrono::milliseconds;
44+
using s = std::chrono::seconds;
45+
using double_ms = std::chrono::duration<double, ms::period>;
4546

46-
std::string input_filepath = TEST_MEDIA_PATH;
47-
input_filepath += "sintel_trailer-720p.mp4";
47+
// Track total time
48+
const auto total_time = double_ms(0.0);
4849

49-
FFmpegReader r9(input_filepath);
50+
// FFmpeg Reader performance test
51+
const auto total_1 = std::chrono::high_resolution_clock::now();
52+
FFmpegReader r9("/home/jonathan/Videos/sintel_trailer-1080p.mp4");
5053
r9.Open();
51-
r9.DisplayInfo();
52-
53-
/* WRITER ---------------- */
54-
FFmpegWriter w9("metadata.mp4");
54+
for (long int frame = 1; frame <= 1000; frame++)
55+
{
56+
const auto time1 = std::chrono::high_resolution_clock::now();
57+
std::shared_ptr<Frame> f = r9.GetFrame(frame);
58+
const auto time2 = std::chrono::high_resolution_clock::now();
59+
std::cout << "FFmpegReader: " << frame << " (" << double_ms(time2 - time1).count() << " ms)" << std::endl;
60+
}
61+
const auto total_2 = std::chrono::high_resolution_clock::now();
62+
auto total_sec = std::chrono::duration_cast<ms>(total_2 - total_1);
63+
std::cout << "FFmpegReader TOTAL: " << total_sec.count() << " ms" << std::endl;
64+
r9.Close();
5565

56-
// Set options
57-
w9.SetAudioOptions(true, "libmp3lame", r9.info.sample_rate, r9.info.channels, r9.info.channel_layout, 128000);
58-
w9.SetVideoOptions(true, "libx264", r9.info.fps, 1024, 576, Fraction(1,1), false, false, 3000000);
5966

60-
w9.info.metadata["title"] = "testtest";
61-
w9.info.metadata["artist"] = "aaa";
62-
w9.info.metadata["album"] = "bbb";
63-
w9.info.metadata["year"] = "2015";
64-
w9.info.metadata["description"] = "ddd";
65-
w9.info.metadata["comment"] = "eee";
66-
w9.info.metadata["comment"] = "comment";
67-
w9.info.metadata["copyright"] = "copyright OpenShot!";
6867

69-
// Open writer
70-
w9.Open();
68+
// Timeline Reader performance test
69+
Timeline tm(r9.info.width, r9.info.height, r9.info.fps, r9.info.sample_rate, r9.info.channels, r9.info.channel_layout);
70+
Clip *c = new Clip(&r9);
71+
tm.AddClip(c);
72+
tm.Open();
7173

72-
for (long int frame = 1; frame <= 100; frame++)
74+
const auto total_3 = std::chrono::high_resolution_clock::now();
75+
for (long int frame = 1; frame <= 1000; frame++)
7376
{
74-
//int frame_number = (rand() % 750) + 1;
75-
int frame_number = frame;
76-
std::shared_ptr<Frame> f = r9.GetFrame(frame_number);
77-
w9.WriteFrame(f);
77+
const auto time1 = std::chrono::high_resolution_clock::now();
78+
std::shared_ptr<Frame> f = tm.GetFrame(frame);
79+
const auto time2 = std::chrono::high_resolution_clock::now();
80+
std::cout << "Timeline: " << frame << " (" << double_ms(time2 - time1).count() << " ms)" << std::endl;
7881
}
82+
const auto total_4 = std::chrono::high_resolution_clock::now();
83+
total_sec = std::chrono::duration_cast<ms>(total_4 - total_3);
84+
std::cout << "Timeline TOTAL: " << total_sec.count() << " ms" << std::endl;
85+
tm.Close();
7986

80-
// Close writer & reader
81-
w9.Close();
82-
83-
// Close timeline
84-
r9.Close();
85-
86-
std::cout << "Completed successfully!" << std::endl;
87+
std::cout << "Completed successfully!" << std::endl;
8788

8889
return 0;
8990
}

‎src/Clip.cpp‎

Lines changed: 87 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ std::shared_ptr<Frame> Clip::GetFrame(int64_t frame_number)
346346
}
347347

348348
// Use an existing openshot::Frame object and draw this Clip's frame onto it
349-
std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> frame, int64_t frame_number)
349+
std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> background_frame, int64_t frame_number)
350350
{
351351
// Check for open reader (or throw exception)
352352
if (!is_open)
@@ -376,7 +376,7 @@ std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> frame, in
376376
int enabled_video = has_video.GetInt(frame_number);
377377
if (enabled_video == -1 && reader && reader->info.has_video)
378378
enabled_video = 1;
379-
else if (enabled_video == -1 && reader && !reader->info.has_audio)
379+
else if (enabled_video == -1 && reader && !reader->info.has_video)
380380
enabled_video = 0;
381381

382382
// Is a time map detected
@@ -386,27 +386,14 @@ std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> frame, in
386386
new_frame_number = time_mapped_number;
387387

388388
// Now that we have re-mapped what frame number is needed, go and get the frame pointer
389-
std::shared_ptr<Frame> original_frame;
390-
original_frame = GetOrCreateFrame(new_frame_number);
391-
392-
// Copy the image from the odd field
393-
if (enabled_video)
394-
frame->AddImage(std::make_shared<QImage>(*original_frame->GetImage()));
395-
396-
// Loop through each channel, add audio
397-
if (enabled_audio && reader->info.has_audio)
398-
for (int channel = 0; channel < original_frame->GetAudioChannelsCount(); channel++)
399-
frame->AddAudio(true, channel, 0, original_frame->GetAudioSamples(channel), original_frame->GetAudioSamplesCount(), 1.0);
389+
std::shared_ptr<Frame> original_frame = GetOrCreateFrame(new_frame_number);
400390

401391
// Get time mapped frame number (used to increase speed, change direction, etc...)
402392
// TODO: Handle variable # of samples, since this resamples audio for different speeds (only when time curve is set)
403-
get_time_mapped_frame(frame, new_frame_number);
404-
405-
// Adjust # of samples to match requested (the interaction with time curves will make this tricky)
406-
// TODO: Implement move samples to/from next frame
393+
get_time_mapped_frame(original_frame, new_frame_number);
407394

408395
// Apply effects to the frame (if any)
409-
apply_effects(frame);
396+
apply_effects(original_frame);
410397

411398
// Determine size of image (from Timeline or Reader)
412399
int width = 0;
@@ -422,13 +409,13 @@ std::shared_ptr<Frame> Clip::GetFrame(std::shared_ptr<openshot::Frame> frame, in
422409
}
423410

424411
// Apply keyframe / transforms
425-
apply_keyframes(frame, width, height);
412+
apply_keyframes(original_frame, background_frame->GetImage());
426413

427-
// Cache frame
428-
cache.Add(frame);
414+
// Cache frame
415+
cache.Add(original_frame);
429416

430417
// Return processed 'frame'
431-
return frame;
418+
return original_frame;
432419
}
433420
else
434421
// Throw error if reader not initialized
@@ -709,7 +696,6 @@ std::shared_ptr<Frame> Clip::GetOrCreateFrame(int64_t number)
709696
// Create a new copy of reader frame
710697
// This allows a clip to modify the pixels and audio of this frame without
711698
// changing the underlying reader's frame data
712-
//std::shared_ptr<Frame> reader_copy(new Frame(number, 1, 1, "#000000", reader_frame->GetAudioSamplesCount(), reader_frame->GetAudioChannelsCount()));
713699
auto reader_copy = std::make_shared<Frame>(*reader_frame.get());
714700
reader_copy->SampleRate(reader_frame->SampleRate());
715701
reader_copy->ChannelsLayout(reader_frame->ChannelsLayout());
@@ -1126,18 +1112,84 @@ bool Clip::isEqual(double a, double b)
11261112
return fabs(a - b) < 0.000001;
11271113
}
11281114

1115+
// Apply keyframes to the source frame (if any)
1116+
void Clip::apply_keyframes(std::shared_ptr<Frame> frame, std::shared_ptr<QImage> background_canvas) {
1117+
// Skip out if video was disabled or only an audio frame (no visualisation in use)
1118+
if (has_video.GetInt(frame->number) == 0 ||
1119+
(!Waveform() && !Reader()->info.has_video))
1120+
// Skip the rest of the image processing for performance reasons
1121+
return;
1122+
1123+
// Get image from clip
1124+
std::shared_ptr<QImage> source_image = frame->GetImage();
1125+
1126+
// Size of final image
1127+
int width = background_canvas->width();
1128+
int height = background_canvas->height();
1129+
1130+
// Get transform from clip's keyframes
1131+
QTransform transform = get_transform(frame, width, height);
1132+
1133+
// Debug output
1134+
ZmqLogger::Instance()->AppendDebugMethod("Clip::ApplyKeyframes (Transform: Composite Image Layer: Prepare)", "frame->number", frame->number);
1135+
1136+
// Load timeline's new frame image into a QPainter
1137+
QPainter painter(background_canvas.get());
1138+
painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
1139+
1140+
// Apply transform (translate, rotate, scale)
1141+
painter.setTransform(transform);
1142+
1143+
// Composite a new layer onto the image
1144+
painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
1145+
painter.drawImage(0, 0, *source_image);
1146+
1147+
if (timeline) {
1148+
Timeline *t = (Timeline *) timeline;
1149+
1150+
// Draw frame #'s on top of image (if needed)
1151+
if (display != FRAME_DISPLAY_NONE) {
1152+
std::stringstream frame_number_str;
1153+
switch (display) {
1154+
case (FRAME_DISPLAY_NONE):
1155+
// This is only here to prevent unused-enum warnings
1156+
break;
1157+
1158+
case (FRAME_DISPLAY_CLIP):
1159+
frame_number_str << frame->number;
1160+
break;
1161+
1162+
case (FRAME_DISPLAY_TIMELINE):
1163+
frame_number_str << (position * t->info.fps.ToFloat()) + frame->number;
1164+
break;
1165+
1166+
case (FRAME_DISPLAY_BOTH):
1167+
frame_number_str << (position * t->info.fps.ToFloat()) + frame->number << " (" << frame->number << ")";
1168+
break;
1169+
}
1170+
1171+
// Draw frame number on top of image
1172+
painter.setPen(QColor("#ffffff"));
1173+
painter.drawText(20, 20, QString(frame_number_str.str().c_str()));
1174+
}
1175+
}
1176+
painter.end();
1177+
1178+
// Add new QImage to frame
1179+
frame->AddImage(background_canvas);
1180+
}
11291181

11301182
// Apply keyframes to the source frame (if any)
1131-
void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
1183+
QTransform Clip::get_transform(std::shared_ptr<Frame> frame, int width, int height)
11321184
{
1133-
// Get actual frame image data
1134-
std::shared_ptr<QImage> source_image = frame->GetImage();
1185+
// Get image from clip
1186+
std::shared_ptr<QImage> source_image = frame->GetImage();
11351187

11361188
/* REPLACE IMAGE WITH WAVEFORM IMAGE (IF NEEDED) */
11371189
if (Waveform())
11381190
{
11391191
// Debug output
1140-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Generate Waveform Image)", "frame->number", frame->number, "Waveform()", Waveform());
1192+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Generate Waveform Image)", "frame->number", frame->number, "Waveform()", Waveform());
11411193

11421194
// Get the color of the waveform
11431195
int red = wave_color.red.GetInt(frame->number);
@@ -1170,7 +1222,7 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
11701222
}
11711223

11721224
// Debug output
1173-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Set Alpha & Opacity)", "alpha_value", alpha_value, "frame->number", frame->number);
1225+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Set Alpha & Opacity)", "alpha_value", alpha_value, "frame->number", frame->number);
11741226
}
11751227

11761228
/* RESIZE SOURCE IMAGE - based on scale type */
@@ -1181,21 +1233,21 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
11811233
source_size.scale(width, height, Qt::KeepAspectRatio);
11821234

11831235
// Debug output
1184-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Scale: SCALE_FIT)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
1236+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Scale: SCALE_FIT)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
11851237
break;
11861238
}
11871239
case (SCALE_STRETCH): {
11881240
source_size.scale(width, height, Qt::IgnoreAspectRatio);
11891241

11901242
// Debug output
1191-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Scale: SCALE_STRETCH)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
1243+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Scale: SCALE_STRETCH)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
11921244
break;
11931245
}
11941246
case (SCALE_CROP): {
11951247
source_size.scale(width, height, Qt::KeepAspectRatioByExpanding);
11961248

11971249
// Debug output
1198-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Scale: SCALE_CROP)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
1250+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Scale: SCALE_CROP)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
11991251
break;
12001252
}
12011253
case (SCALE_NONE): {
@@ -1207,7 +1259,7 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
12071259
source_size.scale(width * source_width_ratio, height * source_height_ratio, Qt::KeepAspectRatio);
12081260

12091261
// Debug output
1210-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Scale: SCALE_NONE)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
1262+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Scale: SCALE_NONE)", "frame->number", frame->number, "source_width", source_size.width(), "source_height", source_size.height());
12111263
break;
12121264
}
12131265
}
@@ -1258,7 +1310,7 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
12581310
}
12591311

12601312
// Debug output
1261-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Gravity)", "frame->number", frame->number, "source_clip->gravity", gravity, "scaled_source_width", scaled_source_width, "scaled_source_height", scaled_source_height);
1313+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Gravity)", "frame->number", frame->number, "source_clip->gravity", gravity, "scaled_source_width", scaled_source_width, "scaled_source_height", scaled_source_height);
12621314

12631315
/* LOCATION, ROTATION, AND SCALE */
12641316
float r = rotation.GetValue(frame->number); // rotate in degrees
@@ -1272,7 +1324,7 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
12721324
QTransform transform;
12731325

12741326
// Transform source image (if needed)
1275-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Build QTransform - if needed)", "frame->number", frame->number, "x", x, "y", y, "r", r, "sx", sx, "sy", sy);
1327+
ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Build QTransform - if needed)", "frame->number", frame->number, "x", x, "y", y, "r", r, "sx", sx, "sy", sy);
12761328

12771329
if (!isEqual(x, 0) || !isEqual(y, 0)) {
12781330
// TRANSLATE/MOVE CLIP
@@ -1297,56 +1349,5 @@ void Clip::apply_keyframes(std::shared_ptr<Frame> frame, int width, int height)
12971349
transform.scale(source_width_scale, source_height_scale);
12981350
}
12991351

1300-
// Debug output
1301-
ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_keyframes (Transform: Composite Image Layer: Prepare)", "frame->number", frame->number);
1302-
1303-
/* COMPOSITE SOURCE IMAGE (LAYER) ONTO FINAL IMAGE */
1304-
auto new_image = std::make_shared<QImage>(QSize(width, height), source_image->format());
1305-
new_image->fill(QColor(QString::fromStdString("#00000000")));
1306-
1307-
// Load timeline's new frame image into a QPainter
1308-
QPainter painter(new_image.get());
1309-
painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
1310-
1311-
// Apply transform (translate, rotate, scale)
1312-
painter.setTransform(transform);
1313-
1314-
// Composite a new layer onto the image
1315-
painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
1316-
painter.drawImage(0, 0, *source_image);
1317-
1318-
if (timeline) {
1319-
Timeline *t = (Timeline *) timeline;
1320-
1321-
// Draw frame #'s on top of image (if needed)
1322-
if (display != FRAME_DISPLAY_NONE) {
1323-
std::stringstream frame_number_str;
1324-
switch (display) {
1325-
case (FRAME_DISPLAY_NONE):
1326-
// This is only here to prevent unused-enum warnings
1327-
break;
1328-
1329-
case (FRAME_DISPLAY_CLIP):
1330-
frame_number_str << frame->number;
1331-
break;
1332-
1333-
case (FRAME_DISPLAY_TIMELINE):
1334-
frame_number_str << (position * t->info.fps.ToFloat()) + frame->number;
1335-
break;
1336-
1337-
case (FRAME_DISPLAY_BOTH):
1338-
frame_number_str << (position * t->info.fps.ToFloat()) + frame->number << " (" << frame->number << ")";
1339-
break;
1340-
}
1341-
1342-
// Draw frame number on top of image
1343-
painter.setPen(QColor("#ffffff"));
1344-
painter.drawText(20, 20, QString(frame_number_str.str().c_str()));
1345-
}
1346-
}
1347-
1348-
painter.end();
1349-
1350-
// Add new QImage to frame
1351-
frame->AddImage(new_image);
1352+
return transform;
13521353
}

0 commit comments

Comments
 (0)