Merge pull request #635 from OpenShot/catch-tests

Switch unit tests to Catch2 framework

Author: ferdnyc

.github/workflows/ci.yml

@@ -35,8 +35,13 @@ jobs:
           libopenshot-audio-dev \
           qtbase5-dev qtbase5-dev-tools \
           libfdk-aac-dev libavcodec-dev libavformat-dev libavdevice-dev libavutil-dev libavfilter-dev libswscale-dev libpostproc-dev libswresample-dev \
-          libzmq3-dev libmagick++-dev libunittest++-dev \
+          libzmq3-dev libmagick++-dev \
           libopencv-dev libprotobuf-dev protobuf-compiler
+        # Install catch2 package from Ubuntu 20.10, since for some reason
+        # even 20.04 only has Catch 1.12.1 available.
+        wget https://launchpad.net/ubuntu/+archive/primary/+files/catch2_2.13.0-1_all.deb
+        sudo dpkg -i catch2_2.13.0-1_all.deb
+
 
     - name: Build libopenshot
       shell: bash
@@ -51,7 +56,7 @@ jobs:
       shell: bash
       run: |
         pushd build
-        cmake --build . --target os_test -- VERBOSE=1
+        cmake --build . --target coverage -- VERBOSE=1
         popd
 
     - name: Install libopenshot
@@ -65,4 +70,3 @@ jobs:
       if: ${{ matrix.compiler == 'clang' }}
       with:
         file: build/coverage.info
-

CMakeLists.txt

@@ -73,10 +73,15 @@ include(FeatureSummary)
 # Optional build settings for libopenshot
 option(USE_SYSTEM_JSONCPP "Use system installed JsonCpp, if found" ON)
 option(DISABLE_BUNDLED_JSONCPP "Don't fall back to bundled JsonCpp" OFF)
+
 option(ENABLE_IWYU "Enable 'Include What You Use' scanner (CMake 3.3+)" OFF)
-option(ENABLE_TESTS "Build unit tests (requires UnitTest++)" ON)
+
+option(ENABLE_TESTS "Build unit tests (requires Catch2)" ON)
+option(ENABLE_PARALLEL_CTEST "Run CTest using multiple processors" ON)
 option(ENABLE_COVERAGE "Scan test coverage using gcov and report" OFF)
+
 option(ENABLE_DOCS "Build API documentation (requires Doxygen)" ON)
+
 option(APPIMAGE_BUILD "Build to install in an AppImage (Linux only)" OFF)
 option(ENABLE_MAGICK "Use ImageMagick, if available" ON)
 option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" ON)
@@ -87,7 +92,7 @@ if (DISABLE_TESTS)
 endif()
 
 if(DEFINED ENABLE_TESTS)
-  set(ENABLE_TESTS ${ENABLE_TESTS} CACHE BOOL "Build unit tests (requires UnitTest++)" FORCE)
+  set(ENABLE_TESTS ${ENABLE_TESTS} CACHE BOOL "Build unit tests (requires Catch2)" FORCE)
 endif()
 
 #### Work around a GCC < 9 bug with handling of _Pragma() in macros
@@ -109,7 +114,7 @@ ENDIF(WIN32)
 ############## Code Coverage #########################
 if (ENABLE_COVERAGE AND NOT ENABLE_TESTS)
   message(WARNING "ENABLE_COVERAGE requires unit tests, forcing ENABLE_TESTS")
-  set(ENABLE_TESTS ON CACHE BOOL "Don't build unit tests" FORCE)
+  set(ENABLE_TESTS ON CACHE BOOL "Build unit tests (requires Catch2 or UnitTest++)" FORCE)
 endif()
 
 if (ENABLE_COVERAGE)
@@ -166,55 +171,87 @@ if (ENABLE_DOCS)
             OPTIONAL )    # No error if the docs aren't found
   endif()
 endif()
-add_feature_info("Documentation" DOCS_ENABLED "Build API documentation with 'make doc'")
 
 ############# PROCESS tests/ DIRECTORY ##############
 if(ENABLE_TESTS)
   set(TESTS_ENABLED TRUE) # May be overridden by tests/CMakeLists.txt
+  find_package(Catch2 REQUIRED)
+  if(ENABLE_PARALLEL_CTEST)
+    # Figure out the amount of parallelism for CTest
+    include(ProcessorCount)
+    ProcessorCount(CPU_COUNT)
+    if(NOT CPU_COUNT EQUAL 0)
+      message(STATUS "Setting up unit tests to use ${CPU_COUNT} processors")
+      set(CTEST_OPTIONS "-j${CPU_COUNT}")
+    endif()
+  endif()
+  include(CTest)
+  include(Catch)
   add_subdirectory(tests)
 endif()
 add_feature_info("Unit tests" TESTS_ENABLED "Compile unit tests for library functions")
 
 ############## COVERAGE REPORTING #################
-if (ENABLE_COVERAGE)
+if (ENABLE_COVERAGE AND DEFINED UNIT_TEST_TARGETS)
   setup_target_for_coverage_lcov(
     NAME coverage
     LCOV_ARGS "--no-external"
-    EXECUTABLE openshot-test
-    DEPENDENCIES openshot openshot-test
+    EXECUTABLE ctest
+    EXECUTABLE_ARGS ${CTEST_OPTIONS}
+    DEPENDENCIES openshot ${UNIT_TEST_TARGETS}
     EXCLUDE
       "bindings/*"
       "examples/*"
       "${CMAKE_CURRENT_BINARY_DIR}/bindings/*"
       "${CMAKE_CURRENT_BINARY_DIR}/src/*_autogen/*"
   )
-  if(NOT TARGET os_test)
-    add_custom_target(os_test)
-    add_dependencies(os_test coverage)
-  endif()
 endif()
 
-# Also hook up 'test' as an alias for the 'os_test' target, if possible
-# This requires CMake 3.11+, where the CMP0037 policy
-# configured to 'NEW' mode will not reserve target names
-# unless the corresponding feature is actually used
-if (POLICY CMP0037)
-	cmake_policy(SET CMP0037 NEW)
+if(TESTS_ENABLED AND NOT TARGET coverage)
+  add_custom_target(coverage
+    COMMAND ctest ${CTEST_OPTIONS}
+    DEPENDS openshot ${UNIT_TEST_TARGETS}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Running unit tests (coverage disabled)"
+  )
 endif()
-if(TARGET os_test)
-  if (CMAKE_VERSION VERSION_GREATER 3.11)
-    message(STATUS "Cmake 3.11+ detected, enabling 'test' target")
-    add_custom_target(test)
-    add_dependencies(test os_test)
-    set(TEST_TARGET_NAME "test")
-  else()
-    set(TEST_TARGET_NAME "os_test")
+
+if(TARGET test AND NOT TARGET os_test)
+  add_custom_target(os_test)
+  add_dependencies(os_test coverage)
+endif()
+
+if(TARGET os_test AND NOT TARGET test AND CMAKE_VERSION VERSION_GREATER 3.11)
+  # Also hook up 'test' as an alias for the 'os_test' target, if possible
+  # This requires CMake 3.11+, where the CMP0037 policy
+  # configured to 'NEW' mode will not reserve target names
+  # unless the corresponding feature is actually used
+  if (POLICY CMP0037)
+  	cmake_policy(SET CMP0037 NEW)
   endif()
-  add_feature_info("Testrunner" ENABLE_TESTS "Run unit tests with 'make ${TEST_TARGET_NAME}'")
+  message(STATUS "Cmake 3.11+ detected, enabling 'test' target")
+  add_custom_target(test)
+  add_dependencies(test os_test)
 endif()
 
+###
+### Add feature-summary details on non-default built targets
+###
+set(optional_targets test os_test coverage doc)
+set(target_test_description "Build and execute unit tests")
+set(target_os_test_description "Build and execute unit tests (legacy target)")
+set(target_coverage_description "Run unit tests and (if enabled) collect coverage data")
+set(target_doc_description "Build formatted API documentation (HTML+SVG)")
+foreach(_tname IN LISTS optional_targets)
+  if(TARGET ${_tname})
+    add_feature_info("Non-default target '${_tname}'" TRUE ${target_${_tname}_description})
+  else()
+    message(DEBUG "No target ${_tname}")
+  endif()
+endforeach()
+
 ########### PRINT FEATURE SUMMARY ##############
 feature_summary(WHAT ALL
   INCLUDE_QUIET_PACKAGES
   FATAL_ON_MISSING_REQUIRED_PACKAGES
-  DESCRIPTION "Displaying feature summary\n\nBuild configuration:")
+  DESCRIPTION "Build configuration:")

tests/CMakeLists.txt

@@ -26,24 +26,6 @@
 
 # Test media path, used by unit tests for input data
 file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/examples/" TEST_MEDIA_PATH)
-add_definitions( -DTEST_MEDIA_PATH="${TEST_MEDIA_PATH}" )
-
-################### UNITTEST++ #####################
-# Find UnitTest++ libraries (used for unit testing)
-find_package(UnitTest++)
-
-if (NOT UnitTest++_FOUND)
-  set(TESTS_ENABLED OFF PARENT_SCOPE)
-  return()
-endif()
-
-# Include UnitTest++ headers (needed for compile)
-include_directories(${UnitTest++_INCLUDE_DIRS})
-
-set_package_properties(UnitTest++ PROPERTIES
-  TYPE RECOMMENDED
-  PURPOSE "Unit testing framework")
-
 
 ################# BLACKMAGIC DECKLINK ###################
 if(ENABLE_BLACKMAGIC)
@@ -56,53 +38,68 @@ if(ENABLE_BLACKMAGIC)
 	endif()
 endif()
 
-###############  SET TEST SOURCE FILES  #################
-set(OPENSHOT_TEST_FILES
-  Cache_Tests.cpp
-  Clip_Tests.cpp
-  Color_Tests.cpp
-  Coordinate_Tests.cpp
-  DummyReader_Tests.cpp
-  ReaderBase_Tests.cpp
-  ImageWriter_Tests.cpp
-  FFmpegReader_Tests.cpp
-  FFmpegWriter_Tests.cpp
-  Fraction_Tests.cpp
-  Frame_Tests.cpp
-  FrameMapper_Tests.cpp
-  KeyFrame_Tests.cpp
-  Point_Tests.cpp
-  QtImageReader_Tests.cpp
-  Settings_Tests.cpp
-  Timeline_Tests.cpp)
+###
+###  TEST SOURCE FILES
+###
+set(OPENSHOT_TESTS
+  CacheDisk
+  CacheMemory
+  Clip
+  Color
+  Coordinate
+  DummyReader
+  ReaderBase
+  ImageWriter
+  FFmpegReader
+  FFmpegWriter
+  Fraction
+  Frame
+  FrameMapper
+  KeyFrame
+  Point
+  QtImageReader
+  Settings
+  Timeline
+)
 
-##########  SET OPENCV RELATED TEST FILES  ###############
+###
+### OPENCV RELATED TEST FILES
+###
 if(ENABLE_OPENCV)
-  list(APPEND OPENSHOT_TEST_FILES
-    CVTracker_Tests.cpp
-    CVStabilizer_Tests.cpp
-    # CVObjectDetection_Tests.cpp
+  list(APPEND OPENSHOT_TESTS
+    CVTracker
+    CVStabilizer
+    # CVObjectDetection
   )
 endif()
 
-################ TESTER EXECUTABLE #################
-# Create unit test executable (openshot-test)
-message (STATUS "Tests enabled, test executable will be built as tests/openshot-test")
+###
+### Catch2 unit tests
+###
+if (TESTS_ENABLED)
+  message (STATUS "Tests enabled, test executables will be compiled")
+  include(Catch)
 
-add_executable(openshot-test
-  tests.cpp
-  ${OPENSHOT_TEST_FILES}
-)
+  include(CTest)
 
-# Link libraries to the new executable
-target_link_libraries(openshot-test
-  openshot
-  ${UnitTest++_LIBRARIES}
-)
+  # Create object library for test executable main(),
+  # to avoid recompiling for every test
+  add_library(catch-main OBJECT catch_main.cpp)
 
-##### RUNNING TESTS (make os_test / make test) #####
-# Hook up the 'make os_test' target to the 'openshot-test' executable,
-# if we aren't defining it as the coverage target
-if(NOT ENABLE_COVERAGE)
-  add_custom_target(os_test COMMAND openshot-test)
+  foreach(tname ${OPENSHOT_TESTS})
+    add_executable(openshot-${tname}-test ${tname}.cpp $<TARGET_OBJECTS:catch-main>)
+    target_compile_definitions(openshot-${tname}-test PRIVATE
+      TEST_MEDIA_PATH="${TEST_MEDIA_PATH}"
+    )
+  	target_link_libraries(openshot-${tname}-test Catch2::Catch2 openshot)
+		# Automatically configure CTest targets from Catch2 test cases
+		catch_discover_tests(
+      openshot-${tname}-test
+      TEST_PREFIX ${tname}:
+    )
+    list(APPEND CATCH2_TEST_TARGETS openshot-${tname}-test)
+	endforeach()
+  # Export target list for coverage use
+  set(UNIT_TEST_TARGETS ${CATCH2_TEST_TARGETS} PARENT_SCOPE)
 endif()
+