diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 0000000000..61bbaf97d5 --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,23 @@ + +comment: + layout: "reach, diff, flags, files" + behavior: default + require_changes: false + require_base: yes + require_head: yes + branches: null + +flag_management: + # this section will govern all default rules of Flags + default_rules: + carryforward: boolean? + ignore: [path]? + paths: [path]? + statuses: [ + name_prefix: string (r"^[\w\-\.]+$") + type: OR("project", "patch", "changes") + target: OR("auto", percent)? + include_changes: OR("auto", percent)? + threshold: percent? + **normal status attributes + ]? diff --git a/.github/actions/tests-gluecode-openfast/action.yml b/.github/actions/tests-gluecode-openfast/action.yml index e295492830..196f05a648 100644 --- a/.github/actions/tests-gluecode-openfast/action.yml +++ b/.github/actions/tests-gluecode-openfast/action.yml @@ -7,7 +7,7 @@ runs: - run: | ctest -VV -L linear -E Ideal ctest -VV -j8 -I 1,1,1,2,3,4,5,6,7,8,10,11,12,13,14,15,17,18,21,22,23,24,25,26,27,28,29 - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash # OpenFAST linearization tests diff --git a/.github/actions/tests-module-aerodyn/action.yml b/.github/actions/tests-module-aerodyn/action.yml index 14ed3a34df..8759e9c58c 100644 --- a/.github/actions/tests-module-aerodyn/action.yml +++ b/.github/actions/tests-module-aerodyn/action.yml @@ -21,5 +21,5 @@ runs: ctest -VV -j7 -R ad_ fi - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/actions/tests-module-beamdyn/action.yml b/.github/actions/tests-module-beamdyn/action.yml index 660a2de9f6..61e13d5c54 100644 --- a/.github/actions/tests-module-beamdyn/action.yml +++ b/.github/actions/tests-module-beamdyn/action.yml @@ -20,5 +20,5 @@ runs: ctest -VV -j7 -R bd_ fi - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/actions/tests-module-hydrodyn/action.yml b/.github/actions/tests-module-hydrodyn/action.yml index 4890c414fc..3e94eba00d 100644 --- a/.github/actions/tests-module-hydrodyn/action.yml +++ b/.github/actions/tests-module-hydrodyn/action.yml @@ -5,5 +5,5 @@ runs: using: "composite" steps: - run: ctest -VV -j7 -R hd_ - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/actions/tests-module-inflowwind/action.yml b/.github/actions/tests-module-inflowwind/action.yml index 4a204980ad..b0d5faa0d9 100644 --- a/.github/actions/tests-module-inflowwind/action.yml +++ b/.github/actions/tests-module-inflowwind/action.yml @@ -5,5 +5,5 @@ runs: using: "composite" steps: - run: ctest -VV -R inflowwind_utest - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/actions/tests-module-nwtclibrary/action.yml b/.github/actions/tests-module-nwtclibrary/action.yml index a8d27e4174..d8fd57ec84 100644 --- a/.github/actions/tests-module-nwtclibrary/action.yml +++ b/.github/actions/tests-module-nwtclibrary/action.yml @@ -5,5 +5,5 @@ runs: using: "composite" steps: - run: ctest -VV -R nwtc_library_utest - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/actions/tests-module-subdyn/action.yml b/.github/actions/tests-module-subdyn/action.yml index 62d76630b9..6e1f5c7b10 100644 --- a/.github/actions/tests-module-subdyn/action.yml +++ b/.github/actions/tests-module-subdyn/action.yml @@ -5,5 +5,5 @@ runs: using: "composite" steps: - run: ctest -VV -j7 -R SD_ - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build shell: bash diff --git a/.github/workflows/automated-dev-tests.yml b/.github/workflows/automated-dev-tests.yml index 285ce78162..d78965f65e 100644 --- a/.github/workflows/automated-dev-tests.yml +++ b/.github/workflows/automated-dev-tests.yml @@ -17,6 +17,9 @@ on: env: FORTRAN_COMPILER: gfortran-10 + CXX_COMPILER: g++-10 + C_COMPILER: gcc-10 + GCOV_EXE: gcov-10 NUM_PROCS: 8 @@ -39,20 +42,22 @@ jobs: pip install numpy Bokeh==1.4 - name: Setup Workspace - run: cmake -E make_directory ${{runner.workspace}}/build + run: cmake -E make_directory ${{runner.workspace}}/openfast/build - name: Configure Build - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake \ - -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install \ + -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/openfast/install \ -DCMAKE_Fortran_COMPILER:STRING=${{env.FORTRAN_COMPILER}} \ + -DCMAKE_CXX_COMPILER:STRING=${{env.CXX_COMPILER}} \ + -DCMAKE_C_COMPILER:STRING=${{env.C_COMPILER}} \ -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ -DBUILD_TESTING:BOOL=ON \ -DCTEST_PLOT_ERRORS:BOOL=ON \ ${GITHUB_WORKSPACE} - name: Build OpenFAST # if: contains(github.event.head_commit.message, 'Action - Test All') || contains(github.event.pull_request.labels.*.name, 'Action - Test All') - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: cmake --build . --target install -- -j ${{env.NUM_PROCS}} - name: Run AeroDyn tests @@ -77,14 +82,14 @@ jobs: with: name: regression-tests-release path: | - ${{runner.workspace}}/build/reg_tests/modules - ${{runner.workspace}}/build/reg_tests/glue-codes/openfast - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/5MW_Baseline - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/AOC - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/AWT27 - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/SWRT - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/UAE_VI - !${{runner.workspace}}/build/reg_tests/glue-codes/openfast/WP_Baseline + ${{runner.workspace}}/openfast/build/reg_tests/modules + ${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/5MW_Baseline + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/AOC + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/AWT27 + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/SWRT + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/UAE_VI + !${{runner.workspace}}/openfast/build/reg_tests/glue-codes/openfast/WP_Baseline regression-tests-debug: runs-on: ${{ matrix.os }} @@ -94,8 +99,10 @@ jobs: include: - os: macOS-11 FORTRAN_COMPILER: gfortran-11 + install_deps: brew install gcovr - os: ubuntu-20.04 FORTRAN_COMPILER: gfortran-10 + install_deps: sudo apt-get install -y gcovr name: regression-test-debug-${{ matrix.os }}-${{ matrix.FORTRAN_COMPILER }} steps: @@ -108,26 +115,30 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install dependencies + - name: Install Dependencies run: | python -m pip install --upgrade pip pip install numpy Bokeh==1.4 + ${{matrix.install_deps}} - name: Setup Workspace - run: cmake -E make_directory ${{runner.workspace}}/build + run: cmake -E make_directory ${{runner.workspace}}/openfast/build - name: Configure Build - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake \ - -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install \ + -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/openfast/install \ -DCMAKE_Fortran_COMPILER:STRING=${{matrix.FORTRAN_COMPILER}} \ + -DCMAKE_CXX_COMPILER:STRING=${{env.CXX_COMPILER}} \ + -DCMAKE_C_COMPILER:STRING=${{env.C_COMPILER}} \ -DCMAKE_BUILD_TYPE:STRING=Debug \ -DBUILD_TESTING:BOOL=ON \ + -DCODECOV:BOOL=ON \ -DCTEST_PLOT_ERRORS:BOOL=ON \ ${GITHUB_WORKSPACE} - name: Build OpenFAST - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake --build . --target aerodyn_driver -- -j ${{env.NUM_PROCS}} cmake --build . --target beamdyn_driver -- -j ${{env.NUM_PROCS}} @@ -147,13 +158,42 @@ jobs: - name: Run SubDyn tests uses: ./.github/actions/tests-module-subdyn + - name: Generate coverage report + working-directory: ${{runner.workspace}}/openfast/build + run: | + find . -type f -name '*.gcno' -not -path "**tests**" -exec ${{env.GCOV_EXE}} -pb {} + + cd .. + gcovr -g -k -r . --xml regressioncov.xml # --html --html-details -o regressioncov.html # -v + + # cp `find . -name *.gcno` . + # cp `find . -name *.gcda` . + # ${{env.GCOV_EXE}} -b -l -p -c *.gcno + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + dry_run: false + # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./regressioncov.xml # optional + flags: regtests # optional + # name: codecov-umbrella # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) + directory: ${{runner.workspace}}/openfast + + - name: Success artifacts + uses: actions/upload-artifact@v2 + if: success() + with: + name: regression-tests-debug + path: | + ${{runner.workspace}}/openfast/regressioncov.xml - name: Failing test artifacts uses: actions/upload-artifact@v2 if: failure() with: name: regression-tests-debug path: | - ${{runner.workspace}}/build/reg_tests/modules + ${{runner.workspace}}/openfast/build/reg_tests/modules fastfarm-regression-test: runs-on: ubuntu-20.04 @@ -194,7 +234,6 @@ jobs: cmake --build . --target regression_tests -- -j ${{env.NUM_PROCS}} - name: Run FAST.Farm tests - # if: contains(github.event.head_commit.message, 'Action - Test All') || contains(github.event.pull_request.labels.*.name, 'Action - Test All') run: | ctest -VV -L fastfarm -j ${{env.NUM_PROCS}} working-directory: ${{runner.workspace}}/build @@ -215,20 +254,36 @@ jobs: uses: actions/checkout@main with: submodules: recursive - - name: Setup - run: cmake -E make_directory ${{runner.workspace}}/build - - name: Configure - working-directory: ${{runner.workspace}}/build + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install numpy Bokeh==1.4 + + sudo apt-get update -y + sudo apt-get install -y gcovr + + - name: Setup Workspace + run: cmake -E make_directory ${{runner.workspace}}/openfast/build + - name: Configure Build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake \ - -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install \ + -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/openfast/install \ -DCMAKE_Fortran_COMPILER:STRING=${{env.FORTRAN_COMPILER}} \ - -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER:STRING=${{env.CXX_COMPILER}} \ + -DCMAKE_C_COMPILER:STRING=${{env.C_COMPILER}} \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ -DBUILD_TESTING:BOOL=ON \ + -DCODECOV:BOOL=ON \ ${GITHUB_WORKSPACE} - name: Build unit tests - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: cmake --build . --target unit_tests -- -j ${{env.NUM_PROCS}} - name: Run NWTC Library tests @@ -244,6 +299,40 @@ jobs: - name: Run InflowWind tests uses: ./.github/actions/tests-module-inflowwind + - name: Generate coverage report + working-directory: ${{runner.workspace}}/openfast/build + run: | + find . -type f -name '*.gcno' -not -path "**tests**" -exec ${{env.GCOV_EXE}} -pb {} + + cd .. + gcovr -g -k -r . --xml unitcov.xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + dry_run: false + # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./unitcov.xml # optional + flags: unittests # optional + # name: codecov-umbrella # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) + directory: ${{runner.workspace}}/openfast + + - name: Success artifacts + uses: actions/upload-artifact@v2 + if: success() + with: + name: unit-tests + path: | + ${{runner.workspace}}/openfast/unitcov.xml + + - name: Failing test artifacts + uses: actions/upload-artifact@v2 + if: failure() + with: + name: unit-tests + path: | + ${{runner.workspace}}/openfast/build/unit_tests + compile-all-single-precision: # Test if single precision compile completes. # Compiles all targets excluding tests. @@ -291,40 +380,71 @@ jobs: run: | python -m pip install --upgrade pip pip install numpy Bokeh==1.4 - sudo apt-get update + + sudo apt-get update -y + sudo apt-get install -y gcovr sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev - name: Setup Workspace - run: cmake -E make_directory ${{runner.workspace}}/build + run: cmake -E make_directory ${{runner.workspace}}/openfast/build - name: Configure Build - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake \ - -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install \ + -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/openfast/install \ -DCMAKE_Fortran_COMPILER:STRING=${{env.FORTRAN_COMPILER}} \ - -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER:STRING=${{env.CXX_COMPILER}} \ + -DCMAKE_C_COMPILER:STRING=${{env.C_COMPILER}} \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ -DBUILD_OPENFAST_CPP_API:BOOL=ON \ -DBUILD_SHARED_LIBS:BOOL=ON \ -DBUILD_TESTING:BOOL=ON \ -DCTEST_PLOT_ERRORS:BOOL=ON \ + -DCODECOV:BOOL=ON \ ${GITHUB_WORKSPACE} + - name: Build OpenFAST C++ API - # if: contains(github.event.head_commit.message, 'Action - Test All') || contains(github.event.pull_request.labels.*.name, 'Action - Test All') - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | cmake --build . --target openfastcpp -- -j ${{env.NUM_PROCS}} cmake --build . --target regression_tests -- -j ${{env.NUM_PROCS}} - name: Run OpenFAST C++ API tests - working-directory: ${{runner.workspace}}/build + working-directory: ${{runner.workspace}}/openfast/build run: | ctest -VV -L cpp + - name: Generate coverage report + working-directory: ${{runner.workspace}}/openfast/build + run: | + find . -type f -name '*.gcno' -not -path "**tests**" -exec ${{env.GCOV_EXE}} -pb {} + + cd .. + gcovr -g -k -r . --xml regressioncov.xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + dry_run: false + # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./regressioncov.xml # optional + flags: regtests # optional + # name: codecov-umbrella # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) + directory: ${{runner.workspace}}/openfast + + - name: Success artifacts + uses: actions/upload-artifact@v2 + if: success() + with: + name: cpp-reg-tests + path: | + ${{runner.workspace}}/openfast/regressioncov.xml + - name: Failing test artifacts uses: actions/upload-artifact@v2 if: failure() with: - name: test-results + name: cpp-reg-tests path: | ${{runner.workspace}}/build/reg_tests/glue-codes/openfast-cpp !${{runner.workspace}}/build/reg_tests/glue-codes/openfast-cpp/5MW_Baseline diff --git a/CMakeLists.txt b/CMakeLists.txt index 998caa12b3..066493ae06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,7 @@ endif() # Do this before configuring modules so that the flags are included option(BUILD_TESTING "Build the testing tree." OFF) if(BUILD_TESTING) + option(CODECOV "Enable infrastructure for measuring code coverage." OFF) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DUNIT_TEST") endif() option(BUILD_OPENFAST_SIMULINK_API "Enable building OpenFAST for use with Simulink" off) @@ -59,6 +60,29 @@ endif() # Setup Fortran Compiler options based on architecture/compiler set_fast_fortran() +# Configure measuring code coverage in tests +if(CODECOV) + # Only supports GNU + if(NOT CMAKE_Fortran_COMPILER_ID MATCHES GNU) + message(WARNING "CODECOV is only support with GNU Compilers. The current Fortran compiler is ${CMAKE_Fortran_COMPILER_ID}.") + endif() + if(NOT CMAKE_CXX_COMPILER_ID MATCHES GNU) + message(WARNING "CODECOV is only support with GNU Compilers. The current C++ compiler is ${CMAKE_CXX_COMPILER_ID}.") + endif() + if(NOT CMAKE_C_COMPILER_ID MATCHES GNU) + message(WARNING "CODECOV is only support with GNU Compilers. The current C compiler is ${CMAKE_C_COMPILER_ID}.") + endif() + + string(TOUPPER ${CMAKE_BUILD_TYPE} _build_type) + if (NOT ${_build_type} STREQUAL "DEBUG") + message(WARNING "CODECOV reporting may be inaccurate with compiler optimizations on. Please run with CMAKE_BUILD_TYPE=DEBUG.") + endif() + + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} --coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage") +endif() + if (USE_DLL_INTERFACE) add_definitions(-DUSE_DLL_INTERFACE) endif (USE_DLL_INTERFACE) diff --git a/README.rst b/README.rst index dff299384a..cd3f0dc277 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,8 @@ OpenFAST .. |actions| image:: https://github.com/openfast/openfast/actions/workflows/automated-dev-tests.yml/badge.svg?branch=dev :target: https://github.com/OpenFAST/openfast/actions/workflows/automated-dev-tests.yml?query=workflow%3A%22Development+Pipeline%22 :alt: Build Status +.. |regtest-coverage| image:: https://codecov.io/gh/openfast/openfast/branch/dev/graph/badge.svg + :target: https://codecov.io/gh/openfast/openfast .. |rtfd| image:: https://readthedocs.org/projects/openfast/badge/?version=dev :target: https://openfast.readthedocs.io/en/dev :alt: Documentation Status diff --git a/docs/source/install/index.rst b/docs/source/install/index.rst index 5ce068af0d..ec6ffcbc78 100644 --- a/docs/source/install/index.rst +++ b/docs/source/install/index.rst @@ -368,20 +368,21 @@ The CMake options specific to OpenFAST and their default settings are: :: BUILD_DOCUMENTATION - Build documentation (Default: OFF) + BUILD_FASTFARM - Enable FAST.Farm capabilities (Default: OFF) BUILD_OPENFAST_CPP_API - Enable building OpenFAST - C++ API (Default: OFF) - BUILD_OPENFAST_SIMULINK_API - Enable building OpenFAST for use with Simulink + BUILD_OPENFAST_SIMULINK_API - Enable building OpenFAST for use with Simulink (Default: OFF) BUILD_SHARED_LIBS - Enable building shared libraries (Default: OFF) BUILD_TESTING - Build the testing tree (Default: OFF) CMAKE_BUILD_TYPE - Choose the build type: Debug Release (Default: Release) CMAKE_Fortran_MODULE_DIRECTORY - Set the Fortran Modules directory CMAKE_INSTALL_PREFIX - Install path prefix, prepended onto install directories. + CODECOV - Enable infrastructure for measuring code coverage (Default: OFF) DOUBLE_PRECISION - Treat REAL as double precision (Default: ON) FPE_TRAP_ENABLED - Enable Floating Point Exception (FPE) trap in compiler options (Default: OFF) - GENERATE_TYPES - Use the openfast-registry to autogenerate types modules + GENERATE_TYPES - Use the openfast-registry to autogenerate types modules (Default: OFF) + OPENMP - Enable OpenMP support (Default: OFF) ORCA_DLL_LOAD - Enable OrcaFlex library load (Default: OFF) USE_DLL_INTERFACE - Enable runtime loading of dynamic libraries (Default: ON) - OPENMP - Enable OpenMP support (Default: OFF) - BUILD_FAST_FARM - Enable FAST.Farm capabilities (Default: OFF) Additional system-specific options may exist for a given system, but those should not impact the OpenFAST configuration. As mentioned above, the diff --git a/docs/source/testing/index.rst b/docs/source/testing/index.rst index a8dad20a20..3facae17d8 100644 --- a/docs/source/testing/index.rst +++ b/docs/source/testing/index.rst @@ -27,7 +27,14 @@ to automatically execute the test suite for new commits and pull requests. This cloud computing resource is available to all GitHub users and is highly recommended as part of the development workflow. After enabling GitHub Actions in an OpenFAST repository, simply -pushing new commits will trigger the tests. +pushing new commits will trigger the tests. The GitHub Actions pipeline +is configured to measure the level of code coverage in the included +tests. These reports are currently available only for the module-level +regression tests and the unit tests, and they are available for viewing +on the `OpenFAST Codecov.io dashboard `_. +For any fork of OpenFAST where the GitHub Actions workflow successfully +completes, a unique Codecov.io dashboard will be available at +`https://app.codecov.io/gh//openfast`. Test specific documentation ---------------------------