diff --git a/.gitignore b/.gitignore index f0568ed816..2855a3a1a5 100644 --- a/.gitignore +++ b/.gitignore @@ -27,9 +27,12 @@ *.exe *.out *.app +*.pyc # CMake specific files build/**/* +install/* +ctest-build/* vs-build build \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index e69de29bb2..d9ccb611aa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "reg_tests/r-test"] + path = reg_tests/r-test + url = https://github.com/openfast/r-test.git + branch = dev diff --git a/CMakeLists.txt b/CMakeLists.txt index c9f1b890cb..36d62ca330 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,7 +105,15 @@ endforeach(IDIR IN ITEMS ${FAST_MODULES_EXTERNAL}) option(BUILD_DOCUMENTATION "Build documentation." OFF) if(BUILD_DOCUMENTATION) - add_subdirectory(docs) + add_subdirectory(docs) +endif() + +option(BUILD_TESTING "Build the testing tree." OFF) +if(BUILD_TESTING) + # add_subdirectory(reg_tests) this should be 'include'. with add_subdirectory, + # the CTestTestlist.cmake file is placed in the reg_tests subdirectory + # and cannot be found by ctest + include(reg_tests/CMakeLists.txt) endif() add_subdirectory(glue-codes) diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 0000000000..037686a5a5 --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,19 @@ +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## +## # The following are required to submit to the CDash dashboard: +## ENABLE_TESTING() +## INCLUDE(CTest) + +set(CTEST_PROJECT_NAME "openfast") +set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC") + +set(CTEST_DROP_METHOD http) +set(CTEST_DROP_SITE "my.cdash.org") +set(CTEST_DROP_LOCATION "/submit.php?project=OpenFAST") +set(CTEST_DROP_SITE_CDASH TRUE) + +# saved for debugging +#set(CTEST_DROP_SITE "localhost") +#set(CTEST_DROP_LOCATION "/cdash/public/submit.php?project=openfast") diff --git a/docs/index.rst b/docs/index.rst index c6b8ddb10d..d32b63ca74 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,16 +1,21 @@ .. OpenFAST documentation master file, created by sphinx-quickstart on Wed Jan 25 13:52:07 2017. -OpenFAST Documentation +OpenFAST Documentation ====================== -This page is under construction. Our intention is that this page will eventually host documents on compiling OpenFAST, theory behind OpenFAST, software-development requirements, reporting issues, etc. +This page is under construction. Our intention is that this page will eventually host +documents on compiling OpenFAST, theory behind OpenFAST, software-development +requirements, reporting issues, etc. -Please contact Michael.A.Sprague@NREL.gov with questions regarding the OpenFAST development plan. +Please contact `Michael.A.Sprague@NREL.gov `_. with questions regarding the OpenFAST +development plan. -For possible bugs, enhancement requests, or code questions, please submit an issues at the OpenFAST Github repository https://github.com/OpenFAST/OpenFAST +For possible bugs, enhancement requests, or code questions, please submit an issues at +the `OpenFAST Github repository `__. -Users may find the established FAST8 through the NWTC Information Portal: https://nwtc.nrel.gov/ +Users may find the established FAST8 through the NWTC Information Portal: +https://nwtc.nrel.gov/ .. toctree:: :maxdepth: 2 @@ -18,7 +23,6 @@ Users may find the established FAST8 through the NWTC Information Portal: https: OpenFAST Github Organization Page OpenFAST Github Repository Installing OpenFAST - Module Documentation + Testing OpenFAST Source Code Documentation Building this Documentation Locally - diff --git a/docs/source/test.rst b/docs/source/test.rst new file mode 100644 index 0000000000..2b6a326776 --- /dev/null +++ b/docs/source/test.rst @@ -0,0 +1,60 @@ +Testing OpenFAST +================ + +The OpenFAST test suite consists of system and module level regression tests and +unit tests. The regression test compares locally generated solutions to a set of +baseline solutions. The unit tests ensure that individual subroutines are +functioning as intended. + +All of the necessary files corresponding to the regression test are contained in +the ``reg_tests`` directory. The unit test framework is housed in ``unit_tests`` +while the actual tests are contained in the directory corresponding to the tested +module. + +Dependencies +------------ +- Python 2.7/3+ +- Numpy +- CMake and CTest (Optional in regression test) + +Configuring the test suite +-------------------------- +A portion of the test suite is linked to the OpenFAST repository through +``git submodule``. Specifically, + +- `r-test `__ +- `pFUnit `__ + +Be sure to clone the repo with the ``--recursive`` flag or execute +``git submodule update --init --recursive`` after cloning. + +The test suite can be built with `CMake `__ similar to +OpenFAST. The default CMake configuration is useful, but may need customization +for particular build environments. CMake variables can be configured in the `CMake +GUI `__ or through the command line interface with +the command ``ccmake``. If the entire OpenFAST package is to be built, the test +related CMake variables can be configured during the OpenFAST CMake +configuration. However, if only the test suite will be built, configure CMake +using the ``reg_tests`` project with ``ccmake path/to/reg_tests`` or selecting +``reg_tests`` as the source directory in the CMake GUI. + +The test specific CMake variables are + +- BUILD_TESTING +- CTEST_OPENFAST_EXECUTABLE +- CTEST_[MODULE]_EXECUTABLE + +Look at the `Installing OpenFAST `__ page for more details on +configuring the CMake targets. + +While the unit tests must be built with CMake due to its external dependencies, +the regression test may be executed without building with CMake. See the +`regression test `__ section for more info. + +Test specific documentation +--------------------------- +.. toctree:: + :maxdepth: 2 + + Unit Test + Regression Test diff --git a/docs/source/test/regression_test.rst b/docs/source/test/regression_test.rst new file mode 100644 index 0000000000..4a42b85a21 --- /dev/null +++ b/docs/source/test/regression_test.rst @@ -0,0 +1,119 @@ +Regression test +=============== + +The regression test executes a series of test cases which fully describe +OpenFAST and its module's capabilities. Each locally computed result is compared +to a static set of baseline results. To account for machine and compiler +differences, the regression test attempts to match the current machine and +compiler type to the appropriate solution set from these combinations + +- macOS with GNU compiler (default) +- Red Hat Enterprise Linux with Intel compiler +- Windows with Intel compiler + +The automated regression test runs +`CTest `__ and can be executed +by running either of the commands ``make test`` or ``ctest`` from the build +directory. If the entire OpenFAST package is to be built, CMake will configure +CTest to find the new binary at ``openfast/build/glue-codes/fast/openfast``. +However, if the intention is to build only the test suite, the OpenFAST binary +should be specified in the CMake configuration under the ``CTEST_OPENFAST_EXECUTABLE`` +flag. There is also a corresponding ``CTEST_[MODULE]_NAME`` flag for each module +included in the regression test. + +The regression test can be executed manually with the included driver +``reg_tests/manualRegressionTest.py``. + +In both modes of execution a subdirectory is created in the build directory +called ``reg_tests`` where all of the input files for the test cases are copied +and all of the locally generated outputs are stored. + +Ultimately, both CTest and the manual execution script call a series of Python +scripts in ``reg_tests`` and ``reg_tests/lib``. One such script is ``lib/pass_fail.py`` +which reads the output files and computes a norm on each channel reported. +If the maximum norm is greater than a predetermined tolerance, that particular +test is reported as failed. The failure criteria is outlined in pseudocode below. + +:: + + for j in range(nChannels) + norm_diff[j] = L2norm(localSolution[j]-baselineSolution[j]) + rms_baseline[j] = L2norm(baselineSolution[j]) + + norm = norm_diff / rms_baseline + + if max(norm) < tolerance: + success + +Running the regression test with CTest +-------------------------------------- +When driven by CTest, the regression test can be executed by running various +forms of the command ``ctest`` from the build directory. The basic commands are + +- ``ctest`` - Run the entire regression test +- ``ctest -V`` - Run the entire regression test with verbose output +- ``ctest -R [TestName]`` - Run a test by name +- ``ctest -j [N]`` - Run all tests with N tests executing in parallel + +Each regression test case contains a series of labels associating all of the +modules used. The labeling can be seen in the test instantiation in +``reg_tests/CTestList.cmake`` and called directly with + +- ``ctest -L [Label]`` + +These flags can be compounded making useful variations of ``ctest`` such as + +- ``ctest -V -L aerodyn14`` - Runs all cases that use AeroDyn14 with verbose output +- ``ctest -j 16 -L aerodyn14`` - Runs all cases that use AeroDyn14 in 16 concurrent processes +- ``ctest -V -R 5MW_DLL_Potential_WTurb`` - Runs the case with name "5MW_DLL_Potential_WTurb" + +Regression test from scratch +---------------------------- + +- Build OpenFAST and the test suite + +:: + + git clone --recursive https://github.com/openfast/openfast.git + cd openfast/reg_tests/r-tests/openfast + python compileDISCON.py + cd ../../ + mkdir build && cd build + # Configure CMake - BUILD_TESTING, CTEST_OPENFAST_EXECUTABLE, CTEST_[MODULE]_EXECUTABLE + cmake .. + make + ctest + + +- Build only the test suite + +:: + + git clone --recursive https://github.com/openfast/openfast.git + cd openfast/reg_tests/r-tests/openfast + python compileDISCON.py + cd ../../ + mkdir build && cd build + # Configure CMake - CTEST_OPENFAST_EXECUTABLE, CTEST_[MODULE]_EXECUTABLE + cmake ../reg_tests + ctest + +- `Windows with Visual Studio regression test `__ + +Follow the link above for a detailed procedure. It is summarized below though +excluding the procedure to build OpenFAST itself. + +:: + + git clone --recursive https://github.com/openfast/openfast.git + cd openfast + + ## Build the ServoDyn external controller libraries + # Open the Visual Studio Solution (DISCON.sln) located in ``openfast\vs-build\DISCON`` + # Choose Release and x64 for the Solutions Configuration and Solutions Platform + # Build Solution + + ## Execute the OpenFAST regression Tests + # Open a command prompt which is configured for Python (like Anaconda) + cd openfast\reg_tests + python manualRegressionTest.py ..\build\bin\FAST_x64.exe Windows Intel diff --git a/docs/source/test/regression_test_windows.rst b/docs/source/test/regression_test_windows.rst new file mode 100644 index 0000000000..d35ecd08bb --- /dev/null +++ b/docs/source/test/regression_test_windows.rst @@ -0,0 +1,81 @@ +Running OpenFAST Regression Tests on Windows +============================================ + +| 1) Clone the openfast repo and initialize the testing database +| a) Open a git command shell window [like git bash] +| b) Change your working directory to the location above where you want your local repo to be located [the repo will be placed into a folder called openfast at this location] +| c) Type: git clone https://github.com/openfast/openfast.git [this creates a local version of the openfast repo on your computer] +| You should see something like this: +| Cloning into 'openfast'... +| remote: Counting objects: 23801, done. +| remote: Compressing objects: 100% (80/80), done. +| remote: Total 23801 (delta 73), reused 102 (delta 50), pack-reused 23670 +| Receiving objects: 100% (23801/23801), 92.10 MiB | 18.99 MiB/s, done. +| Resolving deltas: 100% (13328/13328), done. +| Checking connectivity... done. +| d) Type: cd openfast [change your working directory to the openfast folder] +| e) Type: git checkout dev [this places your local repo on the correct branch of the openfast repo] +| f) Type: git submodule update --init --recursive [this downloads the testing database to your computer] +| You should see something like this: +| Submodule 'reg_tests/r-test' (https://github.com/openfast/r-test.git) registered for path 'reg_tests/r-test' +| Cloning into 'reg_tests/r-test'... +| remote: Counting objects: 3608, done. +| remote: Compressing objects: 100% (121/121), done. +| remote: Total 3608 (delta 22), reused 161 (delta 21), pack-reused 3442 +| Receiving objects: 100% (3608/3608), 154.52 MiB | 26.29 MiB/s, done. +| Resolving deltas: 100% (2578/2578), done. +| Checking connectivity... done. +| Submodule path 'reg_tests/r-test': checked out 'b808f1f3c1331fe5d03c5aaa4167532c2492d378' + + +| 2) Build The Regression Testing DISCON DLLs +| a) Open the Visual Studio Solution (DISCON.sln) located in openfast\vs-build\DISCON folder +| b) Choose Release and x64 for the Solutions Configuration and Solutions Platform, respectively +| c) From the menu bar select Build->Build Solution +| d) You should now see the files DISCON.dll, DISCON_ITIBarge.dll, and DISCON_OC4.dll in your openfast\build\reg_tests\openfast\5MW_Baseline\ServoData folder + + +| 3) Build OpenFAST using Visual Studio +| a) Open the Visual Studio Solution (FAST.sln) located in openfast\vs-build\FAST folder +| b) Choose Release and x64 for the Solutions Configuration and Solutions Platform, respectively +| c) From the menu bar select Build->Build Solution +| i) If this is the first time you have tried to build openfast, you will get build errors!!! [continue to steps (ii) and (iii), otherwise if FAST builds successfully, continue to step (3d) ] +| ii) Cancel build using the menubar Build->Cancel +| [ VS is confused about the build-order/dependency of the project files in FASTlib., but canceling and restarting VS, it somehow as enough info from the partial build to get this right, now] +| iii) Close your Visual Studio and then Repeat Steps (a) through (c) +| d) You should now see the file FAST_x64.exe in your openfast\build\bin folder + + +| 4) Execute the OpenFAST regression Tests +| a) Open a command prompt which is configured for Python [ like Anaconda3 ] +| b) Change your working directory to openfast\reg_tests +| c) Type: python manualRegressionTest.py ..\build\bin\FAST_x64.exe Windows Intel +| You should see this: executing case AWT_YFix_WSt +| d) The tests will continue to execute one-by-one until you finally see something like this: +| ('AWT_YFix_WSt', 'FAIL') +| ('AWT_WSt_StartUp_HighSpShutDown', 'FAIL') +| ('AWT_YFree_WSt', 'FAIL') +| ('AWT_YFree_WTurb', 'FAIL') +| ('AWT_WSt_StartUpShutDown', 'FAIL') +| ('AOC_WSt', 'FAIL') +| ('AOC_YFree_WTurb', 'FAIL') +| ('AOC_YFix_WSt', 'FAIL') +| ('UAE_YRamp_WSt', 'FAIL') +| ('UAE_Rigid_WRamp_PwrCurve', 'FAIL') +| ('WP_VSP_WTurb_PitchFail', 'FAIL') +| ('WP_VSP_ECD', 'FAIL') +| ('WP_VSP_WTurb', 'FAIL') +| ('WP_Stationary_Linear', 'PASS') +| ('SWRT_YFree_VS_EDG01', 'FAIL') +| ('SWRT_YFree_VS_EDC01', 'FAIL') +| ('SWRT_YFree_VS_WTurb', 'FAIL') +| ('5MW_DLL_Potential_WTurb', 'FAIL') +| ('5MW_DLL_Potential_WTurb_WavesIrr', 'FAIL') +| ('5MW_DLL_Potential_WSt_WavesReg', 'FAIL') +| ('5MW_DLL_Potential_WTurb_WavesIrrFixedYawGrowth', 'FAIL') +| ('5MW_DLL_WTurb_WavesIrr', 'FAIL') +| ('5MW_DLL_WTurb_WavesIrr_WavesMulti', 'FAIL') +| ('5MW_DLL_WTurb_WavesIrr', 'FAIL') +| ('5MW_WSt_WhiteNoise_OC4', 'FAIL') +| ('5MW_BD_DLL_Potential_WTurb', 'FAIL') +| e) If an individual test succeeds you will see 'PASS' otherwise you will see 'FAIL' after that test's name diff --git a/docs/source/test/unit_test.rst b/docs/source/test/unit_test.rst new file mode 100644 index 0000000000..3027c3c685 --- /dev/null +++ b/docs/source/test/unit_test.rst @@ -0,0 +1,4 @@ +Unit test +========= + +Coming soon diff --git a/reg_tests/CMakeLists.txt b/reg_tests/CMakeLists.txt new file mode 100644 index 0000000000..a77364bfa9 --- /dev/null +++ b/reg_tests/CMakeLists.txt @@ -0,0 +1,124 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------- +# -- OpenFAST Testing +# ----------------------------------------------------------- + +cmake_minimum_required(VERSION 2.8.12) +project(OpenFAST_RegressionTest Fortran) + +# Store the CTest build directory +set(CTEST_BINARY_DIR "${CMAKE_BINARY_DIR}/reg_tests") + +# Verify that the test data submodule exists +if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/r-test") + message(FATAL_ERROR "CMake cannot find the test data directory, r-test. Did you initialize the git submodule?" ) +endif() + +# Set the OpenFAST executable configuration option and default +set(CTEST_OPENFAST_EXECUTABLE "${CMAKE_BINARY_DIR}/glue-codes/fast/openfast" CACHE FILEPATH "Specify the OpenFAST executable to use in testing.") + +# Set the BeamDyn executable configuration option and default +set(CTEST_BEAMDYN_EXECUTABLE "${CMAKE_BINARY_DIR}/modules-local/beamdyn/beamdyn_driver" CACHE FILEPATH "Specify the BeamDyn driver executable to use in testing.") + +# Set the python executable configuration option and default +if(NOT EXISTS ${PYTHON_EXECUTABLE}) + find_program(PYTHON_EXECUTABLE NAMES python3 python) + if(NOT EXISTS ${PYTHON_EXECUTABLE}) + message(FATAL_ERROR "CMake cannot find a Python interpreter in your path. Python is required to run OpenFAST tests." ) + endif() +endif() + +# Set the testing tolerance +set(CTEST_REGRESSION_TOL "0.0000001" CACHE STRING "Set the tolerance for the regression test. Leave empty to automatically set.") +if(NOT ${CTEST_REGRESSION_TOL} STREQUAL "") + set(TOLERANCE ${CTEST_REGRESSION_TOL}) +else(NOT ${CTEST_REGRESSION_TOL} STREQUAL "") + + if( ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" ) + # default + set(TOLERANCE 0.0000001) + + # compiler specific + if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU" ) + set(TOLERANCE 0.0000001) + elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel" ) + set(TOLERANCE 0.0000001) + endif() + + elseif( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" ) + # default + set(TOLERANCE 0.0000001) + + # compiler specific + if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU" ) + set(TOLERANCE 0.000000000000001) + elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel" ) + set(TOLERANCE 0.0000001) + endif() + + elseif( ${CMAKE_SYSTEM_NAME} MATCHES "Windows" ) + # default + set(TOLERANCE 0.0000001) + + # compiler specific + if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU" ) + set(TOLERANCE 0.0000001) + elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel" ) + set(TOLERANCE 0.0000001) + endif() + + elseif( ${CMAKE_SYSTEM_NAME} MATCHES "CYGWIN" ) + # default + set(TOLERANCE 0.0000001) + + # compiler specific + if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU" ) + set(TOLERANCE 0.0000001) + elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel" ) + set(TOLERANCE 0.0000001) + endif() + + else() + # default for other systems + set(TOLERANCE 0.0000001) + endif() + +endif() +message("-- For system/compiler combination ${CMAKE_SYSTEM_NAME}/${CMAKE_Fortran_COMPILER_ID}, using test tolerance ${TOLERANCE}") + +# include the ServoDyn controller projects +set(SERVODATA_DIR "${CMAKE_CURRENT_LIST_DIR}/r-test/openfast/5MW_Baseline/ServoData") +add_subdirectory("${SERVODATA_DIR}/DISCON" "${SERVODATA_DIR}/DISCON/build") +add_subdirectory("${SERVODATA_DIR}/DISCON_ITI" "${SERVODATA_DIR}/DISCON_ITI/build") +add_subdirectory("${SERVODATA_DIR}/DISCON_OC3" "${SERVODATA_DIR}/DISCON_OC3/build") + +# build and seed the test directories with the data they need to run the tests +file(MAKE_DIRECTORY ${CTEST_BINARY_DIR}) +foreach(regTest openfast beamdyn) + file(MAKE_DIRECTORY ${CTEST_BINARY_DIR}/${regTest}) +endforeach() + +## openfast seed +foreach(turbineDirectory 5MW_Baseline AOC AWT27 SWRT UAE_VI WP_Baseline) + file(COPY "${CMAKE_CURRENT_LIST_DIR}/r-test/openfast/${turbineDirectory}" + DESTINATION "${CTEST_BINARY_DIR}/openfast/") +endforeach() + +# the statement below automatically creates a BUILD_TESTING option +include(CTest) +include(${CMAKE_CURRENT_LIST_DIR}/CTestList.cmake) diff --git a/reg_tests/CTestList.cmake b/reg_tests/CTestList.cmake new file mode 100644 index 0000000000..21e1a4eead --- /dev/null +++ b/reg_tests/CTestList.cmake @@ -0,0 +1,110 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#=============================================================================== +# Functions defining available test types +#=============================================================================== + +# Standard regression test +function(of_regression testname LABELS) + file(TO_NATIVE_PATH "${PYTHON_EXECUTABLE}" PYTHON_EXECUTABLE) + + file(TO_NATIVE_PATH "${CMAKE_CURRENT_LIST_DIR}/executeOpenfastRegressionCase.py" TEST_SCRIPT) + file(TO_NATIVE_PATH "${CTEST_OPENFAST_EXECUTABLE}" OPENFAST_EXECUTABLE) + file(TO_NATIVE_PATH "${CMAKE_CURRENT_LIST_DIR}/.." SOURCE_DIRECTORY) + file(TO_NATIVE_PATH "${CTEST_BINARY_DIR}/openfast" BUILD_DIRECTORY) + + string(REPLACE "\\" "\\\\" TEST_SCRIPT ${TEST_SCRIPT}) + string(REPLACE "\\" "\\\\" OPENFAST_EXECUTABLE ${OPENFAST_EXECUTABLE}) + string(REPLACE "\\" "\\\\" SOURCE_DIRECTORY ${SOURCE_DIRECTORY}) + string(REPLACE "\\" "\\\\" BUILD_DIRECTORY ${BUILD_DIRECTORY}) + + add_test( + ${testname} ${PYTHON_EXECUTABLE} + ${TEST_SCRIPT} + ${testname} + ${OPENFAST_EXECUTABLE} + ${SOURCE_DIRECTORY} # openfast source directory + ${BUILD_DIRECTORY} # build directory for test + ${TOLERANCE} + ${CMAKE_SYSTEM_NAME} # [Darwin,Linux,Windows] + ${CMAKE_Fortran_COMPILER_ID} # [Intel,GNU] + ) + # limit each test to 45 minutes: 2700s + set_tests_properties(${testname} PROPERTIES TIMEOUT 5400 WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" LABELS "${LABELS}") +endfunction(of_regression) + +function(bd_regression testname) + file(TO_NATIVE_PATH "${PYTHON_EXECUTABLE}" PYTHON_EXECUTABLE) + + file(TO_NATIVE_PATH "${CMAKE_CURRENT_LIST_DIR}/executeBeamdynRegressionCase.py" TEST_SCRIPT) + file(TO_NATIVE_PATH "${CTEST_BEAMDYN_EXECUTABLE}" BEAMDYN_EXECUTABLE) + file(TO_NATIVE_PATH "${CMAKE_CURRENT_LIST_DIR}/.." SOURCE_DIRECTORY) + file(TO_NATIVE_PATH "${CTEST_BINARY_DIR}" BUILD_DIRECTORY) + + string(REPLACE "\\" "\\\\" TEST_SCRIPT ${TEST_SCRIPT}) + string(REPLACE "\\" "\\\\" BEAMDYN_EXECUTABLE ${BEAMDYN_EXECUTABLE}) + string(REPLACE "\\" "\\\\" SOURCE_DIRECTORY ${SOURCE_DIRECTORY}) + string(REPLACE "\\" "\\\\" BUILD_DIRECTORY ${BUILD_DIRECTORY}) + + add_test( + ${testname} ${PYTHON_EXECUTABLE} + ${TEST_SCRIPT} + ${testname} + ${BEAMDYN_EXECUTABLE} + ${SOURCE_DIRECTORY} # openfast source directory + ${BUILD_DIRECTORY} # build directory for test + ${TOLERANCE} + ) + # limit each test to 45 minutes: 2700s + set(LABELS beamdyn regression) + set_tests_properties(${testname} PROPERTIES TIMEOUT 5400 WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" LABELS "${LABELS}") +endfunction(bd_regression) + +#=============================================================================== +# Regression tests +#=============================================================================== + +# OpenFAST regression tests +of_Regression("AWT_YFix_WSt" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("AWT_WSt_StartUp_HighSpShutDown" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("AWT_YFree_WSt" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("AWT_YFree_WTurb" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("AWT_WSt_StartUpShutDown" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("AOC_WSt" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("AOC_YFree_WTurb" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("AOC_YFix_WSt" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("UAE_Dnwind_YRamp_WSt" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("UAE_Upwind_Rigid_WRamp_PwrCurve" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("WP_VSP_WTurb_PitchFail" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("WP_VSP_ECD" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("WP_VSP_WTurb" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("WP_Stationary_Linear" "openfast;elastodyn;aerodyn15") +of_Regression("SWRT_YFree_VS_EDG01" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("SWRT_YFree_VS_EDC01" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("SWRT_YFree_VS_WTurb" "openfast;elastodyn;aerodyn14;servodyn") +of_Regression("5MW_Land_DLL_WTurb" "openfast;elastodyn;aerodyn15;servodyn") +of_Regression("5MW_OC3Mnpl_DLL_WTurb_WavesIrr" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;subdyn") +of_Regression("5MW_OC3Trpd_DLL_WSt_WavesReg" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;subdyn") +of_Regression("5MW_OC4Jckt_DLL_WTurb_WavesIrr_MGrowth" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;subdyn") +of_Regression("5MW_ITIBarge_DLL_WTurb_WavesIrr" "openfast;elastodyn;aerodyn14;servodyn;hydrodyn;map") +of_Regression("5MW_TLP_DLL_WTurb_WavesIrr_WavesMulti" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;map") +of_Regression("5MW_OC3Spar_DLL_WTurb_WavesIrr" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;map") +of_Regression("5MW_OC4Semi_WSt_WavesWN" "openfast;elastodyn;aerodyn15;servodyn;hydrodyn;moordyn") +of_Regression("5MW_Land_BD_DLL_WTurb" "openfast;beamdyn;aerodyn15;servodyn") + +# BeamDyn regression tests +bd_regression(bd_isotropic_rollup) diff --git a/reg_tests/README.md b/reg_tests/README.md new file mode 100644 index 0000000000..835a85d355 --- /dev/null +++ b/reg_tests/README.md @@ -0,0 +1,102 @@ +# openfast/reg_tests + +This directory contains the regression test suite for OpenFAST and its modules. Its contents are listed here and further described below. +- [r-test](https://github.com/openfast/r-test), a standalone repository containing the regression test data +- CMake/CTest configuration files +- Module specific regression test execution scripts +- A `lib` subdirectory with lower level python scripts + +Dependencies required to run the regression test suite are +- Python 2.7/3+ +- Numpy +- CMake and CTest + +## Execution +The automated regression test runs CTest and can be executed by running either of the commands `make test` or `ctest` from the build directory. If the entire OpenFAST package is to be built, CMake will configure CTest to find the new binary at `openfast/build/glue-codes/fast/openfast`. However, if the intention is to build only the test suite, the OpenFAST binary should be specified in the CMake configuration under the `CTEST_OPENFAST_EXECUTABLE` flag. There is also a corresponding `CTEST_[MODULE]_NAME` flag for each module that is included in the regression test. + +The regression test can be executed manually with the included driver `manualRegressionTest.py`. + +In both modes of execution a subdirectory is created in the build directory called `reg_tests` where all of the input files for the test cases are copied and all of the locally generated outputs are stored. + +## r-test +This repository serves as a container for regression test data for system level and module level testing of OpenFAST. The repository contains: +- input files for test case execution +- baseline solutions for various machine and compiler combinations +- turbine specific inputs + +The baseline solutions serve as "gold standards" for the regression test suite and are updated periodically as OpenFAST and its modules are improved. + +r-test is brought into OpenFAST as a git submodule and should be initialized after cloning with `git submodule update --init --recursive` or updated with `git submodule update`. + +## CTest/CMake +The configuration files consist of +- CMakeLists.txt +- CTestList.cmake + +#### CMakeLists.txt +This is a CMake file which configures the regression test in the CMake build directory. It should be left untouched unless advanced configuration of CMake or CTest is required. + +#### CTestList.txt +This is the CTest configuration file which lists the test cases that run in the automated test. The test list can be modified as needed by commenting lines with a `#`, but the full regression test consists of all the tests listed in this file. + +## Python Scripts +The included Python scripts are used to execute various parts of the automated regression test, so they should remain in their current location with their current name. Each script can be executed independently. + +#### executeOpenfastRegressionCase.py +This program executes OpenFAST and a regression test for a single test case. +The test data is contained in a git submodule, r-test, which must be initialized +prior to running. r-test can be initialized with +`git submodule update --init --recursive` or updated with `git submodule update`. + +Usage: `python executeOpenfastRegressionCase.py testname openfast_executable source_directory build_directory tolerance system_name compiler_id` +Example: `python executeOpenfastRegressionCase.py Test02 openfast path/to/openfast_repo path/to/openfast_repo/build 0.000001 [Darwin,Linux,Windows] [Intel,GNU]` + +#### executeBeamdynRegressionCase.py +This program executes BeamDyn and a regression test for a single test case. +The test data is contained in a git submodule, r-test, which must be initialized +prior to running. r-test can be initialized with +`git submodule update --init --recursive` or updated with `git submodule update`. + +Usage: `python executeBeamdynRegressionCase.py testname beamdyn_driver source_directory build_directory tolerance system_name compiler_id` +Example: `python executeBeamdynRegressionCase.py Test02 beamdyn_driver path/to/openfast_repo path/to/openfast_repo/build 0.000001 [Darwin,Linux,Windows] [Intel,GNU]` + +#### manualRegressionTest.py +This program executes OpenFAST on all of the CertTest cases. It mimics the +regression test execution through CMake/CTest. All generated data goes into +`openfast/build/reg_tests`. + +Usage: `python manualRegressionTest.py path/to/openfast_executable [Darwin,Linux,Windows] [Intel,GNU]` + +#### lib/executeOpenfastCase.py +This program executes a single OpenFAST case. + +Usage: `python executeOpenfastCase.py input_file openfast_executable` +- `openfast_executable` is an optional argument pointing to the OpenFAST executable of choice. +- if `openfast_executable` is not given, an attempt will be made to find one in $PATH + +Example: `python executeOpenfastCase.py CaseDir/case01.fst` +Example: `python executeOpenfastCase.py CaseDir/case01.fst openfast` +Example: `python executeOpenfastCase.py CaseDir/case01.fst openfast/install/bin/openfast` + +#### lib/executeBeamdynCase.py +This program executes a single BeamDyn case. + +Usage: `python executeBeamdynCase.py input_file beamdyn_executable` +- `beamdyn_executable` is an optional argument pointing to the BeamDyn executable of choice. +- if `beamdyn_executable` is not given, an attempt will be made to find one in $PATH + +Example: `python executeBeamdynCase.py CaseDir/case01.fst` +Example: `python executeBeamdynCase.py CaseDir/case01.fst beamdyn` +Example: `python executeBeamdynCase.py CaseDir/case01.fst openfast/install/bin/beamdyn` + +#### lib/pass_fail.py +This program determines whether a new solution has regressed from the "gold standard" +solution. It reads two OpenFAST binary output files (.outb), and computes the L2 norm +of the two solution files for each output channel. If the max norm is smaller than +the given tolerance, the test case passes. + +Usage: `python pass_fail.py solution1 solution2 tolerance` +Example: `python pass_fail.py output-local/Test01.outb gold-standard/Test01.outb 0.00000001` + +#### lib/fast_io.py +This program reads OpenFAST output files in binary or ascii format and returns the data in a Numpy array. diff --git a/reg_tests/executeBeamdynRegressionCase.py b/reg_tests/executeBeamdynRegressionCase.py new file mode 100644 index 0000000000..bb742bb4b9 --- /dev/null +++ b/reg_tests/executeBeamdynRegressionCase.py @@ -0,0 +1,140 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program executes BeamDyn and a regression test for a single test case. + The test data is contained in a git submodule, r-test, which must be initialized + prior to running. r-test can be initialized with + `git submodule update --init --recursive` or updated with `git submodule update`. + + Usage: `python3 executeBeamdynRegressionCase.py testname beamdyn_driver source_directory build_directory tolerance system_name compiler_id` + Example: `python3 executeBeamdynRegressionCase.py Test02 beamdyn_driver path/to/openfast_repo path/to/openfast_repo/build 0.000001 [Darwin,Linux,Windows] [Intel,GNU]` +""" + +import os +from stat import * +import sys +import shutil +import subprocess + +##### Helper functions + +def exitWithError(error): + print(error) + sys.exit(1) + +def exitWithDirNotFound(dir): + exitWithError("Directory does not exist: {}\n".format(dir)) + +def exitWithFileNotFound(file): + exitWithError("File does not exist: {}\n".format(file)) + +##### Main program + +### Determine python version +if sys.version_info < (3, 0): pythonCommand = "python" +else: pythonCommand = "python3" + +### Verify input arguments +if len(sys.argv) != 6: + exitWithError("Invalid arguments: {}\n".format(" ".join(sys.argv)) + + "Usage: {} executeBeamdynRegressionCase.py testname beamdyn_executable source_directory build_directory tolerance".format(pythonCommand)) + +caseName = sys.argv[1] +executable = sys.argv[2] +sourceDirectory = sys.argv[3] +buildDirectory = sys.argv[4] +tolerance = sys.argv[5] + +# verify that the given executable exists and can be run +if not os.path.isfile(executable): + exitWithError("The given executable, {}, does not exist.".format(executable)) + +permissionsMask = oct(os.stat(executable)[ST_MODE])[-1:] +if not int(permissionsMask)%2 == 1: + exitWithError("The given executable, {}, does not have proper permissions.".format(executable)) + +# verify source directory +if not os.path.isdir(sourceDirectory): + exitWithError("The given source directory, {}, does not exist.".format(sourceDirectory)) + +# verify build directory +if not os.path.isdir(buildDirectory): + os.mkdir(buildDirectory) + +if not os.path.isdir(buildDirectory): + exitWithError("The given build directory, {}, does not exist.".format(buildDirectory)) + +# verify tolerance +try: + float(tolerance) +except ValueError: + exitWithError("The given tolerance, {}, is not a valid number.".format(tolerance)) + +### Build the filesystem navigation variables for running the test case +regtests = os.path.join(sourceDirectory, "reg_tests") +lib = os.path.join(regtests, "lib") +rtest = os.path.join(regtests, "r-test") +modulesLocal = os.path.join(rtest, "modules-local") +targetOutputDirectory = os.path.join(modulesLocal, "beamdyn", caseName) +inputsDirectory = os.path.join(modulesLocal, "beamdyn", caseName) +testBuildDirectory = os.path.join(buildDirectory, "beamdyn", caseName) + +# verify all the required directories exist +if not os.path.isdir(rtest): + exitWithError("The test data directory, {}, does not exist. If you haven't already, run `git submodule update --init --recursive`".format(rtest)) +if not os.path.isdir(targetOutputDirectory): + exitWithError("The test data outputs directory, {}, does not exist. Try running `git submodule update`".format(targetOutputDirectory)) +if not os.path.isdir(inputsDirectory): + exitWithError("The test data inputs directory, {}, does not exist. Verify your local repository is up to date.".format(inputsDirectory)) + +# create the local output directory if it does not already exist +# and initialize it with input files for all test cases +if not os.path.isdir(testBuildDirectory): + os.makedirs(testBuildDirectory) + shutil.copy(os.path.join(inputsDirectory,"bd_driver.inp"), os.path.join(testBuildDirectory,"bd_driver.inp")) + shutil.copy(os.path.join(inputsDirectory,"bd_primary.inp"), os.path.join(testBuildDirectory,"bd_primary.inp")) + shutil.copy(os.path.join(inputsDirectory,"beam_props.inp"), os.path.join(testBuildDirectory,"beam_props.inp")) + +### Run beamdyn on the test case +executionScript = os.path.join(lib, "executeBeamdynCase.py") +executionCommand = " ".join([pythonCommand, executionScript, testBuildDirectory, executable]) + +print("'{}' - running".format(executionCommand)) +sys.stdout.flush() +executionReturnCode = subprocess.call(executionCommand, shell=True) +print("'{}' - finished with exit code {}".format(executionCommand, executionReturnCode)) + +if executionReturnCode != 0: + exitWithError("") + +### Build the filesystem navigation variables for running the regression test +passFailScript = os.path.join(lib, "pass_fail.py") +localOutputFile = os.path.join(testBuildDirectory, "bd_driver.out") +goldStandardFile = os.path.join(targetOutputDirectory, "bd_driver.out") + +if not os.path.isfile(passFailScript): exitWithFileNotFound(passFailScript) +if not os.path.isfile(localOutputFile): exitWithFileNotFound(localOutputFile) +if not os.path.isfile(goldStandardFile): exitWithFileNotFound(goldStandardFile) + +passfailCommand = " ".join([pythonCommand, passFailScript, localOutputFile, goldStandardFile, tolerance]) +print("'{}' - running".format(passfailCommand)) +sys.stdout.flush() +passfailReturnCode = subprocess.call(passfailCommand, shell=True) +print("'{}' - finished with exit code {}".format(passfailCommand, passfailReturnCode)) + +# return pass/fail +sys.exit(passfailReturnCode) diff --git a/reg_tests/executeOpenfastRegressionCase.py b/reg_tests/executeOpenfastRegressionCase.py new file mode 100644 index 0000000000..4d5b9a3753 --- /dev/null +++ b/reg_tests/executeOpenfastRegressionCase.py @@ -0,0 +1,202 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program executes OpenFAST and a regression test for a single test case. + The test data is contained in a git submodule, r-test, which must be initialized + prior to running. r-test can be initialized with + `git submodule update --init --recursive` or updated with `git submodule update`. + + Usage: `python3 executeOpenfastRegressionCase.py testname openfast_executable source_directory build_directory tolerance system_name compiler_id` + Example: `python3 executeOpenfastRegressionCase.py Test02 openfast path/to/openfast_repo path/to/openfast_repo/build 0.000001 [Darwin,Linux,Windows] [Intel,GNU]` +""" + +import os +from stat import * +import sys +import shutil +import subprocess + +##### Helper functions + +def exitWithError(error): + print(error) + sys.exit(1) + +def exitWithDirNotFound(dir): + exitWithError("Directory does not exist: {}\n".format(dir)) + +def exitWithFileNotFound(file): + exitWithError("File does not exist: {}\n".format(file)) + +def ignoreBaselineItems(directory, contents): + itemFilter = ['linux-intel', 'macos-gnu', 'windows-intel'] + caught = [] + for c in contents: + if c in itemFilter: + caught.append(c) + return tuple(caught) + +##### Main program + +### Store the python executable for future python calls +pythonCommand = sys.executable + +### Verify input arguments +if len(sys.argv) < 6 or len(sys.argv) > 8: + exitWithError("Invalid arguments: {}\n".format(" ".join(sys.argv)) + + "Usage: {} executeOpenfastRegressionCase.py testname openfast_executable source_directory build_directory tolerance system_name compiler_id".format(pythonCommand)) + +caseName = sys.argv[1] +executable = sys.argv[2] +sourceDirectory = sys.argv[3] +buildDirectory = sys.argv[4] +tolerance = sys.argv[5] + +# verify executable +if not os.path.isfile(executable): + exitWithError("The given executable, {}, does not exist.".format(executable)) + +permissionsMask = oct(os.stat(executable)[ST_MODE])[-1:] +if not int(permissionsMask)%2 == 1: + exitWithError("The given executable, {}, does not have proper permissions.".format(executable)) + +# verify source directory +if not os.path.isdir(sourceDirectory): + exitWithError("The given source directory, {}, does not exist.".format(sourceDirectory)) + +# verify build directory +if not os.path.isdir(buildDirectory): + os.makedirs(buildDirectory) + +if not os.path.isdir(buildDirectory): + exitWithError("The given build directory, {}, does not exist.".format(buildDirectory)) + +# verify tolerance +try: + float(tolerance) +except ValueError: + exitWithError("The given tolerance, {}, is not a valid number.".format(tolerance)) + +systemcompiler_given = True +try: + systemName = sys.argv[6] +except IndexError: + systemcompiler_given = False + systemName = "not_given" + +try: + compilerId = sys.argv[7] +except IndexError: + systemcompiler_given = False + compilerId = "not_given" + +### Map the system and compiler configurations to a solution set +# Internal names -> Human readable names +systemName_map = { + "darwin": "macos", + "linux": "linux", + "windows": "windows" +} +compilerId_map = { + "gnu": "gnu", + "intel": "intel" +} +# Build the target output directory name or choose the default +if systemName.lower() not in systemName_map or compilerId.lower() not in compilerId_map: + targetSystem = "macos" + targetCompiler = "gnu" +else: + targetSystem = systemName_map.get(systemName.lower()) + targetCompiler = compilerId_map.get(compilerId.lower()) + +outputType = os.path.join(targetSystem+"-"+targetCompiler) +print("-- Using gold standard files with machine-compiler type {}".format(outputType)) + +### Build the filesystem navigation variables for running openfast on the test case +regtests = os.path.join(sourceDirectory, "reg_tests") +lib = os.path.join(regtests, "lib") +rtest = os.path.join(regtests, "r-test") +moduleDirectory = os.path.join(rtest, "openfast") +inputsDirectory = os.path.join(moduleDirectory, caseName) +targetOutputDirectory = os.path.join(inputsDirectory, outputType) +testBuildDirectory = os.path.join(buildDirectory, caseName) + +# verify all the required directories exist +if not os.path.isdir(rtest): + exitWithError("The test data directory, {}, does not exist. If you haven't already, run `git submodule update --init --recursive`".format(rtest)) +if not os.path.isdir(targetOutputDirectory): + exitWithError("The test data outputs directory, {}, does not exist. Try running `git submodule update`".format(targetOutputDirectory)) +if not os.path.isdir(inputsDirectory): + exitWithError("The test data inputs directory, {}, does not exist. Verify your local repository is up to date.".format(inputsDirectory)) + +# create the local output directory if it does not already exist +# and initialize it with input files for all test cases +for data in ["AOC", "AWT27", "SWRT", "UAE_VI", "WP_Baseline"]: + dataDir = os.path.join(buildDirectory, data) + if not os.path.isdir(dataDir): + shutil.copytree(os.path.join(moduleDirectory, data), dataDir) + +# Special copy for the 5MW_Baseline folder because the Windows python-only workflow may have already created data in the subfolder ServoData +dst = os.path.join(buildDirectory, "5MW_Baseline") +src = os.path.join(moduleDirectory, "5MW_Baseline") +if not os.path.isdir(dst): + shutil.copytree(src, dst) +else: + names = os.listdir(src) + for name in names: + if name is "ServoData": + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + if os.path.isdir(srcname): + if not os.path.isdir(dstname): + shutil.copytree(srcname, dstname) + else: + shutil.copy2(srcname, dstname) + +if not os.path.isdir(testBuildDirectory): + shutil.copytree(inputsDirectory, testBuildDirectory, ignore=ignoreBaselineItems) + +### Run openfast on the test case +caseInputFile = os.path.join(testBuildDirectory, caseName + ".fst") +executionScript = os.path.join(lib, "executeOpenfastCase.py") +executionCommand = " ".join([pythonCommand, executionScript, caseInputFile, executable]) +print("'{}' - running".format(executionCommand)) +sys.stdout.flush() +executionReturnCode = subprocess.call(executionCommand, shell=True) +print("'{}' - finished with exit code {}".format(executionCommand, executionReturnCode)) + +if executionReturnCode != 0: + exitWithError("") + +### Build the filesystem navigation variables for running the regression test +passFailScript = os.path.join(lib, "pass_fail.py") +localOutputFile = os.path.join(testBuildDirectory, caseName + ".outb") +goldStandardFile = os.path.join(targetOutputDirectory, caseName + ".outb") + +if not os.path.isfile(passFailScript): exitWithFileNotFound(passFailScript) +if not os.path.isfile(localOutputFile): exitWithFileNotFound(localOutputFile) +if not os.path.isfile(goldStandardFile): exitWithFileNotFound(goldStandardFile) + +passfailCommand = " ".join([pythonCommand, passFailScript, localOutputFile, goldStandardFile, tolerance]) +print("'{}' - running".format(passfailCommand)) +sys.stdout.flush() +passfailReturnCode = subprocess.call(passfailCommand, shell=True) +print("'{}' - finished with exit code {}".format(passfailCommand, passfailReturnCode)) + +# return pass/fail +sys.exit(passfailReturnCode) diff --git a/reg_tests/lib/executeBeamdynCase.py b/reg_tests/lib/executeBeamdynCase.py new file mode 100644 index 0000000000..0a89ef798d --- /dev/null +++ b/reg_tests/lib/executeBeamdynCase.py @@ -0,0 +1,65 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program executes a single BeamDyn case. + + Usage: `python3 executeBeamdynCase.py input_file beamdyn_executable` + - `beamdyn_executable` is an optional argument pointing to the BeamDyn executable of choice. + - if `beamdyn_executable` is not given, an attempt will be made to find one in $PATH + + Example: `python3 executeBeamdynCase.py CaseDir/case01.fst` + Example: `python3 executeBeamdynCase.py CaseDir/case01.fst beamdyn` + Example: `python3 executeBeamdynCase.py CaseDir/case01.fst openfast/install/bin/beamdyn` +""" + +import os +from stat import * +import sys +import shutil +import subprocess + +def exitWithError(error, code=1): + print(error) + sys.exit(code) + +if len(sys.argv) != 3: + exitWithError("Invalid arguments given: {}\n".format(" ".join(sys.argv)) + + "Usage: python3 executeBeamdynCase.py case_directory beamdyn_executable") + +# verify that the given input file exists +caseDirectory = sys.argv[1] +caseInputFile = "bd_driver.inp" +if not os.path.isfile(os.path.join(caseDirectory, caseInputFile)): + exitWithError("The given input file, {}, does not exist.".format(caseInputFile)) + +# verify that the given executable exists and can be run +executable = sys.argv[2] +if not os.path.isfile(executable): + exitWithError("The given beamdyn_driver, {}, does not exist.".format(executable)) + +permissionsMask = oct(os.stat(executable)[ST_MODE])[-1:] +if not int(permissionsMask)%2 == 1: + exitWithError("The given beamdyn_driver, {}, does not executable permission.".format(executable)) + +# execute the given case +os.chdir(caseDirectory) +command = "{} {} > {}.log".format(executable, caseInputFile, caseInputFile.split(".")[0]) +print("'{}' - running".format(command)) +sys.stdout.flush() +return_code = subprocess.call(command, shell=True) +print("'{}' - finished with exit code {}".format(command, return_code)) +sys.exit(return_code) diff --git a/reg_tests/lib/executeOpenfastCase.py b/reg_tests/lib/executeOpenfastCase.py new file mode 100644 index 0000000000..45f8f6a9fd --- /dev/null +++ b/reg_tests/lib/executeOpenfastCase.py @@ -0,0 +1,81 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program executes a single OpenFAST case. + + Usage: `python3 executeOpenfastCase.py input_file openfast_executable` + - `openfast_executable` is an optional argument pointing to the OpenFAST executable of choice. + - if `openfast_executable` is not given, an attempt will be made to find one in $PATH + + Example: `python3 executeOpenfastCase.py CaseDir/case01.fst` + Example: `python3 executeOpenfastCase.py CaseDir/case01.fst openfast` + Example: `python3 executeOpenfastCase.py CaseDir/case01.fst openfast/install/bin/openfast` +""" + +import os +import sys +import shutil +import subprocess + +def exitWithError(error, code=1): + print(error) + sys.exit(code) + +if len(sys.argv) != 2 and len(sys.argv) != 3: + exitWithError("Invalid arguments given: {}\n".format(" ".join(sys.argv)) + + "Usage: python3 executeOpenfastCase.py input_file openfast_executable") + +# verify that the given input file exists +caseInputFile = sys.argv[1] +if not os.path.isfile(caseInputFile): + exitWithError("The given input file, {}, does not exist.".format(caseInputFile)) + +# if no openfast executable was given, search in path +if len(sys.argv) == 2: + try: + devnull = open(os.devnull, 'w') + subprocess.call("openfast", stdout=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + exitWithError("{}: openfast\n".format(e) + + "Usage: python3 executeOpenfastCase.py input_file openfast_executable") + else: + raise + else: + executable = "openfast" + print("Using openfast executable found in path") + +# verify that the given executable exists and can be run +elif len(sys.argv) == 3: + executable = sys.argv[2] + try: + devnull = open(os.devnull, 'w') + subprocess.call(executable, stdout=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + exitWithError("{}: {}".format(e, executable)) + else: + raise + +# execute the given case +command = "{} {} > {}.log".format(executable, caseInputFile, caseInputFile.split(".fst")[0]) +print("'{}' - running".format(command)) +sys.stdout.flush() +return_code = subprocess.call(command, shell=True) +print(return_code) +print("'{}' - finished with exit code {}".format(command, return_code)) +sys.exit(return_code) diff --git a/reg_tests/lib/fast_io.py b/reg_tests/lib/fast_io.py new file mode 100644 index 0000000000..f58729aac8 --- /dev/null +++ b/reg_tests/lib/fast_io.py @@ -0,0 +1,184 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +''' +Created on 03/09/2015 + +@author: MMPE + +Copied from https://github.com/WISDEM/AeroelasticSE/tree/openmdao1/src/AeroelasticSE/old_files on 15 Aug 2016 by Ganesh Vijayakumar +''' +import os +import numpy as np +import struct + +def load_output(filename): + """Load a FAST binary or ascii output file + + Parameters + ---------- + filename : str + filename + + Returns + ------- + data : ndarray + data values + info : dict + info containing: + - name: filename + - description: description of dataset + - attribute_names: list of attribute names + - attribute_units: list of attribute units + """ + + assert os.path.isfile(filename), "File, %s, does not exists" % filename + with open(filename, 'r') as f: + if "outb" in filename: + return load_binary_output(filename) + elif "out" in filename: + try: + print(f.readline()) + except UnicodeDecodeError: + return load_binary_output(filename) + return load_ascii_output(filename) + +def load_ascii_output(filename): + with open(filename) as f: + info = {} + info['name'] = os.path.splitext(os.path.basename(filename))[0] + try: + header = [f.readline() for _ in range(8)] + info['description'] = header[4].strip() + info['attribute_names'] = header[6].split() + info['attribute_units'] = [unit[1:-1] for unit in header[7].split()] #removing "()" + data = np.array([line.split() for line in f.readlines()]).astype(np.float) + + return data, info + except (ValueError, AssertionError): + + raise + + +def load_binary_output(filename): + """Ported from ReadFASTbinary.m by Mads M Pedersen, DTU Wind + Info about ReadFASTbinary.m: + % Author: Bonnie Jonkman, National Renewable Energy Laboratory + % (c) 2012, National Renewable Energy Laboratory + % + % Edited for FAST v7.02.00b-bjj 22-Oct-2012 + """ + def fread(fid, n, type): + fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type] + + return struct.unpack(fmt * n, fid.read(nbytes * n)) + + FileFmtID_WithTime = 1 #% File identifiers used in FAST + FileFmtID_WithoutTime = 2 + LenName = 10 #; % number of characters per channel name + LenUnit = 10 #; % number of characters per unit name + + with open(filename, 'rb') as fid: + FileID = fread(fid, 1, 'int16') #; % FAST output file format, INT(2) + + NumOutChans = fread(fid, 1, 'int32')[0] #; % The number of output channels, INT(4) + NT = fread(fid, 1, 'int32')[0] #; % The number of time steps, INT(4) + + if FileID == FileFmtID_WithTime: + TimeScl = fread(fid, 1, 'float64') #; % The time slopes for scaling, REAL(8) + TimeOff = fread(fid, 1, 'float64') #; % The time offsets for scaling, REAL(8) + else: + TimeOut1 = fread(fid, 1, 'float64') #; % The first time in the time series, REAL(8) + TimeIncr = fread(fid, 1, 'float64') #; % The time increment, REAL(8) + + + + + ColScl = fread(fid, NumOutChans, 'float32') #; % The channel slopes for scaling, REAL(4) + ColOff = fread(fid, NumOutChans, 'float32') #; % The channel offsets for scaling, REAL(4) + + LenDesc = fread(fid, 1, 'int32')[0] #; % The number of characters in the description string, INT(4) + DescStrASCII = fread(fid, LenDesc, 'uint8') #; % DescStr converted to ASCII + DescStr = "".join(map(chr, DescStrASCII)).strip() + + + + ChanName = [] # initialize the ChanName cell array + for iChan in range(NumOutChans + 1): + ChanNameASCII = fread(fid, LenName, 'uint8') #; % ChanName converted to numeric ASCII + ChanName.append("".join(map(chr, ChanNameASCII)).strip()) + + + ChanUnit = [] # initialize the ChanUnit cell array + for iChan in range(NumOutChans + 1): + ChanUnitASCII = fread(fid, LenUnit, 'uint8') #; % ChanUnit converted to numeric ASCII + ChanUnit.append("".join(map(chr, ChanUnitASCII)).strip()[1:-1]) + + + # %------------------------- + # % get the channel time series + # %------------------------- + + nPts = NT * NumOutChans #; % number of data points in the file + + + if FileID == FileFmtID_WithTime: + PackedTime = fread(fid, NT, 'int32') #; % read the time data + cnt = len(PackedTime) + if cnt < NT: + raise Exception('Could not read entire %s file: read %d of %d time values' % (filename, cnt, NT)) + PackedData = fread(fid, nPts, 'int16') #; % read the channel data + cnt = len(PackedData) + if cnt < nPts: + raise Exception('Could not read entire %s file: read %d of %d values' % (filename, cnt, nPts)) + + # %------------------------- + # % Scale the packed binary to real data + # %------------------------- + # + + + data = np.array(PackedData).reshape(NT, NumOutChans) + data = (data - ColOff) / ColScl + + if FileID == FileFmtID_WithTime: + time = (np.array(PackedTime) - TimeOff) / TimeScl; + else: + time = TimeOut1 + TimeIncr * np.arange(NT) + + data = np.concatenate([time.reshape(NT, 1), data], 1) + + info = {'name': os.path.splitext(os.path.basename(filename))[0], + 'description': DescStr, + 'attribute_names': ChanName, + 'attribute_units': ChanUnit} + return data, info + + +if __name__=="__main__": + d,i = load_binary_output('Test18.T1.outb') + types = [] + for j in range(39): + types.append('f8') + print(type(i['attribute_names'])) + + print(np.dtype({'names':tuple(i['attribute_names']), 'formats': tuple(types) })) + print(type(d)) + print(np.array(d,dtype=np.dtype({'names':tuple(i['attribute_names']), 'formats': tuple(types) }))) + + print(i) + print(len(i['attribute_names'])) + print(np.shape(d)) diff --git a/reg_tests/lib/pass_fail.py b/reg_tests/lib/pass_fail.py new file mode 100644 index 0000000000..c8f01f350b --- /dev/null +++ b/reg_tests/lib/pass_fail.py @@ -0,0 +1,87 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program determines whether a new solution has regressed from the "gold standard" + solution. It reads two OpenFAST binary output files (.outb), and computes the variance + of the two solution files for each output channel. If the max variance is less than + the given tolerance, the test case passes. + + Usage: python3 pass_fail.py solution1 solution2 tolerance + Example: python3 pass_fail.py output-local/Test01.outb gold-standard/Test01.outb 0.00000001 +""" +import sys, os +import numpy as np +from numpy import linalg as LA +from fast_io import load_output + +def exitWithError(error): + print(error) + sys.exit(1) + +# validate input arguments +nArgsExpected = 4 +if len(sys.argv) < nArgsExpected: + exitWithError("Error: {} arguments given, expected {}\n".format(len(sys.argv), nArgsExpected) + + "Usage: {} solution1 solution2 tolerance".format(sys.argv[0])) + +solutionFile1 = sys.argv[1] +solutionFile2 = sys.argv[2] +solutionTolerance = sys.argv[3] + +if not os.path.isfile(solutionFile1): + exitWithError("Error: solution file does not exist at {}".format(solutionFile1)) + +if not os.path.isfile(solutionFile2): + exitWithError("Error: solution file does not exist at {}".format(solutionFile2)) + +try: + solutionTolerance = float(solutionTolerance) +except ValueError: + exitWithError("Error: invalid tolerance given, {}".format(solutionTolerance)) + +# parse the FAST solution files +try: + dict1, info1 = load_output(solutionFile1) + dict2, info2 = load_output(solutionFile2) +except Exception as e: + exitWithError("Error: {}".format(e)) + +## gold standard RMS, L2 norm +nColumns = np.size(dict1,1) +diff = np.ones(nColumns) +rms_gold = np.ones(nColumns) +norm_diff = np.ones(nColumns) +for j in range(nColumns): + rms_gold[j] = LA.norm(dict2[:,j], 2) + + diff = dict1[:,j]-dict2[:,j] + norm_diff[j] = LA.norm(diff, 2) + +# replace any 0s with small number before for division +rms_gold[rms_gold == 0] = 1e-16 + +norm = norm_diff / rms_gold + +####### need to reverse inequality to actually see output since test currently passes every time ###### +if max(norm) < solutionTolerance: + print('PASS') + sys.exit(0) +else: + for i in range(len(info1['attribute_names'])): + print(info1['attribute_names'][i], norm[i]) + + sys.exit(1) diff --git a/reg_tests/manualRegressionTest.py b/reg_tests/manualRegressionTest.py new file mode 100644 index 0000000000..d930576245 --- /dev/null +++ b/reg_tests/manualRegressionTest.py @@ -0,0 +1,62 @@ +# +# Copyright 2017 National Renewable Energy Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" + This program executes OpenFAST on all of the CertTest cases. It mimics the + regression test execution through CMake/CTest. All generated data goes into + `openfast/build/reg_tests`. + + Usage: `python manualRegressionTest.py openfast/install/bin/openfast [Darwin,Linux,Windows] [Intel,GNU] tolerance` +""" + +import sys +import os +import subprocess + +def exitWithError(error, code=1): + print(error) + sys.exit(code) + +### Verify input arguments +if len(sys.argv) != 4 and len(sys.argv) != 5: + exitWithError("Invalid arguments: {}\n".format(" ".join(sys.argv)) + + "Usage: python manualRegressionTest.py openfast/install/bin/openfast [Darwin,Linux,Windows] [Intel,GNU] tolerance") + +openfast_executable = sys.argv[1] +sourceDirectory = ".." +buildDirectory = os.path.join("..", "build", "reg_tests", "openfast") +machine = sys.argv[2] +compiler = sys.argv[3] +tolerance = sys.argv[4] if len(sys.argv) == 5 else 0.0000001 +devnull = open(os.devnull, 'w') + +with open(os.path.join("r-test", "openfast", "CaseList.md")) as listfile: + content = listfile.readlines() +casenames = [x.rstrip("\n\r").strip() for x in content if "#" not in x] + +results = [] +for case in casenames: + print("executing case {}".format(case)) + command = "python executeOpenfastRegressionCase.py {} {} {} {} {} {} {}".format(case, openfast_executable, sourceDirectory, buildDirectory, tolerance, machine, compiler) + returnCode = subprocess.call(command, stdout=devnull, shell=True) + if returnCode == 0: + results.append((case, "PASS")) + else: + results.append((case, "FAIL", returnCode)) + +print("Regression test execution completed with these results:") +for r in results: + print(" ".join([str(rr) for rr in r])) diff --git a/reg_tests/r-test b/reg_tests/r-test new file mode 160000 index 0000000000..cb50bbc810 --- /dev/null +++ b/reg_tests/r-test @@ -0,0 +1 @@ +Subproject commit cb50bbc81058a58983e0c23d3dc12e899fbe0920