From 84addf3ad90ed06a96bf6d8fab0284af0894b6e3 Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Tue, 17 May 2022 13:03:26 -0500 Subject: [PATCH 01/25] Clean up in rtestlib --- reg_tests/executeOpenfastRegressionCase.py | 4 +-- reg_tests/lib/pass_fail.py | 38 +++++++++++----------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/reg_tests/executeOpenfastRegressionCase.py b/reg_tests/executeOpenfastRegressionCase.py index 2bb1bb14ca..691736d8d0 100644 --- a/reg_tests/executeOpenfastRegressionCase.py +++ b/reg_tests/executeOpenfastRegressionCase.py @@ -24,7 +24,7 @@ import os import sys -basepath = os.path.dirname(os.path.abspath(__file__)) +basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse import shutil @@ -158,7 +158,7 @@ def ignoreBaselineItems(directory, contents): rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) performance = pass_fail.calculateNorms(testData, baselineData) normalizedNorm = performance[:, 1] diff --git a/reg_tests/lib/pass_fail.py b/reg_tests/lib/pass_fail.py index 3bfdf0062a..61030a6ceb 100644 --- a/reg_tests/lib/pass_fail.py +++ b/reg_tests/lib/pass_fail.py @@ -18,7 +18,7 @@ This library provides tools for comparing a test solution to a baseline solution for any structured output file generated within the OpenFAST framework. """ -import sys, os +import sys import numpy as np from numpy import linalg as LA from fast_io import load_output @@ -72,10 +72,14 @@ def calculateNorms(test_data, baseline_data): relative_norm = calculate_max_norm_over_range(test_data, baseline_data) max_norm = calculate_max_norm(test_data, baseline_data) relative_l2_norm = calculate_relative_norm(test_data, baseline_data) - results = np.hstack(( - relative_norm.reshape(-1, 1), relative_l2_norm.reshape(-1, 1), - max_norm.reshape(-1, 1) - )) + results = np.stack( + ( + relative_norm, + relative_l2_norm, + max_norm + ), + axis=1 + ) return results if __name__=="__main__": @@ -84,22 +88,18 @@ def calculateNorms(test_data, baseline_data): testSolution = sys.argv[1] baselineSolution = sys.argv[2] - tolerance = sys.argv[3] - - try: - tolerance = float(tolerance) - except ValueError: - rtl.exitWithError("Error: invalid tolerance given, {}".format(tolerance)) + tolerance = float(sys.argv[3]) rtl.validateFileOrExit(testSolution) rtl.validateFileOrExit(baselineSolution) testData, testInfo, testPack = readFASTOut(testSolution) - baselineData, baselineInfo, basePack = readFASTOut(baselineSolution) - - normalizedNorm, maxNorm = pass_fail.calculateNorms(testData, baselineData, tolerance) - if passRegressionTest(normalizedNorm, tolerance): - sys.exit(0) - else: - dict1, info1, pack1 = readFASTOut(testSolution) - sys.exit(1) + baselineData, baselineInfo, _ = readFASTOut(baselineSolution) + relative_norm, normalized_norm, max_norm = calculateNorms(testData, baselineData) + print(relative_norm) + print(normalized_norm) + print(max_norm) + + # if not passRegressionTest(normalizedNorm, tolerance): + # dict1, info1, pack1 = readFASTOut(testSolution) + # sys.exit(1) From fa66d5b64434848f024bb5c02304af58276bdfe1 Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Wed, 22 Jun 2022 14:59:56 -0500 Subject: [PATCH 02/25] Determine passing channels instead of norms --- reg_tests/executeOpenfastRegressionCase.py | 43 ++++++++++++---------- reg_tests/lib/errorPlotting.py | 33 +++++++++-------- reg_tests/lib/pass_fail.py | 21 ++++++++--- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/reg_tests/executeOpenfastRegressionCase.py b/reg_tests/executeOpenfastRegressionCase.py index 691736d8d0..f245e780b7 100644 --- a/reg_tests/executeOpenfastRegressionCase.py +++ b/reg_tests/executeOpenfastRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -160,27 +161,29 @@ def ignoreBaselineItems(directory, contents): testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] -# export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - for channel in testInfo["attribute_names"]: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error)) - finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) - - sys.exit(1) +norms = pass_fail.calculateNorms(testData, baselineData) + +# export all case summaries +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/lib/errorPlotting.py b/reg_tests/lib/errorPlotting.py index 1778f3fe75..a68546e610 100644 --- a/reg_tests/lib/errorPlotting.py +++ b/reg_tests/lib/errorPlotting.py @@ -243,40 +243,43 @@ def exportResultsSummary(path, results): html.write( _htmlTail() ) html.close() -def exportCaseSummary(path, case, results, results_max, tolerance): +def exportCaseSummary(path, case, channel_names, passing_channels, norms): + """ + norms: first dimension is the channel and second dimension contains the norms + passing_channels: 1d boolean array for whether a channel passed the comparison + """ + with open(os.path.join(path, case+".html"), "w") as html: html.write( _htmlHead(case + " Summary") ) html.write('\n') html.write('

{}

\n'.format(case + " Summary")) - html.write('

Maximum values for each norm are highlighted and failing norms (norm >= {0}) are highlighted

\n'.format(tolerance)) html.write('
\n') - data = [ - ('{0}'.format(attribute), *norms) - for attribute, *norms in results - ] + channel_tags = [ '{0}'.format(channel) for channel in channel_names ] + cols = [ - 'Channel', 'Relative Max Norm', - 'Relative L2 Norm', 'Infinity Norm' + 'Channel', + 'Relative Max Norm', + 'Relative L2 Norm', + 'Infinity Norm' ] table = _tableHead(cols) body = ' ' + '\n' - for i, d in enumerate(data): + + for i, channel_tag in enumerate(channel_tags): body += ' ' + '\n' body += ' {}'.format(i+1) + '\n' - body += ' {0:s}'.format(d[0]) + '\n' + body += ' {0:s}'.format(channel_tag) + '\n' fmt = '{0:0.4e}' - for j, val in enumerate(d[1]): - if val == results_max[j]: - body += (' ' + fmt + '\n').format(val) - elif val > tolerance: + for val in norms[i]: + if not passing_channels[i]: body += (' ' + fmt + '\n').format(val) else: body += (' ' + fmt + '\n').format(val) - + body += ' ' + '\n' body += ' ' + '\n' table += body diff --git a/reg_tests/lib/pass_fail.py b/reg_tests/lib/pass_fail.py index 61030a6ceb..53ffd5f80a 100644 --- a/reg_tests/lib/pass_fail.py +++ b/reg_tests/lib/pass_fail.py @@ -30,12 +30,21 @@ def readFASTOut(fastoutput): except Exception as e: rtl.exitWithError("Error: {}".format(e)) -def passRegressionTest(norm, tolerance): - if np.any(np.isnan(norm)): - return False - if np.any(np.isinf(norm)): - return False - return True if max(norm) < tolerance else False +def passing_channels(test, baseline) -> np.ndarray: + """ + test, baseline: arrays containing the results from OpenFAST in the following format + [ + channels, + data + ] + So that test[0,:] are the data for the 0th channel and test[:,0] are the 0th entry in each channel. + """ + atol = 1e-4 + rtol = 1e-6 + whereclose = np.isclose( test, baseline, atol=atol, rtol=rtol ) + passing_channels = np.all(whereclose, axis=1) + + return passing_channels def maxnorm(data, axis=0): return LA.norm(data, np.inf, axis=axis) From b2b3d5c7d03f2af41c62f708653c6b0631831f26 Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Tue, 17 May 2022 13:06:41 -0500 Subject: [PATCH 03/25] Plot pass/fail boundary in error plot --- reg_tests/lib/errorPlotting.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reg_tests/lib/errorPlotting.py b/reg_tests/lib/errorPlotting.py index a68546e610..ed686ead36 100644 --- a/reg_tests/lib/errorPlotting.py +++ b/reg_tests/lib/errorPlotting.py @@ -66,6 +66,12 @@ def _plotError(xseries, y1series, y2series, xlabel, title1, title2): p2.grid.grid_line_alpha = 0 p2.xaxis.axis_label = 'Time (s)' p2.line(xseries, abs(y2series - y1series), color='blue') + + atol = 1e-4 + rtol = 1e-6 + passfail_line = atol + rtol * abs(y2series) + p2.line(xseries, passfail_line, color='red') + p2.add_tools(HoverTool(tooltips=[('Time','@x'), ('Error', '@y')], mode='vline')) grid = gridplot([[p1, p2]], plot_width=650, plot_height=375, sizing_mode="scale_both") @@ -117,7 +123,7 @@ def plotOpenfastError(testSolution, baselineSolution, attribute): rtl.exitWithError("Error: Invalid channel name--{}".format(e)) title1 = attribute + " (" + info1["attribute_units"][channel] + ")" - title2 = "Max norm" + title2 = "ABS(Local - Baseline)" xlabel = 'Time (s)' timevec = dict1[:, 0] From 22e8395ae28fea71f7ed8195f4de8b637f3ef0c9 Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Mon, 1 Aug 2022 16:23:01 -0500 Subject: [PATCH 04/25] Unify baselines --- reg_tests/CTestList.cmake | 2 - ...ecuteOpenfastAeroAcousticRegressionCase.py | 79 +++++++------------ reg_tests/executeOpenfastRegressionCase.py | 25 +----- reg_tests/r-test | 2 +- 4 files changed, 30 insertions(+), 78 deletions(-) diff --git a/reg_tests/CTestList.cmake b/reg_tests/CTestList.cmake index 7542433367..9a0c188938 100644 --- a/reg_tests/CTestList.cmake +++ b/reg_tests/CTestList.cmake @@ -57,8 +57,6 @@ function(regression TEST_SCRIPT EXECUTABLE SOURCE_DIRECTORY BUILD_DIRECTORY TEST ${SOURCE_DIRECTORY} # openfast source directory ${BUILD_DIRECTORY} # build directory for test ${TOLERANCE} - ${CMAKE_SYSTEM_NAME} # [Darwin,Linux,Windows] - ${CMAKE_Fortran_COMPILER_ID} # [Intel,GNU] ${PLOT_FLAG} # empty or "-p" ${RUN_VERBOSE_FLAG} # empty or "-v" ) diff --git a/reg_tests/executeOpenfastAeroAcousticRegressionCase.py b/reg_tests/executeOpenfastAeroAcousticRegressionCase.py index 381879eb25..29dfcd926a 100644 --- a/reg_tests/executeOpenfastAeroAcousticRegressionCase.py +++ b/reg_tests/executeOpenfastAeroAcousticRegressionCase.py @@ -28,6 +28,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -56,8 +57,6 @@ def ignoreBaselineItems(directory, contents): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -69,8 +68,6 @@ def ignoreBaselineItems(directory, contents): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] plotError = args.plot noExec = args.noExec verbose = args.verbose @@ -81,37 +78,17 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) - ### Build the filesystem navigation variables for running openfast on the test case regtests = os.path.join(sourceDirectory, "reg_tests") lib = os.path.join(regtests, "lib") rtest = os.path.join(regtests, "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "openfast") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(inputsDirectory, outputType) +targetOutputDirectory = os.path.join(inputsDirectory) testBuildDirectory = os.path.join(buildDirectory, caseName) - + # verify all the required directories exist if not os.path.isdir(rtest): rtl.exitWithError("The test data directory, {}, does not exist. If you haven't already, run `git submodule update --init --recursive`".format(rtest)) @@ -124,14 +101,14 @@ def ignoreBaselineItems(directory, contents): # and initialize it with input files for all test cases if not os.path.isdir(testBuildDirectory): shutil.copytree(inputsDirectory, testBuildDirectory, ignore=ignoreBaselineItems) - + ### Run openfast on the test case if not noExec: caseInputFile = os.path.join(testBuildDirectory, caseName + ".fst") returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable) if returnCode != 0: rtl.exitWithError("") - + ### Build the filesystem navigation variables for running the regression test # testing on file 2. Gives each observer and sweep of frequency ranges localOutFile = os.path.join(testBuildDirectory, caseName + "_2.out") @@ -139,31 +116,31 @@ def ignoreBaselineItems(directory, contents): rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) \ No newline at end of file diff --git a/reg_tests/executeOpenfastRegressionCase.py b/reg_tests/executeOpenfastRegressionCase.py index f245e780b7..39d393d0bd 100644 --- a/reg_tests/executeOpenfastRegressionCase.py +++ b/reg_tests/executeOpenfastRegressionCase.py @@ -56,8 +56,6 @@ def ignoreBaselineItems(directory, contents): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -69,8 +67,6 @@ def ignoreBaselineItems(directory, contents): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] plotError = args.plot noExec = args.noExec verbose = args.verbose @@ -81,25 +77,6 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) ### Build the filesystem navigation variables for running openfast on the test case regtests = os.path.join(sourceDirectory, "reg_tests") @@ -107,7 +84,7 @@ def ignoreBaselineItems(directory, contents): rtest = os.path.join(regtests, "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "openfast") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(inputsDirectory, outputType) +targetOutputDirectory = os.path.join(inputsDirectory) testBuildDirectory = os.path.join(buildDirectory, caseName) # verify all the required directories exist diff --git a/reg_tests/r-test b/reg_tests/r-test index cd9a7a4f27..71bf6ef09b 160000 --- a/reg_tests/r-test +++ b/reg_tests/r-test @@ -1 +1 @@ -Subproject commit cd9a7a4f27f4197dfd3d1ba699fd24a4c7f1aea4 +Subproject commit 71bf6ef09b2bf1961687a0fae5031528f14960ec From 68ed80ae9ecf8c8f4925f0b456cea6063d4e7c0c Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Fri, 20 May 2022 10:46:43 -0500 Subject: [PATCH 05/25] Update reg test driver scripts --- reg_tests/executeAerodynRegressionCase.py | 56 +++++++------- reg_tests/executeBeamdynRegressionCase.py | 49 ++++++------- reg_tests/executeFASTFarmRegressionCase.py | 72 +++++++----------- reg_tests/executeHydrodynRegressionCase.py | 49 ++++++------- .../executeInflowwindPyRegressionCase.py | 51 +++++++------ reg_tests/executeInflowwindRegressionCase.py | 49 ++++++------- reg_tests/executeOpenfastCppRegressionCase.py | 73 +++++++------------ .../executeOpenfastLinearRegressionCase.py | 52 ++++--------- reg_tests/executePythonRegressionCase.py | 71 +++++++----------- reg_tests/executeSubdynRegressionCase.py | 50 ++++++------- 10 files changed, 243 insertions(+), 329 deletions(-) diff --git a/reg_tests/executeAerodynRegressionCase.py b/reg_tests/executeAerodynRegressionCase.py index 1462914737..6cb1da4886 100644 --- a/reg_tests/executeAerodynRegressionCase.py +++ b/reg_tests/executeAerodynRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import glob import subprocess @@ -47,8 +48,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -112,44 +111,45 @@ if returnCode != 0: rtl.exitWithError("") -###Build the filesystem navigation variables for running the regression test +### Build the filesystem navigation variables for running the regression test # For multiple turbines, test turbine 2, for combined cases, test case 4 localOutFile = os.path.join(testBuildDirectory, "ad_driver.outb") localOutFileWT2 = os.path.join(testBuildDirectory, "ad_driver.T2.outb") localOutFileCase4 = os.path.join(testBuildDirectory, "ad_driver.4.outb") if os.path.exists(localOutFileWT2) : - localOutFile=localOutFileWT2 + localOutFile = localOutFileWT2 elif os.path.exists(localOutFileCase4) : - localOutFile=localOutFileCase4 + localOutFile = localOutFileCase4 baselineOutFile = os.path.join(targetOutputDirectory, os.path.basename(localOutFile)) + rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeBeamdynRegressionCase.py b/reg_tests/executeBeamdynRegressionCase.py index 93dbc01f74..0eded5f361 100644 --- a/reg_tests/executeBeamdynRegressionCase.py +++ b/reg_tests/executeBeamdynRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -46,8 +47,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -107,31 +106,31 @@ rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeFASTFarmRegressionCase.py b/reg_tests/executeFASTFarmRegressionCase.py index 273637bb46..4cb0571ee8 100644 --- a/reg_tests/executeFASTFarmRegressionCase.py +++ b/reg_tests/executeFASTFarmRegressionCase.py @@ -24,9 +24,10 @@ import os import sys -basepath = os.path.dirname(os.path.abspath(__file__)) +basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -55,8 +56,6 @@ def ignoreBaselineItems(directory, contents): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -68,8 +67,6 @@ def ignoreBaselineItems(directory, contents): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] plotError = args.plot noExec = args.noExec verbose = args.verbose @@ -80,25 +77,6 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) ### Build the filesystem navigation variables for running openfast on the test case regtests = os.path.join(sourceDirectory, "reg_tests") @@ -106,7 +84,7 @@ def ignoreBaselineItems(directory, contents): rtest = os.path.join(regtests, "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "fast-farm") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(inputsDirectory) #, outputType) +targetOutputDirectory = os.path.join(inputsDirectory) testBuildDirectory = os.path.join(buildDirectory, caseName) # verify all the required directories exist @@ -151,29 +129,31 @@ def ignoreBaselineItems(directory, contents): rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] -# export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - for channel in testInfo["attribute_names"]: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error)) - finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) - - sys.exit(1) +norms = pass_fail.calculateNorms(testData, baselineData) + +# export all case summaries +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeHydrodynRegressionCase.py b/reg_tests/executeHydrodynRegressionCase.py index 2c442eb6ba..0105d73da7 100644 --- a/reg_tests/executeHydrodynRegressionCase.py +++ b/reg_tests/executeHydrodynRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import glob import subprocess @@ -47,8 +48,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", default=False, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include matplotlib plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", default=False, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", default=False, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output") @@ -111,31 +110,31 @@ rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeInflowwindPyRegressionCase.py b/reg_tests/executeInflowwindPyRegressionCase.py index 7d88591ba9..f7b30771c2 100644 --- a/reg_tests/executeInflowwindPyRegressionCase.py +++ b/reg_tests/executeInflowwindPyRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import glob import subprocess @@ -47,8 +48,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -104,38 +103,38 @@ returnCode = openfastDrivers.runInflowwindDriverCase(caseInputFile, executable) if returnCode != 0: rtl.exitWithError("") - + ### Build the filesystem navigation variables for running the regression test localOutFile = os.path.join(testBuildDirectory, "Points.Velocity.dat") baselineOutFile = os.path.join(targetOutputDirectory, "Points.Velocity.dat") rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeInflowwindRegressionCase.py b/reg_tests/executeInflowwindRegressionCase.py index 42c99d6289..521211ac2d 100644 --- a/reg_tests/executeInflowwindRegressionCase.py +++ b/reg_tests/executeInflowwindRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import glob import subprocess @@ -47,8 +48,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -108,31 +107,31 @@ rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeOpenfastCppRegressionCase.py b/reg_tests/executeOpenfastCppRegressionCase.py index 03f9cccc2b..2ebf96b96c 100644 --- a/reg_tests/executeOpenfastCppRegressionCase.py +++ b/reg_tests/executeOpenfastCppRegressionCase.py @@ -16,9 +16,10 @@ import os import sys -basepath = os.path.dirname(os.path.abspath(__file__)) +basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -47,8 +48,6 @@ def ignoreBaselineItems(directory, contents): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -60,8 +59,6 @@ def ignoreBaselineItems(directory, contents): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] plotError = args.plot noExec = args.noExec verbose = args.verbose @@ -72,32 +69,13 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) ### Build the filesystem navigation variables for running openfast on the test case rtest = os.path.join(sourceDirectory, "reg_tests", "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "openfast-cpp") openfast_gluecode_directory = os.path.join(rtest, "glue-codes", "openfast") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(openfast_gluecode_directory, caseName.replace('_cpp', ''), outputType) +targetOutputDirectory = os.path.join(openfast_gluecode_directory, caseName.replace('_cpp', '')) testBuildDirectory = os.path.join(buildDirectory, caseName) # verify all the required directories exist @@ -142,32 +120,35 @@ def ignoreBaselineItems(directory, contents): ### Build the filesystem navigation variables for running the regression test localOutFile = os.path.join(testBuildDirectory, caseName + ".outb") baselineOutFile = os.path.join(targetOutputDirectory, caseName.replace('_cpp', '') + ".outb") + rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] -# export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - for channel in testInfo["attribute_names"]: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error)) - finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) - - sys.exit(1) +norms = pass_fail.calculateNorms(testData, baselineData) + +# export all case summaries +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeOpenfastLinearRegressionCase.py b/reg_tests/executeOpenfastLinearRegressionCase.py index 8e8a9b7015..7d2d9bd051 100644 --- a/reg_tests/executeOpenfastLinearRegressionCase.py +++ b/reg_tests/executeOpenfastLinearRegressionCase.py @@ -24,9 +24,10 @@ import os import sys -basepath = os.path.dirname(os.path.abspath(__file__)) +basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -65,11 +66,9 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") -parser.add_argument("-p", "-plot", dest="plot", default=False, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include plots in failed cases") -parser.add_argument("-n", "-no-exec", dest="noExec", default=False, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases") -parser.add_argument("-v", "-verbose", dest="verbose", default=False, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output") +parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") +parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") +parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") args = parser.parse_args() @@ -78,11 +77,9 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] -plotError = args.plot if args.plot is False else True -noExec = args.noExec if args.noExec is False else True -verbose = args.verbose if args.verbose is False else True +plotError = args.plot +noExec = args.noExec +verbose = args.verbose # validate inputs rtl.validateExeOrExit(executable) @@ -90,33 +87,13 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) - ### Build the filesystem navigation variables for running openfast on the test case regtests = os.path.join(sourceDirectory, "reg_tests") lib = os.path.join(regtests, "lib") rtest = os.path.join(regtests, "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "openfast") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(inputsDirectory, outputType) +targetOutputDirectory = os.path.join(inputsDirectory) testBuildDirectory = os.path.join(buildDirectory, caseName) # verify all the required directories exist @@ -162,11 +139,8 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): if returnCode != 0: rtl.exitWithError("") -### Get a list of all the files in the baseline directory -baselineOutFiles = os.listdir(targetOutputDirectory) -# Drop the log file, if its listed -if caseName + '.log' in baselineOutFiles: - baselineOutFiles.remove(caseName + '.log') +### Get a all the .lin files in the baseline directory +baselineOutFiles = [f for f in os.listdir(targetOutputDirectory) if '.lin' in f] # these should all exist in the local outputs directory localFiles = os.listdir(testBuildDirectory) @@ -240,7 +214,9 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): l_float = float(l_element) b_float = float(b_elements[j]) if not isclose(l_float, b_float, tolerance, tolerance): - print(f"Failed in Jacobian matrix comparison: {l_float} and {b_float}") + print(f"Failed in Jacobian matrix comparison:") + print(f"{l_float} in {local_file}") + print(f"{b_float} in {baseline_file}") sys.exit(1) # skip 2 empty/header lines diff --git a/reg_tests/executePythonRegressionCase.py b/reg_tests/executePythonRegressionCase.py index 58c2599290..753b5c0457 100644 --- a/reg_tests/executePythonRegressionCase.py +++ b/reg_tests/executePythonRegressionCase.py @@ -25,11 +25,12 @@ import os import sys -basepath = os.path.sep.join(sys.argv[0].split(os.path.sep)[:-1]) if os.path.sep in sys.argv[0] else "." +basepath = os.path.dirname(__file__) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) sys.path.insert(0, os.path.sep.join([basepath, "..", "glue-codes", "python"])) import platform import argparse +import numpy as np import shutil import subprocess import rtestlib as rtl @@ -59,8 +60,6 @@ def ignoreBaselineItems(directory, contents): parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', help="bool to include plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', help="bool to include verbose system output") @@ -71,8 +70,6 @@ def ignoreBaselineItems(directory, contents): sourceDirectory = args.sourceDirectory[0] buildDirectory = args.buildDirectory[0] tolerance = args.tolerance[0] -systemName = args.systemName[0] -compilerId = args.compilerId[0] plotError = args.plot noExec = args.noExec verbose = args.verbose @@ -82,25 +79,6 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(buildDirectory): os.makedirs(buildDirectory) -### Map the system and compiler configurations to a solution set -# Internal names -> Human readable names -systemName_map = { - "darwin": "macos", - "linux": "linux", - "windows": "windows" -} -compilerId_map = { - "gnu": "gnu", - "intel": "intel" -} -# Build the target output directory name or choose the default -supportedBaselines = ["macos-gnu", "linux-intel", "linux-gnu", "windows-intel"] -targetSystem = systemName_map.get(systemName.lower(), "") -targetCompiler = compilerId_map.get(compilerId.lower(), "") -outputType = os.path.join(targetSystem+"-"+targetCompiler) -if outputType not in supportedBaselines: - outputType = supportedBaselines[0] -print("-- Using gold standard files with machine-compiler type {}".format(outputType)) ### Build the filesystem navigation variables for running openfast on the test case regtests = os.path.join(sourceDirectory, "reg_tests") @@ -108,7 +86,7 @@ def ignoreBaselineItems(directory, contents): rtest = os.path.join(regtests, "r-test") moduleDirectory = os.path.join(rtest, "glue-codes", "openfast") inputsDirectory = os.path.join(moduleDirectory, caseName) -targetOutputDirectory = os.path.join(inputsDirectory, outputType) +targetOutputDirectory = os.path.join(inputsDirectory) testBuildDirectory = os.path.join(buildDirectory, caseName) # verify all the required directories exist @@ -126,6 +104,7 @@ def ignoreBaselineItems(directory, contents): if not os.path.isdir(dataDir): shutil.copytree(os.path.join(moduleDirectory, data), dataDir) +# Special copy for the 5MW_Baseline folder because the Windows python-only workflow may have already created data in the subfolder ServoData dst = os.path.join(buildDirectory, "5MW_Baseline") src = os.path.join(moduleDirectory, "5MW_Baseline") if not os.path.isdir(dst): @@ -173,27 +152,29 @@ def ignoreBaselineItems(directory, contents): } testData = openfastlib.output_values baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] -# export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - for channel in testInfo["attribute_names"]: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error)) - finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) - - sys.exit(1) +norms = pass_fail.calculateNorms(testData, baselineData) + +# export all case summaries +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) diff --git a/reg_tests/executeSubdynRegressionCase.py b/reg_tests/executeSubdynRegressionCase.py index 94e04a710f..6efa5cbb78 100644 --- a/reg_tests/executeSubdynRegressionCase.py +++ b/reg_tests/executeSubdynRegressionCase.py @@ -27,6 +27,7 @@ basepath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.sep.join([basepath, "lib"])) import argparse +import numpy as np import shutil import glob import subprocess @@ -47,8 +48,6 @@ parser.add_argument("sourceDirectory", metavar="path/to/openfast_repo", type=str, nargs=1, help="The path to the OpenFAST repository.") parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.") parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="Tolerance defining pass or failure in the regression test.") -parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="The current system\'s name: [Darwin,Linux,Windows]") -parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="The compiler\'s id: [Intel,GNU]") parser.add_argument("-p", "-plot", dest="plot", action='store_true', default=False, help="bool to include matplotlib plots in failed cases") parser.add_argument("-n", "-no-exec", dest="noExec", action='store_true', default=False, help="bool to prevent execution of the test cases") parser.add_argument("-v", "-verbose", dest="verbose", action='store_true', default=False, help="bool to include verbose system output") @@ -104,34 +103,35 @@ ### Build the filesystem navigation variables for running the regression test localOutFile = os.path.join(testBuildDirectory, caseName+".SD.out") baselineOutFile = os.path.join(targetOutputDirectory, caseName+".SD.out") + rtl.validateFileOrExit(localOutFile) rtl.validateFileOrExit(baselineOutFile) -testData, testInfo, testPack = pass_fail.readFASTOut(localOutFile) +testData, testInfo, _ = pass_fail.readFASTOut(localOutFile) baselineData, baselineInfo, _ = pass_fail.readFASTOut(baselineOutFile) -performance = pass_fail.calculateNorms(testData, baselineData) -normalizedNorm = performance[:, 1] + +passing_channels = pass_fail.passing_channels(testData.T, baselineData.T) +passing_channels = passing_channels.T + +norms = pass_fail.calculateNorms(testData, baselineData) # export all case summaries -results = list(zip(testInfo["attribute_names"], [*performance])) -results_max = performance.max(axis=0) -exportCaseSummary(testBuildDirectory, caseName, results, results_max, tolerance) +channel_names = testInfo["attribute_names"] +exportCaseSummary(testBuildDirectory, caseName, channel_names, passing_channels, norms) -# failing case -if not pass_fail.passRegressionTest(normalizedNorm, tolerance): - if plotError: - from errorPlotting import finalizePlotDirectory, plotOpenfastError - ixFailChannels = [i for i in range(len(testInfo["attribute_names"])) if normalizedNorm[i] > tolerance] - failChannels = [channel for i, channel in enumerate(testInfo["attribute_names"]) if i in ixFailChannels] - failResults = [res for i, res in enumerate(results) if i in ixFailChannels] - for channel in failChannels: - try: - plotOpenfastError(localOutFile, baselineOutFile, channel) - except: - error = sys.exc_info()[1] - print("Error generating plots: {}".format(error.msg)) - finalizePlotDirectory(localOutFile, failChannels, caseName) - sys.exit(1) - # passing case -sys.exit(0) +if np.all(passing_channels): + sys.exit(0) + +# failing case +if plotError: + from errorPlotting import finalizePlotDirectory, plotOpenfastError + for channel in testInfo["attribute_names"]: + try: + plotOpenfastError(localOutFile, baselineOutFile, channel) + except: + error = sys.exc_info()[1] + print("Error generating plots: {}".format(error)) + finalizePlotDirectory(localOutFile, testInfo["attribute_names"], caseName) + +sys.exit(1) From acf2c89a012d116389a05f241cca40d99c0cc536 Mon Sep 17 00:00:00 2001 From: Rafael M Mudafort Date: Wed, 22 Jun 2022 15:02:03 -0500 Subject: [PATCH 06/25] Update reg test plotting - bug fixes, Bokeh 2.4 --- reg_tests/lib/errorPlotting.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/reg_tests/lib/errorPlotting.py b/reg_tests/lib/errorPlotting.py index ed686ead36..998d2937b9 100644 --- a/reg_tests/lib/errorPlotting.py +++ b/reg_tests/lib/errorPlotting.py @@ -51,7 +51,7 @@ def _plotError(xseries, y1series, y2series, xlabel, title1, title2): from bokeh.embed import components from bokeh.layouts import gridplot from bokeh.plotting import figure - from bokeh.models.tools import HoverTool, BoxZoomTool + from bokeh.models.tools import HoverTool p1 = figure(title=title1) p1.title.align = 'center' @@ -74,7 +74,7 @@ def _plotError(xseries, y1series, y2series, xlabel, title1, title2): p2.add_tools(HoverTool(tooltips=[('Time','@x'), ('Error', '@y')], mode='vline')) - grid = gridplot([[p1, p2]], plot_width=650, plot_height=375, sizing_mode="scale_both") + grid = gridplot([[p1, p2]], width=650, height=375, sizing_mode="scale_both") script, div = components(grid) return script, div @@ -86,7 +86,7 @@ def _replace_id_div(html_string, plot): return html_string def _replace_id_script(html_string, plot): - id_start = html_string.find('var render_items') + id_start = html_string.find('const render_items') id_start += html_string[id_start:].find('roots') id_start += html_string[id_start:].find('":"') + 3 id_end = html_string[id_start:].find('"') + id_start @@ -98,7 +98,8 @@ def _save_plot(script, div, path, attribute): file_name = "_script".join((attribute, ".txt")) with open(os.path.join(path, file_name), 'w') as f: - script = _replace_id_script(script.replace('\n', '\n '), attribute) + script = script.replace('\n', '\n ') + script = _replace_id_script(script, attribute) f.write(script) file_name = "_div".join((attribute, ".txt")) @@ -137,19 +138,25 @@ def plotOpenfastError(testSolution, baselineSolution, attribute): _save_plot(script, div, plotPath, attribute) def _htmlHead(title): + from bokeh.resources import CDN + head = '' + '\n' head += '' + '\n' head += '' + '\n' head += ' {}'.format(title) + '\n' - + + # CSS head += ' ' + '\n' head += ' ' + '\n' head += ' ' + '\n' - + + # JS head += ' ' + '\n' head += ' ' + '\n' - head += ' ' + '\n' - head += ' ' + '\n' + + # JS - Bokeh + head += f' ' + '\n' + head += f' ' + '\n' head += ' ' + '\n' head += '