Skip to content

[144][145] fail fast and pattern #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wyli merged 2 commits into from
Mar 22, 2021
Merged

[144][145] fail fast and pattern #148

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0a5236a
fail fast and pattern
rijobro Mar 22, 2021
f500da9
updates
rijobro Mar 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 49 additions & 27 deletions runner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Stop on error
# During setup, stop on error
set -e

# Notification on finish
Expand Down Expand Up @@ -51,22 +51,26 @@ fi
doChecks=true
doRun=true
autofix=false
failfast=false
pattern="."

function print_usage {
echo "runner.sh [--no-run] [--no-checks] [--autofix] [--file <filename>] [--help] [--version]"
echo "runner.sh [--no-run] [--no-checks] [--autofix] [-f/--failfast] [-p/--pattern <regex pattern>] [-h/--help] [-v/--version]"
echo ""
echo "MONAI tutorials testing utilities. When running the notebooks, we first search for variables, such as `max_epochs` and set them to 1 to reduce testing time."
echo "MONAI tutorials testing utilities. When running the notebooks, we first search for variables, such as max_epochs and set them to 1 to reduce testing time."
echo ""
echo "Examples:"
echo "./runner.sh # run full tests (${green}recommended before making pull requests${noColor})."
echo "./runner.sh --no-run # don't run the notebooks."
echo "./runner.sh --no-checks # don't run code checks."
echo "./runner.sh --pattern \"read|load\" # check files with \"read\" or \"load\" in path."
echo ""
echo "Code style check options:"
echo " --no-run : don't run notebooks"
echo " --no-checks : don't run code checks"
echo " --autofix : autofix where possible"
echo " --file : only run on specified file(s). use as many times as desired"
echo " -f, --failfast : stop on first error"
echo " -p, --pattern : pattern of files to be run (with grep -E \"<pattern>\"). Added on top of: \"*.ipynb\" and not \".ipynb_checkpoints\""
echo " -h, --help : show this help message and exit"
echo " -v, --version : show MONAI and system version information and exit"
echo ""
Expand Down Expand Up @@ -99,8 +103,11 @@ do
--autofix)
autofix=true
;;
--file)
files=("${files[@]}" "$2")
-f|--failfast)
failfast=true
;;
-p|--pattern)
pattern="$2"
shift
;;
-h|--help)
Expand All @@ -120,15 +127,24 @@ do
shift
done

# Initially set exit code to 0. If any tests fail, set to 1
exit_code=0

# if failfast, exit returning code. else set exit code to 1 and continue
function test_fail {
print_style_fail_msg
if [ $failfast = true ]; then exit $1; fi
exit_code=1
}

function check_installed {
set +e # don't want immediate error

command -v $1 &>/dev/null
success=$?
if [ ${success} -ne 0 ]; then
echo "${red}Missing package: $1 (try pip install -r requirements.txt)${noColor}"
print_error_msg "Missing package: $1 (try pip install -r requirements.txt)"
exit $success
fi
set -e
}

# check that packages are installed
Expand All @@ -146,9 +162,9 @@ fi


base_path="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
cd "${base_path}"

function replace_text {
set +e # otherwise blank grep causes error
oldString="${s}\s*=\s*[0-9]\+"
newString="${s} = 1"

Expand All @@ -159,14 +175,19 @@ function replace_text {

after=$(echo "$notebook" | grep "$newString")
[ ! -z "$after" ] && echo After: && echo "$after"
set -e
}

# If files haven't been added individually, get all
if [ -z ${files+x} ]; then
files=($(find . -type f \( -name "*.ipynb" -and -not -iwholename "*.ipynb_checkpoints*" \)))
# Get notebooks (pattern is . unless user specifies otherwise)
files=($(find . -type f -name "*.ipynb" -and ! -wholename "*.ipynb_checkpoints*" | grep -E "${pattern}"))
if [[ $files == "" ]]; then
print_error_msg "No files match pattern"
exit 1
fi
echo "Files to be tested:"
for i in "${files[@]}"; do echo $i; done

# After setup, don't want immediate error
set +e

########################################################################
# #
Expand All @@ -192,25 +213,23 @@ for file in "${files[@]}"; do
echo Applying autofixes...
jupytext "$filename" \
--pipe "autoflake --in-place --remove-unused-variables --imports numpy,monai,matplotlib,torch,ignite {}" \
--pipe "autopep8 - --ignore W291 --max-line-length 120"\
--pipe "autopep8 - --ignore W291 --max-line-length 120" \
--pipe "sed 's/ = list()/ = []/'"
fi

# to check flake8, convert to python script, don't check
# magic cells, and don't check line length for comment
# lines (as this includes markdown), and then run flake8
echo Checking PEP8 compliance...
jupytext "$filename" --to script -o - | \
jupytext "$filename" -w --to script -o - | \
sed 's/\(^\s*\)%/\1pass # %/' | \
sed 's/\(^#.*\)$/\1 # noqa: E501/' | \
flake8 - --show-source --max-line-length 120

success=$?
if [ ${success} -ne 0 ]
then
echo "Try running with autofixes: ${green}--autofix${noColor}"
print_style_fail_msg
exit ${success}
print_error_msg "Try running with autofixes: ${green}--autofix${noColor}"
test_fail ${success}
fi
fi

Expand All @@ -233,9 +252,8 @@ for file in "${files[@]}"; do
done
# then error
if [[ $should_contain_max_epochs == true ]]; then
echo "Couldn't find the keyword \"max_epochs\", and the notebook wasn't on the list of expected exemptions (\"doesnt_contain_max_epochs\")."
print_style_fail_msg
exit 1
print_error_msg "Couldn't find the keyword \"max_epochs\", and the notebook wasn't on the list of expected exemptions (\"doesnt_contain_max_epochs\")."
test_fail 1
fi
fi

Expand All @@ -246,13 +264,17 @@ for file in "${files[@]}"; do
done

# echo "$notebook" > "${base_path}/debug_notebook.ipynb"
out=$(echo "$notebook" | papermill --progress-bar)
time out=$(echo "$notebook" | papermill --progress-bar)
success=$?
if [[ ${success} -ne 0 || "$out" =~ "\"status\": \"failed\"" ]]; then
print_style_fail_msg
exit ${success}
test_fail ${success}
fi
fi
done

if [[ ${exit_code} -eq 0 ]]; then
echo "${green}passed!${noColor}"
done
else
echo "${red}failed!${noColor}"
fi
exit ${exit_code}