From 9fa638c50b9f036bb1e7efdfee7c25edd6433ce0 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 3 Dec 2025 16:11:54 -0500 Subject: [PATCH 1/4] fix: clean up recently added build options --- .gitmodules | 6 +-- build-coatjava.sh | 119 +++++++++++++++++++++++++++++++--------------- 2 files changed, 84 insertions(+), 41 deletions(-) diff --git a/.gitmodules b/.gitmodules index cd6128fa73..0d3a763a6c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "etc/data/magfield"] path = etc/data/magfield - url = https://code.jlab.org/hallb/clas12/magfield + url = https://code.jlab.org/hallb/clas12/magfield.git [submodule "validation/advanced-tests/data"] path = validation/advanced-tests/data - url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests + url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests.git [submodule "etc/nnet"] path = etc/data/nnet - url = https://code.jlab.org/hallb/clas12/coatjava/networks + url = https://code.jlab.org/hallb/clas12/coatjava/networks.git diff --git a/build-coatjava.sh b/build-coatjava.sh index d2c031f82d..9cd1dfb08e 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -14,10 +14,10 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]... --no-progress no download progress printouts --help show this message - OPTIONS FOR MAGNETIC FIELD MAPS - --lfs use git-lfs for field maps and test data - --cvmfs use cvmfs to download field maps - --xrootd use xrootd to download field maps + OPTIONS FOR MAGNETIC FIELD MAPS, NEURAL NETWORK MODELS, etc. + --lfs use Git Large File Storage (requires `git-lfs`) + --cvmfs use CernVM-FS (requires `/cvfms`) + --xrootd use XRootD (requires `xrootd`) --nomaps do not download field maps OPTIONS FOR TESTING @@ -31,6 +31,10 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]... -T4 will build with 4 parallel threads ''' +################################################################################ +# parse arguments +################################################################################ + cleanBuild="no" anaDepends="no" runSpotBugs="no" @@ -85,6 +89,10 @@ if ! [[ $(hostname) == *.jlab.org ]]; then useLfs=true fi +################################################################################ +# setup +################################################################################ + src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" prefix_dir=$src_dir/coatjava clara_home=$src_dir/clara @@ -97,26 +105,31 @@ wgetArgs+=(--timestamping --no-check-certificate) # `--timestamping` only redown mvn="mvn ${mvnArgs[@]:-}" wget="wget ${wgetArgs[@]:-}" +################################################################################ +# download field maps, NN models, etc. +################################################################################ + command_exists () { type "$1" &> /dev/null } -download () { + +download_lfs() { + if command_exists git-lfs ; then + cd $src_dir > /dev/null + git lfs install + git submodule update --init $1 + cd - > /dev/null + else + echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2 + exit 1 + fi +} + +download_map () { ret=0 if $useXrootd; then xrdcp $1 ./ ret=$? - elif $useLfs; then - if command_exists git-lfs ; then - cd $src_dir > /dev/null - git lfs install - git submodule update --init etc/data/magfield - git submodule update --init etc/data/nnet - if $downloadData; then git submodule update --init validation/advanced-tests/data; fi - cd - > /dev/null - else - echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2 - ret=1 - fi elif $useCvmfs; then cp -v $1 ./ ret=$? @@ -141,26 +154,46 @@ source libexec/env.sh --no-classpath magfield_dir=$src_dir/etc/data/magfield if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then echo 'Retrieving field maps ...' - webDir=https://clasweb.jlab.org/clas12offline/magfield - if $useLfs; then webDir=${magfield_dir##$src_dir}; fi - if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi - if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi - mkdir -p $magfield_dir - cd $magfield_dir - for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP - do - download $webDir/$map - if [ $? -ne 0 ]; then - echo "ERROR::::::::::: Could not download field map:" >&2 - echo "$webDir/$map" >&2 - echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2 - exit 1 - fi - $useLfs && break - done - cd - + if $useLfs; then + download_lfs etc/data/magfield + else + webDir=https://clasweb.jlab.org/clas12offline/magfield + if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi + if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi + mkdir -p $magfield_dir + cd $magfield_dir + for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP + do + download_map $webDir/$map + if [ $? -ne 0 ]; then + echo "ERROR::::::::::: Could not download field map:" >&2 + echo "$webDir/$map" >&2 + echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2 + exit 1 + fi + done + cd - + fi +fi + +# download neural networks +if $useLfs; then + download_lfs etc/data/nnet +elif $useCvmfs; then + cp -rv /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ +else + echo "WARNING: neural networks not downloaded" >&2 fi +# download validation data +if $downloadData; then + download_lfs validation/advanced-tests/data +fi + +################################################################################ +# cleaning +################################################################################ + # always clean the installation prefix rm -rf $prefix_dir $clara_home @@ -171,13 +204,20 @@ if [ $cleanBuild == "yes" ]; then echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2 done echo """DONE CLEANING. - NOTE: if you want to remove locally downloaded magnetic field maps, run: - rm $magfield_dir/*.dat + NOTE: + - to remove local magnetic field maps: + rm $magfield_dir/*.dat + - to clear all LFS git submodules: + git submodule deinit --all Now re-run without \`--clean\` to build.""" exit fi +################################################################################ +# build +################################################################################ + # run dependency analysis and exit if [ $anaDepends == "yes" ]; then libexec/dependency-analysis.sh @@ -215,7 +255,10 @@ if [ $runSpotBugs == "yes" ]; then if [ $? != 0 ] ; then echo "spotbugs failure" >&2 ; exit 1 ; fi fi -# installation +################################################################################ +# install +################################################################################ + # NOTE: a maven plugin, such as `maven-assembly-plugin`, would be better, but it seems that they: # - require significantly more repetition of the module names and/or generation of additional XML file(s) # - seem to break thread safety of `mvn install`, i.e., we'd need to run `mvn package` first, then `mvn install` From e8c58434fc2334c71e5d4db1a0b841958877d8bb Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 3 Dec 2025 18:24:37 -0500 Subject: [PATCH 2/4] refactor: use booleans rather than yes/no strings --- build-coatjava.sh | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index 9cd1dfb08e..256ec430f2 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -35,11 +35,11 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]... # parse arguments ################################################################################ -cleanBuild="no" -anaDepends="no" -runSpotBugs="no" -downloadMaps="yes" -runUnitTests="no" +cleanBuild=false +anaDepends=false +runSpotBugs=false +downloadMaps=true +runUnitTests=false useXrootd=false useCvmfs=false useLfs=false @@ -50,12 +50,12 @@ wgetArgs=() for xx in $@ do case $xx in - --spotbugs) runSpotBugs="yes" ;; - -n) runSpotBugs="no" ;; - --nomaps) downloadMaps="no" ;; - --unittests) runUnitTests="yes" ;; - --clean) cleanBuild="yes" ;; - --depana) anaDepends="yes" ;; + --spotbugs) runSpotBugs=true ;; + -n) runSpotBugs=false ;; + --nomaps) downloadMaps=false ;; + --unittests) runUnitTests=true ;; + --clean) cleanBuild=true ;; + --depana) anaDepends=true ;; --quiet) mvnArgs+=(--quiet --batch-mode) wgetArgs+=(--quiet) @@ -152,7 +152,7 @@ download_map () { # (and duplicated in etc/services/reconstruction.yaml): source libexec/env.sh --no-classpath magfield_dir=$src_dir/etc/data/magfield -if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then +if ! $cleanBuild && $downloadMaps; then echo 'Retrieving field maps ...' if $useLfs; then download_lfs etc/data/magfield @@ -182,7 +182,8 @@ if $useLfs; then elif $useCvmfs; then cp -rv /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ else - echo "WARNING: neural networks not downloaded" >&2 + echo "WARNING: neural networks not downloaded; run with '--help' for guidance" >&2 + sleep 1 fi # download validation data @@ -198,7 +199,7 @@ fi rm -rf $prefix_dir $clara_home # clean up any cache copies -if [ $cleanBuild == "yes" ]; then +if $cleanBuild; then $mvn clean for target_dir in $(find $src_dir -type d -name target); do echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2 @@ -219,7 +220,7 @@ fi ################################################################################ # run dependency analysis and exit -if [ $anaDepends == "yes" ]; then +if $anaDepends; then libexec/dependency-analysis.sh libexec/dependency-tree.sh exit 0 @@ -241,13 +242,13 @@ cp external-dependencies/jclara-4.3-SNAPSHOT.jar $prefix_dir/lib/utils # spotbugs, unit tests unset CLAS12DIR -if [ $runUnitTests == "yes" ]; then +if $runUnitTests; then $mvn install # also runs unit tests else $mvn install -DskipTests fi -if [ $runSpotBugs == "yes" ]; then +if $runSpotBugs; then # mvn com.github.spotbugs:spotbugs-maven-plugin:spotbugs # spotbugs goal produces a report target/spotbugsXml.xml for each module $mvn com.github.spotbugs:spotbugs-maven-plugin:check # check goal produces a report and produces build failed if bugs # the spotbugsXml.xml file is easiest read in a web browser From 6940ddb41b409b1ac098ce7f124f934de350a42d Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 3 Dec 2025 18:46:54 -0500 Subject: [PATCH 3/4] refactor: put cleaning and depana before downloading stuff --- build-coatjava.sh | 140 ++++++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 60 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index 256ec430f2..0c0c54b623 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -5,32 +5,35 @@ set -e set -u set -o pipefail -usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]... - - OPTIONS - --clara install clara too - --clean clean up built objects and exit (does not compile) - --quiet run more quietly - --no-progress no download progress printouts - --help show this message - - OPTIONS FOR MAGNETIC FIELD MAPS, NEURAL NETWORK MODELS, etc. - --lfs use Git Large File Storage (requires `git-lfs`) - --cvmfs use CernVM-FS (requires `/cvfms`) - --xrootd use XRootD (requires `xrootd`) - --nomaps do not download field maps - - OPTIONS FOR TESTING - --spotbugs also run spotbugs plugin - --unittests also run unit tests - --depana run dependency analysis (only) - --data download test data (requires lfs) - - MAVEN_OPTIONS +usage='''build-coatjava.sh [OPTIONS]... + + GENERAL OPTIONS + --clara install clara too + --clean clean up built objects and exit (does not compile) + --quiet run more quietly + --no-progress no download progress printouts + --help show this message + + DATA RETRIEVAL OPTIONS + How to retrieve magnetic field maps, neural network models, etc.; + choose only one: + --lfs use Git Large File Storage (requires `git-lfs`) + --cvmfs use CernVM-FS (requires `/cvfms`) + --xrootd use XRootD (requires `xrootd`) + --nomaps do not download field maps + + TESTING OPTIONS + --spotbugs also run spotbugs plugin + --unittests also run unit tests + --depana run dependency analysis (only) + --data download test data (requires `git-lfs`) + + MAVEN OPTIONS all other arguments will be passed to `mvn`; for example, -T4 will build with 4 parallel threads ''' + ################################################################################ # parse arguments ################################################################################ @@ -64,9 +67,9 @@ do mvnArgs+=(--no-transfer-progress) wgetArgs+=(--no-verbose) ;; - --xrootd) useXrootd=true ;; - --cvmfs) useCvmfs=true ;; - --lfs) useLfs=true ;; + --xrootd) useXrootd=true ;; + --cvmfs) useCvmfs=true ;; + --lfs) useLfs=true ;; --clara) installClara=true ;; --data) downloadData=true ;; -h|--help) @@ -89,13 +92,16 @@ if ! [[ $(hostname) == *.jlab.org ]]; then useLfs=true fi + ################################################################################ # setup ################################################################################ +# directories src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" prefix_dir=$src_dir/coatjava clara_home=$src_dir/clara +magfield_dir=$src_dir/etc/data/magfield # working directory should be the source code directory cd $src_dir @@ -105,14 +111,55 @@ wgetArgs+=(--timestamping --no-check-certificate) # `--timestamping` only redown mvn="mvn ${mvnArgs[@]:-}" wget="wget ${wgetArgs[@]:-}" +# environment +source libexec/env.sh --no-classpath + + +################################################################################ +# cleaning, dependency analysis, etc. +################################################################################ + +# function to clean installation prefixes +clean_prefixes() { + rm -rf $prefix_dir $clara_home +} + +# clean up any cache copies +if $cleanBuild; then + clean_prefixes + $mvn clean + for target_dir in $(find $src_dir -type d -name target); do + echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2 + done + echo """DONE CLEANING. + NOTE: + - to remove local magnetic field maps: + rm $magfield_dir/*.dat + - to clear all LFS git submodules: + git submodule deinit --all + + Now re-run without \`--clean\` to build.""" + exit +fi + +# run dependency analysis and exit +if $anaDepends; then + libexec/dependency-analysis.sh + libexec/dependency-tree.sh + exit 0 +fi + + ################################################################################ # download field maps, NN models, etc. ################################################################################ +# check if a command exists command_exists () { - type "$1" &> /dev/null + type "$1" &> /dev/null } +# update an LFS submodule download_lfs() { if command_exists git-lfs ; then cd $src_dir > /dev/null @@ -125,6 +172,7 @@ download_lfs() { fi } +# download a magnetic field map download_map () { ret=0 if $useXrootd; then @@ -150,9 +198,7 @@ download_map () { # download the default field maps, as defined in libexec/env.sh: # (and duplicated in etc/services/reconstruction.yaml): -source libexec/env.sh --no-classpath -magfield_dir=$src_dir/etc/data/magfield -if ! $cleanBuild && $downloadMaps; then +if $downloadMaps; then echo 'Retrieving field maps ...' if $useLfs; then download_lfs etc/data/magfield @@ -191,42 +237,13 @@ if $downloadData; then download_lfs validation/advanced-tests/data fi -################################################################################ -# cleaning -################################################################################ - -# always clean the installation prefix -rm -rf $prefix_dir $clara_home - -# clean up any cache copies -if $cleanBuild; then - $mvn clean - for target_dir in $(find $src_dir -type d -name target); do - echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2 - done - echo """DONE CLEANING. - NOTE: - - to remove local magnetic field maps: - rm $magfield_dir/*.dat - - to clear all LFS git submodules: - git submodule deinit --all - - Now re-run without \`--clean\` to build.""" - exit -fi ################################################################################ # build ################################################################################ -# run dependency analysis and exit -if $anaDepends; then - libexec/dependency-analysis.sh - libexec/dependency-tree.sh - exit 0 -fi - # start new installation tree +clean_prefixes # always clean the installation prefix mkdir -p $prefix_dir cp -r bin $prefix_dir/ cp -r etc $prefix_dir/ @@ -240,7 +257,7 @@ $python etc/bankdefs/util/bankSplit.py $prefix_dir/etc/bankdefs/hipo4 || exit 1 mkdir -p $prefix_dir/lib/utils cp external-dependencies/jclara-4.3-SNAPSHOT.jar $prefix_dir/lib/utils -# spotbugs, unit tests +# build (and test) unset CLAS12DIR if $runUnitTests; then $mvn install # also runs unit tests @@ -248,6 +265,7 @@ else $mvn install -DskipTests fi +# run spotbugs if $runSpotBugs; then # mvn com.github.spotbugs:spotbugs-maven-plugin:spotbugs # spotbugs goal produces a report target/spotbugsXml.xml for each module $mvn com.github.spotbugs:spotbugs-maven-plugin:check # check goal produces a report and produces build failed if bugs @@ -256,6 +274,7 @@ if $runSpotBugs; then if [ $? != 0 ] ; then echo "spotbugs failure" >&2 ; exit 1 ; fi fi + ################################################################################ # install ################################################################################ @@ -287,6 +306,7 @@ for pom in $(find common-tools -name pom.xml); do done echo "installed coatjava to: $prefix_dir" +# install clara if $installClara; then ./install-clara -c $prefix_dir $clara_home; fi echo "COATJAVA SUCCESSFULLY BUILT !" From 0ac247ce572c65ee03fb9a0d6b83e7847c9ba22c Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 3 Dec 2025 19:24:57 -0500 Subject: [PATCH 4/4] feat: notifiers --- build-coatjava.sh | 53 +++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index 0c0c54b623..f0503ae69a 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -7,30 +7,30 @@ set -o pipefail usage='''build-coatjava.sh [OPTIONS]... - GENERAL OPTIONS +GENERAL OPTIONS --clara install clara too --clean clean up built objects and exit (does not compile) --quiet run more quietly --no-progress no download progress printouts --help show this message - DATA RETRIEVAL OPTIONS - How to retrieve magnetic field maps, neural network models, etc.; - choose only one: +DATA RETRIEVAL OPTIONS + How to retrieve magnetic field maps, neural network models, etc.; + choose only one: --lfs use Git Large File Storage (requires `git-lfs`) --cvmfs use CernVM-FS (requires `/cvfms`) --xrootd use XRootD (requires `xrootd`) --nomaps do not download field maps - TESTING OPTIONS +TESTING OPTIONS --spotbugs also run spotbugs plugin --unittests also run unit tests --depana run dependency analysis (only) - --data download test data (requires `git-lfs`) + --data download test data (requires option `--lfs`) - MAVEN OPTIONS - all other arguments will be passed to `mvn`; for example, - -T4 will build with 4 parallel threads +MAVEN OPTIONS + all other arguments will be passed to `mvn`; for example, + -T4 will build with 4 parallel threads ''' @@ -80,12 +80,6 @@ do esac done -if $downloadData && ! $useLfs; then - echo "$usage" - echo "ERROR::::::::::: --data requires --lfs" >&2 - exit 2 -fi - # Currently only git-lfs works from offsite: if ! [[ $(hostname) == *.jlab.org ]]; then echo "INFO: using --lfs for offsite usage" @@ -114,6 +108,10 @@ wget="wget ${wgetArgs[@]:-}" # environment source libexec/env.sh --no-classpath +# install LFS +if $useLfs; then + git lfs install +fi ################################################################################ # cleaning, dependency analysis, etc. @@ -159,11 +157,19 @@ command_exists () { type "$1" &> /dev/null } +# print retrieval notice +notify_retrieval() { + echo "Retrieving $1 from $2 ..." +} + # update an LFS submodule download_lfs() { + if ! $useLfs; then + echo 'ERROR: attempted to use LFS, but option `--lfs` not set' >&2 + exit 1 + fi if command_exists git-lfs ; then cd $src_dir > /dev/null - git lfs install git submodule update --init $1 cd - > /dev/null else @@ -176,15 +182,19 @@ download_lfs() { download_map () { ret=0 if $useXrootd; then + notify_retrieval 'field map' 'xrootd' xrdcp $1 ./ ret=$? elif $useCvmfs; then - cp -v $1 ./ + notify_retrieval 'field map' 'cvmfs' + cp $1 ./ ret=$? elif command_exists wget ; then + notify_retrieval 'field map' 'clasweb via wget' $wget $1 ret=$? elif command_exists curl ; then + notify_retrieval 'field map' 'clasweb via curl' if ! [ -e ${1##*/} ]; then curl $1 -o ${1##*/} ret=$? @@ -199,8 +209,8 @@ download_map () { # download the default field maps, as defined in libexec/env.sh: # (and duplicated in etc/services/reconstruction.yaml): if $downloadMaps; then - echo 'Retrieving field maps ...' if $useLfs; then + notify_retrieval 'field maps' 'lfs' download_lfs etc/data/magfield else webDir=https://clasweb.jlab.org/clas12offline/magfield @@ -224,16 +234,19 @@ fi # download neural networks if $useLfs; then + notify_retrieval 'neural networks' 'lfs' download_lfs etc/data/nnet elif $useCvmfs; then - cp -rv /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ + notify_retrieval 'neural networks' 'cvmfs' + cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ else - echo "WARNING: neural networks not downloaded; run with '--help' for guidance" >&2 + echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2 sleep 1 fi # download validation data if $downloadData; then + notify_retrieval 'validation data' 'lfs' download_lfs validation/advanced-tests/data fi