Add Urania machine

Add no-validate flag to scheduler

Change name of option

Add environment file

Clean script

Fix validate flag

Fix

Author: guilara

support/Environments/urania.sh

@@ -0,0 +1,60 @@
+#!/bin/env sh
+
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+spectre_load_modules() {
+    module load gcc/11
+    module load impi/2021.7
+    module load boost/1.79
+    module load gsl/1.16
+    module load cmake/3.26
+    module load hdf5-serial/1.12.2
+    module load anaconda/3/2021.11
+    module load paraview/5.10
+    # Load Spack environment
+    source /u/guilara/repos/spack/share/spack/setup-env.sh
+    spack env activate env3_spectre_impi
+    # Load python environment
+    source /u/guilara/envs/spectre_env/bin/activate
+    # Define Charm paths
+    export CHARM_ROOT=/u/guilara/charm_impi_3/mpi-linux-x86_64-smp
+    export PATH=$PATH:/u/guilara/charm_impi_3/mpi-linux-x86_64-smp/bin
+}
+
+spectre_unload_modules() {
+    module unload gcc/11
+    module unload impi/2021.7
+    module unload boost/1.79
+    module unload gsl/1.16
+    module unload cmake/3.26
+    module unload hdf5-serial/1.12.2
+    module unload anaconda/3/2021.11
+    module unload paraview/5.10
+    # Unload Spack environment
+    spack env deactivate
+    # Unload python environment
+    deactivate
+}
+
+spectre_run_cmake() {
+    if [ -z ${SPECTRE_HOME} ]; then
+        echo "You must set SPECTRE_HOME to the cloned SpECTRE directory"
+        return 1
+    fi
+    spectre_load_modules
+
+    cmake -D CMAKE_C_COMPILER=gcc \
+          -D CMAKE_CXX_COMPILER=g++ \
+          -D CMAKE_Fortran_COMPILER=gfortran \
+          -D CHARM_ROOT=$CHARM_ROOT \
+          -D CMAKE_BUILD_TYPE=Release \
+          -D DEBUG_SYMBOLS=OFF \
+          -D BUILD_SHARED_LIBS=ON \
+          -D MEMORY_ALLOCATOR=JEMALLOC \
+          -D BUILD_PYTHON_BINDINGS=ON \
+          -D MACHINE=Urania \
+          -D SPEC_ROOT=/u/guilara/repos/spec \
+          -D Catch2_DIR=/u/guilara/repos/Catch2/install_dir/lib64/cmake/Catch2 \
+          "$@" $SPECTRE_HOME
+}

support/Machines/Urania.yaml

@@ -0,0 +1,11 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+Machine:
+  Name: Urania
+  Description: |
+    Supercomputer at the Max Planck Computing Data Facilty.
+  DefaultProcsPerNode: 72
+  DefaultQueue: "p.urania"
+  DefaultTimeLimit: "1-00:00:00"
+  LaunchCommandSingleNode: ["srun", "-n", "1"]

support/Python/Schedule.py

@@ -137,6 +137,7 @@ def schedule(
     submit: Optional[bool] = None,
     clean_output: bool = False,
     force: bool = False,
+    validate: Optional[bool] = True,
     extra_params: dict = {},
     **kwargs,
 ) -> Optional[subprocess.CompletedProcess]:
@@ -307,6 +308,8 @@ def schedule(
         files in the 'run_dir' before scheduling the run. (Default: 'False')
       force: Optional. When 'True', overwrite input file and submit script
         in the 'run_dir' instead of raising an error when they already exist.
+      validate: Optional. When 'True', validate that the input file is parsed
+        correctly. When 'False' skip this step.
       extra_params: Optional. Dictionary of extra parameters passed to input
         file and submit script templates. Parameters can also be passed as
         keyword arguments to this function instead.
@@ -538,9 +541,11 @@ def schedule(
     )
 
     # Validate input file
-    validate_input_file(
-        input_file_path.resolve(), executable=executable, work_dir=run_dir
-    )
+    if validate:
+        validate_input_file(
+            input_file_path.resolve(), executable=executable, work_dir=run_dir
+        )
+
     # - If the input file may request resubmissions, make sure we have a
     #   segments directory
     metadata, input_file = yaml.safe_load_all(rendered_input_file)
@@ -847,6 +852,11 @@ def scheduler_options(f):
             "You may also want to use '--clean-output'."
         ),
     )
+    @click.option(
+        "--validate/--no-validate",
+        default=True,
+        help="Validate or skip the validation of the input file.",
+    )
     # Scheduling options
     @click.option(
         "--scheduler",

support/SubmitScripts/Urania.sh

@@ -0,0 +1,37 @@
+{% extends "SubmitTemplateBase.sh" %}
+
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+# Urania -- HPC cluster of ACR division of MPI for Grav Physics, housed at the
+# Max Planck Computing & Data Facility.
+# https://docs.mpcdf.mpg.de/doc/computing/clusters/systems/Gravitational_Physics_ACR.html
+
+{% block head %}
+{{ super() -}}
+#SBATCH --nodes {{ num_nodes | default(1) }}
+#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks-per-core=1
+#SBATCH --cpus-per-task=72
+#SBATCH -t {{ time_limit | default("1-00:00:00") }}
+#SBATCH -p {{ queue | default("p.urania") }}
+{% endblock %}
+
+{% block charm_ppn %}
+# Two thread for communication
+CHARM_PPN=$(expr ${SLURM_CPUS_PER_TASK} - 2)
+{% endblock %}
+
+{% block list_modules %}
+# Load compiler and MPI modules with explicit version specifications,
+# consistently with the versions used to build the executable.
+source ${SPECTRE_HOME}/support/Environments/urania.sh
+spectre_load_modules
+{% endblock %}
+
+{% block run_command %}
+srun -n ${SLURM_NTASKS} ${SPECTRE_EXECUTABLE} \
+    --input-file ${SPECTRE_INPUT_FILE} \
+    ++ppn ${CHARM_PPN} +pemap 0-34,36-70 +commap 35,71 \
+    ${SPECTRE_CHECKPOINT:+ +restart "${SPECTRE_CHECKPOINT}"}
+{% endblock %}

tests/support/Python/Test_Schedule.py

@@ -230,6 +230,7 @@ def test_schedule(self):
                 extra_option="TestOpt",
                 metadata_option="MetaOpt",
                 force=False,
+                validate=True,
                 input_file="InputFile.yaml",
                 input_file_name="InputFile.yaml",
                 input_file_template=str(self.test_dir / "InputFile.yaml"),