diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 6e453f6f25..1c92a1bd65 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -5,20 +5,24 @@ pipeline { disableConcurrentBuilds() overrideIndexTriggers(false) skipDefaultCheckout(true) + timestamps() + timeout(time: 12, unit: 'HOURS') } parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') - // Uncomment the following line to re-enable comprehensive tests - // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' + // Workflow Wrapper test depth {0..9}, 0=none, 1=simple, 9=all [default] + choice(name: 'SRW_WRAPPER_TASK_DEPTH', choices: ['9', '1', '0'], description: '0=none, 1=simple, 9=all [default]') + // WE2E Tests ? + choice(name: 'SRW_WE2E_SINGLE_TEST', choices: ['coverage', 'none', 'skill-score', 'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0'], description: 'Specify the WE2E test to use') + booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' } stages { @@ -72,6 +76,11 @@ pipeline { // Run on all platform/compiler combinations by default or build and test only on the platform(s) and // compiler(s) specified by SRW_PLATFORM_FILTER and SRW_COMPILER_FILTER when { + beforeAgent true + expression { + return nodesByLabel(env.SRW_PLATFORM).size() > 0 + } + allOf { anyOf { expression { params.SRW_PLATFORM_FILTER == 'all' } @@ -88,8 +97,7 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - // values 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -103,7 +111,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -129,30 +137,33 @@ pipeline { stage('Initialize') { steps { dir ("${env.SRW_PLATFORM}") { - echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + echo "${env.STAGE_NAME} SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" cleanWs() checkout scm - sh '"${WORKSPACE}/${SRW_PLATFORM}/manage_externals/checkout_externals"' + sh '"${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_init.sh"' + sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/disk_usage.sh"' } } - } - - // Run the unittest functional tests that require an HPC platform - stage('Functional UnitTests') { - steps { - dir ("${env.SRW_PLATFORM}") { - echo "Running unittest on retrieve_data.py" - sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"' + + post { + always { + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-srw_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } - + // Run the unified build script; if successful create a tarball of the build and upload to S3 stage('Build') { + options { + timeout(time: 4, unit: 'HOURS') + } + steps { dir ("${env.SRW_PLATFORM}") { - echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + echo "${env.STAGE_NAME} SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_build.sh"' + sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/disk_usage.sh"' } } @@ -161,14 +172,33 @@ pipeline { sh 'cd "${WORKSPACE}/${SRW_PLATFORM}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${SRW_PLATFORM}/${BUILD_NAME}.tgz" *' s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } + always { + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-env.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-srw_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } } } + // Run the unittest functional tests that require an HPC platform + stage('Functional UnitTests') { + steps { + dir ("${env.SRW_PLATFORM}") { + echo "Running unittest on retrieve_data.py" + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"' + } + } + } + // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage stage('Functional WorkflowTaskTests') { + environment { + TASK_DEPTH = "${env.SRW_WRAPPER_TASK_DEPTH}" + } + steps { dir ("${env.SRW_PLATFORM}") { - echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + echo "Running ${TASK_DEPTH} simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/wrapper_srw_ftest.sh"' } } @@ -176,22 +206,22 @@ pipeline { // Run the unified test script stage('Test') { + options { + timeout(time: 8, unit: 'HOURS') + } + environment { SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/${env.SRW_PLATFORM}/expt_dirs" } steps { dir ("${env.SRW_PLATFORM}") { - echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + echo "${env.STAGE_NAME} SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" - // Remove the following line to re-enable comprehensive tests - sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' - - // Uncomment the following block to re-enable comprehensive tests - /* // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set, // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter script { + def single_test = params.SRW_WE2E_SINGLE_TEST def run_we2e_comprehensive_tests = params.SRW_WE2E_COMPREHENSIVE_TESTS def run_we2e_comprehensive_tests_label = 'run_we2e_comprehensive_tests' @@ -203,19 +233,25 @@ pipeline { } } - sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' - } - */ + sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests} SRW_WE2E_SINGLE_TEST=${single_test}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' + + // Archive the test log files + sh "[[ -d ${SRW_WE2E_EXPERIMENT_BASE_DIR} ]] && cd ${SRW_WE2E_EXPERIMENT_BASE_DIR} && tar --create --gzip --verbose --dereference --file ${WORKSPACE}/${SRW_PLATFORM}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/log.* || cat /dev/null > ${WORKSPACE}/${SRW_PLATFORM}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" + } + sh "STAGE_NAME=${env.STAGE_NAME} " + 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/disk_usage.sh"' } } post { + success { + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/we2e_test_logs-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*-skill-score.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + } always { - // Archive the test log files - sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/${SRW_PLATFORM}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/log.*' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-time-srw_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}-*-disk-usage${env.STAGE_NAME}.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] // Remove the data sets from the experiments directory to conserve disk space sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/we2e_test_logs-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } @@ -223,13 +259,23 @@ pipeline { } } } + // end of stages{} - // Uncomment the following block to re-enable PW clusters - /* post { always { - // Stop any Parallel Works clusters that were started during the pipeline execution script { + // Trigger another job to collect all build statistics + CI_JOB_NAME=env.JOB_NAME.replace("/${env.JOB_BASE_NAME}","") + CI_BRANCH_NAME=env.JOB_BASE_NAME.replace("%2F","%252F") + echo "post: Triggering ufs-srweather-app/ufs-srw-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..." + build job: '/ufs-srweather-app/ufs-srw-metrics', parameters: [ + string(name: 'CI_JOB_NAME', value: "${CI_JOB_NAME}"), + string(name: 'CI_BUILD_NUMBER', value: "${CI_BRANCH_NAME}/${env.BUILD_NUMBER}") + ], wait: false + + // Uncomment the following block to re-enable PW clusters + /* + // Stop any Parallel Works clusters that were started during the pipeline execution // def pw_clusters = ['pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'] def pw_clusters = ['pclusternoaav2use1'] def clusters = [] @@ -248,8 +294,8 @@ pipeline { // PW_CLUSTER_NAME parameter build job: 'parallel-works-jenkins-client/stop-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: clusters[i])] } + */ } } } - */ } diff --git a/.cicd/scripts/disk_usage.sh b/.cicd/scripts/disk_usage.sh new file mode 100755 index 0000000000..08a482d70f --- /dev/null +++ b/.cicd/scripts/disk_usage.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Output a CSV report of disk usage on subdirs of some path +# Usage: +# [JOB_NAME=] [BUILD_NUMBER=] [SRW_COMPILER=] [SRW_PLATFORM=] disk_usage path depth size outfile.csv +# +# args: +# directory=$1 +# depth=$2 +# size=$3 +# outfile=$4 + +[[ -n ${WORKSPACE} ]] || WORKSPACE=$(pwd) +[[ -n ${SRW_PLATFORM} ]] || SRW_PLATFORM=$(hostname -s 2>/dev/null) || SRW_PLATFORM=$(hostname 2>/dev/null) +[[ -n ${SRW_COMPILER} ]] || SRW_COMPILER=compiler + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set +# relative to script directory. +declare workspace +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" +fi + +echo "STAGE_NAME=${STAGE_NAME}" # from pipeline +outfile="${4:-${workspace}-${SRW_COMPILER}-disk-usage${STAGE_NAME}.csv}" + +function disk_usage() { + local directory=${1:-${PWD}} + local depth=${2:-1} + local size=${3:-k} + echo "Disk usage: ${JOB_NAME:-ci}/${SRW_PLATFORM}/$(basename $directory)" + ( + cd $directory || exit 1 + echo "Platform,Build,Owner,Group,Inodes,${size:-k}bytes,Access Time,Filename" + du -Px -d ${depth:-1} --inode --exclude='./workspace' | \ + while read line ; do + arr=($line); inode=${arr[0]}; filename=${arr[1]}; + echo "${SRW_PLATFORM}-${SRW_COMPILER:-compiler},${JOB_NAME:-ci}/${BUILD_NUMBER:-0},$(stat -c '%U,%G' $filename),${inode:-0},$(du -Px -s -${size:-k} --time $filename)" | tr '\t' ',' ; + done | sort -t, -k5 -n #-r + ) + echo "" +} + +disk_usage $1 $2 $3 | tee ${outfile} diff --git a/.cicd/scripts/sbatch_srw_ftest.sh b/.cicd/scripts/sbatch_srw_ftest.sh index 5add5368b5..07888d5ae9 100644 --- a/.cicd/scripts/sbatch_srw_ftest.sh +++ b/.cicd/scripts/sbatch_srw_ftest.sh @@ -7,7 +7,7 @@ #SBATCH --account=${SRW_PROJECT} #SBATCH --qos=batch #SBATCH --nodes=1 -#SBATCH --tasks-per-node=24 +#SBATCH --tasks-per-node=12 #SBATCH --cpus-per-task=1 #SBATCH -t 00:30:00 #SBATCH -o log_wrap.%j.log diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index 196d984a05..4733c4a4ca 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -27,7 +27,8 @@ fi # Build and install cd ${workspace}/tests set +e -./build.sh ${platform} ${SRW_COMPILER} +/usr/bin/time -p -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${SRW_PLATFORM}-${SRW_COMPILER}-time-srw_build.json \ + ./build.sh ${platform} ${SRW_COMPILER} build_exit=$? set -e cd - diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 7a787a7137..b77ee767f3 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -46,7 +46,6 @@ fi # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" -nco_dir="${workspace}/nco_dirs" pwd @@ -78,6 +77,9 @@ sed "s|^task_get_extrn_lbcs:|task_get_extrn_lbcs:\n EXTRN_MDL_SOURCE_BASEDIR_LB # Use staged data for HPSS supported machines sed 's|^platform:|platform:\n EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml +# Set OMP_NUM_THREADS_RUN_FCST to 1 in config.yaml +sed 's|^task_run_fcst:|task_run_fcst:\n OMP_NUM_THREADS_RUN_FCST: 1|1' -i ush/config.yaml + # Activate the workflow environment ... source etc/lmod-setup.sh ${platform,,} module use modulefiles @@ -85,11 +87,9 @@ module load build_${platform,,}_${SRW_COMPILER} module load wflow_${platform,,} [[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests -conda activate workflow_tools +conda activate srw_app set -e -u -export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src - # Adjust for strict limitation of stack size sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml @@ -102,13 +102,13 @@ cd ${workspace} cd ${EXPTDIR} pwd -cp ${workspace}/ush/wrappers/* . +cp ${workspace}/ush/wrappers/*.sh . # Set parameters that the task scripts require ... export JOBSdir=${workspace}/jobs export USHdir=${workspace}/ush export OMP_NUM_THREADS=1 -export nprocs=24 +export nprocs=12 [[ -n ${TASKS} ]] || TASKS=( run_make_grid diff --git a/.cicd/scripts/srw_init.sh b/.cicd/scripts/srw_init.sh new file mode 100755 index 0000000000..688255ac98 --- /dev/null +++ b/.cicd/scripts/srw_init.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# A unified init script for the SRW application. This script is expected to +# fetch initial source for the SRW application for all supported platforms. +# +set -e -u -x + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set +# relative to script directory. +declare workspace +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" +fi + +# Normalize Parallel Works cluster platform value. +declare platform +if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then + platform='noaacloud' +else + platform="${SRW_PLATFORM}" +fi + +# Build and install +cd ${workspace} +set +e +/usr/bin/time -p -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${SRW_PLATFORM}-${SRW_COMPILER}-time-srw_init.json \ + ./manage_externals/checkout_externals +init_exit=$? +echo "STAGE_NAME=${STAGE_NAME}" +env | grep = | sort > ${WORKSPACE}/${SRW_PLATFORM}-${SRW_COMPILER}-env.txt +set -e +cd - + +exit $init_exit diff --git a/.cicd/scripts/srw_metric.sh b/.cicd/scripts/srw_metric.sh new file mode 100755 index 0000000000..cbb216c959 --- /dev/null +++ b/.cicd/scripts/srw_metric.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# The goal of this script is to provide an example of performing Indy-Severe-Weather test run and compare results to reference with +# Skill score index that is calculated by MET Stat-Analysis Tools +# +# Required (these options are set in the Jenkins env): +# WORKSPACE= +# SRW_PLATFORM= +# SRW_COMPILER= +# SRW_PROJECT= +# +# Optional: +#[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" +[[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true +set -e -u -x + +BUILD_OPT=false +RUN_WE2E_OPT=false +RUN_STAT_ANLY_OPT=false + +if [[ $# -eq 0 ]]; then + BUILD_OPT=true + RUN_WE2E_OPT=true + RUN_STAT_ANLY_OPT=true +elif [[ $# -ge 4 ]]; then + echo "Too many arguments, expecting three or less" + exit 1 +else + for opt in "$@"; do + case $opt in + build) BUILD_OPT=true ;; + run_we2e) RUN_WE2E_OPT=true ;; + run_stat_anly) RUN_STAT_ANLY_OPT=true ;; + *) echo "Not valid option. Exiting!" ; exit 1 ;; + esac + done +fi + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" + +# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set +# relative to script directory. +declare workspace +if [[ -d "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" +else + workspace="$(cd -- "${script_dir}/../.." && pwd)" +fi + +# Normalize Parallel Works cluster platform value. +declare platform +if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then + platform='noaacloud' +else + platform="${SRW_PLATFORM}" +fi + +# Test directories +we2e_experiment_base_dir="${we2e_experiment_base_dir:=${workspace}/../expt_dirs/metric_test}" +we2e_test_dir="${we2e_test_dir:=${workspace}/tests/WE2E}" +we2e_test_name="${test_type:=grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0}" + +pwd + +# Setup the build environment +declare srw_compiler +srw_compiler=${SRW_COMPILER} +source ${workspace}/etc/lmod-setup.sh ${platform,,} +module use ${workspace}/modulefiles +module load build_${platform,,}_${srw_compiler} + +# Build srw +if [[ ${BUILD_OPT} == true ]]; then + cd ${workspace}/tests + ./build.sh ${platform,,} ${srw_compiler} +fi +cd ${workspace} + +# Activate workflow environment +module load wflow_${platform,,} + +[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but would not necessarily block workflow tests +conda activate srw_app +set -e -u + +# Run test +declare srw_project +srw_project=${SRW_PROJECT} +if [[ ${RUN_WE2E_OPT} == true ]]; then + [[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} + cd ${workspace}/tests/WE2E + ./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${srw_project} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q +fi +cd ${workspace} + +# Run skill-score check +if [[ ${RUN_STAT_ANLY_OPT} == true ]]; then + # Clear out data + rm -rf ${workspace}/Indy-Severe-Weather/ + # Check if metprd data exists locally otherwise get it from S3 + TEST_EXTRN_MDL_SOURCE_BASEDIR=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM}.yaml | awk '{print $NF}') + if [[ -d $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat ]] ; then + mkdir -p Indy-Severe-Weather/metprd/point_stat + cp -rp $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat Indy-Severe-Weather/metprd + elif [[ -f Indy-Severe-Weather.tgz ]]; then + tar xvfz Indy-Severe-Weather.tgz + else + wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz + tar xvfz Indy-Severe-Weather.tgz + fi + [[ -f ${platform,,}-${srw_compiler}-skill-score.txt ]] && rm ${platform,,}-${srw_compiler}-skill-score.txt + # Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. + # It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. + # In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, + # temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. + cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ + # Remove conda for Orion due to conda env conflicts + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|load("conda")|--load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Load met and metplus + module use modulefiles/tasks/${platform,,} + module load run_vx.local + # Reset Orion run_vx.local file + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|--load("conda")|load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Run stat_analysis + stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out ${platform,,}-${srw_compiler}-skill-score.txt + + # check skill-score.txt + cat ${platform,,}-${srw_compiler}-skill-score.txt + + # get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 + # A value greater than 1.0 indicates that the forecast model outperforms the reference, + # while a value less than 1.0 indicates that the reference outperforms the forecast. + tmp_string=$( tail -2 ${platform,,}-${srw_compiler}-skill-score.txt | head -1 ) + SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') + echo "Skill Score: ${SS_INDEX}" + if [[ ${SS_INDEX} < "0.700" ]]; then + echo "Your Skill Score is way smaller than 1.00, better check before merging" + exit 1 + else + echo "Congrats! You pass check!" + fi +fi diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh deleted file mode 100755 index 2018505735..0000000000 --- a/.cicd/scripts/srw_metric_example.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# -# The goal of this script is to provide an example of performing Indy-Severe-Weather test run and compare results to reference with -# Skill score index that is calculated by MET Stat-Analysis Tools -# -# Required: -# WORKSPACE= -# SRW_PLATFORM= -# SRW_COMPILER= -# -# Optional: -[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" -[[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true -set -e -u -x - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" - -# Get repository root from Jenkins WORKSPACE variable if set, otherwise, set -# relative to script directory. -declare workspace -if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then - workspace="${WORKSPACE}/${SRW_PLATFORM}" -else - workspace="$(cd -- "${script_dir}/../.." && pwd)" -fi - -# Normalize Parallel Works cluster platform value. -declare platform -if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then - platform='noaacloud' -else - platform="${SRW_PLATFORM}" -fi - -# Test directories -we2e_experiment_base_dir="${workspace}/../expt_dirs/metric_test" -we2e_test_dir="${workspace}/tests/WE2E" -we2e_test_name="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0" - -pwd - -# Activate the workflow environment ... -source etc/lmod-setup.sh ${platform,,} -module use modulefiles -module load build_${platform,,}_${SRW_COMPILER} -module load wflow_${platform,,} - -[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests -conda activate workflow_tools -set -e -u - -# build srw -cd ${workspace}/tests -./build.sh ${platform,,} ${SRW_COMPILER} -cd ${workspace} - -# run test -[[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} -cd ${workspace}/tests/WE2E -./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${SRW_PROJECT} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q -cd ${workspace} - -# run skill-score check -[[ ! -f Indy-Severe-Weather.tgz ]] && wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz -[[ ! -d Indy-Severe-Weather ]] && tar xvfz Indy-Severe-Weather.tgz -[[ -f skill-score.out ]] && rm skill-score.out -# Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. -# It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. -# In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, -# temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. -cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ -# load met and metplus -module use modulefiles/tasks/${platform,,} -module load run_vx.local -stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out - -# check skill-score.out -cat skill-score.out - -# get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 -# A value greater than 1.0 indicates that the forecast model outperforms the reference, -# while a value less than 1.0 indicates that the reference outperforms the forecast. -tmp_string=$( tail -2 skill-score.out | head -1 ) -SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') -echo "Skill Score: ${SS_INDEX}" -if [[ ${SS_INDEX} < "0.700" ]]; then - echo "Your Skill Score is way smaller than 1.00, better check before merging" - exit 1 -else - echo "Congrats! You pass check!" -fi diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 1bffe083bd..8ed4756987 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -11,7 +11,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then +if [[ -d "${WORKSPACE}/${SRW_PLATFORM}" ]]; then workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" @@ -26,23 +26,30 @@ else fi # Test directories -we2e_experiment_base_dir="${workspace}/expt_dirs" -we2e_test_dir="${workspace}/tests/WE2E" -nco_dir="${workspace}/nco_dirs" +export we2e_experiment_base_dir="${workspace}/expt_dirs" +export we2e_test_dir="${workspace}/tests/WE2E" + +# Clean any stale test logs +rm -f ${workspace}/tests/WE2E/log.* +rm -f ${we2e_experiment_base_dir}/*/log.generate_FV3LAM_wflow ${we2e_experiment_base_dir}/*/log/* WE2E_summary*txt # Run the end-to-end tests. if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then - test_type="comprehensive" + export test_type="comprehensive" else - test_type="coverage" + export test_type=${SRW_WE2E_SINGLE_TEST:-"coverage"} + if [[ "${test_type}" = skill-score ]]; then + export test_type="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0" + fi fi cd ${we2e_test_dir} # Progress file progress_file="${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt" -./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \ - --expt_basedir=${we2e_experiment_base_dir} \ - --opsroot=${nco_dir} | tee ${progress_file} +/usr/bin/time -p -f '{\n "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${SRW_PLATFORM}-${SRW_COMPILER}-time-srw_test.json \ + ./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \ + --expt_basedir=${we2e_experiment_base_dir} | tee ${progress_file}; \ + [[ -f ${we2e_experiment_base_dir}/grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0/log.generate_FV3LAM_wflow ]] && ${workspace}/.cicd/scripts/srw_metric.sh run_stat_anly # Set exit code to number of failures set +e diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh index 2e35b43b89..950ceb7a34 100755 --- a/.cicd/scripts/wrapper_srw_ftest.sh +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -15,22 +15,16 @@ declare arg_1 if [[ "${SRW_PLATFORM}" == cheyenne ]] || [[ "${SRW_PLATFORM}" == derecho ]]; then workflow_cmd=qsub arg_1="" - check_job="qstat -u ${USER} -r ${job_id}" else workflow_cmd=sbatch arg_1="--parsable" - check_job="squeue -u ${USER} -j ${job_id} --noheader" fi # Customize wrapper scripts if [[ "${SRW_PLATFORM}" == gaea ]]; then - sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh - sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh -fi - -if [[ "${SRW_PLATFORM}" == gaea-c5 ]]; then sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh fi if [[ "${SRW_PLATFORM}" == hera ]]; then @@ -39,6 +33,14 @@ if [[ "${SRW_PLATFORM}" == hera ]]; then fi fi +if [[ "${SRW_PLATFORM}" == jet ]]; then + sed -i '15i #SBATCH --partition=xjet' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh +fi + +if [[ "${TASK_DEPTH}" == 0 ]] ; then + exit 0 +fi + # Call job card and return job_id echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) @@ -49,6 +51,11 @@ sleep 10 # Check for job and exit when done while true do + if [[ "${SRW_PLATFORM}" == derecho ]]; then + check_job="qstat -u ${USER} -r ${job_id}" + else + check_job="squeue -u ${USER} -j ${job_id} --noheader" + fi job_id_info=$($check_job) if [ ! -z "$job_id_info" ]; then echo "Job is still running. Check again in two minutes" diff --git a/.github/workflows/python_func_tests.yaml b/.github/workflows/python_func_tests.yaml deleted file mode 100644 index 980b67a77c..0000000000 --- a/.github/workflows/python_func_tests.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Python functional tests -on: - push: - branches: - - develop - - 'release/*' - paths: - - ush/*retrieve_data.py - - parm/data_locations.yml - pull_request: - branches: - - develop - - 'release/*' - workflow_dispatch: - -env: - CI: true - -jobs: - python_functests: - name: Python Functional Tests - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Install dependencies - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pyyaml - - # Run python functional tests - - name: Run python functional tests - run: | - export PYTHONPATH=${PWD}/ush - python3 -m unittest -b tests/test_python/test_retrieve_data.py diff --git a/.github/workflows/python_linter.yaml b/.github/workflows/python_linter.yaml deleted file mode 100644 index b63a2a513c..0000000000 --- a/.github/workflows/python_linter.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Python linting -on: - push: - branches: - - develop - - 'release/*' - pull_request: - branches: - - develop - - 'release/*' - workflow_dispatch: - -defaults: - run: - shell: bash -jobs: - - python_linter: - name: Python unittests - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Install dependencies - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pylint==2.16 - - # Run python unittests - - name: Lint the test directory - run: | - ./manage_externals/checkout_externals workflow-tools - export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src - pylint --ignore-imports=yes tests/test_python/ - pylint ush/create_*.py - pylint ush/generate_FV3LAM_wflow.py diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml new file mode 100644 index 0000000000..fb0de16910 --- /dev/null +++ b/.github/workflows/python_tests.yaml @@ -0,0 +1,60 @@ +name: Python Tests +on: + push: + branches: + - develop + - 'release/*' + pull_request: + branches: + - develop + - 'release/*' + workflow_dispatch: + +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + srw_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Micromamba and srw_app environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment.yml + environment-name: srw_app + init-shell: bash + cache-downloads: true + cache-environment: true + + - name: Checkout externals + run: | + ./manage_externals/checkout_externals ufs-weather-model + + - name: Lint the python code + run: | + micromamba activate srw_app + export PYTHONPATH=$(pwd)/ush + pylint --ignore-imports=yes tests/test_python/ + pylint ush/create_*.py + pylint ush/generate_FV3LAM_wflow.py + pylint ush/set_fv3nml*.py + pylint ush/update_input_nml.py + + - name: Run python unittests + run: | + # exclude test_retrieve_data that is tested in functional test + micromamba activate srw_app + export UNIT_TEST=True + export PYTHONPATH=$(pwd)/ush + python -m unittest -b tests/test_python/*.py + + - name: Run python functional tests + run: | + micromamba activate srw_app + export CI=true + export PYTHONPATH=${PWD}/ush + python3 -m unittest -b tests/test_python/test_retrieve_data.py diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml deleted file mode 100644 index 5e491dea6e..0000000000 --- a/.github/workflows/python_unittests.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Python unittests -on: - push: - branches: - - develop - - 'release/*' - pull_request: - branches: - - develop - - 'release/*' - workflow_dispatch: - -env: - UNIT_TEST: True - -defaults: - run: - shell: bash -jobs: - - python_unittests: - name: Python unittests - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Install dependencies - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pyyaml jinja2==2.11 f90nml - sudo pip3 install numpy matplotlib basemap - - # Run python unittests - - name: Run python unittests - run: | - ./manage_externals/checkout_externals ufs-weather-model workflow-tools - # exclude test_retrieve_data that is tested in functional test - export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src - python3 -m unittest -b tests/test_python/*.py - diff --git a/.gitignore b/.gitignore index b6da1c53a0..ed78ca4182 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ bin/ +conda/ exec/ build/ fix/ include/ lib/ +parm/aqm_utils_parm/ +parm/nexus_config/ +parm/ufs_utils_parm/ +parm/upp_parm/ share/ sorc/*/ tests/WE2E/WE2E_tests_*.yaml @@ -11,10 +16,13 @@ tests/WE2E/*.txt tests/WE2E/*.log tests/WE2E/log.* ush/__pycache__/ +ush/aqm_utils_python/ +ush/nexus_utils/ ush/config.yaml ush/python_utils/__pycache__/ ush/python_utils/workflow-tools/ ush/*.swp +conda_loc *.swp __pycache__ diff --git a/.pylintrc b/.pylintrc index 0cb488e3d5..1d441d141f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -182,6 +182,7 @@ function-naming-style=snake_case good-names=i, j, k, + e, ex, Run, _ @@ -422,7 +423,9 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - logging-fstring-interpolation + logging-fstring-interpolation, + too-many-locals, + similarities # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e0987f8926..c8ce6064b2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,7 +17,7 @@ build: # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/UsersGuide/source/conf.py + configuration: doc/conf.py # If using Sphinx, optionally build your docs in additional formats such as PDF # formats: @@ -26,7 +26,7 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/UsersGuide/requirements.txt + - requirements: doc/requirements.txt submodules: include: diff --git a/Externals.cfg b/Externals.cfg index 8173a486e5..22e67411ed 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -3,7 +3,7 @@ protocol = git repo_url = https://github.com/ufs-community/UFS_UTILS # Specify either a branch name or a hash but not both. #branch = develop -hash = 6a7d534 +hash = 57bd832 local_path = sorc/UFS_UTILS required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = baa7751 +hash = 945cb2c local_path = sorc/UPP required = True @@ -30,7 +30,7 @@ protocol = git repo_url = https://github.com/noaa-oar-arl/NEXUS # Specify either a branch name or a hash but not both. #branch = develop -hash = 6a7a994 +hash = 40346b6 local_path = sorc/arl_nexus required = True @@ -39,18 +39,9 @@ protocol = git repo_url = https://github.com/NOAA-EMC/AQM-utils # Specify either a branch name or a hash but not both. #branch = develop -hash = 694a139 +hash = d953bd1 local_path = sorc/AQM-utils required = True -[workflow-tools] -protocol = git -repo_url = https://github.com/ufs-community/workflow-tools -# Specify either a branch name or a hash but not both. -# branch = develop -hash = e1b3b6f -local_path = ush/python_utils/workflow-tools -required = True - [externals_description] schema_version = 1.0.0 diff --git a/README.md b/README.md index d4268e5e80..3bf56f4c21 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ # UFS Short-Range Weather Application -The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader Weather Enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/. +The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. NOAA's operational model suite for numerical weather prediction (NWP) is quickly transitioning to the UFS from a number of legacy modeling systems. The UFS enables research, development, and contribution opportunities within the broader Weather Enterprise (including government, industry, and academia). For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/. -The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.1.0) represents a snapshot of this continuously evolving system. +The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.2.0) represents a snapshot of this continuously evolving system. -The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.1.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.1.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app. +The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.2.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.2.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app. For instructions on how to clone the repository, build the code, and run the workflow, see: -https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started +- https://ufs-srweather-app.readthedocs.io/en/develop/BuildingRunningTesting/Quickstart.html For a debugging guide for users and developers in the field of Earth System Modeling, please see: https://epic.noaa.gov/wp-content/uploads/2022/12/Debugging-Guide.pdf -UFS Development Team. (2022, Nov. 17). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.1.0). Zenodo. https://doi.org/10.5281/zenodo.7277602 +UFS Development Team. (2023, Oct. 31). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.2.0). Zenodo. https://doi.org/10.5281/zenodo.10015544 [![Python unittests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml) [![Python functional tests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml) diff --git a/aqm_environment.yml b/aqm_environment.yml new file mode 100644 index 0000000000..afd8a7b634 --- /dev/null +++ b/aqm_environment.yml @@ -0,0 +1,13 @@ +name: srw_aqm +channels: + - conda-forge + - ufs-community +dependencies: + - esmpy + - netcdf4=1.6.* + - numpy=1.23.* + - pylint=2.17* + - pytest=7.2* + - scipy=1.10.* + - uwtools=2.1* + - xarray=2022.11.* diff --git a/devbuild.sh b/devbuild.sh index 306f227c25..f79c788a81 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -41,6 +41,8 @@ OPTIONS installation prefix --bin-dir=BIN_DIR installation binary directory name ("exec" by default; any name is available) + --conda-dir=CONDA_DIR + installation location for miniconda (SRW clone conda subdirectory by default) --build-type=BUILD_TYPE build type; defaults to RELEASE (e.g. DEBUG | RELEASE | RELWITHDEBINFO) @@ -52,9 +54,11 @@ OPTIONS build with verbose output TARGETS - default = builds the default list of apps (also not passing any target does the same) - all = builds all apps - Or any combinations of (ufs, ufs_utils, upp) + default = builds the default list of components for the specified application + (also not passing any target does the same) + all = builds all standard components for ATM + conda_only = installs miniconda, but no other + Or any combinations of (ufs, ufs_utils, upp, nexus, aqm_utils) NOTE: See User's Guide for detailed build instructions @@ -99,11 +103,8 @@ usage_error () { # default settings LCL_PID=$$ -SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh -BUILD_DIR="${SRW_DIR}/build" -INSTALL_DIR=${SRW_DIR} BIN_DIR="exec" +CONDA_BUILD_DIR="conda" COMPILER="" APPLICATION="" CCPP_SUITES="" @@ -117,6 +118,7 @@ VERBOSE=false # Turn off all apps to build and choose default later DEFAULT_BUILD=true +BUILD_CONDA="on" BUILD_UFS="off" BUILD_UFS_UTILS="off" BUILD_UPP="off" @@ -164,6 +166,8 @@ while :; do --install-dir|--install-dir=) usage_error "$1 requires argument." ;; --bin-dir=?*) BIN_DIR=${1#*=} ;; --bin-dir|--bin-dir=) usage_error "$1 requires argument." ;; + --conda-dir=?*) CONDA_BUILD_DIR=${1#*=} ;; + --conda-dir|--conda-dir=) usage_error "$1 requires argument." ;; --build-type=?*) BUILD_TYPE=${1#*=} ;; --build-type|--build-type=) usage_error "$1 requires argument." ;; --build-jobs=?*) BUILD_JOBS=$((${1#*=})) ;; @@ -175,6 +179,7 @@ while :; do default) ;; all) DEFAULT_BUILD=false; BUILD_UFS="on"; BUILD_UFS_UTILS="on"; BUILD_UPP="on";; + conda_only) DEFAULT_BUILD=false;; ufs) DEFAULT_BUILD=false; BUILD_UFS="on" ;; ufs_utils) DEFAULT_BUILD=false; BUILD_UFS_UTILS="on" ;; upp) DEFAULT_BUILD=false; BUILD_UPP="on" ;; @@ -188,10 +193,9 @@ while :; do done # Ensure uppercase / lowercase ============================================ -APPLICATION="${APPLICATION^^}" -PLATFORM="${PLATFORM,,}" -COMPILER="${COMPILER,,}" -EXTERNALS="${EXTERNALS^^}" +APPLICATION=$(echo ${APPLICATION} | tr '[a-z]' '[A-Z]') +PLATFORM=$(echo ${PLATFORM} | tr '[A-Z]' '[a-z]') +COMPILER=$(echo ${COMPILER} | tr '[A-Z]' '[a-z]') # check if PLATFORM is set if [ -z $PLATFORM ] ; then @@ -203,6 +207,57 @@ fi MACHINE="${PLATFORM}" printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2 + +# Conda is not used on WCOSS2 +if [ "${PLATFORM}" = "wcoss2" ]; then + BUILD_CONDA="off" +fi + +# build conda and conda environments, if requested. +if [ "${BUILD_CONDA}" = "on" ] ; then + if [ ! -d "${CONDA_BUILD_DIR}" ] ; then + os=$(uname) + test $os == Darwin && os=MacOSX + hardware=$(uname -m) + installer=Miniforge3-${os}-${hardware}.sh + curl -L -O "https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/${installer}" + bash ./${installer} -bfp "${CONDA_BUILD_DIR}" + rm ${installer} + fi + + source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh + # Put some additional packages in the base environment on MacOS systems + if [ "${os}" == "MacOSX" ] ; then + mamba install -y bash coreutils sed + fi + conda activate + if ! conda env list | grep -q "^srw_app\s" ; then + mamba env create -n srw_app --file environment.yml + fi + if ! conda env list | grep -q "^srw_graphics\s" ; then + mamba env create -n srw_graphics --file graphics_environment.yml + fi + if [ "${APPLICATION}" = "ATMAQ" ]; then + if ! conda env list | grep -q "^srw_aqm\s" ; then + mamba env create -n srw_aqm --file aqm_environment.yml + fi + fi + +else + if [ -d "${CONDA_BUILD_DIR}" ] ; then + source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh + conda activate + fi +fi + +# Conda environment should have linux utilities to perform these tasks on macos. +SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) +MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh +BUILD_DIR="${BUILD_DIR:-${SRW_DIR}/build}" +INSTALL_DIR=${INSTALL_DIR:-$SRW_DIR} +CONDA_BUILD_DIR="$(readlink -f "${CONDA_BUILD_DIR}")" +echo ${CONDA_BUILD_DIR} > ${SRW_DIR}/conda_loc + # choose default apps to build if [ "${DEFAULT_BUILD}" = true ]; then BUILD_UFS="on" @@ -229,7 +284,7 @@ set -eu # automatically determine compiler if [ -z "${COMPILER}" ] ; then case ${PLATFORM} in - jet|hera|gaea|gaea-c5) COMPILER=intel ;; + jet|hera|gaea) COMPILER=intel ;; orion) COMPILER=intel ;; wcoss2) COMPILER=intel ;; cheyenne) COMPILER=intel ;; @@ -449,4 +504,38 @@ else fi fi +# Copy config/python directories from component to main directory (EE2 compliance) +if [ "${BUILD_UFS_UTILS}" = "on" ]; then + if [ -d "${SRW_DIR}/parm/ufs_utils_parm" ]; then + rm -rf ${SRW_DIR}/parm/ufs_utils_parm + fi + cp -rp ${SRW_DIR}/sorc/UFS_UTILS/parm ${SRW_DIR}/parm/ufs_utils_parm +fi +if [ "${BUILD_UPP}" = "on" ]; then + if [ -d "${SRW_DIR}/parm/upp_parm" ]; then + rm -rf ${SRW_DIR}/parm/upp_parm + fi + cp -rp ${SRW_DIR}/sorc/UPP/parm ${SRW_DIR}/parm/upp_parm +fi +if [ "${BUILD_NEXUS}" = "on" ]; then + if [ -d "${SRW_DIR}/parm/nexus_config" ]; then + rm -rf ${SRW_DIR}/parm/nexus_config + fi + cp -rp ${SRW_DIR}/sorc/arl_nexus/config ${SRW_DIR}/parm/nexus_config + if [ -d "${SRW_DIR}/ush/nexus_utils" ]; then + rm -rf ${SRW_DIR}/ush/nexus_utils + fi + cp -rp ${SRW_DIR}/sorc/arl_nexus/utils ${SRW_DIR}/ush/nexus_utils +fi +if [ "${BUILD_AQM_UTILS}" = "on" ]; then + if [ -d "${SRW_DIR}/parm/aqm_utils_parm" ]; then + rm -rf ${SRW_DIR}/parm/aqm_utils_parm + fi + cp -rp ${SRW_DIR}/sorc/AQM-utils/parm ${SRW_DIR}/parm/aqm_utils_parm + if [ -d "${SRW_DIR}/ush/aqm_utils_python" ]; then + rm -rf ${SRW_DIR}/ush/aqm_utils_python + fi + cp -rp ${SRW_DIR}/sorc/AQM-utils/python_utils ${SRW_DIR}/ush/aqm_utils_python +fi + exit 0 diff --git a/devclean.sh b/devclean.sh index 36fbc36898..01ace7a7d9 100755 --- a/devclean.sh +++ b/devclean.sh @@ -16,12 +16,16 @@ OPTIONS removes the "build" directory, keeps the "bin", "lib" and other build artifacts intact --clean removes "bin", "build" directories, and other build artifacts (same as "-a", "--all") - --install-dir=INSTALL_DIR + --conda + removes "conda" directory and conda_loc file in SRW + --install-dir=INSTALL_DIR installation directory name (\${SRW_DIR} by default) --build-dir=BUILD_DIR main build directory, absolute path (\${SRW_DIR}/build/ by default) --bin-dir=BIN_DIR binary directory name ("exec" by default); full path is \${INSTALL_DIR}/\${BIN_DIR}) + --conda-dir=CONDA_DIR + directory where conda is installed. caution: if outside the SRW clone, it may have broader use --sub-modules remove sub-module directories. They will need to be checked out again by sourcing "\${SRW_DIR}/manage_externals/checkout_externals" before attempting subsequent builds -v, --verbose @@ -38,6 +42,7 @@ Settings: INSTALL_DIR=${INSTALL_DIR} BUILD_DIR=${BUILD_DIR} BIN_DIR=${BIN_DIR} + CONDA_DIR=${CONDA_DIR} REMOVE=${REMOVE} VERBOSE=${VERBOSE} @@ -61,6 +66,7 @@ SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) INSTALL_DIR=${INSTALL_DIR:-${SRW_DIR}} BUILD_DIR=${BUILD_DIR:-"${SRW_DIR}/build"} BIN_DIR="exec" +CONDA_DIR=${CONDA_DIR:-"${SRW_DIR}/conda"} REMOVE=false VERBOSE=false @@ -74,6 +80,7 @@ if [[ ("$1" == "--help") || ("$1" == "-h") ]]; then usage exit 0 fi + # process optional arguments while :; do case $1 in @@ -82,12 +89,15 @@ while :; do --remove) REMOVE=true ;; --remove=?*|--remove=) usage_error "$1 argument ignored." ;; --clean) CLEAN=true ;; + --conda) REMOVE_CONDA=true ;; --install-dir=?*) INSTALL_DIR=${1#*=} ;; --install-dir|--install-dir=) usage_error "$1 requires argument." ;; --build-dir=?*) BUILD_DIR=${1#*=} ;; --build-dir|--build-dir=) usage_error "$1 requires argument." ;; --bin-dir=?*) BIN_DIR=${1#*=} ;; --bin-dir|--bin-dir=) usage_error "$1 requires argument." ;; + --conda-dir=?*) CONDA_DIR=${1#*=} ;; + --conda-dir|--conda-dir=) usage_error "$1 requires argument." ;; --sub-modules) INCLUDE_SUB_MODULES=true ;; --verbose|-v) VERBOSE=true ;; --verbose=?*|--verbose=) usage_error "$1 argument ignored." ;; @@ -95,7 +105,7 @@ while :; do default) ALL_CLEAN=false ;; # unknown -?*|?*) usage_error "Unknown option $1" ;; - *) usage; break ;; + *) break ;; esac shift done @@ -117,32 +127,49 @@ if [ "${REMOVE}" = true ] && [ "${CLEAN}" = false ] ; then elif [ "${CLEAN}" = true ]; then printf '%s\n' "Remove build directory, bin directory, and other build artifacts " printf '%s\n' " from the installation directory = ${INSTALL_DIR} " - [[ -d "${BUILD_DIR}" ]] && rm -rf ${BUILD_DIR} && printf '%s\n' "rm -rf ${BUILD_DIR}" - [[ -d "${INSTALL_DIR}/${BIN_DIR}" ]] && ( rm -rf ${INSTALL_DIR}/${BIN_DIR} && printf '%s\n' "rm -rf ${INSTALL_DIR}/${BIN_DIR}" ) - [[ -d "${SRW_DIR}/${BIN_DIR}" ]] && ( rm -rf ${SRW_DIR}/${BIN_DIR} && printf '%s\n' "rm -rf ${SRW_DIR}/${BIN_DIR}" ) - [[ -d "${INSTALL_DIR}/share" ]] && ( rm -rf ${INSTALL_DIR}/share && printf '%s\n' "rm -rf ${INSTALL_DIR}/share" ) - [[ -d "${INSTALL_DIR}/include" ]] && ( rm -rf ${INSTALL_DIR}/include && printf '%s\n' "rm -rf ${INSTALL_DIR}/include" ) - [[ -d "${INSTALL_DIR}/lib" ]] && rm -rf ${INSTALL_DIR}/lib && printf '%s\n' "rm -rf ${INSTALL_DIR}/lib" - [[ -d "${INSTALL_DIR}/lib64" ]] && rm -rf ${INSTALL_DIR}/lib && printf '%s\n' "rm -rf ${INSTALL_DIR}/lib64" - [[ -d "${SRW_DIR}/manage_externals/manic" ]] && rm -f ${SRW_DIR}/manage_externals/manic/*.pyc && printf '%s\n' "rm -f ${SRW_DIR}/manage_externals/manic/*.pyc" + + directories=( \ + "${BUILD_DIR}" \ + "${INSTALL_DIR}/${BIN_DIR}" \ + "${INSTALL_DIR}/share" \ + "${INSTALL_DIR}/include" \ + "${INSTALL_DIR}/lib" \ + "${INSTALL_DIR}/lib64" \ + ) + if [ ${#directories[@]} -ge 1 ]; then + for dir in ${directories[@]}; do + [[ -d "${dir}" ]] && rm -rfv ${dir} + done echo " " + fi fi -# Clean all the submodules if requested. NB: Need to check out them again before attempting subsequent builds, by sourcing ${SRW_DIR}/manage_externals/checkout_externals +# Clean all the submodules if requested. Note: Need to check out them again before attempting subsequent builds, by sourcing ${SRW_DIR}/manage_externals/checkout_externals if [ ${INCLUDE_SUB_MODULES} == true ]; then printf '%s\n' "Removing submodules ..." declare -a submodules='()' submodules=(${SRW_DIR}/sorc/*) -# echo " submodules are: ${submodules[@]} (total of ${#submodules[@]}) " +# echo " submodules are: ${submodules[@]} (total of ${#submodules[@]}) " if [ ${#submodules[@]} -ge 1 ]; then for sub in ${submodules[@]}; do [[ -d "${sub}" ]] && ( rm -rf ${sub} && printf '%s\n' "rm -rf ${sub}" ); done fi - printf '%s\n' "NB: Need to check out submodules again for any subsequent builds, " \ + printf '%s\n' "Note: Need to check out submodules again for any subsequent builds, " \ " by sourcing ${SRW_DIR}/manage_externals/checkout_externals " fi # -echo " " + +# Clean conda if requested +if [ "${REMOVE_CONDA}" = true ] ; then + printf '%s\n' "Removing conda location file" + rm -rf ${SRW_DIR}/conda_loc + printf '%s\n' "Removing conda installation" + rm -rf ${CONDA_DIR} +fi + + + +echo " " echo "All the requested cleaning tasks have been completed" -echo " " +echo " " exit 0 diff --git a/doc/ContribGuide/code-configuration-standards.rst b/doc/ContribGuide/code-configuration-standards.rst new file mode 100644 index 0000000000..4cc043a9c2 --- /dev/null +++ b/doc/ContribGuide/code-configuration-standards.rst @@ -0,0 +1,61 @@ +================================== +Code and Configuration Standards +================================== + +General Policies +================== + +* Platform-specific settings should be handled only through configuration and modulefiles, not in code or scripts. +* For changes to the ``scripts``, ``ush``, or ``jobs`` directories, developers should follow the :nco:`NCO Guidelines <>` for what is incorporated into each layer. +* Developers should ensure that their contributions work with the most recent version of the ``ufs-srweather-app``, including all the specific up-to-date hashes of each subcomponent. +* Modifications should not break any existing supported capabilities on any supported platforms. +* Update the RST documentation files where appropriate as part of the PR. If necessary, contributors may update the documentation in a subsequent PR. In these cases, the contributor should :srw-repo:`open an issue ` reflecting the need for documentation and include the issue number and explanation in the Documentation section of their initial PR. +* Binary files will no longer be merged into the ``develop`` branch. A binary file is defined as a "non-text" file and can include ``*.png``, ``*.gif``, ``*.jp*g``, ``*.tiff``, ``*.tar``, ``*.tgz``, ``*.gz``, ``*.mod``, ``*.o``, and executables. If a binary file needs to be staged in the ``ufs-srweather-app`` repository, please add it to the wiki's repository. The command to clone the ``ufs-srweather-app``'s wiki repository is ``git clone https://github.com/ufs-community/ufs-srweather-app.wiki.git``. Users with write access to the wiki repository can add the files here and link them to the documentation as needed. Users who do not have write access to the wiki repository should reach out to @MichaelLueken and/or note this in their pull request so that the files can be added. + +SRW Application Guidelines +============================ + + +**General Coding Standards:** + +* The ``ufs-srweather-app`` repository must not contain source code for compiled programs. Only scripts and configuration files should reside in this repository. +* All bash scripts must explicitly be ``#!/bin/bash`` scripts. They should *not* be login-enabled (i.e., scripts should *not* use the ``-l`` flag). +* MacOS does not have all Linux utilities by default. Developers should ensure that they do not break any MacOS capabilities with their contribution. +* All code must be indented appropriately and conform to the style of existing scripts (e.g., local variables should be lowercase, global variables should be uppercase). + +**External Components** + +* All externals live in a single ``Externals.cfg`` file. +* Only a single hash will be maintained for any given external code base. All externals should point to this static hash (not to the top of a branch). +* All new entries in ``Externals.cfg`` must point only to authoritative repositories. In other words, entries must point to either a `ufs-community GitHub organization `__ repository or another NOAA project organization repository. + + * Temporary exceptions are made for a PR into the ``develop`` branch of ``ufs-srweather-app`` that is dependent on another PR. When the component PR is merged, the contributor must update the corresponding ``ufs-srweather-app`` PR with the hash of the component's authoritative repository. + +**Build System** + +* Each component must build with CMake +* Each component must build with Intel compilers on official :srw-wiki:`Level 1 ` platforms and with GNU or Intel compilers on other platforms. +* Each component must have a mechanism for platform independence (i.e., no hard-coded machine-specific settings outside of established environment, configuration, and modulefiles). +* Each component must build with the standard supported NCEPLIBS environment (currently `spack-stack `__). + +**Modulefiles** + +* All official platforms should have a modulefile that can be sourced to provide the appropriate Python packages and other settings for the platform. +* Each SRW component must build using the common modules located in the ``modulefiles/srw_common`` file. + + +Workflow Coding Standards +-------------------------- + +**Python Coding Standards:** + + * All new Python workflow contributions should come with an appropriate environment YAML file (similar to ``environment.yaml``, ``graphics_environment.yaml``, and ``aqm_environment.yaml``). + * Keep the use of external Python packages to a minimum for necessary workflow tasks. Currently, these include ``f90nml``, ``pyyaml``, and ``Jinja2``. + +**Workflow Design:** Follow the :nco:`NCO Guidelines <>` for what is incorporated in each layer of the workflow. This is particularly important in the ``scripts`` directory. + +**Management of the Configuration File:** New configurable options must be consistent with existing configurable options and be documented in :srw-repo:`UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst `. Add necessary checks on acceptable options where applicable. Add appropriate default values in ``config_defaults.yaml``. + +**Management of Template Files:** If a new configurable option is required in an existing template, it must be handled similarly to its counterparts in the scripts that fill in the template. For example, if a new type of namelist is introduced for a new application component, it should make use of the existing ``jinja`` framework for populating namelist settings. + +**Namelist Management:** Namelists in ``ufs-srweather-app`` are generated using a Python tool and managed by setting YAML configuration parameters. This allows for the management of multiple configuration settings with maximum flexibility and minimum duplication of information. \ No newline at end of file diff --git a/doc/ContribGuide/contributing.rst b/doc/ContribGuide/contributing.rst new file mode 100644 index 0000000000..ed1671363e --- /dev/null +++ b/doc/ContribGuide/contributing.rst @@ -0,0 +1,340 @@ +============================ +Contributing to the SRW App +============================ + +Fork and PR Overview +===================== + +.. note:: + + Thank you to the Unified Workflow (UW) team for allowing us to adapt their Fork and PR Model overview for use in the SRW App. The original can be viewed in the `uwtools` :uw:`documentation `. + +Contributions to the ``ufs-srweather-app`` project are made via a :github-docs:`Fork` and :github-docs:`Pull Request (PR)` model. GitHub provides a thorough description of this contribution model in their `Contributing to a project` :github-docs:`Quickstart`, but the steps, with respect to ``ufs-srweather-app`` contributions, can be summarized as: + +#. :github-docs:`Create an issue ` to document proposed changes. +#. :github-docs:`Fork` the :srw-repo:`ufs-srweather-app repository<>` into your personal GitHub account. +#. :github-docs:`Clone` your fork onto your development system. +#. :github-docs:`Create a branch` in your clone for your changes. All development should take place on a branch, *not* on ``develop``. +#. :github-docs:`Make, commit, and push changes` in your clone / to your fork. +#. When your work is complete, :github-docs:`create a pull request (PR)` to merge your changes. + +For future contributions, you may delete and then recreate your fork or configure the official ``ufs-srweather-app`` repository as a :github-docs:`remote repository` on your clone and :github-docs:`sync upstream changes` to stay up-to-date with the official repository. + + +Development and Testing Process +================================= + +#. **Create issue:** Open an :srw-repo:`issue ` in the ``ufs-srweather-app`` to document proposed changes. See :ref:`Opening an Issue ` for detailed instructions. +#. **Fork & Clone the SRW App:** :github-docs:`Fork` the :srw-repo:`ufs-srweather-app repository<>` into your personal GitHub account and :github-docs:`clone` your fork onto your development system if you have not already done so. +#. **Create a branch:** in your clone for your changes. All development should take place on a branch, not on ``develop``. Branches should be named as follows, where ``[name]`` is a one-word description of the branch: + + * ``bugfix/[name]``: Fixes a demonstrably incorrect portion of code + * ``feature/[name]``: Adds a new feature to the code or improves an existing portion of the code + * ``text/[name]``: Changes elements of the repository that do not impact the compiled code in any way (e.g., changes to README, documentation, comments, changing quoted Registry elements, white space alignment). + +#. **Development:** Perform and test changes in the feature branch (not on ``develop``!). Document work in the issue and mention the issue number in commit messages to link your work to the issue (e.g., ``commit -m "Issue #23 - "``). Document changes to the workflow and capabilities in the RST files so that the SRW App documentation stays up-to-date. +#. **Testing:** Test code modifications on as many platforms as possible, and request help with further testing from the code management team when unable to test on all Level 1 platforms. The bare minimum testing required before opening a PR is to run the fundamental (:srw-repo:`tests/WE2E/machine_suites/fundamental `) tests on at least one supported machine (additional testing from the comprehensive suite might be required, depending on the nature of the change). To run the fundamental tests manually, please use the following command in the ``tests/WE2E`` directory: + + .. code-block:: console + + ./run_WE2E_tests.py -t=fundamental -m=your_machine -a=your_account + + where ``your_machine`` is the Tier-1 machine you are running the tests on, and ``your_account`` is the account you charge your computational resources to. See section :numref:`Section %s ` for more detail on SRW App testing. + +#. **Pull Request:** When your work is complete, :github-docs:`create a pull request` to merge your changes. When a PR is initiated, the :ref:`PR template ` autofills. Developers should use the template to provide information about the PR in the proper fields. See the guidelines in the :ref:`Making a Pull Request ` section for more details on making a good pull request. +#. **Merge** - When review and testing are complete, a code manager will merge the PR into ``develop``. PRs that are not ready for merging should have a "Work in Progress" label on them. Users who lack the permissions required to add the label can request in their PR that a code manager do so. +#. **Cleanup** - After the PR is merged, the code developer should delete the branch on their fork and close the issue. Feature branches are intended to be short-lived, concentrated on code with one sole purpose, and applicable to a single PR. A new feature branch should be created when subsequent code development continues. + +.. note:: + + Communication with code managers and the :ref:`repository code management team ` throughout the process is encouraged. + +.. _open-issue: + +Opening an Issue +================= + +All changes to ``ufs-srweather-app`` should be associated with a :srw-repo:`GitHub Issue `. Developers should search the existing issues in the ``ufs-srweather-app`` repository before beginning their work. If an issue does not exist for the work they are doing, they should create one prior to opening a new pull request. If an issue does exist, developers should be sure to collaborate to avoid duplicative work. + +To open an issue, click on :srw-repo:`"New Issue"` within the ``ufs-srweather-app`` GitHub repository. + +Choose from four options: + +#. :srw-repo:`Bug Report `: Report specific problems ("bugs") in the code using the following template: + + .. code-block:: console + + + + Your bug may already be reported! + Please search on the [Issue tracker](https://github.com/ufs-community/ufs-srweather-app/issues) before creating a new issue. + If an issue already exists, please use that issue to add any additional information. + + ## Expected behavior + + + ## Current behavior + + + ## Machines affected + + + + ## Steps To Reproduce + + + ## Detailed Description of Fix (optional) + + + ## Additional Information (optional) + + + ## Possible Implementation (optional) + + + ## Output (optional) + + +#. :srw-repo:`Feature Request `: New features and feature enhancements fall under this category. Propose features and enhancements using the following template. Optional sections may be deleted. + + .. code-block:: console + + + + Your issue may already be reported! + Please search on the [Issue tracker](https://github.com/ufs-community/ufs-srweather-app/issues) before creating a new issue. If an issue already exists, please use that issue to add any additional information. + + ## Description + + + + + ## Solution + + + ## Requirements** + + + ## Acceptance Criteria (Definition of Done) + + + ## Dependencies (optional) + + + + ## Alternative Solutions (optional) + + + +#. :srw-repo:`Text-Only Changes `: Propose text-only changes using the "Text-only request" template. Optional sections may be deleted. + + .. code-block:: console + + ## Description + + + ## Solution + + + ## Alternatives (optional) + + + ## Related to (optional) + + +#. :srw-repo:`Other `: Open a blank issue, and use the "Feature Request" template above as a starting point to describe the issue. + +For all issue reports, indicate whether this is: + #. A problem that you plan to work on and submit a PR for + #. A problem that you will **not** work on but that requires attention + #. A suggested improvement + +Additionally, please add a priority label to the issue (low, medium, or high priority). If you are unable to add labels to your issues, please request that a code manager add a priority label for you. + + * **High priority:** Issues related to a bug fix, a failing test configuration, or an update required for a release (either an operational implementation or public release). + * **Medium priority:** New features that are not required immediately for either an implementation or release + * **Low priority:** Refactoring work or other work that does not seem to be medium or high priority. + +If you are unable to work on the issue and require assistance through :term:`EPIC`, please make sure to include the ``EPIC Support Requested`` label. If the ``EPIC Support Requested`` label is added to a ``high priority`` issue, it might take some time before EPIC will work on the issue, since EPIC management needs to account for and prioritize these issues. However, after seeing that EPIC is required for high priority issues, management will adapt and allocate the necessary resources to assist. After filling out the issue report, click on "Submit new issue." + + +.. _make-pr: + +Making a Pull Request +====================== + +All changes to the SRW App ``develop`` branch should be handled via GitHub’s "Pull Request" (PR) functionality from a branch in the developer's fork. When creating your PR, please follow these guidelines, specific to the ``ufs-srweather-app`` project: + +* Ensure that your PR is targeting the base repository ``ufs-community/ufs-srweather-app`` and an appropriate base branch (usually ``develop``). +* **Complete PR template.** Your PR will appear pre-populated with a :ref:`template ` that you should complete. Provide an informative synopsis of your contribution, then mark appropriate checklist items by placing an "X" between their square brackets. You may tidy up the description by removing boilerplate text and non-selected checklist items. View :ref:`useful PR template guidance ` and information on :ref:`best practices ` for completing each section below. +* **Create draft PR.** Use the pull-down arrow on the green button below the description to initially create a :github-docs:`draft pull request`. + + * Once your draft PR is open, visit its *Files changed* tab and add comments to any lines of code where you think reviewers will benefit from more explanation. Try to save time by proactively answering questions you suspect reviewers will ask. + +* **Open PR.** Once your draft PR is marked up with your comments and ready for review, return to the *Conversation* tab and click the *Ready for review* button. + + * A default set of reviewers will automatically be added to your PR. You may add or request others, if appropriate. Pull requests will be reviewed and approved by at least two code reviewers, at least one of whom must be a code manager. Reviewers may make comments, ask questions, or request changes on your PR. Respond to these as needed, making commits in your clone and pushing to your fork/branch. Your PR will automatically be updated when commits are pushed to its source branch in your fork, so reviewers will immediately see your updates. When a PR has met the contribution and testing requirements and has been approved by two code reviewers, a code manager will merge the PR. + +.. _pr-template: + +PR Template +------------ + +Here is the template that is provided when developers click "Create pull request": + +.. code-block:: console + + - Update develop to head at ufs-community + + - Use this template to give a detailed message describing the change you want to make to the code. + + - You may delete any sections labeled "optional" and any instructions within . + + - If you are unclear on what should be written here, see https://github.com/wrf-model/WRF/wiki/Making-a-good-pull-request-message for some guidance and review the Code Contributor's Guide at https://github.com/ufs-community/ufs-srweather-app/wiki/Code-Manager's-Guide. + + - Code reviewers will assess the PR based on the criteria laid out in the Code Reviewer's Guide (https://github.com/ufs-community/ufs-srweather-app/wiki/Code-Manager's-Guide). + + - The title of this pull request should be a brief summary (ideally less than 100 characters) of the changes included in this PR. Please also include the branch to which this PR is being issued (e.g., "[develop]: Updated UFS_UTILS hash"). + + - Use the "Preview" tab to see what your PR will look like when you hit "Create pull request" + + + # --- Delete this line and those above before hitting "Create pull request" --- + + ## DESCRIPTION OF CHANGES: + + + ### Type of change + + - [ ] Bug fix (non-breaking change which fixes an issue) + - [ ] New feature (non-breaking change which adds functionality) + - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) + - [ ] This change requires a documentation update + + ## TESTS CONDUCTED: + + + + - [ ] hera.intel + - [ ] orion.intel + - [ ] hercules.intel + - [ ] cheyenne.intel + - [ ] cheyenne.gnu + - [ ] derecho.intel + - [ ] gaea.intel + - [ ] gaeac5.intel + - [ ] jet.intel + - [ ] wcoss2.intel + - [ ] NOAA Cloud (indicate which platform) + - [ ] Jenkins + - [ ] fundamental test suite + - [ ] comprehensive tests (specify *which* if a subset was used) + + ## DEPENDENCIES: + + + ## DOCUMENTATION: + + + ## ISSUE: + + + ## CHECKLIST + + - [ ] My code follows the style guidelines in the Contributor's Guide + - [ ] I have performed a self-review of my own code using the Code Reviewer's Guide + - [ ] I have commented my code, particularly in hard-to-understand areas + - [ ] My changes need updates to the documentation. I have made corresponding changes to the documentation + - [ ] My changes do not require updates to the documentation (explain). + - [ ] My changes generate no new warnings + - [ ] New and existing tests pass with my changes + - [ ] Any dependent changes have been merged and published + + ## LABELS (optional): + + A Code Manager needs to add the following labels to this PR: + - [ ] Work In Progress + - [ ] bug + - [ ] enhancement + - [ ] documentation + - [ ] release + - [ ] high priority + - [ ] run_ci + - [ ] run_we2e_fundamental_tests + - [ ] run_we2e_comprehensive_tests + - [ ] Needs Cheyenne test + - [ ] Needs Jet test + - [ ] Needs Hera test + - [ ] Needs Orion test + - [ ] help wanted + + ## CONTRIBUTORS (optional): + + +.. _pr-template-guidance: + +PR Template Guidance +--------------------- + +**TITLE:** Titles should start with the branch name in brackets and should give code reviewers a clear idea of what the change will do in approximately 5-10 words. Some good examples: + + * [develop] Make thompson_mynn_lam3km ccpp suite available + * [release/public-v2] Add a build_linux_compiler modulefile + * [develop] Fix module loads on Hera + * [develop] Add support for Rocoto with generic LINUX platform + +All of the above examples concisely describe the changes contained in the pull request. The title will not get cut off in emails and web pages. In contrast, here are some made-up (but plausible) examples of BAD pull request titles: + + * Bug fixes (Bug fixes on what part of the code?) + * Changes to surface scheme (What kind of changes? Which surface scheme?) + +**DESCRIPTION OF CHANGES:** The first line of the description should be a single-line "purpose" for this change. Note the type of change (i.e., bug fix, feature/enhancement, text-only). Summarize the problem, proposed solution, and required changes. If this is an enhancement or new feature, describe why the change is important. + +**DOCUMENTATION:** Developers should include documentation on new capabilities and enhancements by updating the appropriate RST documentation files in their fork prior to opening the PR. These documentation updates should be noted in the "Documentation" section of the PR message. If necessary, contributors may submit the RST documentation in a subsequent PR. In these cases, the developers should include any existing documentation in the "Documentation" section of the initial PR message or as a file attachment to the PR. Then, the contributor should open an issue reflecting the need for official RST documentation updates and include the issue number and explanation in the "Documentation" section of the initial PR template. + +.. _tips-best-practices: + +Tips, Best Practices, and Protocols to Follow When Issuing a PR +----------------------------------------------------------------- + +* **Label PR status appropriately.** If the PR is not completely ready to be merged, please add a "Work In Progress" label. Urgent PRs should be marked "high priority." All PRs should have a type label (e.g., "bug," "enhancement"). Labels can be added on the right-hand side of a submitted PR request by clicking on the gear icon beside "Labels" (below the list of reviewers). If users do not have the permissions to add a label to their PR, they should request in their PR description that a code manager add the appropriate labels. +* **Indicate urgency.** If a PR is particularly urgent, this information should be provided in the PR "Description" section, and multiple code management team members should be tagged to draw attention to this PR. After submitting the PR, a "high priority" label should be added to it. +* **Indicate the scope of the PR.** If the PR is extremely minor (e.g., change to the README file), indicate this in the PR message. If it is an extensive PR, the developer should test it on as many platforms as possible and stress the necessity that it be tested on systems for which they do not have access. +* **Clarify in the PR message where the code has been tested.** At a minimum, code should be tested on the platform where code modification has taken place. It should also be tested on machines where code modifications will impact results. If the developer does not have access to these platforms, this should be noted in the PR. +* **Follow separation of concerns.** For example, module loads are only handled in the appropriate modulefiles, Rocoto always sets the work directory, j-jobs make the work directory, and ex-scripts require the work directory to exist. +* **Target subject matter experts (SMEs) among the code management team.** When possible, tag team members who are familiar with the modifications made in the PR so that the code management team can provide effective and streamlined PR reviews and approvals. Developers can tag SMEs by selecting the gear icon next to "Assignees" (under the Reviewers list) and adding the appropriate names. +* **Schedule a live code review** if the PR is exceptionally complex in order to brief members of the code management team on the PR either in-person or through a teleconference. Developers should indicate in the PR message that they are interested in a live code review if they believe that it would be beneficial. + +Merging +======== + +Your PR is ready to merge when: + +#. It has been approved by a required number of ``ufs-srweather-app`` reviewers, including at least one code manager. +#. All conversations have been marked as resolved. +#. All required checks have passed. + +These criteria and their current statuses are detailed in a section at the bottom of your PR's *Conversation* tab. Checks take some time to run, so please be patient. + +In general, the lead code manager will merge the PR when ready. Developers with write permissions should not merge their code themselves unless instructed otherwise by the lead code manager. + +Need Help? +=========== + +See the :ref:`User Support ` section for an overview of user support options. For assistance directly related to a PR, please use comments in the *Conversation* tab of your PR to ask for help with any difficulties you encounter! diff --git a/doc/ContribGuide/documentation.rst b/doc/ContribGuide/documentation.rst new file mode 100644 index 0000000000..9e0bad6bda --- /dev/null +++ b/doc/ContribGuide/documentation.rst @@ -0,0 +1,72 @@ +.. _doc-guidelines: + +Documentation +============= + +.. note:: + + Thank you to the Unified Workflow (UW) team for allowing us to adapt their documentation guidance for use in the SRW App. The original can be viewed in the `uwtools` :uw:`documentation `. + + +Locally Building and Previewing Documentation +--------------------------------------------- + +To locally build the docs: + +#. Install ``sphinx``, ``sphinx-rtd-theme``, and ``sphinxcontrib-bibtex`` on your system if they are not already installed. +#. From the root of your clone: ``cd doc`` +#. Build the docs: ``make doc`` + +The ``make doc`` command will build the documentation under ``doc/build/html``, after which you can preview them in your web browser at the URL: + +.. code-block:: text + + file:///doc/build/html/index.html + +Rerun ``make doc`` and refresh your browser after making and saving changes. + +Viewing Online Documentation +---------------------------- + +Online documentation generation and hosting for the SRW App is provided by :rtd:`Read the Docs<>`. The green *View Docs* button near the upper right of that page links to the official docs for the project. When viewing the documentation, the version selector at the bottom of the navigation column on the left can be used to switch between the latest development code (``develop``), the latest released version (``latest``), and any previously released version. + +Docs are also built and temporarily published when Pull Requests (PRs) targeting the ``develop`` branch are opened. Visit the :rtd:`Builds page` to see recent builds, including those made for PRs. Click a PR-related build marked *Passed*, then the small *View docs* link (**not** the large green *View Docs* button) to see the docs built specifically for that PR. If your PR includes documentation updates, it may be helpful to include the URL of this build in your PR's description so that reviewers can see the rendered HTML docs and not just the modified ``.rst`` files. Note that if commits are pushed to the PR's source branch, Read the Docs will rebuild the PR docs. See the checks section near the bottom of a PR for current status and for another link to the PR docs via the *Details* link. + +.. COMMENT: Technically, docs are built when any PR is opened, regardless of branch. Look into changing. + +Documentation Guidelines +------------------------ + +Please follow these guidelines when contributing to the documentation: + +* Keep formatting consistent across pages. Update all pages when better ideas are discovered. Otherwise, follow the conventions established in existing content. +* Ensure that the ``make doc`` command completes with no errors or warnings. +* If the link-check portion of ``make doc`` reports that a URL is ``permanently`` redirected, update the link in the docs to use the new URL. Non-permanent redirects can be left as-is. +* Do not manually wrap lines in the ``.rst`` files. Insert newlines only as needed to achieve correctly formatted HTML, and let HTML wrap long lines and/or provide a scrollbar. +* Use one blank line between documentation elements (headers, paragraphs, code blocks, etc.) unless additional lines are necessary to achieve correctly formatted HTML. +* Remove all trailing whitespace. +* In general, avoid pronouns like "we" and "you". (Using "we" may be appropriate when synonymous with "The SRW Code Management Team", "The UFS Community", etc., when the context is clear.) Prefer direct, factual statements about what the code does, requires, etc. +* Use the `Oxford Comma `__. +* Follow the :rst:`RST Sections` guidelines, underlining section headings with = characters, subsections with - characters, and subsubsections with ^ characters. If a further level of refinement is needed, use " to underline paragraph headers. +* In [[sub]sub]section titles, capitalize all "principal" words. In practice this usually means all words but articles (a, an, the), logicals (and, etc.), and prepositions (for, of, etc.). Always fully capitalize acronyms (e.g., YAML). +* Never capitalize proper names when their owners do not (e.g., write `"pandas" `__, not "Pandas", even at the start of a sentence) or when referring to a software artifact (e.g., write ``numpy`` when referring to the library, and "NumPy" when referring to the project). +* When referring to YAML constructs, ``block`` refers to an entry whose values is a nested collection of key/value pairs, while ``entry`` is a single key/value pair. +* When using the ``.. code-block::`` directive, align the actual code with the word ``code``. Also, when ``.. code-block::`` directives appear in bulleted or numberd lists, align them with the text following the space to the right of the bullet/number. Include a blank line prior to the coe content. For example: + + .. code-block:: text + + * Lorem ipsum + + .. code-block:: python + + n = 88 + + or + + .. code-block:: text + + #. Lorem ipsum + + .. code-block:: python + + n = 88 \ No newline at end of file diff --git a/doc/ContribGuide/git-submodules.rst b/doc/ContribGuide/git-submodules.rst new file mode 100644 index 0000000000..42e7616cfe --- /dev/null +++ b/doc/ContribGuide/git-submodules.rst @@ -0,0 +1,225 @@ +============================ +Working with Git Submodules +============================ + +.. note:: + + Thank you to Janet Derrico (@jderrico-noaa) [#f1]_ for authoring the summary of Git submodules on which this chapter is based. [#f2]_ It has been adapted slightly for use in the SRW App. + +What Are Git Submodules? +========================= + +Git submodules are pointers to other Git repositories. They enable developers to include external repositories as a subdirectory within their main project. This is particularly useful when a project depends on external libraries or components that are developed and maintained in separate repositories. + +Key Benefits +============= + +* **Version Control:** Submodules link to specific commits in external repositories, ensuring consistency and predictability. Developers can control exactly which version of an external repository their project depends on. +* **Separate Development:** Changes to submodules are tracked separately from the main repository, allowing for independent development of external dependencies. +* **Collaborative Workflows:** Multiple teams can work on different parts of a larger project simultaneously without interference, each with its own repository (e.g. changes to ``ccpp-physics`` can be developed at the same time as changes to ``ufs-weather-model``). + +How Submodules Are Linked +========================== + +Git knows which submodules to check out based on two key pieces of information: the submodule pointer, and the information on where to find that pointer. The pointer is a commit reference---when you add a submodule to your repository, Git doesn't just store the URL; it also records a specific commit hash from that submodule. The commit hash is what Git uses to know which exact state of the submodule to checkout. These commit references are stored in the main repository and are updated whenever a change is committed in the submodule. When you run ``git submodule update``, Git checks out the commit of each submodule according to what is recorded in the main repository. The ``.gitmodules`` file tracks where to find this information, storing the submodule's path within your repository and its corresponding URL. + +If you commit a hash in a submodule but push to a different fork, then Git will add the new submodule hash to the supermodule, which will result in a Git error when trying to recursively check out the supermodule. + +Adding a Submodule +=================== + +You can add a submodule to your repository using ``git submodule add ``. This clones the external repository to the specified path and adds a new entry in a special file named ``.gitmodules``. + +Cloning a Repository with Submodules +===================================== +When cloning a repository that has submodules, use git clone --recursive to ensure that all submodules are also cloned. + +Updating a Submodule +====================== + +To update a submodule, navigate into the submodule directory, check out the desired commit or branch, and then go back to the main repository to commit this change. Here is an example for making a change to ``ccpp-physics``, ``fv3``, and ``ufs-weather-model``. Since ``ccpp-phsyics`` is a submodule of ``fv3atm`` and ``ufs-weather-model``, a change to ``ccpp-physics`` requires PRs to all three repositories. +This method requires two remotes on your local workspace: the authoritative (e.g., ``ufs-community/ufs-weather-model``) and the personal fork you push to (e.g., ``jderrico-noaa/ufs-weather-model``). The steps involved are: + +#. Clone locally +#. Create your working branches +#. Commit your changes +#. Push your working branches to your personal fork +#. Submit PRs from personal fork to authoritative + +Cloning the Authoritative Repository and Adding Your Personal Fork +-------------------------------------------------------------------- + +Clone the authoritative repository to your local workspace: + +.. code-block:: console + + git clone --recursive -b branch-name https://github.com/ufs-community/ufs-weather-model + cd ufs-weather-model + +where ``branch-name`` is the name of the branch you want to clone (usually ``develop``). + +Adding Your Personal Fork as a Remote Repository +-------------------------------------------------- + +.. code-block:: console + + git remote add my-fork + +where ``my-fork`` is the name of your fork. You can name your fork whatever you want as long as you can distinguish it from the authoritative (e.g., janet) https://github.com//ufs-weather-model + +Run: + +.. code-block:: console + + git remote -v + +to show the remote repositories that have been added to your local copy of ``ufs-weather-model``, if should show origin (the authoritative ufs-community repo) and my-fork (your personal fork that you push changes to) +The local repository for ufs-weather-model has been created. This process is repeated for the other submodules (``fv3atm`` and ``ccpp-physics``, where the code will be modified): + +.. code-block:: console + + cd FV3 + git remote add my-fork https://github.com//fv3atm + cd ccpp/physics + git remote add my-fork https://github.com//ccpp-physics + +Create Working Branches +------------------------ + +The next step is to create working branches that will hold your changes until they are merged. From ``ccpp-physics``, navigate up to ``ufs-weather-model``. It is good practice to checkout the main branch (e.g., ``develop``) to ensure that you are working with the latest updates and then create your working branch. You will do this all the way down: + + +Then, navigate from ``ccpp/physics`` back to to ``ufs-weather-model`` and create a new branch to hold your changes: + +.. code-block:: console + + cd ../../.. + git checkout -b working_branch + +This command creates a new branch named ``working_branch``; in practice the branch name should be more descriptive and reflect the development it will be holding. Follow the same process for the Git submodules you will be working in: + +.. code-block:: console + + cd FV3 + git checkout develop + git checkout -b working_branch + cd ccpp/physics + git checkout ufs/dev + git checkout -b working_branch + +Commit Changes and Push Working Branches +------------------------------------------ + +As you make changes to the code, you should commit often. This ensures that all of your development is tracked (so you don't lose anything) and makes it easier to go back to a working version if one of your changes breaks things (it happens!). Commit messages should be descriptive of the changes they contain. + +To push your working branches to your fork from the top down, navigate to the ``ufs-weather model`` directory. Then run: + +.. code-block:: console + + git push -u my-fork working_branch + +The ``-u`` flag here tells Git to set ``my-fork/working_branch`` as the default remote branch for ``working_branch``. After executing this command, you can simply use ``git push`` or ``git pull`` while on ``working_branch``, and Git will automatically know to push or pull from ``my_fork/working_branch``. + +Continue this process with the other submodule repositories: + +.. code-block:: console + + cd FV3 + git push -u my-fork working_branch + cd ccpp/physics + git push -u my-fork working_branch + +All working changes are now in your personal fork. + +Submitting PRs +--------------- + +When working with Git submodules, developers must submit individual pull requests to each repository where changes were made and link them to each other. In this case, developers would submit PRs to ``ufs-weather-model``, ``fv3atm``, and ``ccpp-physics``. There are several steps to this process: opening the PR, updating the submodules, and creating new submodule pointers. Each authoritative repository should have its own PR template that includes space to link to the URLs of related PRs. If for some reason this is not the case, developers should link to the related PRs in the "Description" section of their PR. + +Updating the Submodules +^^^^^^^^^^^^^^^^^^^^^^^^ + +When changes are made to the authoritative repositories while you are developing or while your PR is open, you need to update the PR to include those updates. From your local workspace, navigate to ``ufs-weather-model`` and run: + +.. code-block:: console + + git checkout develop + git pull origin develop + git checkout working_branch + git merge develop + git push -u my-fork working_branch + +This will check out the ``develop`` branch, retrieve the latest updates, then check out the ``working_branch`` and merge the latest changes from ``develop`` into it. After pushing the changes on ``working_branch`` to your personal fork, your PR will update automatically. This process must then be repeated for the other components (e.g., ``fv3`` and ``ccpp-physics``). It is important to check that you are merging the correct branch---for example, the main development branch in ``ufs-community/ccpp-physics`` is ``ufs/dev``, so you would checkout/pull ``ufs/dev`` instead. + +.. note:: + + If you have already pushed ``working_branch`` to ``my-fork`` using the ``-u`` flag, you can omit the flag and fork specification, but it doesn't hurt to use them. + +Add Submodule Pointers +^^^^^^^^^^^^^^^^^^^^^^^ +To create submodule pointers, developers will navigate to the lowest submodule directory (rather than going from the top down) to create pointers linking the submodule to the supermodule. In this example, we are using *ufs-weather-model → fv3 → ccpp-physics*, so developers would start by navigating to ``ccpp-physics``. Once your PR to ``ccpp-physics`` is merged, you then need to update your PRs to ``fv3`` and ``ufs-weather-model`` so that they point to the updated ``ccpp-physics`` submodule. + +First, update the local copy of ``ccpp-physics`` with what was merged to the authoritative (e.g., your changes): + +.. code-block:: console + + git checkout ufs/dev + git pull origin ufs/dev + +Then navigate to ``fv3atm``: + +.. code-block:: console + + cd ../.. + +If you were working with other submodules, you would navigate to submodule above the lowest here. Then create the submodule pointer, commit the change, and push it to your fork of ``fv3atm``: + +.. code-block:: console + + git checkout working_branch + git add ccpp/physics + git commit -m "update submodule pointer for ccpp-physics" + git push -u my-fork working_branch + +Once again, pushing to your personal fork will automatically update the PR that includes ``working_branch``. + +The ``fv3atm`` code managers will then merge your ``fv3atm`` PR, at which point only the ``ufs-weather-model`` PR will require a submodule pointer update. From your local workspace, navigate to the ``fv3`` directory (``ufs-weather-model/FV3``) and update the local copy of ``fv3atm`` with what was just merged into the authoritative: + +.. code-block:: console + + git checkout develop + git pull origin develop + +Then, navigate up to ``ufs-weather model`` directory, check out the working branch, and add the submodule pointer for ``fv3atm``. Commit and push the changes to your personal fork. + +.. code-block:: console + + cd .. + git checkout working_branch + git add FV3 + git commit -m "update submodule pointer for fv3atm" + git push -u my-fork + +The UFS code managers will then test and merge the ``ufs-weather-model`` PR. + +Switching Branches With Submodules +==================================== + +If you are working off a branch that has different versions (or commit references/pointers) of submodules, it is important to synchronize the submodules correctly. From the supermodule, you would switch to your desired branch and then update the submodules. For example, if you want to work on a different branch of the ``ufs-weather-model`` repository: + +.. code-block:: console + + git checkout desired_branch + git submodule update --init --recursive + +Here, ``--init`` initializes any submodules that have not yet been initialized, while ``--recursive`` ensures that all nested submodules (e.g., ``fv3atm``) are updated. If you know there have been upstream changes to a submodule, and you want to incorporate these latest changes, you would go into each submodule directory and pull the changes: + +.. code-block:: console + + cd path/to/submodule + git pull origin + +When working with submodules, it is best practice to always run ``git submodule update --init --recursive`` after switching branches. Changes to submodules need to be committed and pushed separately within their respective repositories (see sections above). + +.. [#f1] of NOAA Global Systems Laboratory (GSL) and Coorperative Institute for Research in Environmental Sciences (CIRES) +.. [#f2] with the assistance of Grant Firl, Joseph Olson, and ChatGPT \ No newline at end of file diff --git a/doc/ContribGuide/index.rst b/doc/ContribGuide/index.rst new file mode 100644 index 0000000000..c4adfc997d --- /dev/null +++ b/doc/ContribGuide/index.rst @@ -0,0 +1,13 @@ +Contributor's Guide +====================== + +.. toctree:: + :maxdepth: 3 + + introduction + contributing + code-configuration-standards + testing + git-submodules + documentation + diff --git a/doc/ContribGuide/introduction.rst b/doc/ContribGuide/introduction.rst new file mode 100644 index 0000000000..687dc2ea25 --- /dev/null +++ b/doc/ContribGuide/introduction.rst @@ -0,0 +1,31 @@ +================= +Introduction +================= + +Background +============ + +Authoritative Branch +---------------------- + +The ``ufs-srweather-app`` repository maintains a main branch for development called ``develop``. The HEAD of ``develop`` reflects the latest development changes. It points to regularly updated hashes for individual subcomponents. Pull requests (PRs) are typically merged to ``develop``. + +The ``develop`` branch is protected by the code management team: + + #. Pull requests for this branch require approval by at least two code reviewers. + #. A code manager should perform at least one of the reviews and the merge, but other contributors are welcome to provide comments/suggestions. + +.. _rcm-team: + +Repository Code Management Team +--------------------------------- + +Scientists and engineers from multiple labs and organizations have volunteered to review pull requests for the ``develop`` branch: + +.. csv-table:: + :file: ../tables/code-managers.csv + :widths: auto + :delim: ; + :header-rows: 1 + + diff --git a/doc/ContribGuide/testing.rst b/doc/ContribGuide/testing.rst new file mode 100644 index 0000000000..b296a3f90a --- /dev/null +++ b/doc/ContribGuide/testing.rst @@ -0,0 +1,70 @@ +.. _pr-testing: + +======== +Testing +======== + +The ``ufs-srweather-app`` repository uses the established workflow end-to-end (WE2E) testing framework (see :ref:`WE2E tests `) to implement two tiers of testing: fundamental and comprehensive. *Fundamental testing* consists of a lightweight set of tests that can be automated and run regularly on each :srw-wiki:`Level 1 ` platform. These tests verify that there are no major, obvious faults in the underlying code when running common combinations of grids, input data, and physics suites. *Comprehensive testing* includes the entire set of WE2E tests and covers a broader range of capabilities, configurations, and components. Eventually, new categories of tests will be added, including regression tests and unit tests. + +Before opening a PR, a minimum set of tests should be run: + + * Developers must run the fundamental test suite manually on at least one supported platform and report on the outcome in the PR template. Developers should test code modifications on as many platforms as possible. + + * To run the fundamental tests manually, run the following command from the ``tests/WE2E`` directory: + + .. code-block:: console + + ./run_WE2E_tests.py -t=fundamental -m=your_machine -a=your_account + + where ``your_machine`` is the Tier-1 machine you are running the tests on, and ``your_account`` is the account you charge your computational resources to. Refer to the :ref:`WE2E Tests ` chapter of the User's Guide for more detail on how to run SRW App tests. + + * Developers will not be required to run tests on *all* supported platforms, but if a failure is pointed out by another reviewer (or by automated testing), then it is expected that the developer will work with reviewers and code managers to ensure that the problem is resolved prior to merging. + + * If the PR impacts functionality contained within comprehensive WE2E tests not included in the fundamental test suite, the developer must run those tests on the PR. + * Any new functionality must be tested explicitly, and any new tests should be described in detail in the PR message. Depending on the impact of this functionality, new tests should be added to the suite of comprehensive WE2E tests, followed by a discussion with code managers on whether they should also be included as fundamental tests. + + * In some cases, it may be possible to modify a current test instead of creating a completely new test. Code developers introducing new capabilities should work with code managers to provide the proper configuration files, data, and other information necessary to create new tests for these capabilities. + + * When the above tests are complete and the PR has been approved by at least one code manager, a code manager will add the ``run_we2e_coverage_tests`` label to initiate fundamental testing on all Level 1 platforms via the Jenkins CI/CD pipeline. + +Testing on Jenkins +=================== + +`Jenkins `__ is an "open source automation server" that automates code testing. For the Jenkins automated testing labels, it should be noted that **ONLY** code managers should apply these labels and only after at least one code manager has given approval to the PR. The PR will not be merged until all Jenkins-based builds and testing have successfully passed. + +The following automated testing labels are available for the SRW App: + + * ``run_we2e_coverage_tests`` + * *Coming Soon:* ``run_we2e_comprehensive_tests`` + +Due to a security issue on Jenkins, where all Jenkins usernames are exposed, access to Jenkins logs through the Jenkins API has been disabled for the public. However, users can visit the `EPIC Health Dashboard `__ and click the *Jenkins Artifacts* tab to access the log files for their PR. On that page, users can identify their PR number, pull the ``we2e_test_logs-{machine}-{compiler}.tgz`` file (where ``{machine}`` is the Tier-1 platform that failed and ``{compiler}`` is the compiler used for the failed test), untar and ungzip the file, and look through the logs from the test that failed. + +Additionally, users can potentially access the directories where the Jenkins tests are run on the various machines so that they can view the tests, monitor progress, and investigate failures. The locations of the experiment directories on the various machines are as follows: + +.. list-table:: + :header-rows: 1 + + * - Tier-1 Platform + - Location of Jenkins experiment directories + * - Derecho + - /glade/derecho/scratch/epicufsrt/jenkins/workspace + * - Gaea + - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea + * - Gaea C5 + - /lustre/f2/dev/wpo/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/gaea-c5 + * - Hera (Intel) + - /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#__2/hera + * - Hera (GNU) + - /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/hera + * - Hercules + - /work/noaa/epic/role-epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/hercules + * - Jet + - /lfs1/NAGAPE/epic/role.epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/jet + * - Orion + - /work/noaa/epic/role-epic/jenkins/workspace/fs-srweather-app_pipeline_PR-#/orion + +where ``#`` is the PR number. + +If the Jenkins tests fail, then the developer will need to make the necessary corrections to their PR. Unfortunately, removing and adding the label back will not kick off the Jenkins test again. Instead, the job will need to be manually re-run through Jenkins (by a member of the EPIC team). + + diff --git a/docs/INSTALL b/doc/INSTALL similarity index 97% rename from docs/INSTALL rename to doc/INSTALL index e53044f6ad..53dc159bbd 100644 --- a/docs/INSTALL +++ b/doc/INSTALL @@ -12,7 +12,7 @@ git clone https://github.com/ufs-community/ufs-srweather-app.git cd ufs-srweather-app/ ./manage_externals/checkout_externals -# We can build ufs-sreweather-app binaries in two ways. +# We can build ufs-srweather-app binaries in two ways. # Method 1 # ======== diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000000..c91f2f147b --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,27 @@ +# Makefile for Sphinx documentation + +SPHINXOPTS = -a -n #-W +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = build +LINKCHECKDIR = $(BUILDDIR)/linkcheck + +.PHONY: help Makefile linkcheck + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + +doc: + make clean + $(MAKE) linkcheck + $(MAKE) html + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) $(SOURCEDIR) $(LINKCHECKDIR) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log \ No newline at end of file diff --git a/docs/UsersGuide/README b/doc/README similarity index 100% rename from docs/UsersGuide/README rename to doc/README diff --git a/docs/RUNTIME b/doc/RUNTIME similarity index 100% rename from docs/RUNTIME rename to doc/RUNTIME diff --git a/doc/UsersGuide/BackgroundInfo/CCPPUpdates.rst b/doc/UsersGuide/BackgroundInfo/CCPPUpdates.rst new file mode 100644 index 0000000000..1726e1747e --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/CCPPUpdates.rst @@ -0,0 +1,94 @@ +:orphan: + +.. _CCPPUpdates: + +================================================ +CCPP Updates for the SRW App v2.2.0 Release +================================================ + +Here is what's new in CCPP Physics for the UFS SRW v2.2.0 public release. + +General Updates +================= + +* Added RAP suite (``FV3_RAP``) as a new supported suite (documentation `here `__) +* Added the Community Land Model (CLM) lake model in the HRRR suite (``FV3_HRRR``) + +Thompson Microphysics Scheme +============================== + +* Reduced ice generation supersaturation requirement from 0.25 to 0.15 to generate more ice at the upper levels and reduce the outgoing longwave radiation bias +* Divided cloud number concentration into two parts (over land and others). Reduced number concentration over ocean to a smaller number (50/L) from its previous default (100/L). Both changes were made to reduce excessive surface downward shortwave radiative flux off coastal regions including the Southeast Pacific +* Implemented small fixes to the minimum size of snow and collision constants + +.. note:: + + The above improvements were tested with the non-aerosol option, so results with the aerosol-aware Thompson (used in the SRW App) may vary. + + +NoahMP Land Surface Model +=========================== + +* Option for using the unified frozen precipitation fraction in NoahMP. +* Diagnostic 2-meter temperature and humidity now based on vegetation and bare-ground tiles (new namelist option ``iopt_diag``) +* Bug fixes for GFS-based thermal roughness length scheme +* New soil color dataset introduced to improve soil albedo to reduce the large warm bias found in the Sahel desert +* Wet leaf contribution factor is included +* Leaf-area index now depends on momentum roughness length + + +RUC Land Surface Model +======================== + +* Initialization of land and ice emissivity and albedo with consideration of partial snow cover +* Initialization of water vapor mixing ratio over land ice +* Initialization of fractions of soil and vegetation types in a grid cell +* Changes in the computation of a flag for sea ice: set to true only if ``flag_cice=.false`` (atmosphere uncoupled from the sea ice model). +* Separate variables for sea ice, for example: ``snowfallac`` is replaced with ``snowfallac_ice`` +* Solar angle dependence of albedo for snow-free land +* Coefficient in soil resistance formulation (Sakaguchi and Zeng, 2009) raised from 0.7 to 1.0 to increase soil resistance to evaporation +* Computation of snow cover fraction and snow thermal conductivity updated + +GFS Scale-Aware TKE-EDMF PBL and Cumulus Schemes +================================================== + +* Parameterization to represent environmental wind shear effect added to reduce excessively high hurricane intensity +* Entrainment rates enhanced proportionally to the sub-cloud or PBL-mean TKE when TKE is larger than a threshold value +* Entrainment rate is increased as a function of vegetation fraction and surface roughness length to enhance underestimated CAPE + +MYNN-EDMF PBL Scheme +====================== + +* Small increase of buoyancy length scale in convective environments +* Patch for ensuring non-zero cloud fractions for all grid cells where cloud mixing ratio is greater than 1e-6 or ice mixing ratio is greater than 1e-9 + +Subgrid-Scale (SGS) Clouds Scheme +=================================== + +* Bug fix for cloud condensate input into RRTMG radiation +* New code section for use with SAS convection scheme +* Cloud fraction now computed as a mix between the area-dependent form and the modified Chaboureau and Bechtold (2005) form +* Adjusted limit for the boundary flux functions + +MYNN Surface-layer Scheme +=========================== + +* Reintroduced friction velocity averaging over water to reduce noise in 10-m winds in the hurricane regime + +Grell-Freitas Scale and Aerosol Aware Convection Scheme +========================================================= + +* Update for aerosol-awareness (experimental) +* Scale-awareness turned off when explicit microphysics is not active anywhere in the column +* Convection is completely suppressed at grid points where the MYNN PBL scheme produces shallow convection +* Radar reflectivity considers mass flux PDF as well as whether scale-awareness is turned on at the grid point in question + +Unified Gravity Wave Physics Scheme +===================================== + +* Optional diagnostic for tendencies computed. They can be switched on by setting the following namelist variables to ``“.true.”``: ``ldiag3d`` and ``ldiag_ugwp`` + + +.. attention:: + + The improvements in Thompson cloud microphysics, NoahMP land surface model, GFS TKE-EDMF and cumulus schemes were tested in the UFS global configuration, so results in the UFS limited-area configuration (SRW) may vary. \ No newline at end of file diff --git a/docs/UsersGuide/source/BackgroundInfo/Components.rst b/doc/UsersGuide/BackgroundInfo/Components.rst similarity index 60% rename from docs/UsersGuide/source/BackgroundInfo/Components.rst rename to doc/UsersGuide/BackgroundInfo/Components.rst index fd362ec691..1ba9349d8d 100644 --- a/docs/UsersGuide/source/BackgroundInfo/Components.rst +++ b/doc/UsersGuide/BackgroundInfo/Components.rst @@ -6,21 +6,21 @@ SRW Application Components The SRW Application assembles a variety of components, including: -* Pre-processor Utilities & Initial Conditions -* UFS Weather Forecast Model +* UFS Utilities +* UFS Weather Model * Unified Post Processor -* METplus Verification Suite -* Unified Workflow Tools -* Build System and Workflow +* METplus verification suite +* Unified Workflow tools +* Build system and workflow -These components are documented within this User's Guide and supported through the `GitHub Discussions `__ forum. +These components are documented within this User's Guide and supported through the `GitHub Discussions `__ forum. .. _Utils: UFS Preprocessing Utilities (UFS_UTILS) ========================================== -The SRW Application includes a number of pre-processing utilities (UFS_UTILS) that initialize and prepare the model. Since the SRW App provides forecast predictions over a limited area (rather than globally), these utilities generate a regional grid (``regional_esg_grid/make_hgrid``) along with :term:`orography` (``orog``) and surface climatology (``sfc_climo_gen``) files on that grid. Grids include a strip, or "halo," of six cells that surround the regional grid and feed in lateral boundary condition data. Since different grid and orography files require different numbers of :term:`halo` cells, additional utilities handle topography filtering and shave the number of halo points (based on downstream workflow component requirements). The pre-processing software :term:`chgres_cube` is used to convert the raw external model data into initial and lateral boundary condition files in :term:`netCDF` format. These are needed as input to the :term:`FV3`-:term:`LAM`. Additional information about the UFS pre-processor utilities can be found in the `UFS_UTILS Technical Documentation `__ and in the `UFS_UTILS Scientific Documentation `__. +The SRW Application includes a number of pre-processing utilities (UFS_UTILS) that initialize and prepare the model. Since the SRW App provides forecast predictions over a limited area (rather than globally), these utilities generate a regional grid (``regional_esg_grid/make_hgrid``) along with :term:`orography` (``orog``) and surface climatology (``sfc_climo_gen``) files on that grid. Grids include a strip, or "halo," of six cells that surround the regional grid and feed in lateral boundary condition data. Since different grid and orography files require different numbers of :term:`halo` cells, additional utilities handle topography filtering and shave the number of halo points (based on downstream workflow component requirements). The pre-processing software :term:`chgres_cube` is used to convert the raw external model data into initial and lateral boundary condition files in :term:`netCDF` format. These are needed as input to the :term:`FV3` limited area model (:term:`LAM`). Additional information about the UFS pre-processing utilities can be found in the :doc:`UFS_UTILS Technical Documentation ` and in the `UFS_UTILS Scientific Documentation `__. The SRW Application can be initialized from a range of operational initial condition files. It is possible to initialize the model from the Global Forecast System (:term:`GFS`), North American Mesoscale (:term:`NAM`) Forecast System, Rapid Refresh (:term:`RAP`), and High-Resolution Rapid Refresh (:term:`HRRR`) files in Gridded Binary v2 (:term:`GRIB2`) format. GFS files also come in :term:`NEMSIO` format for past dates. @@ -31,24 +31,26 @@ The SRW Application can be initialized from a range of operational initial condi Forecast Model ============== -The prognostic atmospheric model in the UFS SRW Application is the Finite-Volume Cubed-Sphere (:term:`FV3`) dynamical core configured with a Limited Area Model (:term:`LAM`) capability (:cite:t:`BlackEtAl2021`). The :term:`dynamical core` is the computational part of a model that solves the equations of fluid motion. A User's Guide for the UFS :term:`Weather Model` can be accessed `here `__. +The prognostic atmospheric model in the UFS SRW Application is the Finite-Volume Cubed-Sphere (:term:`FV3`) dynamical core configured with a Limited Area Model (:term:`LAM`) capability (:cite:t:`BlackEtAl2021`). The :term:`dynamical core` is the computational part of a model that solves the equations of fluid motion. A User's Guide for the UFS :term:`Weather Model` can be accessed :doc:`here `. Supported model resolutions in this release include 3-, 13-, and 25-km predefined contiguous U.S. (:term:`CONUS`) domains, each with 127 vertical levels. Preliminary tools for users to define their own domain are also available in the release with full, formal support of these tools to be provided in future releases. The Extended Schmidt Gnomonic (ESG) grid is used with the FV3-LAM, which features relatively uniform grid cells across the entirety of the domain. Additional information about the FV3 dynamical core can be found in the `scientific documentation `__, the `technical documentation `__, and on the `NOAA Geophysical Fluid Dynamics Laboratory website `__. Model Physics --------------- -The Common Community Physics Package (CCPP), described `here `__, supports interoperable atmospheric physics and land surface model options. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. The most recent SRW App release (v2.1.0) included four supported physics suites, and a fifth has since been added: FV3_RRFS_v1beta, FV3_GFS_v16, FV3_WoFS_v0, FV3_HRRR, and FV3_RAP (new!). The FV3_RRFS_v1beta physics suite is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the FV3_GFS_v16 is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. A detailed list of CCPP updates since the SRW App v2.0.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the `CCPP Technical Documentation `__. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available `here `__. Additionally, a CCPP single-column model (`CCPP-SCM `__) option has also been developed as a child repository. Users can refer to the `CCPP Single Column Model User and Technical Guide `__ for more details. This CCPP-SCM user guide contains a Quick Start Guide with instructions for obtaining the code, compiling, and running test cases, which include five standard test cases and two additional FV3 replay cases (refer to section 5.2 in the CCPP-SCM user guide for more details). Moreover, the CCPP-SCM supports a precompiled version in a docker container, allowing it to be easily executed on NOAA's cloud computing platforms without any issues (see section 2.5 in the CCPP-SCM user guide for more details). +The Common Community Physics Package (CCPP), described `here `__, supports interoperable atmospheric physics and land surface model options. Atmospheric physics are a set of numerical methods describing small-scale processes such as clouds, turbulence, radiation, and their interactions. The most recent SRW App release (|latestr|) included five supported physics suites: FV3_RRFS_v1beta, FV3_GFS_v16, FV3_WoFS_v0, FV3_HRRR, and FV3_RAP. The FV3_RRFS_v1beta physics suite is being tested for use in the future operational implementation of the Rapid Refresh Forecast System (:term:`RRFS`) planned for 2023-2024, and the FV3_GFS_v16 is an updated version of the physics suite used in the operational Global Forecast System (GFS) v16. A detailed list of CCPP updates since the SRW App v2.1.0 release is available :ref:`here `. A full scientific description of CCPP parameterizations and suites can be found in the `CCPP Scientific Documentation `__, and CCPP technical aspects are described in the :doc:`CCPP Technical Documentation `. The model namelist has many settings beyond the physics options that can optimize various aspects of the model for use with each of the supported suites. Additional information on Stochastic Physics options is available :doc:`here `. .. note:: SPP is currently only available for specific physics schemes used in the RAP/HRRR physics suite. Users need to be aware of which physics suite definition file (:term:`SDF`) is chosen when turning this option on. Among the supported physics suites, the full set of parameterizations can only be used with the ``FV3_HRRR`` option for ``CCPP_PHYS_SUITE``. +Additionally, a CCPP single-column model (`CCPP-SCM `__) option has also been developed as a child repository. Users can refer to the `CCPP Single Column Model User and Technical Guide `__ for more details. This CCPP-SCM user guide contains a Quick Start Guide with instructions for obtaining the code, compiling, and running test cases, which include five standard test cases and two additional FV3 replay cases (refer to section 5.2 in the CCPP-SCM user guide for more details). Moreover, the CCPP-SCM supports a precompiled version in a docker container, allowing it to be easily executed on NOAA's cloud computing platforms without any issues (see section 2.5 in the CCPP-SCM user guide for more details). + The SRW App supports the use of both :term:`GRIB2` and :term:`NEMSIO` input data. The UFS Weather Model ingests initial and lateral boundary condition files produced by :term:`chgres_cube` and outputs files in netCDF format on a specific projection (e.g., Lambert Conformal) in the horizontal direction and model levels in the vertical direction. Unified Post Processor (UPP) ============================== -The Unified Post Processor (:term:`UPP`) processes raw output from a variety of numerical weather prediction (:term:`NWP`) models. In the SRW App, the UPP converts model output data from the model's native :term:`netCDF` format to :term:`GRIB2` format on standard isobaric vertical coordinates. The UPP can also be used to compute a variety of useful diagnostic fields, as described in the `UPP User's Guide `__. Output from UPP can be used with visualization, plotting, and verification packages or in further downstream post-processing (e.g., statistical post-processing techniques). +The Unified Post Processor (:term:`UPP`) processes raw output from a variety of numerical weather prediction (:term:`NWP`) models. In the SRW App, the UPP converts model output data from the model's native :term:`netCDF` format to :term:`GRIB2` format on standard isobaric vertical coordinates. The UPP can also be used to compute a variety of useful diagnostic fields, as described in the :doc:`UPP User's Guide `. Output from UPP can be used with visualization, plotting, and verification packages or in further downstream post-processing (e.g., statistical post-processing techniques). .. _MetplusComponent: @@ -59,20 +61,18 @@ The Model Evaluation Tools (MET) package is a set of statistical verification to The METplus verification framework has been integrated into the SRW App to facilitate forecast evaluation. METplus is a verification framework that spans a wide range of temporal scales (warn-on-forecast to climate) and spatial scales (storm to global). It is supported by the `Developmental Testbed Center (DTC) `__. -METplus *installation* is not included as part of the build process for the most recent release of the SRW App. However, METplus is preinstalled on many `Level 1 & 2 `__ systems; existing builds can be viewed `here `__. - -METplus can be installed on other systems individually or as part of :term:`HPC-Stack` installation. Users on systems without a previous installation of METplus can follow the `MET Installation Guide `__ and `METplus Installation Guide `__ for individual installation. Currently, METplus *installation* is not a supported feature for this release of the SRW App. However, METplus *use* is supported on systems with a functioning METplus installation. +METplus comes preinstalled with :term:`spack-stack` but can also be installed on other systems individually or as part of :term:`HPC-Stack` installation. Users on systems without a previous installation of METplus can follow the :ref:`MET Installation Guide ` and :ref:`METplus Installation Guide ` for individual installation. Currently, METplus *installation* is only supported as part of spack-stack installation; users attempting to install METplus individually or as part of HPC-Stack will need to direct assistance requests to the METplus team. However, METplus *use* is supported on any system with a functioning METplus installation. -The core components of the METplus framework include the statistical driver, MET, the associated database and display systems known as METviewer and METexpress, and a suite of Python wrappers to provide low-level automation and examples, also called use cases. MET is a set of verification tools developed for use by the :term:`NWP` community. It matches up grids with either gridded analyses or point observations and applies configurable methods to compute statistics and diagnostics. Extensive documentation is available in the `METplus User's Guide `__ and `MET User's Guide `__. Documentation for all other components of the framework can be found at the Documentation link for each component on the METplus `downloads `__ page. +The core components of the METplus framework include the statistical driver (MET), the associated database and display systems known as METviewer and METexpress, and a suite of Python wrappers to provide low-level automation and examples, also called use cases. MET is a set of verification tools developed for use by the :term:`NWP` community. It matches up gridded forecast fields with either gridded analyses or point observations and applies configurable methods to compute statistics and diagnostics. Extensive documentation is available in the :doc:`METplus User's Guide ` and :doc:`MET User's Guide `. Documentation for all other components of the framework can be found at the *Documentation* link for each component on the METplus `downloads `__ page. -Among other techniques, MET provides the capability to compute standard verification scores for comparing deterministic gridded model data to point-based and gridded observations. It also provides ensemble and probabilistic verification methods for comparing gridded model data to point-based or gridded observations. Verification tasks to accomplish these comparisons are defined in the SRW App in :numref:`Table %s `. Currently, the SRW App supports the use of :term:`NDAS` observation files (which include conventional point-based surface and upper-air data) in `prepBUFR format `__ for point-based verification. It also supports gridded Climatology-Calibrated Precipitation Analysis (:term:`CCPA`) data for accumulated precipitation evaluation and Multi-Radar/Multi-Sensor (:term:`MRMS`) gridded analysis data for composite reflectivity and :term:`echo top` verification. +Among other techniques, MET provides the capability to compute standard verification scores for comparing deterministic gridded model data to point-based and gridded observations. It also provides ensemble and probabilistic verification methods for comparing gridded model data to point-based or gridded observations. Verification tasks to accomplish these comparisons are defined in the SRW App in :numref:`Table %s `. Currently, the SRW App supports the use of :term:`NDAS` observation files (which include conventional point-based surface and upper-air data) `in prepBUFR format `__ for point-based verification. It also supports gridded Climatology-Calibrated Precipitation Analysis (:term:`CCPA`) data for accumulated precipitation evaluation and Multi-Radar/Multi-Sensor (:term:`MRMS`) gridded analysis data for composite reflectivity and :term:`echo top` verification. -METplus is being actively developed by :term:`NCAR`/Research Applications Laboratory (RAL), NOAA/Earth Systems Research Laboratories (ESRL), and NOAA/Environmental Modeling Center (:term:`EMC`), and it is open to community contributions. More details about METplus can be found in :numref:`Chapter %s ` and on the `METplus website `__. +METplus is being actively developed by :term:`NCAR`/Research Applications Laboratory (RAL), NOAA/Earth Systems Research Laboratories (`ESRL `__), and NOAA/Environmental Modeling Center (:term:`EMC`), and it is open to community contributions. More details about METplus can be found on the `METplus website `__. Air Quality Modeling (AQM) Utilities ======================================= -AQM Utilities (AQM-utils) includes the utility executables and python scripts to run SRW-AQM (Online-CMAQ). +AQM Utilities (AQM-utils) include the utility executables and python scripts to run SRW-AQM (Online-:term:`CMAQ`). For more information on AQM-utils, visit the GitHub repository at https://github.com/NOAA-EMC/AQM-utils. .. _nexus: @@ -89,22 +89,22 @@ For more information on NEXUS, visit the GitHub repository at https://github.com Unified Workflow Tools ======================== -The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes templater and config tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about the UW can be found in the `workflow-tools `__ GitHub repository and in the `UW Documentation `__. +The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes templater and configuration (config) tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about UW tools can be found in the `uwtools `__ GitHub repository and in the :uw:`UW Documentation <>`. Build System and Workflow ========================= -The SRW Application has a portable, CMake-based build system that packages together all the components required to build the SRW Application. This build system collects the components necessary for running the end-to-end SRW Application, including the UFS Weather Model and the pre- and post-processing software. Additional libraries necessary for the application (e.g., :term:`NCEPLIBS-external` and :term:`NCEPLIBS`) are not included in the SRW Application build system but are available pre-built on pre-configured platforms. On other systems, they can be installed via the HPC-Stack (see :doc:`HPC-Stack Documentation `). There is a small set of system libraries and utilities that are assumed to be present on the target computer: the CMake build software; a Fortran, C, and C++ compiler; and an :term:`MPI` library. +The SRW Application has a portable, CMake-based build system that packages together all the components required to build the SRW Application. This build system collects the components necessary for running the end-to-end SRW Application, including the UFS Weather Model and the pre- and post-processing software. Additional libraries necessary for the application (e.g., :term:`NCEPLIBS-external` and :term:`NCEPLIBS`) are not included in the SRW Application build system but are available pre-built on pre-configured platforms. On other systems, they can be installed via spack-stack (see :doc:`spack-stack Documentation `). There is a small set of :ref:`prerequisite system libraries ` and utilities that are assumed to be present on the target computer: the CMake build software; a Fortran, C, and C++ compiler; and an :term:`MPI` library. Once built, users can generate a Rocoto-based workflow that will run each task in the proper sequence (see :numref:`Chapter %s ` or the `Rocoto documentation `__ for more information on Rocoto and workflow management). If Rocoto and/or a batch system is not present on the available platform, the individual components can be run in a stand-alone, command line fashion with provided run scripts. The SRW Application allows users to configure various elements of the workflow. For example, users can modify the parameters of the atmospheric model, such as start and end dates, duration, integration time step, and the physics suite used for the simulation. It also allows for configuration of other elements of the workflow; for example, users can choose whether to run some or all of the pre-processing, forecast model, and post-processing steps. More information on how to configure the workflow is available in :numref:`Section %s `. -An optional Python plotting task is also included to create basic visualizations of the model output. The task outputs graphics in PNG format for several standard meteorological variables on the pre-defined :term:`CONUS` domain. A difference plotting option is also included to visually compare two runs for the same domain and resolution. These plots may be used to perform a visual check to verify that the application is producing reasonable results. Configuration instructions are provided in :numref:`Section %s `. +An optional Python plotting task can also be included in the workflow to create basic visualizations of the model output. The task outputs graphics in PNG format for several standard meteorological variables on the pre-defined :term:`CONUS` domain. A difference plotting option is also included to visually compare two runs for the same domain and resolution. These plots may be used to perform a visual check to verify that the application is producing reasonable results. Configuration instructions are provided in :numref:`Section %s `. -The SRW Application has been tested on a variety of platforms widely used by researchers, including NOAA High-Performance Computing (HPC) systems (e.g., Hera, Orion, Jet); the National Center for Atmospheric Research (:term:`NCAR`) Cheyenne system; cloud environment, and generic Linux and MacOS systems using Intel and GNU compilers. Four `levels of support `__ have been defined for the SRW Application, including pre-configured (Level 1), configurable (Level 2), limited-test (Level 3), and build-only (Level 4) platforms. +The SRW Application has been tested on a variety of platforms widely used by researchers, including NOAA High-Performance Computing (HPC) systems (e.g., Hera, Jet); the National Center for Atmospheric Research (:term:`NCAR`) Derecho system; cloud environments; and generic Linux and MacOS systems using Intel and GNU compilers. Four :srw-wiki:`levels of support ` have been defined for the SRW Application, including pre-configured (Level 1), configurable (Level 2), limited-test (Level 3), and build-only (Level 4) platforms. -Preconfigured (Level 1) systems already have the required external libraries available in a central location (via :term:`HPC-Stack` or :term:`spack-stack`). The SRW Application is expected to build and run out-of-the-box on these systems, and users can :ref:`download the SRW App code ` without first installing prerequisites. +Preconfigured (Level 1) systems already have the required external libraries available in a central location (via :term:`spack-stack` or :term:`HPC-Stack`). The SRW Application is expected to build and run out-of-the-box on these systems, and users can :ref:`download the SRW App code ` without first installing prerequisites. Configurable platforms (Level 2) are platforms where all of the required libraries for building the SRW Application are expected to install successfully but are not available in a central location. Users will need to install the required libraries as part of the :ref:`SRW Application build ` process. Applications and models are expected to build and run once the required libraries are built. Release testing is conducted on these systems to ensure that the SRW App runs smoothly there. @@ -112,5 +112,5 @@ Limited-Test (Level 3) and Build-Only (Level 4) computational platforms are thos On all platforms, the SRW App can be :ref:`run within a container ` that includes the prerequisite software. -A complete description of the levels of support, along with a list of preconfigured and configurable platforms can be found in the `SRW Application Wiki `__. +A complete description of the levels of support, along with a list of preconfigured and configurable platforms can be found in the :srw-wiki:`SRW Application Wiki `. diff --git a/docs/UsersGuide/source/BackgroundInfo/Introduction.rst b/doc/UsersGuide/BackgroundInfo/Introduction.rst similarity index 67% rename from docs/UsersGuide/source/BackgroundInfo/Introduction.rst rename to doc/UsersGuide/BackgroundInfo/Introduction.rst index f4aa344fa7..f1a384e025 100644 --- a/docs/UsersGuide/source/BackgroundInfo/Introduction.rst +++ b/doc/UsersGuide/BackgroundInfo/Introduction.rst @@ -8,35 +8,49 @@ The Unified Forecast System (:term:`UFS`) is a community-based, coupled, compreh The UFS includes `multiple applications `__ that support different forecast durations and spatial domains. This documentation describes the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The most recent SRW Application includes a prognostic atmospheric model, pre- and post-processing, and a community workflow for running the system end-to-end. These components are documented within this User's Guide and supported through the `GitHub Discussions `__ forum. The SRW App also includes support for a verification package (METplus) for both deterministic and ensemble simulations and support for four stochastically perturbed physics schemes. -Since the v2.1.0 release, developers have added a variety of features: +Since the last release, developers have added a variety of features: * Bug fixes since the v2.1.0 release - * Pre-implementation Rapid Refresh Forecast System (RRFS) forecast configurations - * Air Quality Modeling (AQM) capabilities - * Updates to :term:`CCPP` that target the top of the ``main`` branch (which is ahead of CCPP v6.0.0). See :ref:`this page ` for a detailed summary of updates that came in ahead of the v2.1.0 release. - * Support for the :term:`UPP` inline post option (see :ref:`here `) - * Documentation updates to reflect the changes above - -The SRW App v2.1.0 citation is as follows and should be used when presenting results based on research conducted with the App: - -UFS Development Team. (2022, Nov. 17). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.1.0). Zenodo. https://doi.org/10.5281/zenodo.7277602 + * Addition of the supported ``FV3_RAP`` physics suite (:srw-repo:`PR #811 `) and support for the ``RRFS_NA_13km`` predefined grid + * Addition of ``FV3_GFS_v17_p8`` physics suite (:srw-repo:`PR #574 `) + * Updates to :term:`CCPP` that target the top of the ``main`` branch (which is ahead of CCPP v6.0.0). See :ref:`this page ` for a detailed summary of updates that came in ahead of the v2.2.0 release. + * Expansion of :srw-wiki:`Level 1 platforms ` to include Derecho, Hercules, and Gaea C5 (PRs :srw-repo:`#894 `, :srw-repo:`#898 `, :srw-repo:`#911 `) + * Transition to spack-stack modulefiles for most supported platforms to align with the UFS WM shift to spack-stack (PRs :srw-repo:`#913 ` and :srw-repo:`#941 `) + * Overhaul of the WE2E testing suite (see, e.g., PRs :srw-repo:`#686 `, :srw-repo:`#732 `, :srw-repo:`#864 `, :srw-repo:`#871 `) + * Improvements to the CI/CD automated testing pipeline (see, e.g., PRs :pull/707>` and :srw-repo:`#847 `) + * Incorporation of additional METplus verification capabilities (PRs :srw-repo:`#552 `, :srw-repo:`#614 `, :srw-repo:`#757 `, :srw-repo:`#853 `) + * Integration of the Unified Workflow's templater tool (:srw-repo:`PR #793 `) + * Ability to create a user-defined custom workflow (:srw-repo:`PR #676 `) + * Option to use a custom vertical coordinate file with different distribution of vertical layers (:srw-repo:`PR #813 `) and :ref:`documentation on how to use this feature ` (:srw-repo:`PR #888 `) + * Incorporation of plotting tasks into the workflow (PR :srw-repo:`#482 `); addition of ability to plot on both CONUS and smaller regional grid (:srw-repo:`PR #560 `) + * Addition of a sample verification case (:srw-repo:`PR #500 `) with :ref:`documentation ` + * A new :ref:`tutorial chapter ` in the documentation (:srw-repo:`PR #584 `) + * Incorporation of `UFS Case Studies `__ within the WE2E framework (PRs :srw-repo:`#736 ` and :srw-repo:`#822 `) + * Air Quality Modeling (AQM) capabilities (unsupported but available; see :srw-repo:`PR #613 `) + * Miscellaneous documentation updates to reflect the changes above + +The SRW App |latestr| citation is as follows and should be used when presenting results based on research conducted with the App: + +UFS Development Team. (2023, Oct. 31). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.2.0). Zenodo. https://doi.org/10.5281/zenodo.10015544 + +.. _ug-organization: User's Guide Organization ============================ -The SRW Application documentation is organized into four sections: *Background Information*; *Building, Running, and Testing the SRW App*; *Customizing the Workflow*; and *Reference*. +The SRW Application documentation is organized into four sections: (1) *Background Information*; (2) *Building, Running, and Testing the SRW App*; (3) *Customizing the Workflow*; and (4) *Reference*. Background Information ------------------------- * This **Introduction** section explains how the SRW App documentation is organized, how to use this guide, and where to find user support and component documentation. - * :numref:`Section %s: Technical Overview ` provides a technical overview, including SRW App prerequisites and an overview of the code directory structure. - * :numref:`Section %s: SRW Application Components ` provides a detailed description of the application components, including optional application components. + * :numref:`Section %s: Technical Overview ` provides technical information about the SRW App, including prerequisites and an overview of the code directory structure. + * :numref:`Section %s: SRW Application Components ` provides a description of the application components, including optional components. Building, Running, and Testing the SRW App -------------------------------------------- - * :numref:`Section %s: Quick Start Guide ` is an overview of the workflow and gives instructions for its use on `Level 1 platforms `__. + * :numref:`Section %s: Quick Start Guide ` is an overview of the workflow and gives instructions for its use on :srw-wiki:`Level 1 platforms `. * :numref:`Section %s: Container-Based Quick Start Guide ` explains how to run the SRW Application in a container. Containers may be run on a broad range of systems and come with SRW App prerequisites already installed. * :numref:`Section %s: Building the SRW App ` provides a *detailed* explanation of how to build the SRW App. * :numref:`Section %s: Running the SRW App ` provides a *detailed* explanation of how to run the SRW App after it has been built/compiled. It includes information on standard workflow tasks, additional optional tasks (e.g., METplus verification, plotting), and different techniques for running the workflow. @@ -48,7 +62,7 @@ Building, Running, and Testing the SRW App .. hint:: * To get started with the SRW App, it is recommended that users try one of the following options: - #. View :numref:`Section %s: Quick Start Guide ` for a quick overview of the workflow steps. Especially helpful for users with access to a `Level 1 platform `__. + #. View :numref:`Section %s: Quick Start Guide ` for a quick overview of the workflow steps. Especially helpful for users with access to a :srw-wiki:`Level 1 platform `. #. To build the application in a container, which provides a more uniform work environment, users can refer to :numref:`Section %s: Container-Based Quick Start Guide `. #. For detailed instructions on building and running the SRW App, users can refer to :numref:`Section %s: Building the SRW App ` and :numref:`Section %s: Running the SRW App `. @@ -57,7 +71,7 @@ Customizing the Workflow * :numref:`Section %s: Workflow Parameters ` documents all of the user-configurable experiment parameters that can be set in the user configuration file (``config.yaml``). * :numref:`Section %s: Input & Output Files ` describes application input and output files, as well as information on where to get publicly available data. - * :numref:`Section %s: Limited Area Model (LAM) Grids ` describes the SRW App predefined grids in detail and explains how to create a custom user-generated grid. + * :numref:`Section %s: Limited Area Model (LAM) Grids ` describes the SRW App predefined grids, explains how to create a custom user-generated grid, and provides information on using a custom distribution of vertical levels. * :numref:`Section %s: Defining an SRW App Workflow ` explains how to build a customized SRW App workflow XML file. * :numref:`Section %s: Template Variables ` explains how to use template variables. @@ -68,6 +82,7 @@ Reference Information * :numref:`Section %s: FAQ ` answers users' frequently asked questions. * :numref:`Section %s: Glossary ` defines important terms related to the SRW App. +.. _doc-conventions: SRW App Documentation Conventions =================================== @@ -84,6 +99,8 @@ Code that includes angle brackets (e.g., ``build__``) indica File or directory paths that begin with ``/path/to/`` should be replaced with the actual path on the user's system. For example, ``/path/to/modulefiles`` might be replaced by ``/Users/Jane.Smith/ufs-srweather-app/modulefiles``. +.. _component-docs: + Component Documentation ========================= @@ -91,16 +108,16 @@ A list of available component documentation is shown in :numref:`Table %s `__, since many "bugs" do not require a code change/fix --- instead, the user may be unfamiliar with the system and/or may have misunderstood some component of the system or the instructions, which is causing the problem. Asking for assistance in a `GitHub Discussion `__ post can help clarify whether there is a simple adjustment to fix the problem or whether there is a genuine bug in the code. Users are also encouraged to search `open issues `__ to see if their bug has already been identified. If there is a genuine bug, and there is no open issue to address it, users can report the bug by filing a `GitHub Issue `__. +If users (especially new users) believe they have identified a bug in the system, it is recommended that they first ask about the problem in :srw-repo:`GitHub Discussions `, since many "bugs" do not require a code change/fix --- instead, the user may be unfamiliar with the system and/or may have misunderstood some component of the system or the instructions, which is causing the problem. Asking for assistance in a :srw-repo:`GitHub Discussion ` post can help clarify whether there is a simple adjustment to fix the problem or whether there is a genuine bug in the code. Users are also encouraged to search :srw-repo:`open issues ` to see if their bug has already been identified. If there is a genuine bug, and there is no open issue to address it, users can report the bug by filing a :srw-repo:`GitHub Issue `. Feature Requests and Enhancements ----------------------------------- @@ -162,20 +181,21 @@ Community Contributions ------------------------- The UFS community is encouraged to contribute to the development efforts of all related -utilities, model code, and infrastructure. As described above, users can post issues in the SRW App to report bugs or to announce upcoming contributions to the code base. Additionally, users can file issues in component repositories for contributions that directly concern those repositories. Contributions to the `ufs-srweather-app `__ repository should follow the guidelines contained in the `SRW App Contributor's Guide `__. For code to be accepted into a component repository, users must follow the code management rules of that component's authoritative repository. These rules are usually outlined in the User's Guide (see :numref:`Table %s `) or GitHub wiki for each respective repository (see :numref:`Table %s `). +utilities, model code, and infrastructure. As described above, users can post issues in the SRW App to report bugs or to announce upcoming contributions to the code base. +Contributions to the `ufs-srweather-app `__ repository should follow the guidelines contained in the :srw-wiki:`SRW App Contributor's Guide `. +Additionally, users can file issues in component repositories for contributions that directly concern those repositories. For code to be accepted into a component repository, users must follow the code management rules of that component's authoritative repository. These rules are usually outlined in the component's User's Guide (see :numref:`Table %s `) or GitHub wiki for each respective repository (see :numref:`Table %s `). + +.. _future-direction: Future Direction ================= Users can expect to see incremental improvements and additional capabilities in upcoming releases of the SRW Application to enhance research opportunities and support operational forecast implementations. Planned enhancements include: +* Inclusion of data assimilation and forecast restart/cycling capabilities. * A more extensive set of supported developmental physics suites. * A larger number of pre-defined domains/resolutions and a *fully supported* capability to create a user-defined domain. -* Add user-defined vertical levels (number and distribution). -* Inclusion of data assimilation and forecast restart/cycling capabilities. - - -.. bibliography:: ../references.bib - +* Incorporation of additional `Unified Workflow `__ tools. +.. bibliography:: ../../references.bib diff --git a/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst b/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst new file mode 100644 index 0000000000..52365a86e5 --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst @@ -0,0 +1,320 @@ +.. _TechOverview: + +==================== +Technical Overview +==================== + +This chapter provides information on SRW App prerequistes, component code repositories, and SRW App directory structure. + +.. _SRWPrerequisites: + +Prerequisites for Using the SRW Application +=============================================== + +Background Knowledge Prerequisites +-------------------------------------- + +The instructions in this documentation assume that users have certain background knowledge: + +* Familiarity with LINUX/UNIX systems +* Command line basics +* System configuration knowledge (e.g., compilers, environment variables, paths, etc.) +* Numerical Weather Prediction (e.g., concepts of parameterizations: physical, microphysical, convective) +* Meteorology (in particular, meteorology at the scales being predicted: 25km, 13km, and 3km resolutions) + +Additional background knowledge in the following areas could be helpful: + +* High-Performance Computing (HPC) Systems (for those running the SRW App on an HPC system) +* Programming (particularly Python and bash scripting) for those interested in contributing to the SRW App code +* Creating an SSH Tunnel to access HPC systems from the command line +* Containerization +* Workflow Managers/Rocoto + +.. _software-prereqs: + +Software/Operating System Requirements +----------------------------------------- +The UFS SRW Application has been designed so that any sufficiently up-to-date machine with a UNIX-based operating system should be capable of running the application. SRW App :srw-wiki:`Level 1 & 2 systems ` already have these prerequisites installed. However, users working on other systems must ensure that the following requirements are installed on their system: + +**Minimum Platform Requirements:** + +* POSIX-compliant UNIX-style operating system + +* >97 GB disk space + + * 53 GB input data for a standard collection of global data, or "fix" file data (topography, climatology, observational data) for a short 12-hour test forecast on the :term:`CONUS` 25km domain. See data download instructions in :numref:`Section %s `. + * ~23 GB for full :term:`spack-stack` installation (or ~8 GB :term:`HPC-Stack`) + * 3 GB for ``ufs-srweather-app`` installation + * 1 GB for boundary conditions for a short 12-hour test forecast on the CONUS 25km domain. See data download instructions in :numref:`Section %s `. + * 17 GB for a 12-hour test forecast on the CONUS 25km domain, with model output saved hourly. + +* Fortran compiler released since 2018 + + * gfortran v9+ or ifort v18+ are the only ones tested, but others may work. + +* C and C++ compilers compatible with the Fortran compiler + + * gcc v9+, ifort v18+, and clang v9+ (macOS, native Apple clang, LLVM clang, GNU) have been tested + +* Python v3.7+ (preferably 3.9+), including prerequisite packages ``jinja2``, ``pyyaml``, and ``f90nml`` + + * Python packages ``scipy``, ``matplotlib``, ``pygrib``, ``cartopy``, and ``pillow`` are required for users who would like to use the provided graphics scripts. + +* Perl 5 + +* git v2.12+ + +* Lmod + +* wget + + * Only required for retrieving data using ``retrieve_data.py``. If data is prestaged, *wget* is not required. If data is retrieved using other means, *curl* may be used as an alternative. + +The following software is also required to run the SRW Application, but the :term:`spack-stack` (which contains the software libraries necessary for building and running the SRW App) can be configured to build these requirements: + +* CMake v3.20+ + +* :term:`MPI` (MPICH, OpenMPI, or other implementation) + + * Only **MPICH** or **OpenMPI** can be built with spack-stack. Other implementations must be installed separately by the user (if desired). + +For MacOS systems, some additional software packages are needed. When possible, it is recommended that users install and/or upgrade this software (along with software listed above) using the `Homebrew `__ package manager for MacOS. See :doc:`HPC-Stack Documentation: Chapter 3 ` for further guidance on installing these prerequisites on MacOS. + +* bash v4.x +* GNU compiler suite v11 or higher with gfortran +* cmake +* make +* coreutils +* gsed + +Optional but recommended prerequisites for all systems: + +* Conda for installing/managing Python packages +* Bash v4+ +* Rocoto Workflow Management System (1.3.1) +* Python packages ``scipy``, ``matplotlib``, ``pygrib``, ``cartopy``, and ``pillow`` for graphics + + +.. _SRWStructure: + +Code Repositories and Directory Structure +========================================= + +.. _HierarchicalRepoStr: + +Hierarchical Repository Structure +----------------------------------- +The :term:`umbrella repository` for the SRW Application is named ``ufs-srweather-app`` and is available on GitHub at https://github.com/ufs-community/ufs-srweather-app. The SRW Application uses the ``manage_externals`` tool and a configuration file called ``Externals.cfg`` to pull in the appropriate versions of the external repositories associated with the SRW App (see :numref:`Table %s `). + +.. _top_level_repos: + +.. list-table:: List of top-level repositories that comprise the UFS SRW Application + :widths: 20 40 + :header-rows: 1 + + * - Repository Description + - Authoritative repository URL + * - Umbrella repository for the UFS Short-Range Weather (SRW) Application + - https://github.com/ufs-community/ufs-srweather-app + * - Repository for the UFS Weather Model + - https://github.com/ufs-community/ufs-weather-model + * - Repository for UFS Utilities, including pre-processing, chgres_cube, and more + - https://github.com/ufs-community/UFS_UTILS + * - Repository for the Unified Post Processor (UPP) + - https://github.com/NOAA-EMC/UPP + * - Repository for Air Quality Modeling (AQM) Utilities + - https://github.com/NOAA-EMC/AQM-utils + * - Repository for the NOAA Emission and eXchange Unified System (NEXUS) + - https://github.com/noaa-oar-arl/NEXUS + * - Repository for the Unified Workflow (UW) Toolkit + - https://github.com/ufs-community/uwtools + +The UFS Weather Model contains a number of sub-repositories, which are documented :doc:`here `. + +.. note:: + The prerequisite libraries (including NCEP Libraries and external libraries) are **not** included in the UFS SRW Application repository. The `spack-stack `__ repository assembles these prerequisite libraries. Spack-stack has already been built on :srw-wiki:`preconfigured (Level 1) platforms `. However, it must be built on other systems. See the :doc:`spack-stack Documentation ` for details on installing spack-stack. + +.. _TopLevelDirStructure: + +Repository Structure +---------------------- +The ``ufs-srweather-app`` :term:`umbrella repository` is an NCO-compliant repository. Its structure follows the standards laid out in :term:`NCEP` Central Operations (NCO) WCOSS :nco:`Implementation Standards `. This structure is implemented using the ``local_path`` settings contained within the ``Externals.cfg`` file. After ``manage_externals/checkout_externals`` is run (see :numref:`Section %s `), the specific GitHub repositories described in :numref:`Table %s ` are cloned into the target subdirectories shown below. Directories that will be created as part of the build process appear in parentheses and will not be visible until after the build is complete. Some directories have been removed for brevity. + +.. code-block:: console + + ufs-srweather-app + ├── (build) + ├── docs + │ └── UsersGuide + ├── etc + ├── (exec) + ├── (include) + ├── jobs + ├── (lib) + ├── manage_externals + ├── modulefiles + │ ├── build__.lua + │ └── wflow_.lua + ├── parm + │ ├── wflow + │ │ └── default_workflow.yaml + │ └── FV3LAM_wflow.xml + ├── (share) + ├── scripts + ├── sorc + │ ├── CMakeLists.txt + │ ├── (UPP) + │ │ ├── parm + │ │ └── sorc + │ │ └── ncep_post.fd + │ ├── (UFS_UTILS) + │ │ ├── sorc + │ │ │ ├── chgres_cube.fd + │ │ │ ├── sfc_climo_gen.fd + │ │ │ └── vcoord_gen.fd + │ │ └── ush + │ └── (ufs-weather-model) + │ └── FV3 + │ ├── atmos_cubed_sphere + │ └── ccpp + ├── tests/WE2E + │ └── run_WE2E_tests.py + ├── ush + │ ├── bash_utils + │ ├── machine + │ ├── wrappers + │ ├── config.community.yaml + │ ├── config_defaults.yaml + │ ├── generate_FV3LAM_wflow.py + │ ├── launch_FV3LAM_wflow.sh + │ ├── setup.py + │ └── valid_param_vals.yaml + └── versions + +SRW App SubDirectories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:numref:`Table %s ` describes the contents of the most important SRW App subdirectories. :numref:`Table %s ` provides a more comprehensive explanation of the ``ufs-srweather-app`` files and subdirectories. Users can reference the :nco:`NCO Implementation Standards ` (p. 19) for additional details on repository structure in NCO-compliant repositories. + +.. _Subdirectories: + +.. list-table:: *Subdirectories of the ufs-srweather-app repository* + :widths: 20 50 + :header-rows: 1 + + * - Directory Name + - Description + * - docs + - Repository documentation + * - jobs + - :term:`J-job ` scripts launched by Rocoto + * - modulefiles + - Files used to load modules needed for building and running the workflow + * - parm + - Parameter files used to configure the model, physics, workflow, and various SRW App components + * - scripts + - Scripts launched by the J-jobs + * - sorc + - External source code used to build the SRW App + * - tests + - Tests for baseline experiment configurations + * - ush + - Utility scripts used by the workflow + +.. _ExperimentDirSection: + +Experiment Directory Structure +-------------------------------- +When the user generates an experiment using the ``generate_FV3LAM_wflow.py`` script (:numref:`Step %s `), a user-defined experiment directory (``$EXPTDIR``) is created based on information specified in the ``config.yaml`` file. :numref:`Table %s ` shows the contents of the experiment directory before running the experiment workflow. + +.. _ExptDirStructure: + +.. list-table:: *Files and subdirectory initially created in the experiment directory* + :widths: 33 67 + :header-rows: 1 + + * - File Name + - Description + * - config.yaml + - User-specified configuration file, see :numref:`Section %s ` + * - data_table + - :term:`Cycle-independent` input file (empty) + * - field_table + - :term:`Tracers ` in the :ref:`forecast model ` + * - FV3LAM_wflow.xml + - Rocoto XML file to run the workflow + * - input.nml + - :term:`Namelist` for the :ref:`UFS Weather Model ` + * - launch_FV3LAM_wflow.sh + - Symlink to the ``ufs-srweather-app/ush/launch_FV3LAM_wflow.sh`` shell script, + which can be used to (re)launch the Rocoto workflow. Each time this script is + called, it appends information to a log file named ``log.launch_FV3LAM_wflow``. + * - log.generate_FV3LAM_wflow + - Log of the output from the experiment generation script (``generate_FV3LAM_wflow.py``) + * - nems.configure + - See :ref:`NEMS configuration file ` + * - suite_{CCPP}.xml + - :term:`CCPP` suite definition file (:term:`SDF`) used by the forecast model + * - var_defns.sh + - Shell script defining the experiment parameters. It contains all of the primary + parameters specified in the default and user-specified configuration files plus + many secondary parameters that are derived from the primary ones by the + experiment generation script. This file is sourced by various other scripts in + order to make all the experiment variables available to these scripts. + * - YYYYMMDDHH + - Cycle directory (empty) + +In addition, running the SRW App in *community* mode creates the ``fix_am`` and ``fix_lam`` directories (see :numref:`Table %s `) in ``$EXPTDIR``. The ``fix_lam`` directory is initially empty but will contain some *fix* (time-independent) files after the grid, orography, and/or surface climatology generation tasks run. + +.. _FixDirectories: + +.. list-table:: *Description of the fix directories* + :widths: 33 67 + :header-rows: 1 + + * - Directory Name + - Description + * - fix_am + - Directory containing the global fix (time-independent) data files. The + experiment generation script symlinks these files from a machine-dependent + system directory. + * - fix_lam + - Directory containing the regional fix (time-independent) data files that + describe the regional grid, orography, and various surface climatology fields, + as well as symlinks to pre-generated files. + +Once the Rocoto workflow is launched, several files and directories are generated. A log file named ``log.launch_FV3LAM_wflow`` will be created (unless it already exists) in ``$EXPTDIR``. The first several workflow tasks (i.e., ``make_grid``, ``make_orog``, ``make_sfc_climo``, ``get_extrn_ics``, and ``get_extrn_lbcs``) are preprocessing tasks, and these tasks also result in the creation of new files and subdirectories, described in :numref:`Table %s `. + +.. _CreatedByWorkflow: + +.. list-table:: *New directories and files created when the workflow is launched* + :widths: 30 70 + :header-rows: 1 + + * - Directory/File Name + - Description + * - YYYYMMDDHH + - This is a “cycle directory” that is updated when the first cycle-specific + workflow tasks (``get_extrn_ics`` and ``get_extrn_lbcs``) are run. These tasks + are launched simultaneously for each cycle in the experiment. Cycle directories + are created to contain cycle-specific files for each cycle that the experiment + runs. If ``DATE_FIRST_CYCL`` and ``DATE_LAST_CYCL`` are different in the + ``config.yaml`` file, more than one cycle directory will be created under the + experiment directory. + * - grid + - Directory generated by the ``make_grid`` task to store grid files for the experiment + * - log + - Contains log files generated by the overall workflow and by its various tasks. View the files in this directory to determine why a task may have failed. + * - orog + - Directory generated by the ``make_orog`` task containing the orography files for the experiment + * - sfc_climo + - Directory generated by the ``make_sfc_climo`` task containing the surface climatology files for the experiment + * - FV3LAM_wflow.db + + FV3LAM_wflow_lock.db + - Database files that are generated when Rocoto is called (by the launch script) to launch the workflow + * - log.launch_FV3LAM_wflow + - The ``launch_FV3LAM_wflow.sh`` script appends its output to this log file each time it is called. View the last several lines of this file to check the status of the workflow. + +The output files for an experiment are described in :numref:`Section %s `. +The workflow tasks are described in :numref:`Section %s `. + diff --git a/docs/UsersGuide/source/BackgroundInfo/index.rst b/doc/UsersGuide/BackgroundInfo/index.rst similarity index 100% rename from docs/UsersGuide/source/BackgroundInfo/index.rst rename to doc/UsersGuide/BackgroundInfo/index.rst diff --git a/docs/UsersGuide/source/BuildingRunningTesting/AQM.rst b/doc/UsersGuide/BuildingRunningTesting/AQM.rst similarity index 93% rename from docs/UsersGuide/source/BuildingRunningTesting/AQM.rst rename to doc/UsersGuide/BuildingRunningTesting/AQM.rst index 4c83bcee2b..6d2ae0f193 100644 --- a/docs/UsersGuide/source/BuildingRunningTesting/AQM.rst +++ b/doc/UsersGuide/BuildingRunningTesting/AQM.rst @@ -4,6 +4,10 @@ Air Quality Modeling (SRW-AQM) ===================================== +.. attention:: + + AQM capabilities are an unsupported feature of the SRW App. This means that it is available for users to experiment with, but assistance for AQM-related issues is limited. + The standard SRW App distribution uses the uncoupled version of the UFS Weather Model (atmosphere-only). However, users have the option to use a coupled version of the SRW App that includes the standard distribution (atmospheric model) plus the Air Quality Model (AQM). The AQM is a UFS Application that dynamically couples the Community Multiscale Air Quality (:term:`CMAQ`) model with the UFS Weather Model (WM) through the :term:`NUOPC` Layer to simulate temporal and spatial variations of atmospheric compositions (e.g., ozone and aerosol compositions). The CMAQ model, treated as a column chemistry model, updates concentrations of chemical species (e.g., ozone and aerosol compositions) at each integration time step. The transport terms (e.g., :term:`advection` and diffusion) of all chemical species are handled by the UFS WM as tracers. @@ -12,17 +16,17 @@ The AQM is a UFS Application that dynamically couples the Community Multiscale A Although this chapter is the primary documentation resource for running the AQM configuration, users may need to refer to :numref:`Chapter %s ` and :numref:`Chapter %s ` for additional information on building and running the SRW App, respectively. +Quick Start Guide (SRW-AQM) +===================================== + .. attention:: These instructions should work smoothly on Hera and WCOSS2, but users on other systems may need to make additional adjustments. -Quick Start Guide (SRW-AQM) -===================================== - Download the Code ------------------- -Clone the ``develop`` branch of the authoritative SRW App repository: +Clone the |branch| branch of the authoritative SRW App repository: .. code-block:: console @@ -49,7 +53,7 @@ On Hera and WCOSS2, users can build the SRW App AQM binaries with the following where ```` is ``hera``, or ``wcoss2``. The ``-a`` argument indicates the configuration/version of the application to build. -Building the SRW App with AQM on other machines, including other `Level 1 `__ platforms, is not currently guaranteed to work, and users may have to make adjustments to the modulefiles for their system. +Building the SRW App with AQM on other machines, including other :srw-wiki:`Level 1 ` platforms, is not currently guaranteed to work, and users may have to make adjustments to the modulefiles for their system. If the SRW-AQM builds correctly, users should see the standard executables listed in :numref:`Table %s `. Additionally, users should see the AQM-specific executables described in :numref:`Table %s ` in the ``ufs-srweather-app/exec`` directory. @@ -68,7 +72,7 @@ If the SRW-AQM builds correctly, users should see the standard executables liste * - nexus - Runs the NOAA Emission and eXchange Unified System (:ref:`NEXUS `) emissions processing system -Load the ``workflow_tools`` Environment +Load the |wflow_env| Environment -------------------------------------------- Load the python environment for the workflow: @@ -88,9 +92,9 @@ If the console outputs a message, the user should run the commands specified in .. code-block:: console Please do the following to activate conda: - > conda activate workflow_tools + > conda activate srw_app -then the user should run ``conda activate workflow_tools``. Otherwise, the user can continue with configuring the workflow. +then the user should run |activate|. Otherwise, the user can continue with configuring the workflow. .. _AQMConfig: @@ -131,7 +135,7 @@ Users may also wish to change :term:`cron`-related parameters in ``config.yaml`` This means that cron will submit the launch script every 3 minutes. Users may choose not to submit using cron or to submit at a different frequency. Note that users should create a crontab by running ``crontab -e`` the first time they use cron. -When using the basic ``config.aqm.community.yaml`` experiment, the AQM pre-processing tasks are automatically turned because ``"parm/wflow/aqm_prep.yaml"`` appears in the list of workflow files in the ``rocoto: tasks: taskgroups:`` section of ``config.yaml`` (see :numref:`Section %s ` for task descriptions). To turn on AQM post-processing tasks in the workflow, include ``"parm/wflow/aqm_post.yaml"`` in the ``rocoto: tasks: taskgroups:`` section, too (see :numref:`Section %s ` for task descriptions). +When using the basic ``config.aqm.community.yaml`` experiment, the AQM pre-processing tasks are automatically turned on because ``"parm/wflow/aqm_prep.yaml"`` appears in the list of workflow files in the ``rocoto: tasks: taskgroups:`` section of ``config.yaml`` (see :numref:`Section %s ` for task descriptions). To turn on AQM *post*-processing tasks in the workflow, include ``"parm/wflow/aqm_post.yaml"`` in the ``rocoto: tasks: taskgroups:`` section, too (see :numref:`Section %s ` for task descriptions). .. attention:: @@ -153,10 +157,10 @@ If ``USE_CRON_TO_RELAUNCH`` is set to true in ``config.yaml`` (see :numref:`Sect .. code-block:: console - cd / + cd ${EXPT_BASEDIR}/${EXPT_SUBDIR} ./launch_FV3LAM_wflow.sh -Repeat the launch command regularly until a SUCCESS or FAILURE message appears on the terminal window. See :numref:`Section %s ` for more on the ```` and ```` variables. +Repeat the launch command regularly until a SUCCESS or FAILURE message appears on the terminal window. See :numref:`Section %s ` for more on the ``${EXPT_BASEDIR}`` and ``${EXPT_SUBDIR}`` variables. Users may check experiment status from the experiment directory with either of the following commands: @@ -235,7 +239,7 @@ Structure of SRW-AQM Workflow .. _FlowProcAQM: -.. figure:: https://github.com/ufs-community/ufs-srweather-app/wiki/SRW-AQM_workflow.png +.. figure:: https://github.com/ufs-community/ufs-srweather-app/wiki/WorkflowImages/SRW-AQM_workflow.png :alt: Flowchart of the SRW-AQM tasks. *Workflow Structure of SRW-AQM (non-DA)* @@ -307,6 +311,7 @@ Add the WE2E test for AQM to the list file: .. code-block:: console + cd /path/to/ufs-srweather-app/tests/WE2E echo "custom_ESGgrid" > my_tests.txt echo "aqm_grid_AQM_NA13km_suite_GFS_v16" >> my_tests.txt diff --git a/docs/UsersGuide/source/BuildingRunningTesting/BuildSRW.rst b/doc/UsersGuide/BuildingRunningTesting/BuildSRW.rst similarity index 61% rename from docs/UsersGuide/source/BuildingRunningTesting/BuildSRW.rst rename to doc/UsersGuide/BuildingRunningTesting/BuildSRW.rst index 5b871fe87f..3076a5f6eb 100644 --- a/docs/UsersGuide/source/BuildingRunningTesting/BuildSRW.rst +++ b/doc/UsersGuide/BuildingRunningTesting/BuildSRW.rst @@ -8,46 +8,53 @@ The Unified Forecast System (:term:`UFS`) Short-Range Weather (SRW) Application .. attention:: - The SRW Application has `four levels of support `__. The steps described in this chapter will work most smoothly on preconfigured (Level 1) systems. This chapter also provides guidance for running the SRW App on other systems (including generic Linux/Mac systems), but the user may need to perform additional steps and/or troubleshooting. + The SRW Application has :srw-wiki:`four levels of support `. The steps described in this chapter will work most smoothly on preconfigured (Level 1) systems. This chapter also provides guidance for running the SRW App on other systems (including generic Linux/Mac systems), but the user may need to perform additional steps and/or troubleshooting. .. note:: The :ref:`container approach ` is recommended for a smoother first-time build and run experience. Building without a container may allow for more customization. However, the non-container approach requires more in-depth system-based knowledge, especially on Level 3 and 4 systems, so it is less appropriate for beginners. To build the SRW App, users will complete the following steps: - #. :ref:`Install prerequisites ` + #. :ref:`Install prerequisites ` #. :ref:`Clone the SRW App from GitHub ` #. :ref:`Check out the external repositories ` #. :ref:`Set up the build environment and build the executables ` .. _AppBuildProc: -.. figure:: https://github.com/ufs-community/ufs-srweather-app/wiki/SRW_build_process.png +.. figure:: https://github.com/ufs-community/ufs-srweather-app/wiki/WorkflowImages/SRW_build_process.png :alt: Flowchart describing the SRW App build process. *Overview of the SRW App Build Process* -.. _HPCstackInfo: +.. _StackInfo: Install the Prerequisite Software Stack ========================================== Users on any sufficiently up-to-date machine with a UNIX-based operating system should be able to install the prerequisite software stack and run the SRW Application. However, a list of prerequisites is available in :numref:`Section %s ` for reference. Users should install or update their system as required before attempting to install the software stack. -Currently, installation of the prerequisite software stack is supported via HPC-Stack. :term:`HPC-Stack` is a :term:`repository` that provides a unified, shell script-based system to build the software stack required for `UFS `__ applications such as the SRW App. +Currently, installation of the prerequisite software stack is supported via spack-stack on most systems. :term:`Spack-stack` is a :term:`repository` that provides a Spack-based system to build the software stack required for `UFS `__ applications such as the SRW App. Spack-stack is the software stack validated by the UFS Weather Model (:term:`WM`), and the SRW App has likewise shifted to spack-stack for most Level 1 systems. -.. Attention:: - Skip the HPC-Stack installation if working on a `Level 1 system `__ (e.g., Cheyenne, Hera, Orion, NOAA Cloud), and :ref:`continue to the next section `. +.. hint:: + Skip the spack-stack installation if working on a :srw-wiki:`Level 1 system ` (e.g., Hera, Jet, Derecho, NOAA Cloud), and :ref:`continue to the next section `. Background ---------------- -SRW App components, including the UFS Weather Model (:term:`WM `), draw on over 50 code libraries to run. These libraries range from libraries developed in-house at NOAA (e.g., NCEPLIBS, FMS) to libraries developed by NOAA's partners (e.g., PIO, ESMF) to truly third party libraries (e.g., netCDF). Individual installation of these libraries is not practical, so the `HPC-Stack `__ was developed as a central installation system to ensure that the infrastructure environment across multiple platforms is as similar as possible. Installation of the HPC-Stack (or :term:`spack-stack`) is required to run the SRW App. +SRW App components, including the UFS :term:`WM`, draw on over 50 code libraries to run. These libraries range from libraries developed in-house at NOAA (e.g., NCEPLIBS, FMS) to libraries developed by NOAA's partners (e.g., PIO, ESMF) to truly third-party libraries (e.g., netCDF). Individual installation of these libraries is not practical, so `spack-stack `__ was developed as a central installation system to ensure that the infrastructure environment across multiple platforms is as similar as possible. Installation of spack-stack (or its predecessor, :term:`HPC-Stack`) is required to run the SRW App. Instructions ------------------------- -Users working on systems that fall under `Support Levels 2-4 `__ will need to install the HPC-Stack the first time they try to build applications (such as the SRW App) that depend on it. Users can either build the HPC-Stack on their local system or use the centrally maintained stacks on each HPC platform if they are working on a Level 1 system. Before installing the HPC-Stack, users on both Linux and MacOS systems should set the stack size to "unlimited" (if allowed) or to the largest possible value: + +.. attention:: + + Spack-stack is the fully-supported software stack validated by the UFS WM as of `PR #1707 `__ on August 24, 2023. UFS applications are therefore shifting to :term:`spack-stack`, too. When all systems have shifted to spack-stack, support for HPC-Stack will be deprecated. Users are encouraged to check out `spack-stack `__ to prepare for this shift in support from HPC-Stack to spack-stack even if their system currently has support for HPC-Stack. + + As of the v2.2.0 release, spack-stack is supported in the SRW App on most Level 1 systems with the exception of Derecho, which uses HPC-Stack. Transition to spack-stack is underway for Derecho. Users on generic MacOS and Linux systems will find HPC-Stack-based modulefiles in the v2.2.0 release but can expect that these will also shift to spack-stack in the ``develop`` branch in the coming months. + +Users working on systems that fall under :srw-wiki:`Support Levels 2-4 ` will need to install spack-stack or HPC-Stack the first time they try to build applications (such as the SRW App) that depend on it. Users can build the stack on their local system or use the centrally maintained stacks on each HPC platform if they are working on a Level 1 system. Before installing spack-stack or HPC-Stack, users on both Linux and MacOS systems should set the stack size to "unlimited" (if allowed) or to the largest possible value: .. code-block:: console @@ -57,10 +64,7 @@ Users working on systems that fall under `Support Levels 2-4 `. - -.. attention:: - Although HPC-Stack is currently the fully-supported software stack option, UFS applications are gradually shifting to :term:`spack-stack`, which is a :term:`Spack`-based method for installing UFS prerequisite software libraries. The spack-stack is currently used on NOAA Cloud platforms and in containers, while HPC-Stack is still used on other Level 1 systems and is the software stack validated by the UFS Weather Model. Users are encouraged to check out `spack-stack `__ to prepare for the upcoming shift in support from HPC-Stack to spack-stack. +For a detailed description of installation options, see :doc:`spack-stack instructions for configuring the stack on a new platform ` or :ref:`HPC-Stack installation instructions `. After completing installation, continue to the :ref:`next section ` to download the UFS SRW Application Code. @@ -68,7 +72,7 @@ After completing installation, continue to the :ref:`next section ` of this User's Guide. Each component has its own repository. Users must run the ``checkout_externals`` script to collect the individual components of the SRW App from their respective GitHub repositories. The ``checkout_externals`` script uses the configuration file ``Externals.cfg`` in the top level directory of the SRW App to clone the correct tags (code versions) of the external repositories listed in :numref:`Section %s ` into the appropriate directories (e.g., ``ush``, ``sorc``). +The SRW App relies on a variety of components (e.g., UFS_UTILS, ufs-weather-model, and UPP) detailed in :numref:`Section %s ` of this User's Guide. Each component has its own repository. Users must run the ``checkout_externals`` script to collect the individual components of the SRW App from their respective GitHub repositories. The ``checkout_externals`` script uses the configuration file ``Externals.cfg`` in the top-level directory of the SRW App to clone the correct tags (code versions) of the external repositories listed in :numref:`Section %s ` into the appropriate directories (e.g., ``ush``, ``sorc``). Run the executable that pulls in SRW App components from external repositories: @@ -174,6 +155,17 @@ Run the executable that pulls in SRW App components from external repositories: The script should output dialogue indicating that it is retrieving different code repositories. It may take several minutes to download these repositories. +.. hint:: + + Some systems (e.g., Hercules, Gaea) may have difficulty finding prerequisite software, such as python. If users run into this issue but know that the software exists on their system, they can run ``module load `` followed by ``module save``. For example: + + .. code-block:: console + + /usr/bin/env: ‘python’: No such file or directory + hercules-login-1[10] username$ module load python + hercules-login-1[11] username$ module save + Saved current collection of modules to: "default", for system: "hercules" + To see more options for the ``checkout_externals`` script, users can run ``./manage_externals/checkout_externals -h``. For example: * ``-S``: Outputs the status of the repositories managed by ``checkout_externals``. By default, only summary information is provided. Use with the ``-v`` (verbose) option to see details. @@ -198,7 +190,20 @@ On Level 1 systems for which a modulefile is provided under the ``modulefiles`` ./devbuild.sh --platform= -where ```` is replaced with the name of the platform the user is working on. Valid values include: ``cheyenne`` | ``gaea`` | ``hera`` | ``jet`` | ``linux`` | ``macos`` | ``noaacloud`` | ``orion`` +where ```` is replaced with the name of the platform the user is working on. See :numref:`Section %s ` for all valid ``MACHINE`` options. + +Directly following the release of SRW v2.2.0, the App will install miniconda and SRW environments as part +of the build process. The location defaults to inside the SRW clone in ``ufs-srweather-app/conda``, +however users can set any path on their system using the ``--conda-dir`` flag. If conda is already +installed in that location, conda installation will be skipped. The following example uses a +pre-installed conda installation at ``/path/to/conda`` + +.. code-block:: console + + ./devbuild.sh --platform= --conda-dir /path/to/conda + +Running ``./devbuild.sh`` without any arguments will show the usage statement for all available +flags and targets for this script. .. note:: Although build modulefiles exist for generic Linux and MacOS machines, users will need to alter these according to the instructions in Sections :numref:`%s ` & :numref:`%s `. Users on these systems may have more success building the SRW App with the :ref:`CMake Approach ` instead. @@ -211,9 +216,15 @@ If compiler auto-detection fails for some reason, specify it using the ``--compi where valid values are ``intel`` or ``gnu``. -The last line of the console output should be ``[100%] Built target ufs-weather-model``, indicating that the UFS Weather Model executable has been built successfully. +The last few lines of the console output should include ``[100%] Built target ufs-weather-model``, indicating that the UFS Weather Model executable has been built successfully. + +After running ``devbuild.sh``, the executables listed in :numref:`Table %s ` should appear in the ``ufs-srweather-app/exec`` directory. If the application built properly, users may configure and run an experiment. Users have a few options: + +#. Proceed to :numref:`Section %s: Quick Start Guide ` for a quick overview of the workflow steps. +#. Try the :ref:`SRW App Tutorials ` (good for new users!). +#. For detailed information on running the SRW App, including optional tasks like plotting and verification, users can refer to :numref:`Section %s: Running the SRW App `. -After running ``devbuild.sh``, the executables listed in :numref:`Table %s ` should appear in the ``ufs-srweather-app/exec`` directory. If the ``devbuild.sh`` build method does not work, or if users are not on a supported machine, they will have to manually set up the environment and build the SRW App binaries with CMake as described in :numref:`Section %s `. +If the ``devbuild.sh`` build method did *not* work, or if users are not on a supported machine, they will have to manually set up the environment and build the SRW App binaries with CMake as described in :numref:`Section %s `. .. _ExecDescription: @@ -287,7 +298,7 @@ Set Up the Build Environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. attention:: - * If users successfully built the executables in :numref:`Table %s `, they should skip to step :numref:`Chapter %s: Running the SRW App `. + * If users successfully built the executables listed in :numref:`Table %s `, they can skip to step :numref:`Section %s: Running the SRW App `. * Users who want to build the SRW App on MacOS or generic Linux systems should skip to :numref:`Section %s ` and follow the approach there. If the ``devbuild.sh`` approach failed, users need to set up their environment to run a workflow on their specific platform. First, users should make sure ``Lmod`` is the app used for loading modulefiles. This is the case on most Level 1 systems; however, on systems such as Gaea/Odin, the default modulefile loader is from Cray and must be switched to Lmod. For example, on Gaea, users can run one of the following two commands depending on whether they have a bash or csh shell, respectively: @@ -305,10 +316,10 @@ From here, ``Lmod`` is ready to load the modulefiles needed by the SRW App. Thes .. code-block:: console - module use /path/to/modulefiles + module use /path/to/ufs-srweather-app/modulefiles module load build__ -where ``/path/to/modulefiles/`` is the full path to the ``modulefiles`` directory. +where ``/path/to/ufs-srweather-app/modulefiles/`` is the full path to the ``modulefiles`` directory. This will work on Level 1 systems, where a modulefile is available in the ``modulefiles`` directory. Users on Level 2-4 systems (including generic Linux/MacOS systems) will need to modify an appropriate ``build__`` modulefile. One of the current ``build__`` modulefiles can be copied and used as a template. However, users will need to adjust certain environment variables in their modulefile, such as the path to HPC-Stack, so that the SRW App can find and load the appropriate modules. @@ -342,7 +353,7 @@ From the build directory, run the following commands to build the pre-processing cmake .. -DCMAKE_INSTALL_PREFIX=.. -DCMAKE_INSTALL_BINDIR=exec .. make -j 4 >& build.out & -``-DCMAKE_INSTALL_PREFIX`` specifies the location where the ``exec``, ``include``, ``lib``, and ``share`` directories will be created. These directories will contain various components of the SRW App. Its recommended value ``..`` denotes one directory up from the build directory. In the next line, the ``make`` argument ``-j 4`` indicates that the build will run in parallel with four threads. Although users can specify a larger or smaller number of threads (e.g., ``-j 8``, ``-j 2``), it is highly recommended to use at least four parallel threads to prevent overly long installation times. +``-DCMAKE_INSTALL_PREFIX`` specifies the location where the ``exec``, ``include``, ``lib``, and ``share`` directories will be created. These directories will contain various components of the SRW App. Its recommended value ``..`` denotes one directory up from the ``build`` directory. In the next line, the ``make`` argument ``-j 4`` indicates that the build will run in parallel with four threads. Although users can specify a larger or smaller number of threads (e.g., ``-j 8``, ``-j 2``), it is highly recommended to use at least four parallel threads to prevent overly long installation times. The build will take a few minutes to complete. When it starts, a random number is printed to the console, and when it is done, a ``[1]+ Done`` message is printed to the console. ``[1]+ Exit`` indicates an error. Output from the build will be in the ``ufs-srweather-app/build/build.out`` file. When the build completes, users should see the forecast model executable ``ufs_model`` and several pre- and post-processing executables in the ``ufs-srweather-app/exec`` directory. These executables are described in :numref:`Table %s `. @@ -356,17 +367,16 @@ Additional Details for Building on MacOS or Generic Linux ------------------------------------------------------------ .. note:: - Users who are **not** building the SRW App on MacOS or generic Linux platforms may skip to :numref:`Section %s ` to finish building the SRW App or continue to :numref:`Section %s ` to configure and run an experiment. + Users who are **not** building the SRW App on MacOS or generic Linux platforms may skip to :numref:`Section %s ` to finish building the SRW App or continue to :numref:`Section %s ` to configure and run an experiment if they have already built the App. The SRW App can be built on MacOS and generic Linux machines after the prerequisite software has been installed on these systems (via :term:`HPC-Stack` or :term:`spack-stack`). The installation for MacOS is architecture-independent and has been tested using both x86_64 and M1 chips (running natively). The following configurations for MacOS have been tested: - #. MacBookPro 2019, 2.4 GHz 8-core Intel Core i9 (x86_64), Monterey Sur 12.1, GNU compiler suite v.11.3.0 (gcc, gfortran, g++); mpich 3.3.2 or openmpi/4.1.2 - #. MacBookAir 2020, M1 chip (arm64, running natively), 4+4 cores, Big Sur 11.6.4, GNU compiler suite v.11.3.0 (gcc, gfortran, g++); mpich 3.3.2 or openmpi/4.1.2 - #. MacBook Pro 2015, 2.8 GHz Quad-Core Intel Core i7 (x86_64), Catalina OS X 10.15.7, GNU compiler suite v.11.2.0_3 (gcc, gfortran, g++); mpich 3.3.2 or openmpi/4.1.2 + #. MacBookPro 2019, 2.4 GHz 8-core Intel Core i9 (x86_64), OS Monterey 12.6.1, 32 GB RAM; GNU compiler suite v.12.3.0 (gcc, gfortran, g++); openmpi/4.1.5 + #. MacBookAir 2020, M1 chip (arm64, running natively), 4+4 cores, OS Ventura 13.0.1, 16 GB RAM; GNU compiler suite v.12.3.0 (gcc, gfortran, g++); openmpi/4.1.5 Several Linux builds have been tested on systems with x86_64 architectures. -The ``$SRW/modulefiles/build__gnu.lua`` modulefile (where ```` is ``macos`` or ``linux``) is written as a Lmod module in the Lua language. It can be loaded once the Lmod module environment has been initialized (which should have happened even prior to :ref:`installing HPC-Stack `). The ``build__gnu`` modulefile lists the location of the HPC-Stack modules, loads the meta-modules and modules, sets serial and parallel compilers, additional flags, and any environment variables needed for building the SRW App. The modulefile must be modified to include the absolute path to the user's HPC-Stack installation: +The ``$SRW/modulefiles/build__gnu.lua`` modulefile (where ```` is ``macos`` or ``linux``) is written as a Lmod module in the Lua language. It can be loaded once the Lmod module environment has been initialized (which should have happened even prior to :ref:`installing HPC-Stack `). The ``build__gnu`` modulefile lists the location of the HPC-Stack modules, loads the meta-modules and modules, sets serial and parallel compilers, additional flags, and any environment variables needed for building the SRW App. The modulefile must be modified to include the absolute path to the user's HPC-Stack installation: .. code-block:: console @@ -391,4 +401,8 @@ Proceed to building the executables using the process outlined in :numref:`Step Run an Experiment ===================== -To configure and run an experiment, users should proceed to :numref:`Chapter %s `. +To configure and run an experiment, users have a few options: + +#. Proceed to :numref:`Section %s: Quick Start Guide ` for a quick overview of the workflow steps. +#. Try the :ref:`SRW App Tutorials ` (good for new users!). +#. For detailed information on running the SRW App, including optional tasks like plotting and verification, users can refer to :numref:`Section %s: Running the SRW App `. diff --git a/docs/UsersGuide/source/BuildingRunningTesting/ContainerQuickstart.rst b/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst similarity index 78% rename from docs/UsersGuide/source/BuildingRunningTesting/ContainerQuickstart.rst rename to doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst index bcc17e73ce..9e4f58f0bd 100644 --- a/docs/UsersGuide/source/BuildingRunningTesting/ContainerQuickstart.rst +++ b/doc/UsersGuide/BuildingRunningTesting/ContainerQuickstart.rst @@ -10,7 +10,7 @@ The basic "out-of-the-box" case described in this User's Guide builds a weather .. attention:: - * The SRW Application has `four levels of support `__. The steps described in this chapter will work most smoothly on preconfigured (Level 1) systems. However, this guide can serve as a starting point for running the SRW App on other systems, too. + * The SRW Application has :srw-wiki:`four levels of support `. The steps described in this chapter will work most smoothly on preconfigured (Level 1) systems. However, this guide can serve as a starting point for running the SRW App on other systems, too. * This chapter of the User's Guide should **only** be used for container builds. For non-container builds, see :numref:`Section %s ` for a Quick Start Guide or :numref:`Section %s ` for a detailed guide to building the SRW App **without** a container. .. _DownloadCodeC: @@ -21,78 +21,73 @@ Download the Container Prerequisites ------------------- -Users must have an **Intel** compiler and :term:`MPI` (available for free `here `__) in order to run the SRW App in the container provided using the method described in this chapter. Additionally, it is recommended that users install the `Rocoto workflow manager `__ on their system in order to take advantage of automated workflow options. Although it is possible to run an experiment without Rocoto, and some tips are provided, the only fully-supported and tested container option assumes that Rocoto is pre-installed. +**Intel Compiler and MPI** -Install Singularity/Apptainer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Users must have an **Intel** compiler and :term:`MPI` (`available for free here `__) in order to run the SRW App in the container provided using the method described in this chapter. Additionally, it is recommended that users install the `Rocoto workflow manager `__ on their system in order to take advantage of automated workflow options. Although it is possible to run an experiment without Rocoto, and some tips are provided, the only fully-supported and tested container option assumes that Rocoto is preinstalled. -.. note:: +**Install Singularity/Apptainer** + +To build and run the SRW App using a Singularity/Apptainer container, first install the software according to the `Apptainer Installation Guide `__. This will include the installation of all dependencies. - As of November 2021, the Linux-supported version of Singularity has been `renamed `__ to *Apptainer*. Apptainer has maintained compatibility with Singularity, so ``singularity`` commands should work with either Singularity or Apptainer (see compatibility details `here `__.) +.. note:: -To build and run the SRW App using a Singularity/Apptainer container, first install the software according to the `Apptainer Installation Guide `__. This will include the installation of all dependencies. + As of November 2021, the Linux-supported version of Singularity has been `renamed `__ to *Apptainer*. Apptainer has maintained compatibility with Singularity, so ``singularity`` commands should work with either Singularity or Apptainer (see compatibility details `here `__.) -.. warning:: +.. attention:: Docker containers can only be run with root privileges, and users cannot have root privileges on :term:`HPCs `. Therefore, it is not possible to build the SRW App, which uses the spack-stack, inside a Docker container on an HPC system. However, a Singularity/Apptainer image may be built directly from a Docker image for use on the system. -.. COMMENT: Update reference to HPC-Stack --> spack-stack? +.. _work-on-hpc: Working in the Cloud or on HPC Systems ----------------------------------------- -For users working on systems with limited disk space in their ``/home`` directory, it is recommended to set the ``SINGULARITY_CACHEDIR`` and ``SINGULARITY_TMPDIR`` environment variables to point to a location with adequate disk space. For example: +Users working on systems with limited disk space in their ``/home`` directory may need to set the ``SINGULARITY_CACHEDIR`` and ``SINGULARITY_TMPDIR`` environment variables to point to a location with adequate disk space. For example: .. code-block:: export SINGULARITY_CACHEDIR=/absolute/path/to/writable/directory/cache export SINGULARITY_TMPDIR=/absolute/path/to/writable/directory/tmp -where ``/absolute/path/to/writable/directory/`` refers to a writable directory (usually a project or user directory within ``/lustre``, ``/work``, ``/scratch``, or ``/glade`` on NOAA Level 1 systems). If the ``cache`` and ``tmp`` directories do not exist already, they must be created with a ``mkdir`` command. - -On NOAA Cloud systems, the ``sudo su`` command may also be required: - -.. code-block:: - - mkdir /lustre/cache - mkdir /lustre/tmp - sudo su - export SINGULARITY_CACHEDIR=/lustre/cache - export SINGULARITY_TMPDIR=/lustre/tmp - exit - -.. note:: - ``/lustre`` is a fast but non-persistent file system used on NOAA Cloud systems. To retain work completed in this directory, `tar the files `__ and move them to the ``/contrib`` directory, which is much slower but persistent. +where ``/absolute/path/to/writable/directory/`` refers to the absolute path to a writable directory with sufficient disk space. If the ``cache`` and ``tmp`` directories do not exist already, they must be created with a ``mkdir`` command. See :numref:`Section %s ` to view an example of how this can be done. .. _BuildC: Build the Container ------------------------ +* :ref:`On Level 1 Systems ` (see :srw-wiki:`list `) +* :ref:`On Level 2-4 Systems ` + .. hint:: If a ``singularity: command not found`` error message appears when working on Level 1 platforms, try running: ``module load singularity`` or (on Derecho) ``module load apptainer``. +.. _container-L1: + Level 1 Systems ^^^^^^^^^^^^^^^^^^ On most Level 1 systems, a container named ``ubuntu20.04-intel-ue-1.4.1-srw-dev.img`` has already been built at the following locations: -.. table:: Locations of pre-built containers - - +--------------+--------------------------------------------------------+ - | Machine | File location | - +==============+========================================================+ - | Cheyenne | /glade/scratch/epicufsrt/containers | - +--------------+--------------------------------------------------------+ - | Gaea | /lustre/f2/dev/role.epic/containers | - +--------------+--------------------------------------------------------+ - | Hera | /scratch1/NCEPDEV/nems/role.epic/containers | - +--------------+--------------------------------------------------------+ - | Jet | /mnt/lfs4/HFIP/hfv3gfs/role.epic/containers | - +--------------+--------------------------------------------------------+ - | NOAA Cloud | /contrib/EPIC/containers | - +--------------+--------------------------------------------------------+ - | Orion | /work/noaa/epic-ps/role-epic-ps/containers | - +--------------+--------------------------------------------------------+ +.. list-table:: Locations of pre-built containers + :widths: 20 50 + :header-rows: 1 + + * - Machine + - File Location + * - Derecho [#fn]_ + - /glade/work/epicufsrt/contrib/containers + * - Gaea [#fn]_ + - /lustre/f2/dev/role.epic/containers + * - Hera + - /scratch1/NCEPDEV/nems/role.epic/containers + * - Jet + - /mnt/lfs4/HFIP/hfv3gfs/role.epic/containers + * - NOAA Cloud + - /contrib/EPIC/containers + * - Orion/Hercules [#fn]_ + - /work/noaa/epic/role-epic/contrib/containers + +.. [#fn] On these systems, container testing shows inconsistent results. .. note:: * On Gaea, Singularity/Apptainer is only available on the C5 partition, and therefore container use is only supported on Gaea C5. @@ -104,7 +99,7 @@ Users can simply set an environment variable to point to the container: export img=/path/to/ubuntu20.04-intel-ue-1.4.1-srw-dev.img -Users may convert the container ``.img`` file to a writable sandbox. This step is required when running on Cheyenne but is optional on other systems: +Users may convert the container ``.img`` file to a writable sandbox: .. code-block:: console @@ -119,6 +114,8 @@ When making a writable sandbox on Level 1 systems, the following warnings common WARNING: integrity: signature not found for object group 1 WARNING: Bootstrap image could not be verified, but build will continue. +.. _container-L2-4: + Level 2-4 Systems ^^^^^^^^^^^^^^^^^^^^^ @@ -131,11 +128,11 @@ On non-Level 1 systems, users should build the container in a writable sandbox: Some users may prefer to issue the command without the ``sudo`` prefix. Whether ``sudo`` is required is system-dependent. .. note:: - Users can choose to build a release version of the container (SRW App v2.1.0) using a similar command: + Users can choose to build a release version of the container using a similar command: .. code-block:: console - sudo singularity build --sandbox ubuntu20.04-intel-srwapp docker://noaaepic/ubuntu20.04-intel-srwapp:release-public-v2.1.0 + sudo singularity build --sandbox ubuntu20.04-intel-srwapp docker://noaaepic/ubuntu20.04-intel-srwapp:release-public-v2.2.0 For easier reference, users can set an environment variable to point to the container: @@ -143,37 +140,6 @@ For easier reference, users can set an environment variable to point to the cont export img=/path/to/ubuntu20.04-intel-srwapp - -.. _WorkOnHPC: - -Allocate a Compute Node --------------------------- - -Users working on HPC systems that do **not** have Rocoto installed must `install Rocoto `__ or allocate a compute node. All other users may skip to the :ref:`next step `. - -.. note:: - - All NOAA Level 1 systems have Rocoto pre-installed. - -The appropriate commands for allocating a compute node will vary based on the user's system and resource manager (e.g., Slurm, PBS). If the user's system has the Slurm resource manager, the allocation command will follow this pattern: - -.. code-block:: console - - salloc -N 1 -n -A -t