From a8b40699cefa2c692502d60fc1ee5a065a31f12b Mon Sep 17 00:00:00 2001 From: johnhg Date: Thu, 27 Jan 2022 15:53:41 -0700 Subject: [PATCH] Update develop-ref after #2010 and #2029 (#2030) Co-authored-by: Julie Prestopnik Co-authored-by: johnhg Co-authored-by: Seth Linden Co-authored-by: John Halley Gotway Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway Co-authored-by: jprestop Co-authored-by: Seth Linden Co-authored-by: hsoh-u Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: John Halley Gotway Co-authored-by: MET Tools Test Account Co-authored-by: mo-mglover <78152252+mo-mglover@users.noreply.github.com> Co-authored-by: Howard Soh --- .github/jobs/Dockerfile.truth | 17 + .github/jobs/bash_functions.sh | 27 + .github/jobs/build_and_push_docker_image.sh | 40 - .github/jobs/build_docker_image.sh | 14 + .github/jobs/copy_diff_files.py | 80 ++ .github/jobs/create_docker_truth.sh | 18 + .github/jobs/get_branch_name.sh | 15 + .github/jobs/get_test_input_data.sh | 7 + .github/jobs/get_test_truth_data.sh | 11 + .github/jobs/pull_docker_image.sh | 7 + .github/jobs/push_docker_image.sh | 15 + .github/jobs/run_diff_docker.sh | 48 + .github/jobs/run_diff_tests.sh | 33 + .github/jobs/run_unit_docker.sh | 37 + .github/jobs/run_unit_tests.sh | 26 + .github/jobs/set_job_controls.sh | 94 ++ .github/jobs/test_env_vars.sh | 13 + .../build_docker_and_trigger_metplus.yml | 20 +- .github/workflows/unit_tests.yml | 377 +++++++ .gitignore | 3 + met/Make-include | 2 + met/configure.ac | 3 +- met/docs/Users_Guide/appendixF.rst | 50 + met/docs/Users_Guide/ensemble-stat.rst | 2 +- met/docs/Users_Guide/plotting.rst | 15 + met/docs/Users_Guide/point-stat.rst | 2 +- met/docs/Users_Guide/reformat_point.rst | 19 + met/scripts/python/Makefile.am | 4 +- met/scripts/python/met_point_obs.py | 189 ++++ met/scripts/python/read_met_point_obs.py | 121 +++ met/src/basic/vx_util/ascii_table.cc | 2 +- met/src/libcode/Makefile.am | 1 + met/src/libcode/vx_gis/dbf_file.cc | 2 +- met/src/libcode/vx_nc_obs/Makefile.am | 1 + met/src/libcode/vx_nc_obs/met_point_data.cc | 437 ++++++++ met/src/libcode/vx_nc_obs/met_point_data.h | 179 ++++ met/src/libcode/vx_nc_obs/nc_obs_util.cc | 161 +-- met/src/libcode/vx_nc_obs/nc_obs_util.h | 60 +- met/src/libcode/vx_nc_obs/nc_point_obs.cc | 92 +- met/src/libcode/vx_nc_obs/nc_point_obs.h | 42 +- met/src/libcode/vx_nc_obs/nc_point_obs_in.cc | 4 +- .../libcode/vx_pointdata_python/Makefile.am | 19 + .../pointdata_from_array.cc | 659 +++++++++++++ .../pointdata_from_array.h | 59 ++ .../vx_pointdata_python/pointdata_python.cc | 290 ++++++ .../vx_pointdata_python/pointdata_python.h | 112 +++ .../vx_pointdata_python/python_pointdata.cc | 588 +++++++++++ .../vx_pointdata_python/python_pointdata.h | 80 ++ .../libcode/vx_python3_utils/python3_util.cc | 36 +- .../libcode/vx_python3_utils/python3_util.h | 2 + .../tools/core/ensemble_stat/ensemble_stat.cc | 114 ++- met/src/tools/core/point_stat/point_stat.cc | 115 ++- met/src/tools/other/madis2nc/madis2nc.cc | 5 + .../tools/other/mode_graphics/cgraph_main.cc | 4 - met/src/tools/other/pb2nc/pb2nc.cc | 141 +-- .../other/plot_point_obs/plot_point_obs.cc | 143 ++- .../other/plot_point_obs/plot_point_obs.h | 5 +- met/src/tools/other/point2grid/point2grid.cc | 254 ++++- scripts/docker/Dockerfile | 154 +-- scripts/docker/Dockerfile.copy | 46 + scripts/docker/Dockerfile.minimum | 138 +++ scripts/docker/Dockerfile.test | 26 + scripts/docker/README.md | 40 + scripts/docker/build_met_docker.sh | 37 + scripts/environment/development.seneca | 5 + scripts/sonarqube/run_nightly.sh | 75 ++ scripts/sonarqube/run_sonarqube.sh | 144 +++ scripts/sonarqube/sonar-project.properties | 18 + test/bin/unit_test.sh | 16 +- test/xml/unit_ascii2nc.xml | 319 +----- test/xml/unit_ascii2nc_indy.xml | 340 +++++++ test/xml/unit_climatology.xml | 361 ------- test/xml/unit_climatology_1.0deg.xml | 252 +++++ test/xml/unit_climatology_1.5deg.xml | 98 ++ test/xml/unit_climatology_2.5deg.xml | 53 + test/xml/unit_pb2nc.xml | 136 +-- test/xml/unit_pb2nc_indy.xml | 157 +++ test/xml/unit_pcp_combine.xml | 4 +- test/xml/unit_plot_data_plane.xml | 4 +- test/xml/unit_python.xml | 119 ++- test/xml/unit_ref_config.xml | 931 +----------------- test/xml/unit_ref_config_lead_00.xml | 178 ++++ test/xml/unit_ref_config_lead_12.xml | 230 +++++ test/xml/unit_ref_config_lead_24.xml | 231 +++++ test/xml/unit_ref_config_lead_36.xml | 274 ++++++ test/xml/unit_ref_config_lead_48.xml | 231 +++++ test/xml/unit_stat_analysis.xml | 331 ------- test/xml/unit_stat_analysis_es.xml | 182 ++++ test/xml/unit_stat_analysis_gs.xml | 45 + test/xml/unit_stat_analysis_ps.xml | 120 +++ test/xml/unit_stat_analysis_ws.xml | 38 + 91 files changed, 7423 insertions(+), 2826 deletions(-) create mode 100644 .github/jobs/Dockerfile.truth create mode 100755 .github/jobs/bash_functions.sh delete mode 100755 .github/jobs/build_and_push_docker_image.sh create mode 100755 .github/jobs/build_docker_image.sh create mode 100755 .github/jobs/copy_diff_files.py create mode 100755 .github/jobs/create_docker_truth.sh create mode 100755 .github/jobs/get_branch_name.sh create mode 100755 .github/jobs/get_test_input_data.sh create mode 100755 .github/jobs/get_test_truth_data.sh create mode 100755 .github/jobs/pull_docker_image.sh create mode 100755 .github/jobs/push_docker_image.sh create mode 100755 .github/jobs/run_diff_docker.sh create mode 100755 .github/jobs/run_diff_tests.sh create mode 100755 .github/jobs/run_unit_docker.sh create mode 100755 .github/jobs/run_unit_tests.sh create mode 100755 .github/jobs/set_job_controls.sh create mode 100755 .github/jobs/test_env_vars.sh create mode 100644 .github/workflows/unit_tests.yml create mode 100755 met/scripts/python/met_point_obs.py create mode 100755 met/scripts/python/read_met_point_obs.py create mode 100644 met/src/libcode/vx_nc_obs/met_point_data.cc create mode 100644 met/src/libcode/vx_nc_obs/met_point_data.h create mode 100644 met/src/libcode/vx_pointdata_python/Makefile.am create mode 100644 met/src/libcode/vx_pointdata_python/pointdata_from_array.cc create mode 100644 met/src/libcode/vx_pointdata_python/pointdata_from_array.h create mode 100644 met/src/libcode/vx_pointdata_python/pointdata_python.cc create mode 100644 met/src/libcode/vx_pointdata_python/pointdata_python.h create mode 100644 met/src/libcode/vx_pointdata_python/python_pointdata.cc create mode 100644 met/src/libcode/vx_pointdata_python/python_pointdata.h create mode 100644 scripts/docker/Dockerfile.copy create mode 100644 scripts/docker/Dockerfile.minimum create mode 100644 scripts/docker/Dockerfile.test create mode 100644 scripts/docker/README.md create mode 100755 scripts/docker/build_met_docker.sh create mode 100755 scripts/sonarqube/run_nightly.sh create mode 100755 scripts/sonarqube/run_sonarqube.sh create mode 100644 scripts/sonarqube/sonar-project.properties create mode 100644 test/xml/unit_ascii2nc_indy.xml delete mode 100644 test/xml/unit_climatology.xml create mode 100644 test/xml/unit_climatology_1.0deg.xml create mode 100644 test/xml/unit_climatology_1.5deg.xml create mode 100644 test/xml/unit_climatology_2.5deg.xml create mode 100644 test/xml/unit_pb2nc_indy.xml create mode 100644 test/xml/unit_ref_config_lead_00.xml create mode 100644 test/xml/unit_ref_config_lead_12.xml create mode 100644 test/xml/unit_ref_config_lead_24.xml create mode 100644 test/xml/unit_ref_config_lead_36.xml create mode 100644 test/xml/unit_ref_config_lead_48.xml delete mode 100644 test/xml/unit_stat_analysis.xml create mode 100644 test/xml/unit_stat_analysis_es.xml create mode 100644 test/xml/unit_stat_analysis_gs.xml create mode 100644 test/xml/unit_stat_analysis_ps.xml create mode 100644 test/xml/unit_stat_analysis_ws.xml diff --git a/.github/jobs/Dockerfile.truth b/.github/jobs/Dockerfile.truth new file mode 100644 index 0000000000..6366360e13 --- /dev/null +++ b/.github/jobs/Dockerfile.truth @@ -0,0 +1,17 @@ +FROM centos:7 +MAINTAINER George McCabe + +ENV OUTPUT_DIR /data/output +RUN mkdir -p ${OUTPUT_DIR} + +ARG TRUTH_DIR + +COPY ${TRUTH_DIR} ${OUTPUT_DIR}/ + +ARG TRUTH_DIR + +# Define the volume mount point +VOLUME ${OUTPUT_DIR}/${TRUTH_DIR} + +USER root +CMD ["true"] \ No newline at end of file diff --git a/.github/jobs/bash_functions.sh b/.github/jobs/bash_functions.sh new file mode 100755 index 0000000000..5205753780 --- /dev/null +++ b/.github/jobs/bash_functions.sh @@ -0,0 +1,27 @@ +#! /bin/bash + +# utility function to run command get log the time it took to run +# if CMD_LOGFILE is set, send output to that file and unset var +function time_command { + local start_seconds=$SECONDS + echo "RUNNING: $*" + + local error + # pipe output to log file if set + if [ "x$CMD_LOGFILE" == "x" ]; then + "$@" + error=$? + else + echo "Logging to ${CMD_LOGFILE}" + "$@" &>> $CMD_LOGFILE + error=$? + unset CMD_LOGFILE + fi + + local duration=$(( SECONDS - start_seconds )) + echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" + if [ ${error} -ne 0 ]; then + echo "ERROR: '$*' exited with status = ${error}" + fi + return $error +} diff --git a/.github/jobs/build_and_push_docker_image.sh b/.github/jobs/build_and_push_docker_image.sh deleted file mode 100755 index 766b67cc41..0000000000 --- a/.github/jobs/build_and_push_docker_image.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /bin/bash - -# utility function to run command get log the time it took to run -function time_command { - local start_seconds=$SECONDS - echo "RUNNING: $*" - "$@" - local error=$? - - local duration=$(( SECONDS - start_seconds )) - echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" - if [ ${error} -ne 0 ]; then - echo "ERROR: '$*' exited with status = ${error}" - fi - return $error -} - -prefix=refs/heads/ -branch_name=${GITHUB_REF#"$prefix"} -DOCKERHUB_TAG=dtcenter/met:${branch_name} - -DOCKERFILE_DIR=${GITHUB_WORKSPACE}/scripts/docker - -echo "::group::Docker Build Command" -time_command docker build -t ${DOCKERHUB_TAG} \ - --build-arg SOURCE_BRANCH=$branch_name \ - $DOCKERFILE_DIR -echo "::endgroup::" - -# skip docker push if credentials are not set -if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then - echo "DockerHub credentials not set. Skipping docker push" - exit 0 -fi - -echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - -echo "::group::Docker Push Command" -time_command docker push ${DOCKERHUB_TAG} -echo "::endgroup::" diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh new file mode 100755 index 0000000000..5a9b2e8cf7 --- /dev/null +++ b/.github/jobs/build_docker_image.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH} + +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/scripts/docker/Dockerfile.copy + +CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build.log + +time_command docker build -t ${DOCKERHUB_TAG} \ + --build-arg SOURCE_BRANCH \ + --build-arg MET_BASE_IMAGE \ + -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE} diff --git a/.github/jobs/copy_diff_files.py b/.github/jobs/copy_diff_files.py new file mode 100755 index 0000000000..eed5b6a339 --- /dev/null +++ b/.github/jobs/copy_diff_files.py @@ -0,0 +1,80 @@ +#! /usr/bin/env python3 + +import os +import shutil + +OUTPUT_DIR = os.environ['MET_TEST_OUTPUT'] +TRUTH_DIR = os.environ['MET_TEST_TRUTH'] +DIFF_DIR = os.environ['MET_TEST_DIFF'] + +LOG_DIR = '/met/logs' + +def get_files_with_diffs(log_file): + files_to_copy = set() + + with open(log_file, 'r') as file_handle: + file_content = file_handle.read() + + missing_section, *test_sections = file_content.split( + '\n# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #\n' + ) + + # parse list of missing files + if 'ERROR:' in missing_section: + for missing_group in missing_section.split('ERROR:')[1:]: + dir_str, *rel_paths = missing_group.splitlines() + dir_str = dir_str.split()[1] + if OUTPUT_DIR in dir_str: + top_dir = dir_str.replace(OUTPUT_DIR, TRUTH_DIR) + elif TRUTH_DIR in dir_str: + top_dir = dir_str.replace(TRUTH_DIR, OUTPUT_DIR) + else: + print("ERROR: SOMETHING WENT WRONG PARSING COMP_DIR OUTPUT") + continue + for rel_path in rel_paths: + files_to_copy.add(os.path.join(top_dir, rel_path.strip())) + + # parse file paths out of sections that have errors + error_sections = [item for item in test_sections if 'ERROR:' in item] + for error_section in error_sections: + for line in error_section.splitlines(): + for item in line.split(): + if OUTPUT_DIR in item or TRUTH_DIR in item: + files_to_copy.add(item) + + return files_to_copy + +def copy_files_to_diff_dir(files_to_copy): + + print(f"Found {len(files_to_copy)} diff files") + + # add extension for output/truth and copy files to diff directory + for filename in files_to_copy: + output_path, extension = os.path.splitext(filename) + if OUTPUT_DIR in output_path: + output_path = f'{output_path}_OUTPUT{extension}' + output_path = output_path.replace(OUTPUT_DIR, DIFF_DIR) + elif TRUTH_DIR in output_path: + output_path = f'{output_path}_TRUTH{extension}' + output_path = output_path.replace(TRUTH_DIR, DIFF_DIR) + else: + continue + + # change bad char - this can be removed once test output is changed + output_path = output_path.replace(':', '_') + + print(f"Copy {filename} to {output_path}") + output_dir = os.path.dirname(output_path) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + shutil.copyfile(filename, output_path) + +def main(): + log_file = os.path.join(LOG_DIR, 'comp_dir.log') + print(f"Parsing {log_file}") + all_files_to_copy = get_files_with_diffs(log_file) + + copy_files_to_diff_dir(all_files_to_copy) + +if __name__ == "__main__": + main() diff --git a/.github/jobs/create_docker_truth.sh b/.github/jobs/create_docker_truth.sh new file mode 100755 index 0000000000..298852f80a --- /dev/null +++ b/.github/jobs/create_docker_truth.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +image_name=dtcenter/met-data-output:${TRUTH_DATA_VERSION} + +time_command docker build -t ${image_name} \ + --build-arg TRUTH_DIR=met_test_truth \ + -f ${GITHUB_WORKSPACE}/.github/jobs/Dockerfile.truth \ + ${RUNNER_WORKSPACE} +if [ $? != 0 ]; then + echo "ERROR: Docker build failed" + exit 1 +fi + +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + +time_command docker push ${image_name} diff --git a/.github/jobs/get_branch_name.sh b/.github/jobs/get_branch_name.sh new file mode 100755 index 0000000000..ec42a94e38 --- /dev/null +++ b/.github/jobs/get_branch_name.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +# If pull request, use GitHub head ref and add -PR to end +# Otherwise use GitHub ref + +if [ "${GITHUB_EVENT_NAME}" == "pull_request" ] ; then + branch_name=${GITHUB_HEAD_REF}-PR +else + branch_name=${GITHUB_REF} +fi + +branch_name=${branch_name#"refs/heads/"} + +echo ::set-output name=branch_name::$branch_name +echo branch_name: $branch_name diff --git a/.github/jobs/get_test_input_data.sh b/.github/jobs/get_test_input_data.sh new file mode 100755 index 0000000000..d11c651265 --- /dev/null +++ b/.github/jobs/get_test_input_data.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DATA_VERSION=$1 + +time_command docker create --name met_input dtcenter/met-data-dev:${DATA_VERSION} diff --git a/.github/jobs/get_test_truth_data.sh b/.github/jobs/get_test_truth_data.sh new file mode 100755 index 0000000000..3b4055b4be --- /dev/null +++ b/.github/jobs/get_test_truth_data.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DATA_VERSION=$1 + +time_command docker create --name met_truth dtcenter/met-data-output:${DATA_VERSION} +if [ $? != 0 ]; then + echo "Image tag ${DATA_VERSION} does not exist. Using develop..." + time_command docker create --name met_truth dtcenter/met-data-output:develop +fi diff --git a/.github/jobs/pull_docker_image.sh b/.github/jobs/pull_docker_image.sh new file mode 100755 index 0000000000..e97c6a1f07 --- /dev/null +++ b/.github/jobs/pull_docker_image.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=$1 + +time_command docker pull ${DOCKERHUB_TAG} diff --git a/.github/jobs/push_docker_image.sh b/.github/jobs/push_docker_image.sh new file mode 100755 index 0000000000..83a8e4fc5f --- /dev/null +++ b/.github/jobs/push_docker_image.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH} + +# skip docker push if credentials are not set +if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then + echo "DockerHub credentials not set. Skipping docker push" + exit 0 +fi + +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + +time_command docker push ${DOCKERHUB_TAG} diff --git a/.github/jobs/run_diff_docker.sh b/.github/jobs/run_diff_docker.sh new file mode 100755 index 0000000000..b4095fb21e --- /dev/null +++ b/.github/jobs/run_diff_docker.sh @@ -0,0 +1,48 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH} + +# Get truth output data +${GITHUB_WORKSPACE}/.github/jobs/get_test_truth_data.sh ${TRUTH_DATA_VERSION} + +# Set up directories to mount +LOCAL_OUTPUT_DIR=${RUNNER_WORKSPACE}/output +DOCKER_OUTPUT_DIR=/data/output/met_test_output + +LOCAL_DIFF_DIR=${RUNNER_WORKSPACE}/diff +DOCKER_DIFF_DIR=/data/output/met_test_diff + +LOCAL_LOG_DIR=${RUNNER_WORKSPACE}/logs +DOCKER_LOG_DIR=/met/logs + +# Create local directories to store output +mkdir -p ${LOCAL_LOG_DIR} +mkdir -p ${LOCAL_DIFF_DIR} + +# mount output and log dirs, mount GitHub files into MET_REPO_DIR +mount_args="-v ${LOCAL_OUTPUT_DIR}:${DOCKER_OUTPUT_DIR} -v ${LOCAL_DIFF_DIR}:${DOCKER_DIFF_DIR} -v ${LOCAL_LOG_DIR}:${DOCKER_LOG_DIR}" + +# Set up data volumes +volumes_from="--volumes-from met_truth" + +# run unit test script inside Docker, mount MET output and truth data +# set MET_REPO_DIR env var in Docker to mounted directory +cmd="\${MET_REPO_DIR}/.github/jobs/run_diff_tests.sh" +time_command docker run ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\" +if [ $? != 0 ]; then + exit 1 +fi + +if [ "$(ls -A ${LOCAL_DIFF_DIR})" ]; then + echo "ERROR: Differences exist in the output" + + # only exit non-zero (job fails) if not updating truth data + # this makes difference output available when updating truth data + # so it is easier to see what changed with the update + if [ "${RUN_UPDATE_TRUTH}" != "true" ]; then + exit 1 + fi + +fi diff --git a/.github/jobs/run_diff_tests.sh b/.github/jobs/run_diff_tests.sh new file mode 100755 index 0000000000..fd84a6953f --- /dev/null +++ b/.github/jobs/run_diff_tests.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +source ${MET_REPO_DIR}/.github/jobs/bash_functions.sh + +### +# Set environment variables needed to run unit tests +### + +source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh + +### +# Run comparison of MET unit test output +### + +echo "Running comparison on test output" +CMD_LOGFILE=/met/logs/comp_dir.log +time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH} ${MET_TEST_OUTPUT} +if [ $? != 0 ]; then + echo "ERROR: Test output comparison failed" + cat /met/logs/comp_dir.log + exit 1 +fi + +echo "Running copy_diff_files.py" +CMD_LOGFILE=/met/logs/copy_diff_files.log +time_command ${MET_REPO_DIR}/.github/jobs/copy_diff_files.py +if [ $? != 0 ]; then + echo "ERROR: Copy diff files script failed" + cat /met/logs/copy_diff_files.log + exit 1 +fi + +echo "Success" diff --git a/.github/jobs/run_unit_docker.sh b/.github/jobs/run_unit_docker.sh new file mode 100755 index 0000000000..95f4b2c626 --- /dev/null +++ b/.github/jobs/run_unit_docker.sh @@ -0,0 +1,37 @@ +#! /bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH} + +# Pull MET Image from DockerHub +${GITHUB_WORKSPACE}/.github/jobs/pull_docker_image.sh ${DOCKERHUB_TAG} + +# Get test input data if needed +volumes_from="" +if [ "${INPUT_DATA_VERSION}" != "none" ]; then + ${GITHUB_WORKSPACE}/.github/jobs/get_test_input_data.sh ${INPUT_DATA_VERSION} + volumes_from=${volumes_from}"--volumes-from met_input" +fi + +# Set up directories to mount +LOCAL_OUTPUT_DIR=${RUNNER_WORKSPACE}/output +DOCKER_OUTPUT_DIR=/data/output/met_test_output + +LOCAL_LOG_DIR=${RUNNER_WORKSPACE}/logs +DOCKER_LOG_DIR=/met/logs + +# Create local directories to store output +mkdir -p ${LOCAL_LOG_DIR} +mkdir -p ${LOCAL_OUTPUT_DIR} + +mount_args="-v ${LOCAL_OUTPUT_DIR}:${DOCKER_OUTPUT_DIR} -v ${LOCAL_LOG_DIR}:${DOCKER_LOG_DIR}" + +export TESTS_TO_RUN=$TESTS + +# run unit test script inside Docker, mount MET input and truth data +cmd="\${MET_REPO_DIR}/.github/jobs/run_unit_tests.sh" +time_command docker run -e TESTS_TO_RUN ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\" +if [ $? != 0 ]; then + exit 1 +fi diff --git a/.github/jobs/run_unit_tests.sh b/.github/jobs/run_unit_tests.sh new file mode 100755 index 0000000000..e866b2cc61 --- /dev/null +++ b/.github/jobs/run_unit_tests.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +source ${MET_REPO_DIR}/.github/jobs/bash_functions.sh + +### +# Set environment variables needed to run unit tests +### + +source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh + +### +# Run MET unit tests +### + +echo "Running MET unit tests..." +for testname in $TESTS_TO_RUN; do + CMD_LOGFILE=/met/logs/unit_${testname}.log + time_command ${MET_TEST_BASE}/perl/unit.pl ${MET_TEST_BASE}/xml/unit_${testname}.xml + if [ $? != 0 ]; then + echo "ERROR: Unit test ${testname} failed" + cat /met/logs/unit_${testname}.log + exit 1 + fi +done + +echo "Success" diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh new file mode 100755 index 0000000000..6a4e4bdf82 --- /dev/null +++ b/.github/jobs/set_job_controls.sh @@ -0,0 +1,94 @@ +#! /bin/bash + +run_compile=true +run_push=false +run_unit_tests=false +run_diff=false +run_update_truth=false +met_base_image=minimum +input_data_version=develop +truth_data_version=develop + +if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then + + # only run diff logic if pull request INTO + # branches not ending with -ref + if [ "${GITHUB_BASE_REF: -4}" != "-ref" ]; then + + run_diff=true + + fi + +elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then + + branch_name=`cut -d "/" -f3 <<< "${GITHUB_REF}"` + + # if branch ends with -ref, update truth data from unit tests + if [ "${branch_name: -4}" == -ref ]; then + + run_update_truth=true + run_diff=true + truth_data_version=${branch_name: -4} + + else + + # if develop or main_vX.Y branch, run diff tests using branch's truth data + if [ "$branch_name" == "develop" ] || + [ "${branch_name:0:6}" == "main_v" ]; then + + run_diff=true + truth_data_version=${branch_name} + + fi + + # check commit messages for ci-skip or ci-run keywords + if grep -q "ci-skip-compile" <<< "$commit_msg"; then + + run_compile=false + + fi + + if grep -q "ci-run-unit" <<< "$commit_msg"; then + + run_diff=true + + fi + fi + +fi + +# if updating truth or running diff, run unit tests +if [ "$run_update_truth" == "true" ] || [ "$run_diff" == "true" ]; then + + run_unit_tests=true + +fi + +# if running unit tests, use unit_test MET base image and push image +if [ "$run_unit_tests" == "true" ]; then + + met_base_image=unit_test + run_push=true + +fi + +echo ::set-output name=run_compile::$run_compile +echo ::set-output name=run_push::$run_push +echo ::set-output name=run_unit_tests::$run_unit_tests +echo ::set-output name=run_diff::$run_diff +echo ::set-output name=run_update_truth::$run_update_truth +echo ::set-output name=met_base_image::$met_base_image +echo ::set-output name=input_data_version::$input_data_version +echo ::set-output name=truth_data_version::$truth_data_version + +echo run_compile: $run_compile +echo run_push: $run_push +echo run_unit_tests: $run_unit_tests +echo run_diff: $run_diff +echo run_update_truth: $run_update_truth +echo met_base_image: $met_base_image +echo input_data_version: $input_data_version +echo truth_data_version: $truth_data_version + +# get name of branch +.github/jobs/get_branch_name.sh diff --git a/.github/jobs/test_env_vars.sh b/.github/jobs/test_env_vars.sh new file mode 100755 index 0000000000..5eaaa44dd9 --- /dev/null +++ b/.github/jobs/test_env_vars.sh @@ -0,0 +1,13 @@ +export MET_BASE=/usr/local/share/met + +export MET_BUILD_BASE=${MET_REPO_DIR}/met +export MET_TEST_BASE=${MET_REPO_DIR}/test +export PERL5LIB=${MET_TEST_BASE}/lib + +export MET_TEST_INPUT=/data/input/MET_test_data/unit_test +export MET_TEST_OUTPUT=/data/output/met_test_output +export MET_TEST_TRUTH=/data/output/met_test_truth +export MET_TEST_DIFF=/data/output/met_test_diff + +export MET_TEST_RSCRIPT=/usr/bin/Rscript +export MET_TEST_MET_PYTHON_EXE=/usr/bin/python3 diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index 070fa071a6..88f2a66ebe 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -7,17 +7,33 @@ on: paths-ignore: - 'met/docs/**' +env: + DOCKERHUB_REPO: dtcenter/met-dev + jobs: + build_met_docker: name: Handle Docker Image runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Build and Push Docker Image - run: .github/jobs/build_and_push_docker_image.sh + + - name: Get branch name + id: get_branch_name + run: echo ::set-output name=branch_name::${GITHUB_REF#"refs/heads/"} + + - name: Build Docker Image + run: .github/jobs/build_docker_image.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + MET_BASE_IMAGE: minimum + + - name: Push Docker Image + run: .github/jobs/push_docker_image.sh env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + trigger_metplus: name: Trigger METplus testing workflow runs-on: ubuntu-latest diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000000..c114c61028 --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,377 @@ +name: Unit Tests + +# Compile MET and run unit tests +# for pull requests into develop branch + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - develop + - 'main_v*' + push: + branches: + - 'feature_*' + - 'bugfix_*' + - 'develop' + - 'develop-ref' + - 'main_v*' + +env: + DOCKERHUB_REPO: dtcenter/met-dev + +jobs: + + job_control: + name: Determine which jobs to run + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set job controls + id: job_status + run: .github/jobs/set_job_controls.sh + env: + commit_msg: ${{ github.event.head_commit.message }} + + outputs: + run_compile: ${{ steps.job_status.outputs.run_compile }} + run_push: ${{ steps.job_status.outputs.run_push }} + run_unit_tests: ${{ steps.job_status.outputs.run_unit_tests }} + run_diff: ${{ steps.job_status.outputs.run_diff }} + run_update_truth: ${{ steps.job_status.outputs.run_update_truth }} + met_base_image: ${{ steps.job_status.outputs.met_base_image }} + branch_name: ${{ steps.job_status.outputs.branch_name }} + truth_data_version: ${{ steps.job_status.outputs.truth_data_version }} + input_data_version: ${{ steps.job_status.outputs.input_data_version }} + + compile: + name: Compile MET + runs-on: ubuntu-latest + needs: job_control + if: ${{ needs.job_control.outputs.run_compile == 'true' }} + steps: + - uses: actions/checkout@v2 + + - name: Create directories to store output + run: mkdir -p ${RUNNER_WORKSPACE}/logs + + - name: Compile MET in Docker + run: .github/jobs/build_docker_image.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + MET_BASE_IMAGE: ${{ needs.job_control.outputs.met_base_image }} + + - name: Copy Docker build log into logs directory + if: always() + run: cp ${GITHUB_WORKSPACE}/docker_build.log ${RUNNER_WORKSPACE}/logs/ + + - name: Push Docker Image + run: .github/jobs/push_docker_image.sh + if: ${{ needs.job_control.outputs.run_push == 'true' }} + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_1a: + name: MET Unit Tests 1a + runs-on: ubuntu-latest + needs: [job_control, compile] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'ascii2nc' + - 'pb2nc madis2nc pcp_combine' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }} + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_1a + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_1b: + name: MET Unit Tests 1b + runs-on: ubuntu-latest + needs: [job_control, compile] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_gen' + - 'met_test_scripts mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }} + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_1b + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_ref_config_leads: + name: MET Unit Tests ref_config leads + runs-on: ubuntu-latest + needs: [job_control, compile] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'ref_config_lead_00 ref_config_lead_12' + - 'ref_config_lead_24 ref_config_lead_48' + - 'ref_config_lead_36' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }} + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_ref_config_leads + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_ref_config: + name: MET Unit Tests ref_config + runs-on: ubuntu-latest + needs: [job_control, unit_tests_ref_config_leads] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'ref_config' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Download ref_config_leads output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_ref_config_leads + path: ${{ runner.workspace }}/output + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: 'none' + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_ref_config + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_2a: + name: MET Unit Tests 2a + runs-on: ubuntu-latest + needs: [job_control, unit_tests_1a] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'point_stat stat_analysis_ps' + - 'grid_stat stat_analysis_gs' + - 'wavelet_stat stat_analysis_ws' + - 'ensemble_stat stat_analysis_es' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Download 1a output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_1a + path: ${{ runner.workspace }}/output + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }} + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_2a + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_2b: + name: MET Unit Tests 2b + runs-on: ubuntu-latest + needs: [job_control, unit_tests_1a] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'climatology_1.0deg' + - 'climatology_1.5deg' + - 'climatology_2.5deg' + - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Download 1a output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_1a + path: ${{ runner.workspace }}/output + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TESTS: ${{ matrix.tests }} + INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }} + + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_2b + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + run_diffs: + name: Check for Differences + runs-on: ubuntu-latest + needs: [job_control, unit_tests_1b, unit_tests_2a, unit_tests_2b, unit_tests_ref_config] + if: ${{ needs.job_control.outputs.run_diff == 'true' }} + steps: + - name: Download data from previous jobs + uses: actions/download-artifact@v2 + + - name: Copy test output into single directory + run: | + mkdir ${RUNNER_WORKSPACE}/output + cp -r unit_tests_*/* ${RUNNER_WORKSPACE}/output/ + + - uses: actions/checkout@v2 + + - name: Run Diff Tests in Docker + run: .github/jobs/run_diff_docker.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + RUN_UPDATE_TRUTH: ${{ needs.job_control.outputs.run_update_truth }} + TRUTH_DATA_VERSION: ${{ needs.job_control.outputs.truth_data_version }} + + - name: Upload diff files as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: diff + path: ${{ runner.workspace }}/diff + if-no-files-found: ignore + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + update_truth: + name: Update Truth Data + runs-on: ubuntu-latest + needs: [job_control, unit_tests_1b, unit_tests_2a, unit_tests_2b, unit_tests_ref_config] + if: ${{ needs.job_control.outputs.run_update_truth == 'true' }} + steps: + - uses: actions/checkout@v2 + + - name: Download data from previous jobs + uses: actions/download-artifact@v2 + + - name: Copy test output into single directory + run: | + mkdir ${RUNNER_WORKSPACE}/met_test_truth + cp -r unit_tests_*/* ${RUNNER_WORKSPACE}/met_test_truth/ + + - name: Create Docker Data Volume + run: .github/jobs/create_docker_truth.sh + env: + TRUTH_DATA_VERSION: ${{ needs.job_control.outputs.truth_data_version }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.gitignore b/.gitignore index f2060b9284..bbd74b866f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ met/docs/_build/ # tilda files generated by emacs *~ + +# log file created in Docker image +make_install.log diff --git a/met/Make-include b/met/Make-include index dfa6bd5faa..42f1d5f3b3 100644 --- a/met/Make-include +++ b/met/Make-include @@ -25,6 +25,7 @@ MET_CPPFLAGS = -I${top_builddir}/src/basic/vx_cal \ -I${top_builddir}/src/libcode/vx_nc_util \ -I${top_builddir}/src/libcode/vx_pb_util \ -I${top_builddir}/src/libcode/vx_plot_util \ + -I${top_builddir}/src/libcode/vx_pointdata_python \ -I${top_builddir}/src/libcode/vx_ps \ -I${top_builddir}/src/libcode/vx_pxm \ -I${top_builddir}/src/libcode/vx_render \ @@ -67,6 +68,7 @@ MET_LDFLAGS = -L${top_builddir}/src/basic/vx_cal \ -L${top_builddir}/src/libcode/vx_nc_util \ -L${top_builddir}/src/libcode/vx_pb_util \ -L${top_builddir}/src/libcode/vx_plot_util \ + -L${top_builddir}/src/libcode/vx_pointdata_python \ -L${top_builddir}/src/libcode/vx_ps \ -L${top_builddir}/src/libcode/vx_pxm \ -L${top_builddir}/src/libcode/vx_render \ diff --git a/met/configure.ac b/met/configure.ac index 7008748079..df6a93a428 100644 --- a/met/configure.ac +++ b/met/configure.ac @@ -1043,7 +1043,7 @@ if test "x$ENABLE_PYTHON" = "xyes"; then AC_DEFINE([ENABLE_PYTHON], [], ["build python embedding"]) AC_MSG_NOTICE([python embedding will be compiled]) CPPFLAGS="${CPPFLAGS} -DWITH_PYTHON" - PYTHON_LIBS="-lvx_data2d_python -lvx_python3_utils ${MET_PYTHON_LD}" + PYTHON_LIBS="-lvx_data2d_python -lvx_pointdata_python -lvx_python3_utils ${MET_PYTHON_LD}" else AC_MSG_NOTICE([python embedding will not be compiled]) PYTHON_LIBS= @@ -1240,6 +1240,7 @@ AC_CONFIG_FILES([Makefile src/libcode/vx_summary/Makefile src/libcode/vx_python3_utils/Makefile src/libcode/vx_data2d_python/Makefile + src/libcode/vx_pointdata_python/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile diff --git a/met/docs/Users_Guide/appendixF.rst b/met/docs/Users_Guide/appendixF.rst index 3a4046e9ea..586c264c9e 100644 --- a/met/docs/Users_Guide/appendixF.rst +++ b/met/docs/Users_Guide/appendixF.rst @@ -257,3 +257,53 @@ The **read_ascii_mpr.py** sample script can be found in: • MET installation directory in *MET_BASE/python*. • `MET GitHub repository `_ in *met/scripts/python*. + + +Python Embedding for Point Observations as input +________________________________________________ + + +The point2grid, plot_point_obs, ensemble_stat, and point_stat tools use MET point observation NetCDF. They support the python embedding by the prefix 'PYTHON_NUMPY=" and followed by a python script name instead of the MET point observastion NetCDF filename. The customized python script is expected to extend MET_BASE/python/met_point_obs.py and to produce the python variable, **met_point_data**, which is the dictionary of the MET point observation data. They are defined at MET_BASE/python/met_point_obs.py. + + +.. _pyembed-point-obs-data: + + +.. code-block:: none + + met_point_data = { + + 'use_var_id': Trur/False, # obs_vid are variable index if True, otherwise GRIB codes + + # Header data + 'nhdr': integer_value, # number of headers + 'pbhdr': integer_value, # number of PREPBUFR specific headers + 'nhdr_typ': integer_value, # number of message types + 'nhdr_sid': integer_value, # number of station IDs + 'nhdr_vld': integer_value, # number of valid times + 'hdr_typ': nympy_integer_array, # index of message type + 'hdr_sid': nympy_integer_array, # index of station ID + 'hdr_vld': nympy_integer_array, # index of valid time + 'hdr_lat': nympy_float_array, # latitude + 'hdr_lon': nympy_float_array, # longitude + 'hdr_elv': nympy_float_array, # station elevation + 'hdr_typ_table': string_value, # message types + 'hdr_sid_table': string_value, # station IDs + 'hdr_vld_table': string_value, # valid times "yyyymmdd_hhmmss" + 'hdr_prpt_typ': nympy_integer_array, # optional + 'hdr_irpt_typ': nympy_integer_array, # optional + 'hdr_inst_typ': nympy_integer_array, # optional + + # Observation data + 'nobs': integer_value, # number of observation + 'nobs_qty': integer_value # number of quality marks + 'nobs_var': integer_value # number of variable names + 'obs_qty': nympy_integer_array, # index of quality mark + 'obs_hid': nympy_integer_array, # index of header + 'obs_vid': nympy_integer_array, # index of veriable or GRIB code + 'obs_lvl': nympy_float_array, # pressure level + 'obs_hgt': nympy_float_array, # height of observation data + 'obs_val' nympy_float_array, # observatin value + 'obs_qty_table': string_array, # quality marks + 'obs_var_table': string_array, # variable names + } diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index 5c6277b179..fd1dbf535a 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -93,7 +93,7 @@ Optional arguments for ensemble_stat 4. To produce ensemble statistics using gridded observations, use the **-grid_obs file** option to specify a gridded observation file. This option may be used multiple times if your observations are in several files. -5. To produce ensemble statistics using point observations, use the **-point_obs file** option to specify a NetCDF point observation file. This option may be used multiple times if your observations are in several files. +5. To produce ensemble statistics using point observations, use the **-point_obs file** option to specify a NetCDF point observation file. This option may be used multiple times if your observations are in several files. The python embedding will be activated if the **file** begines with 'PYTHON_NUMPY=" and followed by a python script name. 6. To override the simple ensemble mean value of the input ensemble members for the ECNT, SSVAR, and ORANK line types, the **-ens_mean file** option specifies an ensemble mean model data file. This option replaces the **-ssvar_mean file** option from earlier versions of MET. diff --git a/met/docs/Users_Guide/plotting.rst b/met/docs/Users_Guide/plotting.rst index 7f748e7885..a96dca605a 100644 --- a/met/docs/Users_Guide/plotting.rst +++ b/met/docs/Users_Guide/plotting.rst @@ -8,6 +8,9 @@ __________________ This section describes how to check your data files using plotting utilities. Point observations can be plotted using the Plot-Point-Obs utility. A single model level can be plotted using the plot_data_plane utility. For object based evaluations, the MODE objects can be plotted using plot_mode_field. Occasionally, a post-processing or timing error can lead to errors in MET. These tools can assist the user by showing the data to be verified to ensure that times and locations match up as expected. +MET version 10.1 adds support for the passing point observations to plot_point_obs using a Python scriptAn example of running plot_point_obs with Python embedding is included below. + + plot_point_obs usage ~~~~~~~~~~~~~~~~~~~~ @@ -66,6 +69,18 @@ An example of the plot_point_obs calling sequence is shown below: In this example, the Plot-Point-Obs tool will process the input sample_pb.nc file and write a postscript file containing a plot to a file named sample_pb.ps. +This is an equivalent command with the python embedding. This is an example for the python embeddingt. + +.. code-block:: none + + plot_point_obs 'PYTHON_NUMPY=MET_BASE/python/read_met_point_obs.py sample_pb.nc' sample_data.ps + + +The user should replace the python script with the customized python script for the custom point observation data. + +Please refer to :numref:`Appendix F, Section %s ` for more details about Python embedding in MET. + + plot_point_obs configuration file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/met/docs/Users_Guide/point-stat.rst b/met/docs/Users_Guide/point-stat.rst index 7a9966511c..ce7b74e905 100644 --- a/met/docs/Users_Guide/point-stat.rst +++ b/met/docs/Users_Guide/point-stat.rst @@ -286,7 +286,7 @@ Required arguments for point_stat Optional arguments for point_stat ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -4. The **-point_obs** file may be used to pass additional NetCDF point observation files to be used in the verification. +4. The **-point_obs** file may be used to pass additional NetCDF point observation files to be used in the verification. The python embedding will be activated if the **file** begines with 'PYTHON_NUMPY=" and followed by a python script name. 5. The **-obs_valid_beg** time option in YYYYMMDD[_HH[MMSS]] format sets the beginning of the observation matching time window, overriding the configuration file setting. diff --git a/met/docs/Users_Guide/reformat_point.rst b/met/docs/Users_Guide/reformat_point.rst index 999393b72d..9e41ca80dc 100644 --- a/met/docs/Users_Guide/reformat_point.rst +++ b/met/docs/Users_Guide/reformat_point.rst @@ -981,6 +981,8 @@ _______________ The Point2Grid tool takes point observations from a NetCDF output file from one of the four previously mentioned MET tools (ascii2nc, madis2nc, pb2nc, lidar2nc) and creates a gridded NetCDF file. The other point observations are GOES-16/17 input files in NetCDF format (especially, Aerosol Optical Depth. Future development will include support for reading input files not produced from MET tools. +MET version 10.1 adds support for the passing point observations to point2grid using a Python script with "input_filename" which begins with the "PYTHON_NUMPY= [arguments]" instead of MET point observation NetCDF input. An example of running point2grid with Python embedding is included below. + point2grid usage ~~~~~~~~~~~~~~~~ @@ -1013,6 +1015,8 @@ Required arguments for point2grid 1. The **input_filename** argument indicates the name of the input NetCDF file to be processed. Currently, only NetCDF files produced from the ascii2nc, madis2nc, pb2nc, and lidar2nc are supported. And AOD dataset from GOES16/17 are supported, too. Support for additional file types will be added in future releases. +The MET point observation NetCDF file name as **input_filename** argument is equivalent with "PYTHON_NUMPY=MET_BASE/python/read_met_point_obs.py netcdf_file name'. + 2. The **to_grid** argument defines the output grid as: (1) a named grid, (2) the path to a gridded data file, or (3) an explicit grid specification string. 3. The **output_filename** argument is the name of the output NetCDF file to be written. @@ -1065,6 +1069,21 @@ For the GOES-16 and GOES-17 data, the computing lat/long is time consuming. So t When processing GOES-16 data, the **-qc** option may also be used to specify the acceptable quality control flag values. The example above regrids the GOES-16 AOD values to NCEP Grid number 212 (which QC flags are high, medium, and low), writing to the output the maximum AOD value falling inside each grid box. +Here is an example of processing the same set of observations but using Python embedding instead: + + +.. code-block:: none + + point2grid \ + 'PYTHON_NUMPY=MET_BASE/python/read_met_point_obs.py ascii2nc_edr_hourly.20130827.nc' \ + G212 python_gridded_ascii_python.nc -config Point2GridConfig_edr \ + -field 'name="200"; level="*"; valid_time="20130827_205959";' -method MAX -v 1 + +The user should replace the python script with the customized python script for the custom point observation data. This is an example for the python embedding. + +Please refer to :numref:`Appendix F, Section %s ` for more details about Python embedding in MET. + + point2grid output ~~~~~~~~~~~~~~~~~ diff --git a/met/scripts/python/Makefile.am b/met/scripts/python/Makefile.am index 78dc7d88bc..368da3da7f 100644 --- a/met/scripts/python/Makefile.am +++ b/met/scripts/python/Makefile.am @@ -26,11 +26,13 @@ pythonscriptsdir = $(pkgdatadir)/python pythonscripts_DATA = \ + met_point_obs.py \ read_ascii_numpy.py \ read_ascii_numpy_grid.py \ read_ascii_xarray.py \ read_ascii_point.py \ - read_ascii_mpr.py + read_ascii_mpr.py \ + read_met_point_obs.py EXTRA_DIST = ${pythonscripts_DATA} diff --git a/met/scripts/python/met_point_obs.py b/met/scripts/python/met_point_obs.py new file mode 100755 index 0000000000..c306839226 --- /dev/null +++ b/met/scripts/python/met_point_obs.py @@ -0,0 +1,189 @@ +''' +Created on Nov 10, 2021 + +@author: hsoh + +- This is the base class and the customized script should extend the met_point_obs. +- The customized script (for example "custom_reader") must implement "def read_data(self, args)" + which fills the array (list) variables at __init__(). +- The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments. +- The variable "met_point_data" must be set for MET tools +- The customized script is expected to include following codes: + + # prepare arguments for the customized script + args = {'input', sys.argv[1]} # or args = [] + point_obs_data = custom_reader() + point_obs_data.read_data() + met_point_data = point_obs_data.get_point_data() +''' + +from abc import ABC, abstractmethod +import numpy as np + +COUNT_SHOW = 30 + +MET_PYTHON_OBS_ARGS = "MET_POINT_PYTHON_ARGS" + +class met_point_obs(ABC): + ''' + classdocs + ''' + + python_prefix = 'PYTHON_POINT_RAW' + + def __init__(self, use_var_id=True): + ''' + Constructor + ''' + self.use_var_id = use_var_id # True if variable index, False if GRIB code + + # Header + self.nhdr = 0 + self.npbhdr = 0 + self.nhdr_typ = 0 # type table + self.nhdr_sid = 0 #station_uid table + self.nhdr_vld = 0 # valid time strings + self.hdr_typ = [] # (nhdr) + self.hdr_sid = [] # (nhdr) + self.hdr_vld = [] # (nhdr) + self.hdr_lat = [] # (nhdr) + self.hdr_lon = [] # (nhdr) + self.hdr_elv = [] # (nhdr) + self.hdr_typ_table = [] # (nhdr_typ, mxstr2) + self.hdr_sid_table = [] # (nhdr_sid, mxstr2) + self.hdr_vld_table = [] # (nhdr_vld, mxstr) + self.hdr_prpt_typ = [] # optional + self.hdr_irpt_typ = [] # optional + self.hdr_inst_typ = [] # optional + + #Observation data + self.nobs = 0 + self.nobs_qty = 0 + self.nobs_var = 0 + self.obs_qty = [] # (nobs_qty ) + self.obs_hid = [] # (nobs) + self.obs_vid = [] # (nobs) veriable index or GRIB code + self.obs_lvl = [] # (nobs) + self.obs_hgt = [] # (nobs) + self.obs_val = [] # (nobs) + self.obs_qty_table = [] # (nobs_qty, mxstr) + self.obs_var_table = [] # (nobs_var, mxstr2) optional if GRIB code at self.obs_vid + + @abstractmethod + def read_data(self, args): + # args should be list or dictionary + pass + + def get_point_data(self): + if self.nhdr <= 0: + self.nhdr = len(self.hdr_lat) + if self.nobs <= 0: + self.nobs = len(self.obs_val) + if self.nhdr_typ <= 0: + self.nhdr_typ = len(self.hdr_typ_table) + if self.nhdr_sid <= 0: + self.nhdr_sid = len(self.hdr_sid_table) + if self.nhdr_vld <= 0: + self.nhdr_vld = len(self.hdr_vld_table) + if self.npbhdr <= 0: + self.npbhdr = len(self.hdr_prpt_typ) + if self.nobs_qty <= 0: + self.nobs_qty = len(self.obs_qty_table) + if self.nobs_var <= 0: + self.nobs_var = len(self.obs_var_table) + return self.__dict__ + + @staticmethod + def is_python_script(arg_value): + return arg_value.startswith(met_point_obs.python_prefix) + + @staticmethod + def get_python_script(arg_value): + return arg_value[len(met_point_obs.python_prefix)+1:] + + @staticmethod + def print_data(key, data_array, show_count=COUNT_SHOW): + if isinstance(data_array, list): + data_len = len(data_array) + if show_count >= data_len: + print(" {k:10s}: {v}".format(k=key, v= data_array)) + else: + end_offset = int(show_count/2) + print(" {k:10s}: count={v}".format(k=key, v=data_len)) + print(" {k:10s}[0:{o}] {v}".format(k=key, v=data_array[:end_offset], o=end_offset)) + print(" {k:10s}[{s}:{e}]: {v}".format(k=key, v='...', s=end_offset+1, e=data_len-end_offset-1)) + print(" {k:10s}[{s}:{e}]: {v}".format(k=key, v= data_array[-end_offset:], s=(data_len-end_offset), e=(data_len-1))) + else: + print(" {k:10s}: {v}".format(k=key, v= data_array)) + + @staticmethod + def print_point_data(met_point_data, print_subset=True): + print(' === MET point data by python embedding ===') + if print_subset: + met_point_obs.print_data('nhdr', met_point_data['nhdr']) + met_point_obs.print_data('nobs' , met_point_data['nobs']) + met_point_obs.print_data('use_var_id',met_point_data['use_var_id']) + met_point_obs.print_data('hdr_typ',met_point_data['hdr_typ']) + met_point_obs.print_data('obs_hid',met_point_data['obs_hid']) + met_point_obs.print_data('obs_vid',met_point_data['obs_vid']) + met_point_obs.print_data('obs_val',met_point_data['obs_val']) + met_point_obs.print_data('obs_var_table',met_point_data['obs_var_table']) + met_point_obs.print_data('obs_qty_table',met_point_data['obs_qty_table']) + met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table']) + met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table']) + met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table']) + else: + print('All',met_point_data) + print(" nhdr: ",met_point_data['nhdr']) + print(" nobs: ",met_point_data['nobs']) + print('use_var_id: ',met_point_data['use_var_id']) + print('hdr_typ: ',met_point_data['hdr_typ']) + print('obs_vid: ',met_point_data['obs_vid']) + print('obs_val: ',met_point_data['obs_val']) + print('obs_var_table: ',met_point_data['obs_var_table']) + print('obs_qty_table: ',met_point_data['obs_qty_table']) + print('hdr_typ_table: ',met_point_data['hdr_typ_table']) + print('hdr_sid_table: ',met_point_data['hdr_sid_table']) + print('hdr_vld_table: ',met_point_data['hdr_vld_table']) + print(' === MET point data by python embedding ===') + + +# This is a sample drived class +class sample_met_point_obs(met_point_obs): + + #@abstractmethod + def read_data(self, arg_map={}): + self.hdr_typ = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + self.hdr_sid = np.array([ 0, 0, 0, 0, 0, 1, 2, 3, 3, 1, 2, 2, 3, 0, 0, 0, 0, 0, 1, 2, 3, 3, 1, 2, 2, 3 ]) + self.hdr_vld = np.array([ 0, 1, 2, 3, 4, 4, 3, 4, 3, 4, 5, 4, 3, 0, 1, 2, 3, 4, 4, 3, 4, 3, 4, 5, 4, 3 ]) + self.hdr_lat = np.array([ 43., 43., 43., 43., 43., 43., 43., 43., 43., 46., 46., 46., 46., 43., 43., 43., 43., 43., 43., 43., 43., 43., 46., 46., 46., 46. ]) + self.hdr_lon = np.array([ -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92., -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92. ]) + self.hdr_elv = np.array([ 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220. ]) + + self.obs_qid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + self.obs_hid = np.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25 ]) + self.obs_vid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]) + self.obs_lvl = np.array([ 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000. ]) + self.obs_hgt = np.array([ 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2. ]) + self.obs_val = np.array([ 292., 292.5, 293., 293.5, 294., 294.5, 295., 295.5, 296., 292., 293.4, 293., 296., 294., 92., 92.5, 93., 93.5, 94., 94.5, 95., 95.5, 96., 92., 93.4, 93., 96., 94. ]) + + self.hdr_typ_table = [ "ADPSFC" ] + self.hdr_sid_table = [ "001", "002", "003", "004" ] + self.hdr_vld_table = [ + "20120409_115000", "20120409_115500", "20120409_120100", "20120409_120500", "20120409_121000", + "20120409_120000" ] + self.obs_var_table = [ "TMP", "RH" ] + self.obs_qty_table = [ "NA" ] + + +def main(): + args = {} # or args = [] + point_obs_data = sample_met_point_obs() + point_obs_data.read_data(args) + met_point_data = point_obs_data.get_point_data() + + point_obs_data.print_point_data(met_point_data, print_subset=False) + +if __name__ == '__main__': + main() + print('Done python scripot') diff --git a/met/scripts/python/read_met_point_obs.py b/met/scripts/python/read_met_point_obs.py new file mode 100755 index 0000000000..60f2c62065 --- /dev/null +++ b/met/scripts/python/read_met_point_obs.py @@ -0,0 +1,121 @@ +''' +Created on Nov 10, 2021 + +@author: hsoh + +This script reads the MET point observation NetCDF file like MET tools do. +''' + +import os +import sys +from datetime import datetime +import numpy as np +import netCDF4 as nc + +from met_point_obs import met_point_obs, sample_met_point_obs + +DO_PRINT_DATA = False +ARG_PRINT_DATA = 'show_data' + +class nc_point_obs(met_point_obs): + + def read_data(self, args): + # args should be list or dictionaryu + if isinstance(args, dict): + nc_filename = args.get('nc_name',None) + elif isinstance(args, list): + nc_filename = args[0] + else: + nc_filename = args + if nc_filename is None: + print("==ERROR== The input NetCDF filename is missing") + elif not os.path.exists(nc_filename): + print("==ERROR== input NetCDF file ({f}) does not exist".format(f=nc_filename)) + else: + dataset = nc.Dataset(nc_filename, 'r') + # Header + self.hdr_typ = dataset['hdr_typ'][:] + self.hdr_sid = dataset['hdr_sid'][:] + self.hdr_vld = dataset['hdr_vld'][:] + self.hdr_lat = dataset['hdr_lat'][:] + self.hdr_lon = dataset['hdr_lon'][:] + self.hdr_elv = dataset['hdr_elv'][:] + self.hdr_typ_table = nc_tools.get_string_array(dataset, 'hdr_typ_table') + self.hdr_sid_table = nc_tools.get_string_array(dataset, 'hdr_sid_table') + self.hdr_vld_table = nc_tools.get_string_array(dataset, 'hdr_vld_table') + + nc_var = dataset.variables.get('hdr_prpt_typ', None) + if nc_var: + self.hdr_prpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_irpt_typ', None) + if nc_var: + self.hdr_irpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_inst_typ', None) + if nc_var: + self.hdr_inst_typ =nc_var[:] + + #Observation data + self.hdr_sid = dataset['hdr_sid'][:] + self.obs_qty = np.array(dataset['obs_qty'][:]) + self.obs_hid = np.array(dataset['obs_hid'][:]) + self.obs_lvl = np.array(dataset['obs_lvl'][:]) + self.obs_hgt = np.array(dataset['obs_hgt'][:]) + self.obs_val = np.array(dataset['obs_val'][:]) + nc_var = dataset.variables.get('obs_vid', None) + if nc_var is None: + self.use_var_id = False + nc_var = dataset.variables.get('obs_gc', None) + else: + self.obs_var_table = nc_tools.get_string_array(dataset, 'obs_var') + if nc_var: + self.obs_vid = np.array(nc_var[:]) + + self.obs_qty_table = nc_tools.get_string_array(dataset, 'obs_qty_table') + + +class nc_tools(): + + @staticmethod + def get_num_array(dataset, var_name): + nc_var = dataset.variables.get(var_name, None) + return nc_var[:].filled(nc_var._FillValue if '_FillValue' in nc_var.ncattrs() else -9999) if nc_var else [] + + @staticmethod + def get_ncbyte_array_to_str(nc_var): + nc_str_data = nc_var[:] + if nc_var.datatype.name == 'bytes8': + nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:]] + return nc_str_data + + @staticmethod + def get_string_array(dataset, var_name): + nc_var = dataset.variables.get(var_name, None) + return nc_tools.get_ncbyte_array_to_str(nc_var) if nc_var else [] + + +start_time = datetime.now() + +point_obs_data = None +if len(sys.argv) == 1: + point_obs_data = sample_met_point_obs() + point_obs_data.read_data([]) +else: + netcdf_filename = sys.argv[1] + args = [ netcdf_filename ] + #args = { 'nc_name': netcdf_filename } + point_obs_data = nc_point_obs() + point_obs_data.read_data(args) + + if ARG_PRINT_DATA == sys.argv[-1]: + DO_PRINT_DATA = True + +if point_obs_data is not None: + met_point_data = point_obs_data.get_point_data() + met_point_data['met_point_data'] = point_obs_data + + if DO_PRINT_DATA: + met_point_obs.print_point_data(met_point_data) + +run_time = datetime.now() - start_time + +print('Done python script {s} took {t}'.format(s=sys.argv[0], t=run_time)) diff --git a/met/src/basic/vx_util/ascii_table.cc b/met/src/basic/vx_util/ascii_table.cc index 28fb9004d6..e5b917c377 100644 --- a/met/src/basic/vx_util/ascii_table.cc +++ b/met/src/basic/vx_util/ascii_table.cc @@ -1336,7 +1336,7 @@ const char fill_char = ' '; const int r_start = 1; // skip the header row -for (r=0; r= 0 ) ::close(fd); fd = -1; -memset(&Header, 0, sizeof(Header)); +Header.clear(); Filename.clear(); diff --git a/met/src/libcode/vx_nc_obs/Makefile.am b/met/src/libcode/vx_nc_obs/Makefile.am index ba2fb8c910..22f0871fab 100644 --- a/met/src/libcode/vx_nc_obs/Makefile.am +++ b/met/src/libcode/vx_nc_obs/Makefile.am @@ -12,6 +12,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_nc_obs.a libvx_nc_obs_a_SOURCES = \ + met_point_data.cc met_point_data.h \ nc_obs_util.cc nc_obs_util.h \ nc_point_obs.cc nc_point_obs.h \ nc_point_obs_in.cc nc_point_obs_in.h \ diff --git a/met/src/libcode/vx_nc_obs/met_point_data.cc b/met/src/libcode/vx_nc_obs/met_point_data.cc new file mode 100644 index 0000000000..6cc15a7e81 --- /dev/null +++ b/met/src/libcode/vx_nc_obs/met_point_data.cc @@ -0,0 +1,437 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include + +#include "vx_log.h" +#include "is_bad_data.h" + +#include "met_point_data.h" +//#include "nc_point_obs.h" + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetPointData + // + + +//////////////////////////////////////////////////////////////////////// + +MetPointData::MetPointData() { + // Derived class should set obs_data + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +MetPointData::~MetPointData() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +void MetPointData::init_from_scratch() { + nobs = 0; + nhdr = 0; + qty_length = 0; + + use_var_id = false; + use_arr_vars = false; +} + + +//////////////////////////////////////////////////////////////////////// + +//void MetPointData::allocate() { +// obs_data.allocate(); +//} + +//////////////////////////////////////////////////////////////////////// + +void MetPointData::clear() { + if (obs_data) obs_data->clear(); + header_data.clear(); +} + +//////////////////////////////////////////////////////////////////////// + +bool MetPointData::get_header(int header_offset, float hdr_arr[HDR_ARRAY_LEN], + ConcatString &hdr_typ_str, ConcatString &hdr_sid_str, + ConcatString &hdr_vld_str) { + int hdr_idx; + + // Read the corresponding header array for this observation + + hdr_arr[0] = header_data.lat_array[header_offset]; + hdr_arr[1] = header_data.lon_array[header_offset]; + hdr_arr[2] = header_data.elv_array[header_offset]; + + // Read the corresponding header type for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.typ_idx_array[header_offset]; + hdr_typ_str = header_data.typ_array[hdr_idx]; + + // Read the corresponding header Station ID for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.sid_idx_array[header_offset]; + hdr_sid_str = header_data.sid_array[hdr_idx]; + + // Read the corresponding valid time for this observation + hdr_idx = use_arr_vars ? header_offset : header_data.vld_idx_array[header_offset]; + hdr_vld_str = header_data.vld_array[hdr_idx]; + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetPointData::get_header_type(int header_offset, int hdr_typ_arr[HDR_TYPE_ARR_LEN]) { + int hdr_idx; + // Read the header integer types + hdr_typ_arr[0] = (header_data.prpt_typ_array.n() > header_offset ? + header_data.prpt_typ_array[header_offset] : bad_data_int); + hdr_typ_arr[1] = (header_data.irpt_typ_array.n() > header_offset ? + header_data.irpt_typ_array[header_offset] : bad_data_int); + hdr_typ_arr[2] = (header_data.inst_typ_array.n() > header_offset ? + header_data.inst_typ_array[header_offset] : bad_data_int); + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +bool MetPointData::get_lats(float *hdr_lats) { + for (int idx=0; idxobs_cnt = obs_cnt; +} + + + + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetPointDataPython + // + + +//////////////////////////////////////////////////////////////////////// + +MetPointDataPython::MetPointDataPython() { + obs_data = new MetPointObsData(); + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +MetPointDataPython::MetPointDataPython(MetPointDataPython &d) { + init_from_scratch(); + //obs_data = d.get_point_obs_data(); + //header_data = d.get_header_data(); + obs_data->assign(*d.get_point_obs_data()); + header_data.assign(*d.get_header_data()); +cout << " DEBUG HS MetPointData(MetPointData &d) is called \n"; +cout << " DEBUG HS MetPointData(MetPointData &d) &header_data.lat_array=" << &(header_data.lat_array) << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.lat_array.n()=" << header_data.lat_array.n() << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.lon_array.n()=" << header_data.lon_array.n() << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.elv_array.n()=" << header_data.elv_array.n() << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.typ_idx_array.n()=" << header_data.typ_idx_array.n() << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.sid_idx_array.n()=" << header_data.sid_idx_array.n() << "\n"; +cout << " DEBUG HS MetPointData(MetPointData &d) header_data.vld_idx_array.n()=" << header_data.vld_idx_array.n() << "\n"; +} + + +//////////////////////////////////////////////////////////////////////// + +MetPointDataPython::~MetPointDataPython() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +//void MetPointDataPython::init_from_scratch() { +// nobs = 0; +// nhdr = 0; +// qty_length = 0; +// +// use_var_id = false; +// use_arr_vars = false; +//} + + +//////////////////////////////////////////////////////////////////////// + +void MetPointDataPython::allocate(int obs_cnt) { + set_obs_cnt(obs_cnt); + obs_data->allocate(); +} + +//////////////////////////////////////////////////////////////////////// + + + +/////////////////////////////////////////////////////////////////////////////// +// struct MetPointObsData + +MetPointObsData::MetPointObsData(): + obs_cnt(0), + obs_ids((int *)0), + obs_hids((int *)0), + obs_qids((int *)0), + obs_lvls((float *)0), + obs_hgts((float *)0), + obs_vals((float *)0), + obs_arr((float *)0), + is_obs_array(false) +{ +} + + +/////////////////////////////////////////////////////////////////////////////// + +void MetPointObsData::allocate() { + if (is_obs_array) obs_arr = new float[obs_cnt*OBS_ARRAY_LEN]; // nobs * 5 + else { + obs_ids = new int[obs_cnt]; // grib_code or var_id + obs_hids = new int[obs_cnt]; + obs_qids = new int[obs_cnt]; + obs_lvls = new float[obs_cnt]; + obs_hgts = new float[obs_cnt]; + obs_vals = new float[obs_cnt]; + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void MetPointObsData::assign(MetPointObsData &o) { + obs_cnt = o.obs_cnt; + is_obs_array = o.is_obs_array; + + clear(); + allocate(); + if (is_obs_array) + for (int idx=0; idx + +#include "nc_utils.h" + + +//////////////////////////////////////////////////////////////////////// + +struct MetPointHeader { + int typ_len; + int sid_len; + int vld_len; + int strl_len; + int strll_len; + int min_vld_time; + int max_vld_time; + int hdr_count; + int hdr_type_count; + + StringArray typ_array; + StringArray sid_array; + StringArray vld_array; // string array for valid time + IntArray vld_num_array; // number array for valid time + IntArray typ_idx_array; + IntArray sid_idx_array; + IntArray vld_idx_array; // index to vld_array/vld_num_array + NumArray lat_array; + NumArray lon_array; + NumArray elv_array; + IntArray prpt_typ_array; + IntArray irpt_typ_array; + IntArray inst_typ_array; + + MetPointHeader(); + void assign(MetPointHeader &h); + void clear(); + void reset_counters(); +}; + +//////////////////////////////////////////////////////////////////////// + +struct MetPointObsData { + int obs_cnt; + bool is_obs_array; + + int *obs_ids; // grib_code or var_id + int *obs_hids; + int *obs_qids; + float *obs_lvls; + float *obs_hgts; + float *obs_vals; + float *obs_arr; // nobs * 5 + StringArray var_names; + StringArray qty_names; + + MetPointObsData(); + void allocate(); + void assign(MetPointObsData &d); + void clear(); + void clear_numbers(); + void clear_strings(); + bool fill_obs_buf(int buf_size, int offset, float *obs_arr, int *qty_idx_arr); + float get_obs_val(int index); +}; + + +class MetPointData { + + protected: + + int nobs; + int nhdr; + int qty_length; + bool use_var_id; + bool use_arr_vars; + + MetPointHeader header_data; + MetPointObsData *obs_data = 0; + + void init_from_scratch(); + + public: + + MetPointData(); + ~MetPointData(); + + //void close(); + void clear(); + + int get_buf_size(); + int get_hdr_cnt(); + int get_grib_code_or_var_index(const float obs_arr[OBS_ARRAY_LEN]); + MetPointHeader *get_header_data(); + bool get_header(int header_offset, float hdr_arr[HDR_ARRAY_LEN], + ConcatString &hdr_typ_str, ConcatString &hdr_sid_str, + ConcatString &hdr_vld_str); + int get_header_offset(const float obs_arr[OBS_ARRAY_LEN]); + bool get_header_type(int header_offset, int hdr_typ_arr[HDR_TYPE_ARR_LEN]); + bool get_lats(float *hdr_lats); + bool get_lons(float *hdr_lons); + int get_obs_cnt(); + MetPointObsData *get_point_obs_data(); + StringArray get_qty_data(); + StringArray get_var_names(); + + bool is_same_obs_values(const float obs_arr1[OBS_ARRAY_LEN], const float obs_arr2[OBS_ARRAY_LEN]); + bool is_using_var_id(); + + void set_grib_code_or_var_index(float obs_arr[OBS_ARRAY_LEN], int grib_code); + void set_hdr_cnt(int hdr_cnt); + void set_obs_cnt(int obs_cnt); + // variables + + // data + +}; // MetPointDataBase + + + +class MetPointDataPython : public MetPointData { + + protected: + + public: + + MetPointDataPython(); + MetPointDataPython(MetPointDataPython &d); + ~MetPointDataPython(); + + void allocate(int obs_cnt); + void set_use_var_id(bool new_use_var_id); + + // variables + + // data + +}; // MetPointData + +//////////////////////////////////////////////////////////////////////// + +inline MetPointHeader *MetPointData::get_header_data() { return &header_data; } +inline int MetPointData::get_buf_size() { return OBS_BUFFER_SIZE; } +inline int MetPointData::get_grib_code_or_var_index(const float obs_arr[OBS_ARRAY_LEN]) { return obs_arr[1]; }; +inline int MetPointData::get_hdr_cnt() { return nhdr; } +inline int MetPointData::get_header_offset(const float obs_arr[OBS_ARRAY_LEN]) { return obs_arr[0]; }; +inline int MetPointData::get_obs_cnt() { return nobs; } +inline MetPointObsData *MetPointData::get_point_obs_data() { return obs_data; } +inline StringArray MetPointData::get_qty_data() { return obs_data->qty_names; } +inline StringArray MetPointData::get_var_names() { return obs_data->var_names; } +inline bool MetPointData::is_using_var_id() { return use_var_id; } +inline void MetPointData::set_grib_code_or_var_index(float obs_arr[OBS_ARRAY_LEN], int grib_code) { obs_arr[1] = grib_code; } +inline void MetPointDataPython::set_use_var_id(bool new_use_var_id) { use_var_id = new_use_var_id; } + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __MET_POINT_DATA_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.cc b/met/src/libcode/vx_nc_obs/nc_obs_util.cc index 70ea844d49..efab7d2384 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -68,78 +68,12 @@ void NcDataBuffer::reset_counters() { /////////////////////////////////////////////////////////////////////////////// // struct NcPointObsData -NcPointObsData::NcPointObsData(): - obs_cnt(0), - obs_ids((int *)0), - obs_hids((int *)0), - obs_qids((int *)0), - obs_lvls((float *)0), - obs_hgts((float *)0), - obs_vals((float *)0), - obs_arr((float *)0), - is_obs_array(false) +NcPointObsData::NcPointObsData() { } /////////////////////////////////////////////////////////////////////////////// -void NcPointObsData::clear() { - obs_cnt = 0; - is_obs_array = false; - - clear_numbers(); - clear_strings(); -} - -/////////////////////////////////////////////////////////////////////////////// - -void NcPointObsData::clear_numbers() { - if (0 != obs_ids) { - delete [] obs_ids; - obs_ids = (int *)0; - } - if (0 != obs_hids) { - delete [] obs_hids; - obs_hids = (int *)0; - } - if (0 != obs_qids) { - delete [] obs_qids; - obs_qids = (int *)0; - } - if (0 != obs_lvls) { - delete [] obs_lvls; - obs_lvls = (float *)0; - } - if (0 != obs_hgts) { - delete [] obs_hgts; - obs_hgts = (float *)0; - } - if (0 != obs_vals) { - delete [] obs_vals; - obs_vals = (float *)0; - } - if (0 != obs_arr) { - delete [] obs_arr; - obs_arr = (float *)0; - } -} - -/////////////////////////////////////////////////////////////////////////////// - -void NcPointObsData::clear_strings() { - var_names.clear(); - qty_names.clear(); -} - -/////////////////////////////////////////////////////////////////////////////// - -float NcPointObsData::get_obs_val(int index) { - float obs_val = (is_obs_array ? obs_arr[index*obs_cnt+4] : obs_vals[index]); - return obs_val; -} - -/////////////////////////////////////////////////////////////////////////////// - bool NcPointObsData::read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop) { bool succeed = true; const char* method_name = "NcPointObsData::read_obs_data_numbers()"; @@ -434,7 +368,7 @@ void NetcdfObsVars::create_pb_hdrs (NcFile *f_out, const int hdr_count) { /////////////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, +void NetcdfObsVars::create_table_vars (NcFile *f_out, MetPointHeader &hdr_data, NcDataBuffer &data_buffer) { const string method_name = " create_table_vars()"; @@ -603,7 +537,7 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { //////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::read_header_data(NcHeaderData &hdr_data) { +void NetcdfObsVars::read_header_data(MetPointHeader &hdr_data) { bool is_valid_obs_nc = true; long nhdr_count = get_dim_size(&hdr_dim); int strl_len = get_dim_size(&strl_dim); @@ -970,7 +904,7 @@ bool NetcdfObsVars::read_obs_data(int buf_size, int offset, //////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::read_pb_hdr_data(NcHeaderData &hdr_data) { +void NetcdfObsVars::read_pb_hdr_data(MetPointHeader &hdr_data) { const char *method_name = "get_nc_pb_hdr_data() -> "; int pb_hdr_count = get_dim_size(&pb_hdr_dim); @@ -1132,26 +1066,28 @@ void NetcdfObsVars::write_header_to_nc(NcDataBuffer &data_buf, else if (data_buf.hdr_data_offset == data_buf.pb_hdr_data_offset) { int save_len = lengths[0]; int pb_hdr_len = raw_hdr_cnt - offsets[0]; - if (pb_hdr_len > buf_size) pb_hdr_len = buf_size; - - lengths[0] = pb_hdr_len; - if(IS_VALID_NC(hdr_prpt_typ_var) && !put_nc_data((NcVar *)&hdr_prpt_typ_var, - data_buf.hdr_prpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the pb message type to the netCDF file\n\n"; - exit(1); - } - if(IS_VALID_NC(hdr_irpt_typ_var) && !put_nc_data((NcVar *)&hdr_irpt_typ_var, - data_buf.hdr_irpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the in message type to the netCDF file\n\n"; - exit(1); + if (pb_hdr_len > 0) { + if (pb_hdr_len > buf_size) pb_hdr_len = buf_size; + + lengths[0] = pb_hdr_len; + if(IS_VALID_NC(hdr_prpt_typ_var) && !put_nc_data((NcVar *)&hdr_prpt_typ_var, + data_buf.hdr_prpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the pb message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_irpt_typ_var) && !put_nc_data((NcVar *)&hdr_irpt_typ_var, + data_buf.hdr_irpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the in message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_inst_typ_var) && !put_nc_data((NcVar *)&hdr_inst_typ_var, + data_buf.hdr_inst_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the instrument type to the netCDF file\n\n"; + exit(1); + } + lengths[0] = save_len; + data_buf.pb_hdr_data_offset += pb_hdr_len; } - if(IS_VALID_NC(hdr_inst_typ_var) && !put_nc_data((NcVar *)&hdr_inst_typ_var, - data_buf.hdr_inst_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the instrument type to the netCDF file\n\n"; - exit(1); - } - lengths[0] = save_len; - data_buf.pb_hdr_data_offset += pb_hdr_len; } else { mlog << Debug(6) << method_name @@ -1166,7 +1102,7 @@ void NetcdfObsVars::write_header_to_nc(NcDataBuffer &data_buf, /////////////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::write_table_vars (NcHeaderData &hdr_data, NcDataBuffer &data_buffer) +void NetcdfObsVars::write_table_vars (MetPointHeader &hdr_data, NcDataBuffer &data_buffer) { mlog << Debug(5) << "write_table_vars() is called. valid hdr_typ_tbl_var: " << !IS_INVALID_NC(hdr_typ_tbl_var) << "\n"; @@ -1261,51 +1197,6 @@ void NetcdfObsVars::write_obs_var_descriptions(StringArray &descriptions) { //////////////////////////////////////////////////////////////////////// // End of NetcdfObsVars -/////////////////////////////////////////////////////////////////////////////// - -// struct NcHeaderData - -NcHeaderData::NcHeaderData() -{ - reset_counters(); -} - -/////////////////////////////////////////////////////////////////////////////// - -void NcHeaderData::clear() { - reset_counters(); - - typ_array.clear(); - sid_array.clear(); - vld_array.clear(); - vld_num_array.clear(); - typ_idx_array.clear(); - sid_idx_array.clear(); - vld_idx_array.clear(); - lat_array.clear(); - lon_array.clear(); - elv_array.clear(); - prpt_typ_array.clear(); - irpt_typ_array.clear(); - inst_typ_array.clear(); -} - -/////////////////////////////////////////////////////////////////////////////// - -void NcHeaderData::reset_counters() { - valid_point_obs = false; - typ_len = 0; - sid_len = 0; - vld_len = 0; - strl_len = 0; - strll_len = 0; - hdr_count = 0; - - min_vld_time = -1; - max_vld_time = -1; -} - - /////////////////////////////////////////////////////////////////////////////// long count_nc_headers(vector< Observation > &observations) diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.h b/met/src/libcode/vx_nc_obs/nc_obs_util.h index 4985639c09..d3d07f3007 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.h +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.h @@ -17,6 +17,7 @@ using namespace netCDF; #include "nc_summary.h" +#include "met_point_data.h" //////////////////////////////////////////////////////////////////////// @@ -78,39 +79,6 @@ struct NcDataBuffer { //////////////////////////////////////////////////////////////////////// -struct NcHeaderData { - bool valid_point_obs; - int typ_len; - int sid_len; - int vld_len; - int strl_len; - int strll_len; - int min_vld_time; - int max_vld_time; - int hdr_count; - int hdr_type_count; - - StringArray typ_array; - StringArray sid_array; - StringArray vld_array; - IntArray vld_num_array; - IntArray typ_idx_array; - IntArray sid_idx_array; - IntArray vld_idx_array; - NumArray lat_array; - NumArray lon_array; - NumArray elv_array; - IntArray prpt_typ_array; - IntArray irpt_typ_array; - IntArray inst_typ_array; - - NcHeaderData(); - void clear(); - void reset_counters(); -}; - -//////////////////////////////////////////////////////////////////////// - struct NetcdfObsVars { bool attr_agl ; bool attr_pb2nc ; @@ -167,7 +135,7 @@ struct NetcdfObsVars { void create_hdr_vars (NcFile *f_out, const int hdr_count); void create_obs_vars (NcFile *f_out); void create_obs_name_vars (NcFile *f_out, const int var_count, const int unit_count); - void create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, NcDataBuffer &data_buffer); + void create_table_vars (NcFile *f_out, MetPointHeader &hdr_data, NcDataBuffer &data_buffer); void create_pb_hdrs (NcFile *f_out, const int hdr_count); NcDim create_var_obs_var (NcFile *f_out, int var_count); @@ -175,41 +143,25 @@ struct NetcdfObsVars { int get_obs_index(); void read_dims_vars(NcFile *f_in); - void read_header_data(NcHeaderData &hdr_data); + void read_header_data(MetPointHeader &hdr_data); bool read_obs_data(int buf_size, int offset, int qty_len, float *obs_arr, int *qty_idx_arr, char *obs_qty_buf); - void read_pb_hdr_data(NcHeaderData &hdr_data); + void read_pb_hdr_data(MetPointHeader &hdr_data); void write_header_to_nc(NcDataBuffer &data_buf, const int buf_size, const bool is_pb = false); void write_obs_buffer(NcDataBuffer &data_buffer, const int buf_size); void write_obs_var_names(StringArray &obs_names); void write_obs_var_units(StringArray &units); void write_obs_var_descriptions(StringArray &descriptions); - void write_table_vars(NcHeaderData &hdr_data, NcDataBuffer &data_buffer); + void write_table_vars(MetPointHeader &hdr_data, NcDataBuffer &data_buffer); }; // NetcdfObsVars //////////////////////////////////////////////////////////////////////// -struct NcPointObsData { - int obs_cnt; - bool is_obs_array; - - int *obs_ids; // grib_code or var_id - int *obs_hids; - int *obs_qids; - float *obs_lvls; - float *obs_hgts; - float *obs_vals; - float *obs_arr; // nobs * 5 - StringArray var_names; - StringArray qty_names; +struct NcPointObsData : public MetPointObsData { NcPointObsData(); - void clear(); - void clear_numbers(); - void clear_strings(); - float get_obs_val(int index); bool read_obs_data_numbers(NetcdfObsVars obs_vars, bool stop=true); bool read_obs_data_table_lookups(NetcdfObsVars obs_vars, bool stop=true); }; diff --git a/met/src/libcode/vx_nc_obs/nc_point_obs.cc b/met/src/libcode/vx_nc_obs/nc_point_obs.cc index b665e9d2e9..0869f3a31e 100644 --- a/met/src/libcode/vx_nc_obs/nc_point_obs.cc +++ b/met/src/libcode/vx_nc_obs/nc_point_obs.cc @@ -36,6 +36,7 @@ using namespace std; //////////////////////////////////////////////////////////////////////// MetNcPointObs::MetNcPointObs() { + obs_data = new NcPointObsData(); init_from_scratch(); } @@ -48,104 +49,21 @@ MetNcPointObs::~MetNcPointObs() { //////////////////////////////////////////////////////////////////////// void MetNcPointObs::init_from_scratch() { - obs_nc = (NcFile *) 0; + MetPointData::init_from_scratch(); - nobs = 0; - nhdr = 0; - qty_length = 0; keep_nc = false; - use_var_id = false; - use_arr_vars = false; - - return; + obs_nc = (NcFile *) 0; } //////////////////////////////////////////////////////////////////////// void MetNcPointObs::close() { + MetPointData::clear(); + if ( !keep_nc && obs_nc ) { delete obs_nc; obs_nc = (NcFile *) 0; } - - obs_data.clear(); - header_data.clear(); - return; -} - -//////////////////////////////////////////////////////////////////////// - -int MetNcPointObs::get_qty_length() { - qty_length = get_nc_string_length(&obs_vars.obs_qty_tbl_var); - return qty_length; -} - -//////////////////////////////////////////////////////////////////////// - -bool MetNcPointObs::get_header(int header_offset, float hdr_arr[HDR_ARRAY_LEN], - ConcatString &hdr_typ_str, ConcatString &hdr_sid_str, ConcatString &hdr_vld_str) { - int hdr_idx; - - // Read the corresponding header array for this observation - hdr_arr[0] = header_data.lat_array[header_offset]; - hdr_arr[1] = header_data.lon_array[header_offset]; - hdr_arr[2] = header_data.elv_array[header_offset]; - - // Read the corresponding header type for this observation - hdr_idx = use_arr_vars ? header_offset : header_data.typ_idx_array[header_offset]; - hdr_typ_str = header_data.typ_array[hdr_idx]; - - // Read the corresponding header Station ID for this observation - hdr_idx = use_arr_vars ? header_offset : header_data.sid_idx_array[header_offset]; - hdr_sid_str = header_data.sid_array[hdr_idx]; - - // Read the corresponding valid time for this observation - hdr_idx = use_arr_vars ? header_offset : header_data.vld_idx_array[header_offset]; - hdr_vld_str = header_data.vld_array[hdr_idx]; - - return true; -} - -//////////////////////////////////////////////////////////////////////// - -bool MetNcPointObs::get_header_type(int header_offset, int hdr_typ_arr[HDR_TYPE_ARR_LEN]) { - int hdr_idx; - // Read the header integer types - hdr_typ_arr[0] = (header_data.prpt_typ_array.n() > header_offset ? - header_data.prpt_typ_array[header_offset] : bad_data_int); - hdr_typ_arr[1] = (header_data.irpt_typ_array.n() > header_offset ? - header_data.irpt_typ_array[header_offset] : bad_data_int); - hdr_typ_arr[2] = (header_data.inst_typ_array.n() > header_offset ? - header_data.inst_typ_array[header_offset] : bad_data_int); - - return true; -} - -//////////////////////////////////////////////////////////////////////// - -bool MetNcPointObs::get_lats(float *hdr_lats) { - for (int idx=0; idxread_obs_data_numbers(obs_vars); } return status; } @@ -104,7 +104,7 @@ bool MetNcPointObsIn::read_obs_data_numbers() { bool MetNcPointObsIn::read_obs_data_table_lookups() { bool status = false; if( IS_VALID_NC_P(obs_nc) ) { - status = obs_data.read_obs_data_table_lookups(obs_vars); + status = ((NcPointObsData *)obs_data)->read_obs_data_table_lookups(obs_vars); } return status; } diff --git a/met/src/libcode/vx_pointdata_python/Makefile.am b/met/src/libcode/vx_pointdata_python/Makefile.am new file mode 100644 index 0000000000..c48c07e2df --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/Makefile.am @@ -0,0 +1,19 @@ +## @start 1 +## Makefile.am -- Process this file with automake to produce Makefile.in +## @end 1 + +MAINTAINERCLEANFILES = Makefile.in + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# The library + +noinst_LIBRARIES = libvx_pointdata_python.a +libvx_pointdata_python_a_SOURCES = \ + pointdata_python.h pointdata_python.cc \ + pointdata_from_array.h pointdata_from_array.cc \ + python_pointdata.h python_pointdata.cc +libvx_pointdata_python_a_CPPFLAGS = ${MET_CPPFLAGS} -I../vx_python2_utils ${MET_PYTHON_CC} $(MET_PYTHON_LD) + diff --git a/met/src/libcode/vx_pointdata_python/pointdata_from_array.cc b/met/src/libcode/vx_pointdata_python/pointdata_from_array.cc new file mode 100644 index 0000000000..b3d6cd84fe --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/pointdata_from_array.cc @@ -0,0 +1,659 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#include "string.h" + +#include "vx_python3_utils.h" +#include "vx_statistics.h" +#include "check_endian.h" + +#include "pointdata_from_array.h" + +//////////////////////////////////////////////////////////////////////// + +static const int api_delug_level = 11; + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + float * out) +{ + +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j, x, y, r, c; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + int * out) +{ + +const char *method_name = "load_numpy(int *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy_int (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + IntArray *out) +{ + +const char *method_name = "load_numpy_int(IntArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((int)value); + + mlog << Debug(api_delug_level) << method_name << " [" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy_num (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + NumArray *out) +{ + +const char *method_name = "load_numpy_num(NumArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((float)value); + + mlog << Debug(api_delug_level) << method_name << "[" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool pointdata_from_np_array(Python3_Numpy & np, float * data_out) +{ + +const char *method_name = "pointdata_from_np_array(float) -> "; + + // + // make sure it's a 1D array + // + +if ( np.n_dims() != 1 ) { + + mlog << Error << "\n" << method_name + << "data array is not 1-dimensional! ... " + << "(dim = " << (np.n_dims()) << ")\n\n"; + + exit ( 1 ); + + +} + +int n = np.dim(0); + + + // + // load the data + // + +const ConcatString dtype = np.dtype(); + + // 1 byte integers + + if ( dtype == "|i1" ) load_numpy (np.buffer(), n, little_endian, 0, data_out); +else if ( dtype == "|u1" ) load_numpy (np.buffer(), n, little_endian, 0, data_out); + + // 2 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); + +else if ( dtype == ">i2" ) load_numpy (np.buffer(), n, big_endian, shuffle_2, data_out); +else if ( dtype == ">u2" ) load_numpy (np.buffer(), n, big_endian, shuffle_2, data_out); + + // 4 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); + +else if ( dtype == ">i4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); +else if ( dtype == ">u4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); + + // 8 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); + +else if ( dtype == ">i8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); +else if ( dtype == ">u8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); + + // single precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == ">f4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); + + // double precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == ">f8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); + + // + // nope ... the only other numerical data type for numpy arrays + // is single or double precision complex numbers, and + // we're not supporting those at this time + // + +else { + + mlog << Error << "\n" << method_name + << "unsupported data type \"" << dtype << "\"\n\n"; + + exit ( 1 ); + +} + + //////////////////// + + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool pointdata_from_np_array(Python3_Numpy & np, int * data_out) +{ + +const char *method_name = "pointdata_from_np_array(int) -> "; + + // + // make sure it's a 1D array + // + +if ( np.n_dims() != 1 ) { + + mlog << Error << "\n" << method_name + << "data array is not 1-dimensional! ... " + << "(dim = " << (np.n_dims()) << ")\n\n"; + + exit ( 1 ); + + +} + +int n = np.dim(0); + + + // + // load the data + // + +const ConcatString dtype = np.dtype(); + + // 1 byte integers + + if ( dtype == "|i1" ) load_numpy (np.buffer(), n, little_endian, 0, data_out); +else if ( dtype == "|u1" ) load_numpy (np.buffer(), n, little_endian, 0, data_out); + + // 2 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); + +else if ( dtype == ">i2" ) load_numpy (np.buffer(), n, big_endian, shuffle_2, data_out); +else if ( dtype == ">u2" ) load_numpy (np.buffer(), n, big_endian, shuffle_2, data_out); + + // 4 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); + +else if ( dtype == ">i4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); +else if ( dtype == ">u4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); + + // 8 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); + +else if ( dtype == ">i8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); +else if ( dtype == ">u8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); + + // single precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == ">f4" ) load_numpy (np.buffer(), n, big_endian, shuffle_4, data_out); + + // double precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == ">f8" ) load_numpy (np.buffer(), n, big_endian, shuffle_8, data_out); + + // + // nope ... the only other numerical data type for numpy arrays + // is single or double precision complex numbers, and + // we're not supporting those at this time + // + +else { + + mlog << Error << "\n" << method_name + << "unsupported data type \"" << dtype << "\"\n\n"; + + exit ( 1 ); + +} + + //////////////////// + + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool pointdata_from_np_array(Python3_Numpy & np, IntArray *data_out) +{ + +const char *method_name = "pointdata_from_np_array(IntArray) -> "; + + // + // make sure it's a 1D array + // + +if ( np.n_dims() != 1 ) { + + mlog << Error << "\n" << method_name + << "data array is not 1-dimensional! ... " + << "(dim = " << (np.n_dims()) << ")\n\n"; + + exit ( 1 ); + + +} + +int n = np.dim(0); + + + // + // load the data + // + +IntArray data; +const ConcatString dtype = np.dtype(); + + // 1 byte integers + + if ( dtype == "|i1" ) load_numpy_int (np.buffer(), n, little_endian, 0, data_out); +else if ( dtype == "|u1" ) load_numpy_int (np.buffer(), n, little_endian, 0, data_out); + + // 2 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); + +else if ( dtype == ">i2" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_2, data_out); +else if ( dtype == ">u2" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_2, data_out); + + // 4 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); + +else if ( dtype == ">i4" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_4, data_out); +else if ( dtype == ">u4" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_4, data_out); + + // 8 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); + +else if ( dtype == ">i8" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_8, data_out); +else if ( dtype == ">u8" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_8, data_out); + + // single precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == ">f4" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_4, data_out); + + // double precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == ">f8" ) load_numpy_int (np.buffer(), n, big_endian, shuffle_8, data_out); + + // + // nope ... the only other numerical data type for numpy arrays + // is single or double precision complex numbers, and + // we're not supporting those at this time + // + +else { + + mlog << Error << "\n" << method_name + << "unsupported data type \"" << dtype << "\"\n\n"; + + exit ( 1 ); + +} + + //////////////////// + + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool pointdata_from_np_array(Python3_Numpy & np, NumArray *data_out) +{ + +const char *method_name = "pointdata_from_np_array(NumArray) -> "; + + // + // make sure it's a 1D array + // + +if ( np.n_dims() != 1 ) { + + mlog << Error << "\n" << method_name + << "data array is not 1-dimensional! ... " + << "(dim = " << (np.n_dims()) << ")\n\n"; + + exit ( 1 ); + + +} + +int n = np.dim(0); + + + // + // load the data + // + +NumArray data; +const ConcatString dtype = np.dtype(); + + // 1 byte integers + + if ( dtype == "|i1" ) load_numpy_num (np.buffer(), n, little_endian, 0, data_out); +else if ( dtype == "|u1" ) load_numpy_num (np.buffer(), n, little_endian, 0, data_out); + + // 2 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_2, data_out); + +else if ( dtype == ">i2" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_2, data_out); +else if ( dtype == ">u2" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_2, data_out); + + // 4 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); + +else if ( dtype == ">i4" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_4, data_out); +else if ( dtype == ">u4" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_4, data_out); + + // 8 byte integers + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); + +else if ( dtype == ">i8" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_8, data_out); +else if ( dtype == ">u8" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_8, data_out); + + // single precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_4, data_out); +else if ( dtype == ">f4" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_4, data_out); + + // double precision floats + +else if ( dtype == " (np.buffer(), n, little_endian, shuffle_8, data_out); +else if ( dtype == ">f8" ) load_numpy_num (np.buffer(), n, big_endian, shuffle_8, data_out); + + // + // nope ... the only other numerical data type for numpy arrays + // is single or double precision complex numbers, and + // we're not supporting those at this time + // + +else { + + mlog << Error << "\n" << method_name + << "unsupported data type \"" << dtype << "\"\n\n"; + + exit ( 1 ); + +} + + //////////////////// + + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool pointdata_from_str_array(PyObject *data_array, StringArray *data_out) +{ + +//const char *method_name = "pointdata_from_str_array(StringArray) -> "; + +StringArray a = pyobject_as_string_array(data_array); +data_out->clear(); +data_out->add(a); + + //////////////////// + + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + + // + // we just grab the numpy array and the attributes dictionary + // + // from the xarray DataArray object, and then hand them + // + // off to pointdata_from_numpy_array + // + +bool pointdata_from_xarray(PyObject * data_array, float *data_out) +{ + +Python3_Numpy np; +PyObject *numpy_array = PyObject_GetAttrString(data_array, data_attr_name); + + ///////////////////// + +np.set(numpy_array); + +bool status = pointdata_from_np_array(np, data_out); + + // + // done + // + +return ( status ); + +} + + +//////////////////////////////////////////////////////////////////////// + + // + // we just grab the numpy array and the attributes dictionary + // + // from the xarray DataArray object, and then hand them + // + // off to pointdata_from_numpy_array + // + +bool pointdata_from_xarray(PyObject * data_array, int *data_out) +{ + +Python3_Numpy np; +PyObject *numpy_array = PyObject_GetAttrString(data_array, data_attr_name); + + + ///////////////////// + + +np.set(numpy_array); + +bool status = pointdata_from_np_array(np, data_out); + + // + // done + // + +return ( status ); + +} + + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_pointdata_python/pointdata_from_array.h b/met/src/libcode/vx_pointdata_python/pointdata_from_array.h new file mode 100644 index 0000000000..bec9380705 --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/pointdata_from_array.h @@ -0,0 +1,59 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MET_POINTDATA_FROM_ARRAY_H__ +#define __MET_POINTDATA_FROM_ARRAY_H__ + + +//////////////////////////////////////////////////////////////////////// + + +#include "met_point_data.h" + +#include "python3_dict.h" +#include "python3_numpy.h" + + +extern "C" { + +#include "Python.h" + +} + + +//////////////////////////////////////////////////////////////////////// + +static const char data_attr_name [] = "values"; + +//////////////////////////////////////////////////////////////////////// + + +extern bool pointdata_from_np_array(Python3_Numpy & np, int * data_out); +extern bool pointdata_from_np_array(Python3_Numpy & np, float * data_out); +extern bool pointdata_from_np_array(Python3_Numpy & np, IntArray *data_out); +extern bool pointdata_from_np_array(Python3_Numpy & np, NumArray *data_out); +extern bool pointdata_from_str_array(PyObject *data_array, StringArray *data_out); + + +extern bool pointdata_from_xarray(PyObject *data_array, int *data_out); +extern bool pointdata_from_xarray(PyObject *data_array, float *data_out); +extern bool pointdata_from_xarray(PyObject *data_array, IntArray &data_out); +extern bool pointdata_from_xarray(PyObject *data_array, NumArray &data_out); +extern bool pointdata_from_xarray(PyObject *data_array, StringArray &data_out); + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __MET_POINTDATA_FROM_NUMPY_ARRAY_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_pointdata_python/pointdata_python.cc b/met/src/libcode/vx_pointdata_python/pointdata_python.cc new file mode 100644 index 0000000000..846966a6f1 --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/pointdata_python.cc @@ -0,0 +1,290 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +using namespace std; + +#include +#include +#include +#include + +#include "pointdata_python.h" +#include "pointdata_from_array.h" +#include "vx_python3_utils.h" + +#include "vx_math.h" +#include "vx_log.h" + + +//////////////////////////////////////////////////////////////////////// + + + // + // Code for class MetPythonPointDataFile + // + + +//////////////////////////////////////////////////////////////////////// + + +MetPythonPointDataFile::MetPythonPointDataFile() + +{ + +python_init_from_scratch(); + +} + + +//////////////////////////////////////////////////////////////////////// + + +MetPythonPointDataFile::~MetPythonPointDataFile() + +{ + +close(); + +} + + +//////////////////////////////////////////////////////////////////////// + + +MetPythonPointDataFile::MetPythonPointDataFile(const MetPythonPointDataFile &) + +{ + +mlog << Error << "\nMetPythonPointDataFile::MetPythonPointDataFile(const MetPythonPointDataFile &) -> " + << "should never be called!\n\n"; + +exit ( 1 ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +MetPythonPointDataFile & MetPythonPointDataFile::operator=(const MetPythonPointDataFile &) + +{ + +mlog << Error << "\nMetPythonPointDataFile::operator=(const MetPythonPointDataFile &) -> " + << "should never be called!\n\n"; + +exit ( 1 ); + +return ( * this ); + +} + + +//////////////////////////////////////////////////////////////////////// + + +void MetPythonPointDataFile::python_init_from_scratch() + +{ + +PythonCommand.clear(); +//met_data.clear(); + +close(); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +void MetPythonPointDataFile::close() + +{ + +met_data.clear(); + + // + // Don't reset the Type field + // Don't reset the PythonCommand + // + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + +/* +void MetPythonPointDataFile::set_type(const GrdFileType t) + +{ + +Type = t; + +return; + +} +*/ + +//////////////////////////////////////////////////////////////////////// + + +bool MetPythonPointDataFile::open(const char * cur_command, bool use_xarray) + +{ + +close(); + +ConcatString full_path, file_name; +int i, file_argc; +char **file_argv = (char **) 0; // allocated +StringArray sa; +const char *method_name = "MetPythonPointDataFile::open() "; + + // + // Store the PythonCommand that is being run + // + +PythonCommand = cur_command; + + // + // parse and store argc and argv + // + +sa = PythonCommand.split(" "); + +file_argc = sa.n_elements(); + +if ( file_argc > 0 ) { + file_argv = new char * [ file_argc ]; + char a_var_name[512+1]; + + for ( i=0; idump(out, depth + 1); + +} else { + + out << prefix << "No Grid!\n"; + +} +*/ + // + // done + // + +out.flush(); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// +/* + +bool MetPythonPointDataFile::data_ok(int x, int y) const + +{ + +//const double value = get(x, y); + +//return ( !is_bad_data(value) ); +return true; + +} + + +//////////////////////////////////////////////////////////////////////// + + +void MetPythonPointDataFile::data_minmax(double & data_min, double & data_max) const + +{ + +//Plane.data_range(data_min, data_max); + +return; + +} +*/ + +//////////////////////////////////////////////////////////////////////// + + +MetPointDataPython *MetPythonPointDataFile::get_met_point_data() + +{ + +return &met_data; + +} + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_pointdata_python/pointdata_python.h b/met/src/libcode/vx_pointdata_python/pointdata_python.h new file mode 100644 index 0000000000..22b32c7f52 --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/pointdata_python.h @@ -0,0 +1,112 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MET_VX_POINTDATA_PYTHON_H__ +#define __MET_VX_POINTDATA_PYTHON_H__ + + +//////////////////////////////////////////////////////////////////////// + + +#include "data_plane.h" +#include "data_class.h" +#include "two_to_one.h" + +#include "python_pointdata.h" // the main access point +#include "pointdata_from_array.h" // takes an NumPy array or xarray DataArray + +#include "global_python.h" + + +//////////////////////////////////////////////////////////////////////// + + +//class MetPythonPointDataFile : public Met2dDataFile { +class MetPythonPointDataFile { + + private: + + void python_init_from_scratch(); + + MetPythonPointDataFile(const MetPythonPointDataFile &); + MetPythonPointDataFile & operator=(const MetPythonPointDataFile &); + + ConcatString PythonCommand; + + MetPointDataPython met_data; + + //VarInfoPython VInfo; + + //GrdFileType Type; // FileType_Python_Xarray or FileType_Python_Numpy + + public: + + MetPythonPointDataFile(); + ~MetPythonPointDataFile(); + + + // + // set stuff + // + + //void set_type(const GrdFileType); + + // + // get stuff + // + +// GrdFileType file_type() const; + +// double operator () (int x, int y) const; + + //double get (int x, int y) const; + +// bool data_ok (int x, int y) const; + +// void data_minmax (double & data_min, double & data_max) const; + + // + // do stuff + // + + bool open(const char * cur_command, bool use_xarray); + + void close(); + + + void dump (ostream &, int depth = 0) const; + + //MetPointData get_met_point_data(); + MetPointDataPython *get_met_point_data(); + + //int data_plane_array(VarInfo &, DataPlaneArray &); + + //bool met_point_data(MetPointData &); + +}; + + +//////////////////////////////////////////////////////////////////////// + + +//inline double MetPythonPointDataFile::operator () (int x, int y) const { return ( get(x, y) ); } +//inline GrdFileType MetPythonPointDataFile::file_type () const { return ( Type ); } + + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __MET_VX_POINTDATA_PYTHON_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_pointdata_python/python_pointdata.cc b/met/src/libcode/vx_pointdata_python/python_pointdata.cc new file mode 100644 index 0000000000..43196acc5f --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -0,0 +1,588 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#include "vx_python3_utils.h" +#include "python_pointdata.h" +#include "pointdata_from_array.h" +#include "vx_util.h" + + +#include "global_python.h" +#include "wchar_argv.h" + +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// + + +static bool straight_python_point_data(const char * script_name, + int script_argc, char ** script_argv, + const bool use_xarray, MetPointDataPython &met_pd_out); + +//////////////////////////////////////////////////////////////////////// + +template +static void set_array_from_python(PyObject *python_data, const char *python_key, T *out, bool required=true) { + const char *method_name = "set_array_from_python(T *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else { + if (required) { + mlog << Error << "\n" << method_name + << "error getting the point data by the key (" << python_key << ") from python object\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; + } +} + +/* +static void set_array_from_python(PyObject *python_data, const char *python_key, int *out, bool required=true) { + const char *method_name = "set_array_from_python(T *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; +mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object - before\n"; + np.set(numpy_array_obj); +mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object - after np.set\n"; + pointdata_from_np_array(np, out); + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else { + if (required) { + mlog << Error << "\n" << method_name + << "error getting the point data by the key (" << python_key << ") from python object\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; + } +} +*/ + +/* +static void set_array_from_python(PyObject *python_data, const char *python_key, int *out, bool required=true) { + const char *method_name = "set_array_from_python(int *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + } + else if (required) { + mlog << Error << "\nset_array_from_python(int *) -> " + << "error getting member (" << python_key << ") from python object\"\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; +} + +//////////////////////////////////////////////////////////////////////// + +static void set_array_from_python(PyObject *python_data, const char *python_key, float *out, bool required=true) { + const char *method_name = "set_array_from_python(float *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else if (required) { + mlog << Error << "\n" << method_name + << "error getting member (" << python_key << ") from python object\"\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; +} +*/ +//////////////////////////////////////////////////////////////////////// + +/* +static void set_met_array_from_python(PyObject *python_data, const char *python_key, IntArray *out, bool required=true) { + const char *method_name = "set_met_array_from_python(IntArray *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else if (required) { + mlog << Error << "\n" << method_name + << "error getting member (" << python_key << ") from python object\"\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; +} +*/ +/* +//////////////////////////////////////////////////////////////////////// + +static void set_met_array_from_python(PyObject *python_data, const char *python_key, NumArray &out) { + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + } + else { + mlog << Error << "\nset_array_from_python(NumArray) -> " + << "error getting member (" << python_key << ") from python object\"\n\n"; + exit (1); + } +} +*/ +//////////////////////////////////////////////////////////////////////// + +static void set_str_array_from_python(PyObject *python_data, const char *python_key, StringArray *out) { + const char *method_name = "set_met_array_from_python(StringArray *) -> "; + PyObject *str_array_obj = PyDict_GetItemString (python_data, python_key); + if (str_array_obj) { + pointdata_from_str_array(str_array_obj, out); + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else { + mlog << Error << "\n" << method_name + << "error getting member (" << python_key << ") from python object\"\n\n"; + exit (1); + } +} + + +//////////////////////////////////////////////////////////////////////// + + +bool python_point_data(const char * script_name, int script_argc, char ** script_argv, + const bool use_xarray, MetPointDataPython &met_pd_out) + +{ + +bool status = straight_python_point_data(script_name, script_argc, script_argv, + use_xarray, met_pd_out); + +return ( status ); + +} + +//////////////////////////////////////////////////////////////////////// + + +bool straight_python_point_data(const char * script_name, int script_argc, char ** script_argv, + const bool use_xarray, MetPointDataPython &met_pd_out) +{ + +int int_value; +PyObject * module_obj = 0; +PyObject * module_dict_obj = 0; +PyObject * python_key = 0; +PyObject * python_value = 0; +PyObject * numpy_array_obj = 0; +PyObject * python_met_point_data = 0; +ConcatString cs, user_dir, user_base; +const char *method_name = "straight_python_point_data -> "; +const char *method_name_s = "straight_python_point_data()"; + +mlog << Debug(3) << "Running user's python script (" + << script_name << ").\n"; + +cs = script_name; +user_dir = cs.dirname(); +user_base = cs.basename(); + +Wchar_Argv wa; + +wa.set(script_argc, script_argv); + + // + // if the global python object has already been initialized, + // we need to reload the module + // + +bool do_reload = GP.is_initialized; + +GP.initialize(); + + // + // start up the python interpreter + // + +if ( PyErr_Occurred() ) { + + PyErr_Print(); + + mlog << Warning << "\n" << method_name + << "an error occurred initializing python\n\n"; + + return ( false ); + +} + + // + // set the arguments + // + +run_python_string("import os"); +run_python_string("import sys"); + +ConcatString command; + +command << cs_erase + << "sys.path.append(\"" + << user_dir + << "\")"; + +run_python_string(command.text()); + +if ( script_argc > 0 ) { + + PySys_SetArgv (wa.wargc(), wa.wargv()); + +} + + // + // import the python script as a module + // + +module_obj = PyImport_ImportModule (user_base.c_str()); + + // + // if needed, reload the module + // + +if ( do_reload ) { + + module_obj = PyImport_ReloadModule (module_obj); + +} + +if ( PyErr_Occurred() ) { + + PyErr_Print(); + + mlog << Warning << "\n" << method_name + << "an error occurred importing module \"" + << script_name << "\"\n\n"; + + return ( false ); + +} + +if ( ! module_obj ) { + + mlog << Warning << "\n" << method_name + << "error running python script \"" + << script_name << "\"\n\n"; + + return ( false ); + +} + + // + // get the namespace for the module (as a dictionary) + // + +module_dict_obj = PyModule_GetDict (module_obj); + + // + // get handles to the objects of interest from the module_dict + // + +python_met_point_data = PyDict_GetItemString (module_dict_obj, python_key_point_data); + +python_value = PyDict_GetItemString (python_met_point_data, python_use_var_id); + +bool use_var_id = pyobject_as_bool(python_value); +met_pd_out.set_use_var_id(use_var_id); +mlog << Debug(9) << method_name << "use_var_id: \"" << use_var_id << "\" from python. is_using_var_id(): " << met_pd_out.is_using_var_id() << "\n"; + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nhdr); + +int_value = pyobject_as_int(python_value); +met_pd_out.set_hdr_cnt(int_value); + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nobs); +int_value = pyobject_as_int(python_value); + +met_pd_out.allocate(int_value); + +MetPointObsData *obs_data = met_pd_out.get_point_obs_data(); +MetPointHeader *header_data = met_pd_out.get_header_data(); + +if ( use_xarray ) { + + PyObject * data_array_obj = 0; + + // look up the data array variable name from the dictionary + + data_array_obj = PyDict_GetItemString (python_met_point_data, numpy_array_hdr_typ); + + if ( ! data_array_obj ) { + + mlog << Warning << "\n" << method_name + << "trouble reading data from \"" + << script_name << "\"\n\n"; + + return ( false ); + } + + //pointdata_from_xarray(data_array_obj, &header_data.hdr_typ); + +} else { // numpy array & dict + + // look up the data array variable name from the dictionary + +/* + set_met_array_from_python(python_met_point_data, numpy_array_hdr_typ, header_data->typ_idx_array); + set_met_array_from_python(python_met_point_data, numpy_array_hdr_sid, header_data->sid_idx_array); + set_met_array_from_python(python_met_point_data, numpy_array_hdr_vld, header_data->vld_idx_array); + //IntArray vld_num_array; // number array for valid time + + set_met_array_from_python(python_met_point_data, numpy_array_hdr_lat, header_data->lat_array); + set_met_array_from_python(python_met_point_data, numpy_array_hdr_lon, header_data->lon_array); + set_met_array_from_python(python_met_point_data, numpy_array_hdr_elv, header_data->elv_array); +*/ + set_array_from_python(python_met_point_data, numpy_array_hdr_typ, &header_data->typ_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_sid, &header_data->sid_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_vld, &header_data->vld_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lat, &header_data->lat_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lon, &header_data->lon_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_elv, &header_data->elv_array); + + set_str_array_from_python(python_met_point_data, numpy_array_hdr_typ_table, &header_data->typ_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_sid_table, &header_data->sid_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_vld_table, &header_data->vld_array); + set_array_from_python(python_met_point_data, numpy_array_prpt_typ_table, &header_data->prpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_irpt_typ_table, &header_data->irpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_inst_typ_table, &header_data->inst_typ_array, false); + + +// if ( !numpy_array_obj ) { +// +// mlog << Warning << "\n" << method_name +// << "trouble reading data from \"" +// << script_name << "\"\n\n"; +// +//// return ( false ); +// } +/* +*/ + set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids); + set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); + set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); + set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); + set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); + set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); + + set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); + set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); + + if(mlog.verbosity_level()>=point_data_debug_level) print_met_data(obs_data, header_data, method_name_s); + +} + + // + // done + // + +return ( true ); + +} + + +//////////////////////////////////////////////////////////////////////// + +void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_data, + const char *caller, int debug_level) { + int log_count, count; + const int min_count = 20; + const char *method_name = "print_met_data() "; + + mlog << Debug(debug_level) << "\n" << method_name << "by " << caller << "\n" + << " obs_data.obs_cnt = " << obs_data->obs_cnt << "\n" + << " header_data.hdr_count = " << header_data->hdr_count << " type=" + << header_data->typ_idx_array.n() << ", sid=" + << header_data->sid_idx_array.n() << ", valid=" + << header_data->vld_idx_array.n() << ", lat=" + << header_data->lat_array.n() << ", lon=" + << header_data->lon_array.n() << ", elv=" + << header_data->elv_array.n() << ", message_type=" + << header_data->typ_array.n() << ", station_id=" + << header_data->sid_array.n() << ", valid_time=" + << header_data->vld_array.n() << ", prpt=" + << header_data->prpt_typ_array.n() << ", irpt=" + << header_data->irpt_typ_array.n() << ", inst=" + << header_data->inst_typ_array.n() << " &header_data=" << header_data + << "\n"; + + log_count = (header_data->hdr_count > min_count) ? min_count : header_data->hdr_count; + mlog << Debug(debug_level) << method_name + << "header_data: message_type,station_id,time_time,lat,lon.elv\n"; + for (int idx=0; idxsid_idx_array[idx] << ", " + << header_data->vld_idx_array[idx] << ", " + << header_data->lat_array[idx] << ", " + << header_data->lon_array[idx] << ", " + << header_data->elv_array[idx] << "\n"; + } + if (header_data->hdr_count > log_count) { + log_count = header_data->hdr_count - min_count; + if (log_count < min_count) log_count = min_count; + else mlog << Debug(debug_level) + << " header_data[...] = ...\n"; + for (int idx=log_count; idxhdr_count; idx++) { + mlog << Debug(debug_level) + << " header_data[" << idx << "] = " + << header_data->typ_idx_array[idx] << ", " + << header_data->sid_idx_array[idx] << ", " + << header_data->vld_idx_array[idx] << ", " + << header_data->lat_array[idx] << ", " + << header_data->lon_array[idx] << ", " + << header_data->elv_array[idx] << "\n"; + } + } + if (header_data->typ_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + count = header_data->typ_array.n(); + for (int idx=0; idxsid_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + count = header_data->sid_array.n(); + log_count = (count > min_count) ? min_count : count; + for (int idx=0; idx log_count) { + log_count = count - min_count; + if (log_count < min_count) log_count = min_count; + else mlog << Debug(debug_level) + << " station_id[...] = ...\n"; + for (int idx=log_count; idxvld_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + count = header_data->vld_array.n(); + log_count = (count > min_count) ? min_count : count; + for (int idx=0; idx log_count) { + log_count = count - min_count; + if (log_count < min_count) log_count = min_count; + else mlog << Debug(debug_level) + << " valid time[...] = ...\n"; + for (int idx=log_count; idxprpt_typ_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + for (int idx=0; idxprpt_typ_array.n(); idx++) + mlog << Debug(debug_level) + << " prpt_type[" << idx << "] = " << header_data->prpt_typ_array[idx] << "\n"; + } + if (header_data->irpt_typ_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + for (int idx=0; idxirpt_typ_array.n(); idx++) + mlog << Debug(debug_level) + << " irpt_type[" << idx << "] = " << header_data->irpt_typ_array[idx] << "\n"; + } + if (header_data->inst_typ_array.n() > 0) { + mlog << Debug(debug_level) << "\n"; + for (int idx=0; idxinst_typ_array.n(); idx++) + mlog << Debug(debug_level) + << " inst_type[" << idx << "] = " << header_data->inst_typ_array[idx] << "\n"; + } + + log_count = (obs_data->obs_cnt > min_count) ? min_count : obs_data->obs_cnt; + mlog << Debug(debug_level) << "\n" << method_name + << "obs_data: hid,vid.level,height,value,qty\n"; + for (int idx=0; idxobs_ids[idx] << ", " + << obs_data->obs_lvls[idx] << ", " + << obs_data->obs_hgts[idx] << ", " + << obs_data->obs_vals[idx] << ", " + << obs_data->obs_qids[idx] << "\n"; + } + if (obs_data->obs_cnt > log_count) { + log_count = obs_data->obs_cnt - min_count; + if (log_count < min_count) log_count = min_count; + else mlog << Debug(debug_level) + << " obs_data[...] = ...\n"; + for (int idx=log_count; idxobs_cnt; idx++) { + mlog << Debug(debug_level) + << " obs_data[" << idx << "] = " + << obs_data->obs_hids[idx] << ", " + << obs_data->obs_ids[idx] << ", " + << obs_data->obs_lvls[idx] << ", " + << obs_data->obs_hgts[idx] << ", " + << obs_data->obs_vals[idx] << ", " + << obs_data->obs_qids[idx] << "\n"; + } + } + + if (obs_data->var_names.n() > 0) { + mlog << Debug(debug_level) << "\n"; + for (int idx=0; idxvar_names.n(); idx++) + mlog << Debug(debug_level) + << " var_names[" << idx << "] = " << obs_data->var_names[idx] << "\n"; + } + else mlog << Debug(debug_level) + << " var_names is empty!!!\n"; + if (obs_data->qty_names.n() > 0) { + mlog << Debug(debug_level) << "\n"; + for (int idx=0; idxqty_names.n(); idx++) + mlog << Debug(debug_level) + << " qty_names[" << idx << "] = " << obs_data->qty_names[idx] << "\n"; + } + else mlog << Debug(debug_level) + << " qty_names is empty!!!\n"; + + mlog << Debug(debug_level) << "Done " << method_name << "by " << caller << "\n\n"; + +} + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_pointdata_python/python_pointdata.h b/met/src/libcode/vx_pointdata_python/python_pointdata.h new file mode 100644 index 0000000000..5be19111f0 --- /dev/null +++ b/met/src/libcode/vx_pointdata_python/python_pointdata.h @@ -0,0 +1,80 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __PYTHON_POINTDATA__ +#define __PYTHON_POINTDATA__ + + +//////////////////////////////////////////////////////////////////////// + + +#include "met_point_data.h" + + +extern "C" { + +#include "Python.h" + +} + + +//////////////////////////////////////////////////////////////////////// + +static const char python_key_point_data [] = "met_point_data"; + +static const char python_key_nhdr [] = "nhdr"; +//static const char python_key_npbhdr [] = "npbhdr"; +static const char python_use_var_id [] = "use_var_id"; +static const char numpy_array_hdr_typ [] = "hdr_typ"; // message type IDs +static const char numpy_array_hdr_sid [] = "hdr_sid"; // station IDs +static const char numpy_array_hdr_vld [] = "hdr_vld"; // valid time IDs +static const char numpy_array_hdr_lat [] = "hdr_lat"; +static const char numpy_array_hdr_lon [] = "hdr_lon"; +static const char numpy_array_hdr_elv [] = "hdr_elv"; +static const char numpy_array_hdr_typ_table [] = "hdr_typ_table"; // message type list +static const char numpy_array_hdr_sid_table [] = "hdr_sid_table"; // station ID list +static const char numpy_array_hdr_vld_table [] = "hdr_vld_table"; // valid time list +static const char numpy_array_prpt_typ_table[] = "prpt_typ_table"; +static const char numpy_array_irpt_typ_table[] = "irpt_typ_table"; +static const char numpy_array_inst_typ_table[] = "inst_typ_table"; + +static const char python_key_nobs [] = "nobs"; +static const char numpy_array_obs_qty [] = "obs_qty"; // quality_id +static const char numpy_array_obs_hid [] = "obs_hid"; // header id +static const char numpy_array_obs_vid [] = "obs_vid"; // variable id or grib code +static const char numpy_array_obs_lvl [] = "obs_lvl"; +static const char numpy_array_obs_hgt [] = "obs_hgt"; +static const char numpy_array_obs_val [] = "obs_val"; +static const char numpy_array_obs_qty_table [] = "obs_qty_table"; +static const char numpy_array_obs_var_table [] = "obs_var_table"; // variable names or grib codes as string + +static const int point_data_debug_level = 10; + + +//////////////////////////////////////////////////////////////////////// + + +extern bool python_point_data(const char * script_name, int script_argc, char ** script_argv, + const bool use_xarray, MetPointDataPython &met_pd_out); +//extern bool python_point_data(const char *python_command, const bool use_xarray, +// MetPointData & po_out); +extern void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_data, + const char *caller, int debug_level=point_data_debug_level); + + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __PYTHON_POINTDATA__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/met/src/libcode/vx_python3_utils/python3_util.cc b/met/src/libcode/vx_python3_utils/python3_util.cc index fe56b6f6b2..ab17acd7b8 100644 --- a/met/src/libcode/vx_python3_utils/python3_util.cc +++ b/met/src/libcode/vx_python3_utils/python3_util.cc @@ -12,7 +12,6 @@ using namespace std; #include #include "vx_log.h" -#include "concat_string.h" #include "vx_math.h" #include "python3_util.h" @@ -125,6 +124,18 @@ return ( k ); //////////////////////////////////////////////////////////////////////// +bool pyobject_as_bool (PyObject * obj) + +{ + +return ( 1 == PyObject_IsTrue(obj) ); + +} + + +//////////////////////////////////////////////////////////////////////// + + double pyobject_as_double (PyObject * obj) { @@ -215,6 +226,29 @@ return ( s ); //////////////////////////////////////////////////////////////////////// +StringArray pyobject_as_string_array (PyObject * obj) + +{ + +StringArray a; +ConcatString s; +PyObject *item = 0; + +int size = PyList_Size (obj); +for (int idx=0; idxget_hdr_cnt(); + int obs_count = met_point_obs->get_obs_cnt(); mlog << Debug(2) << "Searching " << (obs_count) << " observations from " << (hdr_count) << " header messages.\n"; - int buf_size = ((obs_count > DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : (obs_count)); + const int buf_size = ((obs_count > DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : (obs_count)); int obs_qty_idx_block[buf_size]; float obs_arr_block[buf_size][OBS_ARRAY_LEN]; @@ -1107,19 +1146,25 @@ void process_point_obs(int i_nc) { ConcatString obs_qty_str; ConcatString var_name; StringArray var_names; - StringArray obs_qty_array = nc_point_obs.get_qty_data(); - if(use_var_id) var_names = nc_point_obs.get_var_names(); + StringArray obs_qty_array = met_point_obs->get_qty_data(); + if(use_var_id) var_names = met_point_obs->get_var_names(); for(int i_start=0; i_start DEF_NC_BUFFER_SIZE) - ? DEF_NC_BUFFER_SIZE : (obs_count-i_start); + int buf_size2 = (obs_count-i_start); + if (buf_size2 > buf_size) buf_size2 = buf_size; + +#ifdef WITH_PYTHON + if (use_python) + status = met_point_obs->get_point_obs_data()->fill_obs_buf( + buf_size2, i_start, (float *)obs_arr_block, obs_qty_idx_block); + else +#endif + status = nc_point_obs.read_obs_data(buf_size2, i_start, (float *)obs_arr_block, + obs_qty_idx_block, (char *)0); + if (!status) exit(1); - if (!nc_point_obs.read_obs_data(buf_size, i_start, (float *)obs_arr_block, - obs_qty_idx_block, (char *)0)) { - exit(1); - } // Process each observation in the file - for(int i_offset=0; i_offsetget_header_offset(obs_arr); // Range check the header offset if(headerOffset < 0 || headerOffset >= hdr_count) { @@ -1144,16 +1189,21 @@ void process_point_obs(int i_nc) { // Read the corresponding header array for this observation // - the corresponding header type, header Station ID, and valid time - nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, - hdr_sid_str, hdr_vld_str); +#ifdef WITH_PYTHON + if (use_python) + met_point_obs->get_header(headerOffset, hdr_arr, hdr_typ_str, hdr_sid_str, hdr_vld_str); + else +#endif + nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, + hdr_sid_str, hdr_vld_str); // Read the header integer types - nc_point_obs.get_header_type(headerOffset, hdr_typ_arr); + met_point_obs->get_header_type(headerOffset, hdr_typ_arr); // Convert string to a unixtime hdr_ut = timestring_to_unix(hdr_vld_str.c_str()); - int grib_code = nc_point_obs.get_grib_code_or_var_index(obs_arr); + int grib_code = met_point_obs->get_grib_code_or_var_index(obs_arr); if (use_var_id && grib_code < var_names.n()) { var_name = var_names[grib_code]; } @@ -1176,6 +1226,10 @@ void process_point_obs(int i_nc) { } // end for i_start // Deallocate and clean up +#ifdef WITH_PYTHON + if (use_python) met_point_file.close(); + else +#endif nc_point_obs.close(); return; diff --git a/met/src/tools/core/point_stat/point_stat.cc b/met/src/tools/core/point_stat/point_stat.cc index f62f3ae511..48ffd21c6e 100644 --- a/met/src/tools/core/point_stat/point_stat.cc +++ b/met/src/tools/core/point_stat/point_stat.cc @@ -127,6 +127,11 @@ using namespace std; #include "nc_obs_util.h" #include "nc_point_obs_in.h" +#ifdef WITH_PYTHON +#include "data2d_nc_met.h" +#include "pointdata_python.h" +#endif + //////////////////////////////////////////////////////////////////////// #define BUFFER_SIZE (DEF_NC_BUFFER_SIZE/2) @@ -672,49 +677,88 @@ void process_obs_file(int i_nc) { // Open the observation file as a NetCDF file. // The observation file must be in NetCDF format as the // output of the PB2NC or ASCII2NC tool. + bool status; + bool use_var_id = true; + bool use_arr_vars = false; + bool use_python = false; MetNcPointObsIn nc_point_obs; - if( !nc_point_obs.open(obs_file[i_nc].c_str()) ) { - nc_point_obs.close(); + MetPointData *met_point_obs = 0; +#ifdef WITH_PYTHON + MetPythonPointDataFile met_point_file; + string python_command = obs_file[i_nc]; + bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); + use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); + if (use_python) { + int offset = python_command.find("="); + if (offset == std::string::npos) { + mlog << Error << "\n" << method_name + << "trouble parsing the python command " << python_command << ".\n\n"; + exit(1); + } - mlog << Warning << "\n" << method_name - << "can't open observation netCDF file: " - << obs_file[i_nc] << "\n\n"; - return; - } + if(!met_point_file.open(python_command.substr(offset+1).c_str(), use_xarray)) { + met_point_file.close(); + mlog << Error << "\n" << method_name + << "trouble getting point observation file from python command " + << python_command << ".\n\n"; + exit(1); + } - nc_point_obs.read_dim_headers(); - nc_point_obs.check_nc(obs_file[i_nc].c_str(), method_name); - nc_point_obs.read_obs_data_table_lookups(); + met_point_obs = met_point_file.get_met_point_data(); + } + else { +#endif + if( !nc_point_obs.open(obs_file[i_nc].c_str()) ) { + nc_point_obs.close(); + + mlog << Warning << "\n" << method_name + << "can't open observation netCDF file: " + << obs_file[i_nc] << "\n\n"; + return; + } + + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(obs_file[i_nc].c_str(), method_name); + nc_point_obs.read_obs_data_table_lookups(); + met_point_obs = (MetPointData *)&nc_point_obs; + use_var_id = nc_point_obs.is_using_var_id(); + use_arr_vars = nc_point_obs.is_using_obs_arr(); +#ifdef WITH_PYTHON + } +#endif - bool use_var_id = nc_point_obs.is_using_var_id(); - int hdr_count = nc_point_obs.get_hdr_cnt(); - int obs_count = nc_point_obs.get_obs_cnt(); + int hdr_count = met_point_obs->get_hdr_cnt(); + int obs_count = met_point_obs->get_obs_cnt(); mlog << Debug(2) << "Searching " << obs_count << " observations from " << hdr_count << " messages.\n"; ConcatString var_name(""); - bool use_arr_vars = nc_point_obs.is_using_obs_arr(); StringArray var_names; - StringArray obs_qty_array = nc_point_obs.get_qty_data(); - if( use_var_id ) var_names = nc_point_obs.get_var_names(); + StringArray obs_qty_array = met_point_obs->get_qty_data(); + if( use_var_id ) var_names = met_point_obs->get_var_names(); - int buf_size = ((obs_count > BUFFER_SIZE) ? BUFFER_SIZE : (obs_count)); + const int buf_size = ((obs_count > BUFFER_SIZE) ? BUFFER_SIZE : (obs_count)); int obs_qty_idx_block[buf_size]; float obs_arr_block[buf_size][OBS_ARRAY_LEN]; // Process each observation in the file int str_length, block_size; - for(int i_block_start_idx=0; i_block_start_idx BUFFER_SIZE) block_size = BUFFER_SIZE; - - if (!nc_point_obs.read_obs_data(block_size, i_block_start_idx, - (float *)obs_arr_block, - obs_qty_idx_block, (char *)0)) { - exit(1); - } + if (block_size > buf_size) block_size = buf_size; + +#ifdef WITH_PYTHON + if (use_python) + status = met_point_obs->get_point_obs_data()->fill_obs_buf( + block_size, i_block_start_idx, (float *)obs_arr_block, obs_qty_idx_block); + else +#endif + status = nc_point_obs.read_obs_data(block_size, i_block_start_idx, + (float *)obs_arr_block, + obs_qty_idx_block, (char *)0); + if (!status) exit(1); int hdr_idx; for(int i_block_idx=0; i_block_idxget_header_offset(obs_arr); // Range check the header offset if(headerOffset < 0 || headerOffset >= hdr_count) { @@ -741,11 +785,16 @@ void process_obs_file(int i_nc) { // Read the corresponding header array for this observation // - the corresponding header type, header Station ID, and valid time - nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, - hdr_sid_str, hdr_vld_str); +#ifdef WITH_PYTHON + if (use_python) + met_point_obs->get_header(headerOffset, hdr_arr, hdr_typ_str, hdr_sid_str, hdr_vld_str); + else +#endif + nc_point_obs.get_header(headerOffset, hdr_arr, hdr_typ_str, + hdr_sid_str, hdr_vld_str); // Store the variable name - int org_grib_code = nc_point_obs.get_grib_code_or_var_index(obs_arr); + int org_grib_code = met_point_obs->get_grib_code_or_var_index(obs_arr); int grib_code = org_grib_code; if (use_var_id && grib_code < var_names.n()) { var_name = var_names[grib_code]; @@ -774,7 +823,7 @@ void process_obs_file(int i_nc) { // and at the same vertical level. if(vflag && is_vgrd) { - if(!nc_point_obs.is_same_obs_values(obs_arr, prev_obs_arr)) { + if(!met_point_obs->is_same_obs_values(obs_arr, prev_obs_arr)) { mlog << Error << "\n" << method_name << "for observation index " << i_obs << ", when computing VL1L2 and/or VAL1L2 vector winds " @@ -802,12 +851,16 @@ void process_obs_file(int i_nc) { grid, var_name.c_str()); } - nc_point_obs.set_grib_code_or_var_index(obs_arr, org_grib_code); + met_point_obs->set_grib_code_or_var_index(obs_arr, org_grib_code); } } // end for i_block_start_idx // Deallocate and clean up +#ifdef WITH_PYTHON + if (use_python) met_point_file.close(); + else +#endif nc_point_obs.close(); return; diff --git a/met/src/tools/other/madis2nc/madis2nc.cc b/met/src/tools/other/madis2nc/madis2nc.cc index 6f48a30ccc..0f8164945a 100644 --- a/met/src/tools/other/madis2nc/madis2nc.cc +++ b/met/src/tools/other/madis2nc/madis2nc.cc @@ -3410,6 +3410,11 @@ void process_madis_acarsProfiles(NcFile *&f_in) { levelsQty[i_idx], GET_NC_NAME(in_var).c_str()); nlvl = levels[i_idx]; + if (nlvl > maxLevels) { + mlog << Warning << "\n" << method_name << " The level (" << nlvl + << ") at nLevels variable can not exceed dimension maxLevels (" << maxLevels<< ")\n\n"; + nlvl = maxLevels; + } obs_arr[2] = levels[i_idx]; // diff --git a/met/src/tools/other/mode_graphics/cgraph_main.cc b/met/src/tools/other/mode_graphics/cgraph_main.cc index d9963263fc..5020cfe719 100644 --- a/met/src/tools/other/mode_graphics/cgraph_main.cc +++ b/met/src/tools/other/mode_graphics/cgraph_main.cc @@ -2223,14 +2223,10 @@ int my_conic (const FT_Vector * control, const FT_Vector * to, void * u) { -ft_user_info * info = (ft_user_info *) u; - mlog << Error << "\n\n my_conic() -> should never be called!\n\n"; exit ( 1 ); -info->have_path = true; - return ( 0 ); } diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index dd0ba183d8..50a8fa9ab8 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -367,7 +367,7 @@ static int combine_tqz_and_uv(map, static float compute_pbl(map pqtzuv_map_tq, map pqtzuv_map_uv); static void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all=true); -static void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, +static bool insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, const float pbl_p, const float pbl_h, const float pbl_qm, const float hdr_lat, const float hdr_lon, const float hdr_elv, const time_t hdr_vld_ut, @@ -807,6 +807,7 @@ void process_pbfile(int i_pb) { int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly; int rej_elv, rej_pb_rpt, rej_in_rpt, rej_itp, rej_nobs; int lv, ev, ev_temp, kk, len1, len2; + int n_derived_obs; double x, y; @@ -844,7 +845,8 @@ void process_pbfile(int i_pb) { IntArray diff_file_times; int diff_file_time_count; StringArray variables_big_nlevels; - static const char *method_name = "process_pbfile()"; + static const char *method_name_s = "process_pbfile()"; + static const char *method_name = "process_pbfile() -> "; bool apply_grid_mask = (conf_info.grid_mask.nx() > 0 && conf_info.grid_mask.ny() > 0); @@ -881,7 +883,7 @@ void process_pbfile(int i_pb) { // Check for multiple PrepBufr files if(pbfile.n_elements() > 1) { - mlog << Error << "\n" << method_name << " -> " + mlog << Error << "\n" << method_name << "the \"-dump\" and \"-pbfile\" options may not be " << "used together. Only one Bufr file may be dump " << "to ASCII at a time.\n\n"; @@ -894,7 +896,7 @@ void process_pbfile(int i_pb) { unit = dump_unit+i_pb; if (unit > MAX_FORTRAN_FILE_ID || unit < MIN_FORTRAN_FILE_ID) { - mlog << Error << "\n" << method_name << " -> " + mlog << Error << "\n" << method_name << "Invalid file ID [" << unit << "] between 1 and 99.\n\n"; } prefix = get_short_name(pbfile[i_pb].c_str()); @@ -907,7 +909,7 @@ void process_pbfile(int i_pb) { // Open the blocked temp PrepBufr file for reading unit = file_unit + i_pb; if (unit > MAX_FORTRAN_FILE_ID || unit < MIN_FORTRAN_FILE_ID) { - mlog << Error << "\n" << method_name << " -> " + mlog << Error << "\n" << method_name << "Invalid file ID [" << unit << "] between 1 and 99.\n\n"; } openpb_(blk_file.c_str(), &unit); @@ -925,7 +927,7 @@ void process_pbfile(int i_pb) { // Check for zero messages to process if(npbmsg <= 0 || npbmsg_total <= 0) { - mlog << Warning << "\n" << method_name << " -> " + mlog << Warning << "\n" << method_name << "No Bufr messages to process in file: " << pbfile[i_pb] << "\n\n"; @@ -936,6 +938,7 @@ void process_pbfile(int i_pb) { } // Initialize counts + n_derived_obs = 0; i_ret = n_file_obs = i_msg = 0; rej_typ = rej_sid = rej_vld = rej_grid = rej_poly = 0; rej_elv = rej_pb_rpt = rej_in_rpt = rej_itp = rej_nobs = 0; @@ -946,7 +949,7 @@ void process_pbfile(int i_pb) { bool is_prepbufr = is_prepbufr_file(&event_names); if(mlog.verbosity_level() >= debug_level_for_performance) { end_t = clock(); - mlog << Debug(debug_level_for_performance) << " PERF: " << method_name << " " + mlog << Debug(debug_level_for_performance) << " PERF: " << method_name_s << " " << (end_t-start_t)/double(CLOCKS_PER_SEC) << " seconds for preparing\n"; start_t = clock(); @@ -1011,8 +1014,8 @@ void process_pbfile(int i_pb) { if (cal_pbl) { is_same_header = false; prev_hdr_vld_ut = -1; - m_strncpy(prev_hdr_typ, not_assigned, m_strlen(not_assigned), method_name, "prev_hdr_typ"); - m_strncpy(prev_hdr_sid, not_assigned, m_strlen(not_assigned), method_name, "prev_hdr_sid"); + m_strncpy(prev_hdr_typ, not_assigned, m_strlen(not_assigned), method_name_s, "prev_hdr_typ"); + m_strncpy(prev_hdr_sid, not_assigned, m_strlen(not_assigned), method_name_s, "prev_hdr_sid"); } IMM = JMM =1; @@ -1023,6 +1026,14 @@ void process_pbfile(int i_pb) { diff_file_time_count = 0; cycle_minute = missing_cycle_minute; // initialize + // Derive quantities which can be derived from + // P, Q, T, Z, U, V + if (n_derive_gc > bufr_derive_cfgs.size()) { + mlog << Debug(3) << "\n" << method_name + << "Skip the derived variables because of not requested (" + << bufr_derive_cfgs.size() << ").\n\n"; + } + for (int idx=0; idx " + mlog << Debug(6) << "\n" << method_name << "Switching report type \"" << hdr_typ << "\" to message type \"" << mappedMessageType << "\".\n"; if (mappedMessageType.length() > HEADER_STR_LEN) max_buf = HEADER_STR_LEN; - m_strncpy(modified_hdr_typ, mappedMessageType.c_str(), max_buf, method_name, "modified_hdr_typ1"); + m_strncpy(modified_hdr_typ, mappedMessageType.c_str(), max_buf, method_name_s, "modified_hdr_typ1"); } else { - m_strncpy(modified_hdr_typ, hdr_typ, sizeof(modified_hdr_typ), method_name, "modified_hdr_typ2"); + m_strncpy(modified_hdr_typ, hdr_typ, sizeof(modified_hdr_typ), method_name_s, "modified_hdr_typ2"); } if (max_buf >= max_str_len) max_buf--; modified_hdr_typ[max_buf] = '\0'; @@ -1307,7 +1318,7 @@ void process_pbfile(int i_pb) { buf_nlev = mxr8lv; for(kk=0; kk " + mlog << Warning << "\n" << method_name << "Too many vertical levels (" << nlev << ") for " << bufr_obs_name_arr[kk] << "). Ignored the vertical levels above " << mxr8lv << ".\n\n"; @@ -1520,40 +1531,36 @@ void process_pbfile(int i_pb) { // Derive quantities which can be derived from // P, Q, T, Z, U, V - if (n_derive_gc > bufr_derive_cfgs.size()) { - mlog << Debug(3) << "\n" << method_name << " -> " - << "Skip the derived variables because of not requested (" - << bufr_derive_cfgs.size() << ").\n\n"; - } - else { + if (n_derive_gc <= bufr_derive_cfgs.size()) { for(i=0; i MAX_CAPE_LEVEL) cape_level = MAX_CAPE_LEVEL; reverse_levels = (cape_data_pres[0] > cape_data_pres[cape_level-1]); if (reverse_levels) { @@ -1618,11 +1625,11 @@ void process_pbfile(int i_pb) { swap_value = cape_data_pres[idx]; cape_data_pres[idx] = cape_data_pres[buf_idx]; cape_data_pres[buf_idx] = swap_value; - + swap_value = cape_data_temp[idx]; cape_data_temp[idx] = cape_data_temp[buf_idx]; cape_data_temp[buf_idx] = swap_value; - + swap_value = cape_data_spfh[idx]; cape_data_spfh[idx] = cape_data_spfh[buf_idx]; cape_data_spfh[buf_idx] = swap_value; @@ -1638,7 +1645,7 @@ void process_pbfile(int i_pb) { cape_data_spfh[idx] = r8bfms * 10; } } - + //p1d = cape_p; //t1d = cape_data_temp[cape_level-1]; //q1d = cape_data_spfh[cape_level-1]; @@ -1646,15 +1653,15 @@ void process_pbfile(int i_pb) { &p1d,&t1d,&q1d, static_dummy_201, &cape_level, &IMM,&JMM, &cape_level, &cape_val, &cin_val, &PLCL, &PEQL, static_dummy_200); - + if(mlog.verbosity_level() >= 7) { mlog << Debug(7) << method_name << " index,P,T,Q to compute CAPE from " << i_read << "-th message\n" ; for (int idx=0; idx MAX_CAPE_VALUE) { cape_cnt_too_big++; - mlog << Debug(5) << method_name + mlog << Debug(5) << method_name << " Ignored cape_value: " << cape_val << " cape_level: " << cape_level << ", cin_val: " << cin_val << ", PLCL: " << PLCL << ", PEQL: " << PEQL << "\n"; @@ -1681,6 +1688,7 @@ void process_pbfile(int i_pb) { hdr_lat, hdr_lon, hdr_elv, cape_qm, OBS_BUFFER_SIZE); cape_count++; + n_derived_obs++; if (is_eq(cape_val, 0.)) cape_cnt_zero_values++; } else cape_cnt_missing_values++; @@ -1747,7 +1755,7 @@ void process_pbfile(int i_pb) { if (nlev2 > mxr8lv) { buf_nlev = mxr8lv; if (!variables_big_nlevels.has(var_name, false)) { - mlog << Warning << "\n" << method_name << " -> " + mlog << Warning << "\n" << method_name << "Too many vertical levels (" << nlev2 << ") for " << var_name << ". Ignored the vertical levels above " << mxr8lv << ".\n\n"; @@ -1833,9 +1841,9 @@ void process_pbfile(int i_pb) { has_pbl_data = (pqtzuv_map_tq.size() > 0 && pqtzuv_map_uv.size() > 0); if (is_same_header && has_pbl_data) { float pbl_value = compute_pbl(pqtzuv_map_tq, pqtzuv_map_uv); - - insert_pbl(obs_arr, pbl_value, pbl_code, pbl_p, pbl_h, pbl_qm, - hdr_lat, hdr_lon, hdr_elv, hdr_vld_ut, hdr_typ, hdr_sid); + + if (insert_pbl(obs_arr, pbl_value, pbl_code, pbl_p, pbl_h, pbl_qm, + hdr_lat, hdr_lon, hdr_elv, hdr_vld_ut, hdr_typ, hdr_sid)) n_derived_obs++; for(vector::iterator it = pqtzuv_list.begin(); it != pqtzuv_list.end(); ++it) { @@ -1850,8 +1858,8 @@ void process_pbfile(int i_pb) { prev_hdr_lat = hdr_lat; prev_hdr_lon = hdr_lon; prev_hdr_elv = hdr_elv; - m_strncpy(prev_hdr_typ, hdr_typ, m_strlen(not_assigned), method_name, "prev_hdr_typ"); - m_strncpy(prev_hdr_sid, hdr_sid.c_str(), m_strlen(not_assigned), method_name, "prev_hdr_sid"); + m_strncpy(prev_hdr_typ, hdr_typ, m_strlen(not_assigned), method_name_s, "prev_hdr_typ"); + m_strncpy(prev_hdr_sid, hdr_sid.c_str(), m_strlen(not_assigned), method_name_s, "prev_hdr_sid"); } // If the number of observations for this header is non-zero, @@ -1874,8 +1882,8 @@ void process_pbfile(int i_pb) { has_pbl_data = (pqtzuv_map_tq.size() > 0 || pqtzuv_map_uv.size() > 0); if (do_pbl && has_pbl_data) { float pbl_value = compute_pbl(pqtzuv_map_tq, pqtzuv_map_uv); - insert_pbl(obs_arr, pbl_value, pbl_code, pbl_p, pbl_h, pbl_qm, - hdr_lat, hdr_lon, hdr_elv, hdr_vld_ut, hdr_typ, hdr_sid); + if (insert_pbl(obs_arr, pbl_value, pbl_code, pbl_p, pbl_h, pbl_qm, + hdr_lat, hdr_lon, hdr_elv, hdr_vld_ut, hdr_typ, hdr_sid)) n_derived_obs++; for(vector::iterator it = pqtzuv_list.begin(); it != pqtzuv_list.end(); ++it) { @@ -1898,7 +1906,7 @@ void process_pbfile(int i_pb) { if(0 < diff_file_time_count && 0 < diff_file_times.n_elements()) { mlog << Warning << "\n" << method_name - << " -> The observation time should remain the same for " + << "The observation time should remain the same for " << "all " << (is_prepbufr ? "PrepBufr" : "Bufr") << " messages\n"; mlog << Warning << method_name << " " << diff_file_time_count << " messages with different reference time (" @@ -1938,7 +1946,7 @@ void process_pbfile(int i_pb) { << "Total Messages retained\t\t= " << i_msg << "\n" << "Total observations retained or derived\t= " - << n_file_obs << "\n"; + << (n_file_obs + n_derived_obs) << "\n"; if (cal_cape) { mlog << Debug(3) << "\nDerived CAPE = " << cape_count @@ -1952,7 +1960,7 @@ void process_pbfile(int i_pb) { if (npbmsg == rej_vld && 0 < rej_vld) { - mlog << Warning << "\n" << method_name << " -> " + mlog << Warning << "\n" << method_name << "All messages were filtered out by valid time.\n" << "\tPlease adjust time range with \"-valid_beg\" and \"-valid_end\".\n" << "\tmin/max obs time from BUFR file: " << min_time_str @@ -1981,17 +1989,17 @@ void process_pbfile(int i_pb) { remove_temp_file(blk_file); if(mlog.verbosity_level() >= debug_level_for_performance) { method_end = clock(); - cout << " PERF: " << method_name << " " + cout << " PERF: " << method_name_s << " " << (method_end-method_start)/double(CLOCKS_PER_SEC) << " seconds\n"; } if(i_msg <= 0) { - mlog << Warning << "\n" << method_name << " -> " + mlog << Warning << "\n" << method_name + << ((n_derived_obs > 0) ? "Saved the derived variables only. " : " ") << "No " << (is_prepbufr ? "PrepBufr" : "Bufr") << " messages retained from file: " << pbfile[i_pb] << "\n\n"; - return; } return; @@ -2306,7 +2314,7 @@ void process_pbfile_metadata(int i_pb) { readpbint_(&unit, &i_ret, &nlev2, bufr_obs, (char*)var_name.c_str(), &var_name_len, &nlev_max_req); if (0 >= nlev2) continue; - + // Search through the vertical levels has_valid_data = false; for(lv=0; lv pqtzuv_map_tq, } interpolate_pqtzuv(prev_pqtzuv, pqtzuv_merged, next_pqtzuv); } - float first_pres = (pqtzuv_merged == 0 ? bad_data_float : pqtzuv_merged[0]); + float first_pres = (pqtzuv_merged[0] == 0 ? bad_data_float : pqtzuv_merged[0]); pqtzuv_map_merged[first_pres] = pqtzuv_merged; mlog << Debug(9) << method_name << "Added " << first_pres << " to merged records\n"; @@ -3094,11 +3102,12 @@ float compute_pbl(map pqtzuv_map_tq, //////////////////////////////////////////////////////////////////////// -void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, +bool insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, const float pbl_p, const float pbl_h, const float pbl_qm, const float hdr_lat, const float hdr_lon, - const float hdr_elv, const time_t hdr_vld_ut, + const float hdr_elv, const time_t hdr_vld_ut, const ConcatString &hdr_typ, const ConcatString &hdr_sid) { + bool added = false; ConcatString hdr_info; hdr_info << unix_to_yyyymmdd_hhmmss(hdr_vld_ut) << " " << hdr_typ << " " << hdr_sid; @@ -3124,9 +3133,13 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, << pbl_value << ") because of the MAX PBL " << MAX_PBL << " (" << hdr_info<< ")\n"; } - else addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut, - hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE); + else { + addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut, + hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE); + added = true; + } } + return added; } //////////////////////////////////////////////////////////////////////// @@ -3189,7 +3202,7 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu if ((nint(prev_pqtzuv[0]) == nint(cur_pqtzuv[0])) || (nint(next_pqtzuv[0]) == nint(cur_pqtzuv[0])) || (nint(prev_pqtzuv[0]) == nint(next_pqtzuv[0]))) { - mlog << Error << "\n" << method_name + mlog << Error << "\n" << method_name << " Can't interpolate because of same pressure levels. prev: " << prev_pqtzuv[0] << ", cur: " << cur_pqtzuv[0] << ", next: " << prev_pqtzuv[0] << "\n\n"; @@ -3284,7 +3297,7 @@ void merge_records(float *first_pqtzuv, map pqtzuv_map_pivot, break; } } - + if (it_aux->first == cur_pres) { copy_pqtzuv(pqtzuv_merged, it_aux->second, false); prev_pqtzuv = pqtzuv_merged; @@ -3383,16 +3396,18 @@ void log_pbl_input(int pbl_level, const char *method_name) { for (int idx=0; idx=0; idx--) { - mlog << Debug(PBL_DEBUG_LEVEL) << method_name << " input to calpbl_: " + mlog << Debug(PBL_DEBUG_LEVEL) << method_name << " " << offset++ << "\t" << log_array[idx] << "\n"; } log_array.clear(); diff --git a/met/src/tools/other/plot_point_obs/plot_point_obs.cc b/met/src/tools/other/plot_point_obs/plot_point_obs.cc index 1ba1b9938b..243f4d0c29 100644 --- a/met/src/tools/other/plot_point_obs/plot_point_obs.cc +++ b/met/src/tools/other/plot_point_obs/plot_point_obs.cc @@ -45,26 +45,11 @@ using namespace std; #include "plot_point_obs.h" -#include "nc_utils.h" -#include "vx_log.h" -#include "data_plane.h" #include "data_plane_plot.h" -#include "write_netcdf.h" -#include "vx_data2d.h" -#include "vx_data2d_factory.h" -#include "vx_data2d_grib.h" -#include "vx_data2d_nc_met.h" -#include "vx_data2d_nc_pinterp.h" -#include "vx_util.h" -#include "vx_cal.h" -#include "vx_grid.h" -#include "vx_math.h" -#include "vx_color.h" #include "vx_ps.h" #include "vx_pxm.h" #include "vx_render.h" #include "vx_plot_util.h" -#include "nc_obs_util.h" #include "nc_point_obs_in.h" //////////////////////////////////////////////////////////////////////// @@ -149,42 +134,75 @@ int main(int argc, char *argv[]) { void process_point_obs(const char *point_obs_filename) { int h, v, i_obs; - const char *method_name = "\nprocess_point_obs() -> "; - const char *method_name_s = "\nprocess_point_obs() "; + const char *method_name = "process_point_obs() -> "; + const char *method_name_s = "process_point_obs() "; // Open the netCDF point observation file mlog << Debug(1) << "Reading point observation file: " << point_obs_filename << "\n"; + bool status; + bool use_var_id = true; + bool use_obs_arr = false; + + bool use_python = false; MetNcPointObsIn nc_point_obs; - if(!nc_point_obs.open(point_obs_filename)) { - nc_point_obs.close(); + MetPointData *met_point_obs = 0; +#ifdef WITH_PYTHON + MetPythonPointDataFile met_point_file; + string python_command = point_obs_filename; + bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); + use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); + if (use_python) { + int offset = python_command.find("="); + if (offset == std::string::npos) { + mlog << Error << "\n" << method_name + << "trouble parsing the python command " << python_command << ".\n\n"; + exit(1); + } - mlog << Error << "\n" << method_name - << "trouble opening point observation file " - << point_obs_filename << ".\n\n"; + if(!met_point_file.open(python_command.substr(offset+1).c_str(), use_xarray)) { + met_point_file.close(); + mlog << Error << "\n" << method_name + << "trouble getting point observation file from python command " + << python_command << ".\n\n"; + exit(1); + } - exit(1); + met_point_obs = met_point_file.get_met_point_data(); } + else +#endif + { + if(!nc_point_obs.open(point_obs_filename)) { + nc_point_obs.close(); - // Read the dimensions and variables - nc_point_obs.read_dim_headers(); - nc_point_obs.check_nc(point_obs_filename, method_name_s); // exit if missing dims/vars - nc_point_obs.read_obs_data(); + mlog << Error << "\n" << method_name + << "trouble opening point observation file " + << point_obs_filename << ".\n\n"; - bool use_var_id = nc_point_obs.is_using_var_id(); + exit(1); + } - // Print warning about ineffective command line arguments - if(use_var_id && ivar.n() != 0) { - mlog << Warning << "\n" << method_name << "-gc option is ignored!\n\n"; - } - if(!use_var_id && svar.n() != 0) { - mlog << Warning << "\n" << method_name << "-obs_var option is ignored!\n\n"; - } + // Read the dimensions and variables + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(point_obs_filename, method_name_s); // exit if missing dims/vars + nc_point_obs.read_obs_data(); + use_obs_arr = nc_point_obs.is_using_obs_arr(); - long nhdr_count = nc_point_obs.get_hdr_cnt(); - long nobs_count = nc_point_obs.get_obs_cnt(); + // Print warning about ineffective command line arguments + if(use_var_id && ivar.n() != 0) { + mlog << Warning << "\n" << method_name << "-gc option is ignored!\n\n"; + } + if(!use_var_id && svar.n() != 0) { + mlog << Warning << "\n" << method_name << "-obs_var option is ignored!\n\n"; + } + met_point_obs = (MetPointData *)&nc_point_obs; + } + + long nhdr_count = met_point_obs->get_hdr_cnt(); + long nobs_count = met_point_obs->get_obs_cnt(); mlog << Debug(2) << "Processing " << nobs_count << " observations at " << nhdr_count << " locations.\n"; @@ -192,8 +210,6 @@ void process_point_obs(const char *point_obs_filename) { // Get the corresponding header: // message type, staton_id, valid_time, and lat/lon/elv - bool use_obs_arr = nc_point_obs.is_using_obs_arr(); - int buf_size = (nobs_count > DEF_NC_BUFFER_SIZE) ? DEF_NC_BUFFER_SIZE : nobs_count; // Allocate space to store the data @@ -202,8 +218,18 @@ void process_point_obs(const char *point_obs_filename) { int obs_qty_block[buf_size]; float obs_arr_block[buf_size][OBS_ARRAY_LEN]; - if(use_var_id) var_list = nc_point_obs.get_var_names(); - qty_list = nc_point_obs.get_qty_data(); + use_var_id = met_point_obs->is_using_var_id(); + if(use_var_id) var_list = met_point_obs->get_var_names(); + qty_list = met_point_obs->get_qty_data(); + + if (0 == qty_list.n()) { + mlog << Error << "\n" << method_name << "Missing quality marks\n\n"; + exit (1); + } + if (use_var_id && (0 == var_list.n())) { + mlog << Debug(1) << method_name << "Missing variable names\ns\n"; + exit (1); + } ConcatString hdr_typ_str; ConcatString hdr_sid_str; @@ -211,16 +237,21 @@ void process_point_obs(const char *point_obs_filename) { ConcatString obs_qty_str; for(int i_start=0; i_start DEF_NC_BUFFER_SIZE) ? - DEF_NC_BUFFER_SIZE : (nobs_count-i_start); - - if (!nc_point_obs.read_obs_data(buf_size, i_start, (float *)obs_arr_block, - obs_qty_block, (char *)0)) { - exit(1); - } + int buf_size2 = ((nobs_count-i_start) > DEF_NC_BUFFER_SIZE) ? + DEF_NC_BUFFER_SIZE : (nobs_count-i_start); + +#ifdef WITH_PYTHON + if (use_python) + status = met_point_obs->get_point_obs_data()->fill_obs_buf( + buf_size2, i_start, (float *)obs_arr_block, obs_qty_block); + else +#endif + status = nc_point_obs.read_obs_data(buf_size2, i_start, (float *)obs_arr_block, + obs_qty_block, (char *)0); + if (!status) exit(1); int typ_idx, sid_idx, vld_idx; - for(int i_offset=0; i_offsetget_header(h, hdr_arr, hdr_typ_str, hdr_sid_str, hdr_vld_str); + else +#endif + nc_point_obs.get_header(h, hdr_arr, hdr_typ_str, hdr_sid_str, hdr_vld_str); // Store data in an observation object Observation cur_obs( @@ -263,12 +299,15 @@ void process_point_obs(const char *point_obs_filename) { } // end for i_start // Clean up +#ifdef WITH_PYTHON + if (use_python) met_point_file.close(); + else +#endif nc_point_obs.close(); return; } - //////////////////////////////////////////////////////////////////////// void create_plot() { @@ -470,7 +509,7 @@ void create_plot() { if(use_flate) plot.end_flate(); mlog << Debug(2) - << "Finished plotting " << plot_count << " locations.\n" + << "Finished plotting " << plot_count << " locations for " << ps_file << ".\n" << "Skipped " << skip_count << " locations off the grid.\n"; plot.showpage(); diff --git a/met/src/tools/other/plot_point_obs/plot_point_obs.h b/met/src/tools/other/plot_point_obs/plot_point_obs.h index 7fec0e19f0..0bf33b0679 100644 --- a/met/src/tools/other/plot_point_obs/plot_point_obs.h +++ b/met/src/tools/other/plot_point_obs/plot_point_obs.h @@ -43,12 +43,13 @@ using namespace netCDF; #include "plot_point_obs_conf_info.h" -#include "vx_data2d_factory.h" -#include "vx_grid.h" #include "vx_util.h" #include "vx_stat_out.h" #include "vx_gsl_prob.h" #include "nc_utils.h" +#ifdef WITH_PYTHON +#include "pointdata_python.h" +#endif //////////////////////////////////////////////////////////////////////// // diff --git a/met/src/tools/other/point2grid/point2grid.cc b/met/src/tools/other/point2grid/point2grid.cc index 7b840f08d9..f806023c11 100644 --- a/met/src/tools/other/point2grid/point2grid.cc +++ b/met/src/tools/other/point2grid/point2grid.cc @@ -43,6 +43,11 @@ using namespace std; #include "point2grid_conf_info.h" +#ifdef WITH_PYTHON +#include "data2d_nc_met.h" +#include "pointdata_python.h" +#endif + //////////////////////////////////////////////////////////////////////// static ConcatString program_name; @@ -53,6 +58,7 @@ static const int TYPE_OBS = 1; // MET Point Obs NetCDF (from xxx2nc) static const int TYPE_NCCF = 2; // CF NetCDF with time and lat/lon variables static const int TYPE_GOES = 5; static const int TYPE_GOES_ADP = 6; +static const int TYPE_PYTHON = 7; // MET Point Obs NetCDF from PYTHON static const InterpMthd DefaultInterpMthd = InterpMthd_UW_Mean; static const int DefaultInterpWdth = 2; @@ -120,14 +126,19 @@ static NcDim lon_dim ; static void process_command_line(int, char **); static void process_data_file(); static void process_point_file(NcFile *nc_in, MetConfig &config, - VarInfo *, const Grid fr_grid, const Grid to_grid); + VarInfo *, const Grid to_grid); +#ifdef WITH_PYTHON +static void process_point_python(string python_command, MetConfig &config, + VarInfo *vinfo, const Grid to_grid, bool use_xarray); +#endif static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, - VarInfo *, Met2dDataFile *fr_mtddf, const Grid to_grid); + VarInfo *, Met2dDataFile *fr_mtddf, + const Grid to_grid); static void open_nc(const Grid &grid, const ConcatString run_cs); static void write_nc(const DataPlane &dp, const Grid &grid, const VarInfo *vinfo, const char *vname); static void write_nc_int(const DataPlane &dp, const Grid &grid, - const VarInfo *vinfo, const char *vname); + const VarInfo *vinfo, const char *vname); static void close_nc(); static void usage(); static void set_field(const StringArray &); @@ -265,11 +276,26 @@ void process_command_line(int argc, char **argv) { OutputFilename = cline[2]; // Check if the input file - if ( !file_exists(InputFilename.c_str()) ) { - mlog << Error << "\n" << method_name - << "file does not exist \"" << InputFilename << "\"\n\n"; - exit(1); + bool use_python = false; +#ifdef WITH_PYTHON + string python_command = InputFilename; + bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); + use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); + if (use_python) { + int offset = python_command.find("="); + if (offset == std::string::npos) { + mlog << Error << "\n" << method_name + << "trouble parsing the python command " << python_command << ".\n\n"; + exit(1); + } } + else +#endif + if ( !file_exists(InputFilename.c_str()) ) { + mlog << Error << "\n" << method_name + << "file does not exist \"" << InputFilename << "\"\n\n"; + exit(1); + } // Check for at least one configuration string if(FieldSA.n() < 1) { @@ -347,18 +373,42 @@ void process_data_file() { // Open the input file mlog << Debug(1) << "Reading data file: " << InputFilename << "\n"; - nc_in = open_ncfile(InputFilename.c_str()); - - // Get the obs type before opening NetCDF - int obs_type = get_obs_type(nc_in); - bool goes_data = (obs_type == TYPE_GOES || obs_type == TYPE_GOES_ADP); - - if (obs_type == TYPE_NCCF) setenv(nc_att_met_point_nccf, "yes", 1); - - // Read the input data file + bool goes_data = false; + bool use_python = false; + int obs_type = TYPE_UNKNOWN; Met2dDataFileFactory m_factory; Met2dDataFile *fr_mtddf = (Met2dDataFile *) 0; - fr_mtddf = m_factory.new_met_2d_data_file(InputFilename.c_str(), ftype); +#ifdef WITH_PYTHON + string python_command = InputFilename; + MetPointData *met_point_obs = 0; + bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); + use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); + if (use_python) { + int offset = python_command.find("="); + if (offset == std::string::npos) { + mlog << Error << "\n" << method_name + << "trouble parsing the python command " << python_command << ".\n\n"; + exit(1); + } + + python_command = python_command.substr(offset+1); + obs_type = TYPE_PYTHON; + fr_mtddf = new MetNcMetDataFile(); + } + else +#endif + { + nc_in = open_ncfile(InputFilename.c_str()); + + // Get the obs type before opening NetCDF + obs_type = get_obs_type(nc_in); + goes_data = (obs_type == TYPE_GOES || obs_type == TYPE_GOES_ADP); + + if (obs_type == TYPE_NCCF) setenv(nc_att_met_point_nccf, "yes", 1); + + // Read the input data file + fr_mtddf = m_factory.new_met_2d_data_file(InputFilename.c_str(), ftype); + } if(!fr_mtddf) { mlog << Error << "\n" << method_name @@ -384,7 +434,11 @@ void process_data_file() { // For python types read the first field to set the grid // Determine the "from" grid +#ifdef WITH_PYTHON + if (!use_python) fr_grid = fr_mtddf->grid(); +#else fr_grid = fr_mtddf->grid(); +#endif // Determine the "to" grid to_grid = parse_vx_grid(RGInfo, &fr_grid, &fr_grid); @@ -410,12 +464,17 @@ void process_data_file() { process_goes_file(nc_in, config, vinfo, fr_grid, to_grid); } else if (TYPE_OBS == obs_type) { - process_point_file(nc_in, config, vinfo, fr_grid, to_grid); + process_point_file(nc_in, config, vinfo, to_grid); } else if (TYPE_NCCF == obs_type) { process_point_nccf_file(nc_in, config, vinfo, fr_mtddf, to_grid); unsetenv(nc_att_met_point_nccf); } +#ifdef WITH_PYTHON + else if (TYPE_PYTHON == obs_type) { + process_point_python(python_command, config, vinfo, to_grid, use_xarray); + } +#endif else { mlog << Error << "\n" << method_name << "Please check the input file. Only supports GOES, MET point obs, " @@ -599,8 +658,8 @@ IntArray prepare_qc_array(const IntArray qc_flags, StringArray qc_tables) { //////////////////////////////////////////////////////////////////////// -void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, - const Grid fr_grid, const Grid to_grid) { +void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarInfo *vinfo, + const Grid to_grid) { int nhdr, nobs; int nx, ny, var_count, to_count, var_count2; int idx, hdr_idx; @@ -616,8 +675,8 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); unixtime requested_valid_time, valid_time; - static const char *method_name = "process_point_file() -> "; - static const char *method_name_s = "process_point_file()"; + static const char *method_name = "process_point_met_data() -> "; + static const char *method_name_s = "process_point_met_data()"; // Check for at least one configuration string if(FieldSA.n() < 1) { @@ -626,35 +685,29 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, usage(); } - MetNcPointObsIn nc_point_obs; - nc_point_obs.set_netcdf(nc_in, true); - // Read the dimensions and variables - nc_point_obs.read_dim_headers(); - nc_point_obs.check_nc(GET_NC_NAME_P(nc_in).c_str(), method_name_s); // exit if missing dims/vars - // Read all obs data to compute the cell mapping - nc_point_obs.read_obs_data(); - NcHeaderData header_data = nc_point_obs.get_header_data(); - NcPointObsData obs_data = nc_point_obs.get_point_obs_data(); +// MetNcPointObsIn nc_point_obs; + MetPointHeader *header_data = met_point_obs->get_header_data(); + MetPointObsData *obs_data = met_point_obs->get_point_obs_data(); - nhdr = nc_point_obs.get_hdr_cnt(); - nobs = nc_point_obs.get_obs_cnt(); + nhdr = met_point_obs->get_hdr_cnt(); + nobs = met_point_obs->get_obs_cnt(); bool empty_input = (nhdr == 0 && nobs == 0); - bool use_var_id = nc_point_obs.is_using_var_id(); + bool use_var_id = met_point_obs->is_using_var_id(); float *hdr_lats = new float[nhdr]; float *hdr_lons = new float[nhdr]; IntArray var_index_array; IntArray valid_time_array; - StringArray qc_tables = nc_point_obs.get_qty_data(); - StringArray var_names = nc_point_obs.get_var_names(); - StringArray hdr_valid_times = header_data.vld_array; + StringArray qc_tables = met_point_obs->get_qty_data(); + StringArray var_names = met_point_obs->get_var_names(); + StringArray hdr_valid_times = header_data->vld_array; hdr_valid_times.sort(); - nc_point_obs.get_lats(hdr_lats); - nc_point_obs.get_lons(hdr_lons); + met_point_obs->get_lats(hdr_lats); + met_point_obs->get_lons(hdr_lons); // Check the message types - prepare_message_types(header_data.typ_array); + prepare_message_types(header_data->typ_array); // Check and read obs_vid and obs_var if exists bool success_to_read = true; @@ -732,7 +785,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, else { bool not_found_grib_code = true; for (idx=0; idxobs_ids[idx]) { not_found_grib_code = false; break; } @@ -757,10 +810,10 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, log_msg << "GRIB codes: "; IntArray grib_codes; for (idx=0; idxobs_ids[idx])) { + grib_codes.add(obs_data->obs_ids[idx]); if (0 < idx) log_msg << ", "; - log_msg << obs_data.obs_ids[idx]; + log_msg << obs_data->obs_ids[idx]; } } } @@ -836,29 +889,29 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, var_count = var_count2 = to_count = 0; filtered_by_time = filtered_by_msg_type = filtered_by_qc = 0; for (idx=0; idx < nobs; idx++) { - if (var_idx_or_gc == obs_data.obs_ids[idx]) { + if (var_idx_or_gc == obs_data->obs_ids[idx]) { var_count2++; - hdr_idx = obs_data.obs_hids[idx]; + hdr_idx = obs_data->obs_hids[idx]; if (0 < valid_time_array.n() && - !valid_time_array.has(header_data.vld_idx_array[hdr_idx])) { + !valid_time_array.has(header_data->vld_idx_array[hdr_idx])) { filtered_by_time++; continue; } - if(!keep_message_type(header_data.typ_idx_array[hdr_idx])) { + if(!keep_message_type(header_data->typ_idx_array[hdr_idx])) { filtered_by_msg_type++; continue; } // Filter by QC flag - if (has_qc_flags && !qc_idx_array.has(obs_data.obs_qids[idx])) { + if (has_qc_flags && !qc_idx_array.has(obs_data->obs_qids[idx])) { filtered_by_qc++; continue; } var_index_array.add(idx); var_count++; - if (is_eq(obs_data.obs_vals[idx], 0.)) obs_count_zero_from++; + if (is_eq(obs_data->obs_vals[idx], 0.)) obs_count_zero_from++; else obs_count_non_zero_from++; } } @@ -869,7 +922,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, } cellMapping = new IntArray[nx * ny]; if( get_grid_mapping(to_grid, cellMapping, var_index_array, - obs_data.obs_hids, hdr_lats, hdr_lons) ) { + obs_data->obs_hids, hdr_lats, hdr_lons) ) { int from_index; IntArray cellArray; NumArray dataArray; @@ -905,7 +958,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, dataArray.extend(cellArray.n()); for (int dIdx=0; dIdxget_obs_val(from_index); if (is_bad_data(data_value)) continue; if(mlog.verbosity_level() >= 4) { @@ -1073,13 +1126,108 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, delete [] hdr_lats; delete [] hdr_lons; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, + const Grid to_grid) { + int nhdr, nobs; + int nx, ny, var_count, to_count, var_count2; + int idx, hdr_idx; + int var_idx_or_gc; + int filtered_by_time, filtered_by_msg_type, filtered_by_qc; + ConcatString vname, vname_cnt, vname_mask; + DataPlane fr_dp, to_dp; + DataPlane cnt_dp, mask_dp; + DataPlane prob_dp, prob_mask_dp; + NcVar var_obs_gc, var_obs_var; + + clock_t start_clock = clock(); + bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); + + unixtime requested_valid_time, valid_time; + static const char *method_name = "process_point_file() -> "; + static const char *method_name_s = "process_point_file()"; + + // Check for at least one configuration string + if(FieldSA.n() < 1) { + mlog << Error << "\n" << method_name + << "The -field option must be used at least once!\n\n"; + usage(); + } + + MetNcPointObsIn nc_point_obs; + nc_point_obs.set_netcdf(nc_in, true); + // Read the dimensions and variables + nc_point_obs.read_dim_headers(); + nc_point_obs.check_nc(GET_NC_NAME_P(nc_in).c_str(), method_name_s); // exit if missing dims/vars + // Read all obs data to compute the cell mapping + nc_point_obs.read_obs_data(); + process_point_met_data(&nc_point_obs, config, vinfo, to_grid); + nc_point_obs.close(); mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; +} + +//////////////////////////////////////////////////////////////////////// + +#ifdef WITH_PYTHON + +void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, + const Grid to_grid, bool use_xarray) { + int nhdr, nobs; + int nx, ny, var_count, to_count, var_count2; + int idx, hdr_idx; + int var_idx_or_gc; + int filtered_by_time, filtered_by_msg_type, filtered_by_qc; + ConcatString vname, vname_cnt, vname_mask; + DataPlane fr_dp, to_dp; + DataPlane cnt_dp, mask_dp; + DataPlane prob_dp, prob_mask_dp; + NcVar var_obs_gc, var_obs_var; + + clock_t start_clock = clock(); + bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); + + unixtime requested_valid_time, valid_time; + static const char *method_name = "process_point_python() -> "; + static const char *method_name_s = "process_point_python()"; + + // Check for at least one configuration string + if(FieldSA.n() < 1) { + mlog << Error << "\n" << method_name + << "The -field option must be used at least once!\n\n"; + usage(); + } + + MetPythonPointDataFile met_point_file; + if(!met_point_file.open(python_command.c_str(), use_xarray)) { + met_point_file.close(); + + mlog << Error << "\n" << method_name + << "trouble getting point observation file from python command " + << python_command << "\n\n"; + + exit(1); + } + + MetPointData *met_point_obs = met_point_file.get_met_point_data(); + process_point_met_data(met_point_obs, config, vinfo, to_grid); + + met_point_file.close(); + + mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " + << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; + return; } +#endif //////////////////////////////////////////////////////////////////////// @@ -1097,7 +1245,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, bool opt_all_attrs = false; Grid fr_grid = fr_mtddf->grid(); int from_size = fr_grid.nx() * fr_grid.ny(); - static const char *method_name = "process_point_file_with_latlon() -> "; + static const char *method_name = "process_point_nccf_file() -> "; NcVar var_lat = get_nc_var_lat(nc_in); NcVar var_lon = get_nc_var_lon(nc_in); diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 330e0bbadd..ddeb6b0a81 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM centos:7 +ARG MET_BASE_IMAGE=minimum + +FROM dtcenter/met-base:${MET_BASE_IMAGE} MAINTAINER John Halley Gotway # @@ -17,163 +19,21 @@ RUN if [ "x${SOURCE_BRANCH}" = "x" ]; then \ fi ENV MET_GIT_NAME ${SOURCE_BRANCH} +ENV MET_REPO_DIR /met/MET-${MET_GIT_NAME} ENV MET_GIT_URL https://github.com/dtcenter/MET ENV MET_DEVELOPMENT true -# -# Define the compilers. -# -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV FC /usr/bin/gfortran -ENV F77 /usr/bin/gfortran - -# -# Define package URL's. -# -ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz -ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z - -ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/v4.4.1.1.zip -ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz - -ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar -ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz - -# -# Install the required packages. -# -RUN yum -y update \ - && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ - libpng-devel jasper jasper-devel zlib zlib-devel \ - cairo-devel freetype-devel epel-release \ - hostname m4 make tar tcsh ksh time wget which \ - flex flex-devel bison bison-devel unzip \ - && yum -y install git g2clib-devel hdf5-devel.x86_64 gsl-devel \ - && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ - && yum -y install python3 python3-devel python3-pip \ - && pip3 install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - # # Set the working directory. # WORKDIR /met -# -# Setup the environment for interactive bash/csh container shells. -# -RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ - && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ - && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ - && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc \ - && echo export LD_LIBRARY_PATH=/usr/local/lib >> /etc/bashrc \ - && echo setenv LD_LIBRARY_PATH /usr/local/lib >> /etc/csh.cshrc -ENV LD_LIBRARY_PATH /usr/local/lib - -# -# Download and install BUFRLIB. -# -RUN mkdir -p /met/logs \ - && mkdir -p /met/external_libs/BUFRLIB \ - && cd /met/external_libs/BUFRLIB \ - && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ - && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ - && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ - && chmod +x preproc_patch.sh \ - && LOG_FILE=/met/logs/BUFRLIB_build.log \ - && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ - && ./preproc_patch.sh *.F > ${LOG_FILE} \ - && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ - && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ - && ar crv libbufr.a *.o >> ${LOG_FILE} \ - && rm -f /usr/local/lib/libbufr.a \ - && cp *.a /usr/local/lib \ - && cd /met/external_libs \ - && rm -rf BUFRLIB - -# -# Download and install NetCDF4 (C and C++). -# -RUN mkdir -p /met/external_libs/netcdf \ - && cd /met/external_libs/netcdf \ - && echo "Downloading netcdf-c-4.4.1.1 from ${NETCDF4C_URL}" \ - && wget ${NETCDF4C_URL} \ - && unzip v4.4.1.1.zip \ - && cd netcdf-c-4.4.1.1 \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_configure.log \ - && echo "Configuring netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_make_install.log \ - && echo "Compiling netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading from ${NETCDF4CXX_URL}" \ - && cd /met/external_libs/netcdf \ - && wget ${NETCDF4CXX_URL} \ - && tar -xzf v4.3.0.tar.gz \ - && cd netcdf-cxx4-4.3.0 \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_configure.log \ - && echo "Configuring netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_make_install.log \ - && echo "Compiling netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf netcdf - -# -# Download and install HDF4 and HDFEOS. -# -RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ - && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/HDF4.2r3 \ - && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ - && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ - && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ - && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ - && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ - && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading hdfeos from ${HDFEOS_URL}" \ - && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/hdfeos \ - && LOG_FILE=/met/logs/hdfeos_configure.log \ - && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdfeos_make_install.log \ - && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && mkdir /usr/local/hdfeos/include \ - && cp include/*.h /usr/local/hdfeos/include/. \ - && cd /met/external_libs \ - && rm -rf HDF4.2r3 hdfeos - # # Download and install MET and GhostScript fonts. # Delete the MET source code for tagged releases matching "v"*. # RUN echo "Checking out MET ${MET_GIT_NAME} from ${MET_GIT_URL}" \ - && git clone ${MET_GIT_URL} /met/MET-${MET_GIT_NAME} \ - && cd /met/MET-${MET_GIT_NAME}/met \ + && git clone ${MET_GIT_URL} ${MET_REPO_DIR} \ + && cd ${MET_REPO_DIR}/met \ && git checkout ${MET_GIT_NAME} \ - && LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log \ - && echo "Running bootstrap" \ - && ./bootstrap \ - && echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" \ - && ./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python \ - MET_HDF=/usr/local/hdf MET_HDFEOS=/usr/local/hdfeos \ - MET_FREETYPEINC=/usr/include/freetype2 MET_FREETYPELIB=/usr/lib \ - MET_CAIROINC=/usr/include/cairo MET_CAIROLIB=/usr/lib \ - MET_PYTHON_CC='-I/usr/include/python3.6m' MET_PYTHON_LD='-lpython3.6m' > ${LOG_FILE} \ - && LOG_FILE=/met/MET-${MET_GIT_NAME}/make_install.log \ - && echo "Compiling MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_test.log \ - && echo "Testing MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" \ - && make test > ${LOG_FILE} 2>&1 \ - && if [[ $MET_GIT_NAME == "v"* ]]; then cd /met; rm -rf MET-*; fi \ - && echo "Downloading GhostScript fonts from ${GSFONT_URL}" \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met + && ../scripts/docker/build_met_docker.sh diff --git a/scripts/docker/Dockerfile.copy b/scripts/docker/Dockerfile.copy new file mode 100644 index 0000000000..18a71e4185 --- /dev/null +++ b/scripts/docker/Dockerfile.copy @@ -0,0 +1,46 @@ +ARG MET_BASE_IMAGE=minimum + +FROM dtcenter/met-base:${MET_BASE_IMAGE} +MAINTAINER John Halley Gotway + +# +# This Dockerfile checks out MET from GitHub and compiles the specified branch or tag from source. +# +ARG SOURCE_BRANCH + +# +# SOURCE_BRANCH is not defined when built via Docker Hub. +# +RUN if [ "x${SOURCE_BRANCH}" = "x" ]; then \ + echo "ERROR: SOURCE_BRANCH undefined! Rebuild with \"--build-arg SOURCE_BRANCH={branch name}\""; \ + exit 1; \ + else \ + echo "Build Argument SOURCE_BRANCH=${SOURCE_BRANCH}"; \ + fi + +ENV MET_GIT_NAME ${SOURCE_BRANCH} +ENV MET_REPO_DIR /met/MET-${MET_GIT_NAME} +ENV MET_GIT_URL https://github.com/dtcenter/MET +ENV MET_DEVELOPMENT true + +# +# Set the working directory. +# +WORKDIR /met + +# +# Download and install MET and GhostScript fonts. +# Delete the MET source code for tagged releases matching "v"*. +# +RUN echo "Copying MET into ${MET_REPO_DIR}" \ + && mkdir -p ${MET_REPO_DIR} + +COPY . ${MET_REPO_DIR} + +RUN if [ ! -e "${MET_REPO_DIR}/met/configure.ac" ]; then \ + echo "ERROR: docker build must be run from the MET directory"; \ + exit 1; \ + fi + +RUN cd ${MET_REPO_DIR}/met \ + && ../scripts/docker/build_met_docker.sh diff --git a/scripts/docker/Dockerfile.minimum b/scripts/docker/Dockerfile.minimum new file mode 100644 index 0000000000..028871196b --- /dev/null +++ b/scripts/docker/Dockerfile.minimum @@ -0,0 +1,138 @@ +FROM centos:7 +MAINTAINER John Halley Gotway + +# +# Define the compilers. +# +ENV CC /usr/bin/gcc +ENV CXX /usr/bin/g++ +ENV FC /usr/bin/gfortran +ENV F77 /usr/bin/gfortran + +# +# Define package URL's. +# +ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz +ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z + +ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/v4.4.1.1.zip +ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz + +ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar +ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz + +# +# Install the required packages. +# +RUN yum -y update \ + && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ + libpng-devel jasper jasper-devel zlib zlib-devel \ + cairo-devel freetype-devel epel-release \ + hostname m4 make tar tcsh ksh time wget which \ + flex flex-devel bison bison-devel unzip \ + && yum -y install git g2clib-devel hdf5-devel.x86_64 gsl-devel \ + && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ + && yum -y install python3 python3-devel python3-pip \ + && pip3 install --upgrade pip \ + && python3 -m pip install numpy xarray netCDF4 + +# +# Set the working directory. +# +WORKDIR /met + +# +# Setup the environment for interactive bash/csh container shells. +# +RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ + && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ + && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ + && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ + && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ + && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc \ + && echo export LD_LIBRARY_PATH=/usr/local/lib >> /etc/bashrc \ + && echo setenv LD_LIBRARY_PATH /usr/local/lib >> /etc/csh.cshrc +ENV LD_LIBRARY_PATH /usr/local/lib +ENV MET_FONT_DIR /usr/local/share/met/fonts + +# +# Download and install BUFRLIB. +# +RUN mkdir -p /met/logs \ + && mkdir -p /met/external_libs/BUFRLIB \ + && cd /met/external_libs/BUFRLIB \ + && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ + && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ + && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ + && chmod +x preproc_patch.sh \ + && LOG_FILE=/met/logs/BUFRLIB_build.log \ + && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ + && ./preproc_patch.sh *.F > ${LOG_FILE} \ + && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ + && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ + && ar crv libbufr.a *.o >> ${LOG_FILE} \ + && rm -f /usr/local/lib/libbufr.a \ + && cp *.a /usr/local/lib \ + && cd /met/external_libs \ + && rm -rf BUFRLIB + +# +# Download and install NetCDF4 (C and C++). +# +RUN mkdir -p /met/external_libs/netcdf \ + && cd /met/external_libs/netcdf \ + && echo "Downloading netcdf-c-4.4.1.1 from ${NETCDF4C_URL}" \ + && wget ${NETCDF4C_URL} \ + && unzip v4.4.1.1.zip \ + && cd netcdf-c-4.4.1.1 \ + && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_configure.log \ + && echo "Configuring netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ + && ./configure > ${LOG_FILE} \ + && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_make_install.log \ + && echo "Compiling netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && echo "Downloading from ${NETCDF4CXX_URL}" \ + && cd /met/external_libs/netcdf \ + && wget ${NETCDF4CXX_URL} \ + && tar -xzf v4.3.0.tar.gz \ + && cd netcdf-cxx4-4.3.0 \ + && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_configure.log \ + && echo "Configuring netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ + && ./configure > ${LOG_FILE} \ + && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_make_install.log \ + && echo "Compiling netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && cd /met/external_libs \ + && rm -rf netcdf + +# +# Download and install HDF4 and HDFEOS. +# +RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ + && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ + && cd /met/external_libs/HDF4.2r3 \ + && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ + && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ + && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ + && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ + && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ + && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ + && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && echo "Downloading hdfeos from ${HDFEOS_URL}" \ + && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ + && cd /met/external_libs/hdfeos \ + && LOG_FILE=/met/logs/hdfeos_configure.log \ + && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ + && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ + && LOG_FILE=/met/logs/hdfeos_make_install.log \ + && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && mkdir /usr/local/hdfeos/include \ + && cp include/*.h /usr/local/hdfeos/include/. \ + && cd /met/external_libs \ + && rm -rf HDF4.2r3 hdfeos + +RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ + && mkdir -p /usr/local/share/met \ + && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met diff --git a/scripts/docker/Dockerfile.test b/scripts/docker/Dockerfile.test new file mode 100644 index 0000000000..bc7963cb91 --- /dev/null +++ b/scripts/docker/Dockerfile.test @@ -0,0 +1,26 @@ +ARG MET_BASE_IMAGE=minimum + +FROM dtcenter/met-base:${MET_BASE_IMAGE} +MAINTAINER John Halley Gotway + +# +# Set the working directory. +# +WORKDIR /met + +# +# Download and install MET and GhostScript fonts. +# Delete the MET source code for tagged releases matching "v"*. +# +RUN echo "Installing tools needed for running MET unit tests..." \ + && echo "Installing Perl XML Parser..." \ + && yum makecache \ + && yum -y install perl-XML-Parser \ + && echo "Installing R..." \ + && yum -y install R \ + && echo "Installing R ncdf4 1.19..." \ + && wget https://cran.r-project.org/src/contrib/ncdf4_1.19.tar.gz \ + && R CMD INSTALL ncdf4_1.19.tar.gz \ + && echo "Installing NCO (for ncdiff)..." \ + && yum -y install nco \ + && echo "Finished installing unit test tools" diff --git a/scripts/docker/README.md b/scripts/docker/README.md new file mode 100644 index 0000000000..80c6acaa08 --- /dev/null +++ b/scripts/docker/README.md @@ -0,0 +1,40 @@ +# How to Use Dockerfiles + +Run all of the Docker commands from the top-level directory of the MET repository + +## Build image with minimum requirements needed to build MET + +```docker build -t dtcenter/met-base:minimum -f scripts/docker/Dockerfile.minimum . +docker push dtcenter/met-base:minimum``` + +## Build image with requirements to build MET and run MET unit tests + +```docker build -t dtcenter/met-base:unit_test -f scripts/docker/Dockerfile.test . +docker push dtcenter/met-base:unit_test``` + +## Build MET from clone + +```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} scripts/docker +docker push dtcenter/met:${TAG_NAME}``` + +where: +* TAG_NAME is the name of the DockerHub tag to create +* BRANCH_NAME is the MET branch to checkout + +## Build MET from local source code with minimum requirements + +```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} -f scripts/docker/Dockerfile.copy . +docker push dtcenter/met:${TAG_NAME}``` + +where: +* TAG_NAME is the name of the DockerHub tag to create +* BRANCH_NAME is the identifier to use for $MET_GIT_NAME inside image + +## Build MET from local source code with unit test requirements + +```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} --build-arg MET_BASE_IMAGE=unit_test -f scripts/docker/Dockerfile.copy . +docker push dtcenter/met:${TAG_NAME}``` + +where: +* TAG_NAME is the name of the DockerHub tag to create +* BRANCH_NAME is the identifier to use for $MET_GIT_NAME inside image diff --git a/scripts/docker/build_met_docker.sh b/scripts/docker/build_met_docker.sh new file mode 100755 index 0000000000..fae3682113 --- /dev/null +++ b/scripts/docker/build_met_docker.sh @@ -0,0 +1,37 @@ +#! /bin/bash + +echo "Running script to build MET in Docker" + +LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log + +echo "Running bootstrap" +./bootstrap + +echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" +./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python \ + MET_HDF=/usr/local/hdf MET_HDFEOS=/usr/local/hdfeos \ + MET_FREETYPEINC=/usr/include/freetype2 MET_FREETYPELIB=/usr/lib \ + MET_CAIROINC=/usr/include/cairo MET_CAIROLIB=/usr/lib \ + MET_PYTHON_CC='-I/usr/include/python3.6m' MET_PYTHON_LD='-lpython3.6m' > ${LOG_FILE} +if [ $? != 0 ]; then + exit 1 +fi + +LOG_FILE=/met/MET-${MET_GIT_NAME}/make_install.log +echo "Compiling MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" +make install > ${LOG_FILE} +if [ $? != 0 ]; then + exit 1 +fi + +LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_test.log +echo "Testing MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" +make test > ${LOG_FILE} 2>&1 +if [ $? != 0 ]; then + exit 1 +fi + + +if [[ $MET_GIT_NAME == "v"* ]]; then + cd /met; rm -rf MET-*; +fi diff --git a/scripts/environment/development.seneca b/scripts/environment/development.seneca index 650257b3ab..ccbf506dbf 100644 --- a/scripts/environment/development.seneca +++ b/scripts/environment/development.seneca @@ -49,3 +49,8 @@ export MET_TEST_RSCRIPT=/usr/local/R-4.1.2/bin/Rscript export PATH="/usr/local/nco/bin:/usr/local/netcdf/bin:\ /usr/local/sbin:/usr/local/bin:/usr/sbin:\ /usr/bin:/sbin:/bin:/usr/bin/X11:/opt/bin:$PATH" + +# SonarQube +export SONARQUBE_DIR=/d1/projects/SonarQube/ +export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86 +export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin diff --git a/scripts/sonarqube/run_nightly.sh b/scripts/sonarqube/run_nightly.sh new file mode 100755 index 0000000000..6fcc166950 --- /dev/null +++ b/scripts/sonarqube/run_nightly.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Run nightly SonarQube scan +#======================================================================= +# +# This run_nightly.sh script calls the run_sonarqube.sh script. +# It is intented to be run nightly through cron. Output should be +# directed to the LOGFILE, per cron convention. To run this script, use +# the following commands: +# +# git clone https://github.com/dtcenter/MET +# MET/scripts/sosnarqube/run_nightly.sh name +# +# Usage: run_nightly.sh name +# where "name" specifies a branch, tag, or hash +# +# For example, scan the develop branch: +# run_nightly.sh develop +# +#======================================================================= + +# Constants +#EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu linden@ucar.edu mccabe@ucar.edu" +EMAIL_LIST="hsoh@ucar.edu" +KEEP_DAYS=5 + +function usage { + echo + echo "USAGE: run_nightly.sh name" + echo " where \"name\" specifies a branch, tag, or hash." + echo +} + +# Check for arguments +if [ $# -lt 1 ]; then usage; exit 1; fi + +# Store the full path to the scripts directory +SCRIPT_DIR=`dirname $0` +if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi + +# Define the development environment +ENV_FILE=${SCRIPT_DIR}/../environment/development.`hostname` +if [[ ! -e ${ENV_FILE} ]]; then + echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}" + exit 1 +fi +source ${ENV_FILE} + +# Delete old directories +find ${MET_PROJ_DIR}/MET_regression/sonarqube \ + -mtime +${KEEP_DAYS} -name "NB*" | \ + xargs rm -rf + +# Create and switch to a run directory +TODAY=`date +%Y%m%d` +YESTERDAY=`date -d "1 day ago" +%Y%m%d` +RUN_DIR=${MET_PROJ_DIR}/MET_regression/sonarqube/NB${TODAY} +if [[ -e ${RUN_DIR} ]]; then rm -rf ${RUN_DIR}; fi +mkdir -p ${RUN_DIR} +cd ${RUN_DIR} + +# Create a logfile +LOGFILE=${RUN_DIR}/run_sonarqube_${TODAY}.log + +# Run scan and check for bad return status +${SCRIPT_DIR}/run_sonarqube.sh ${1} > ${LOGFILE} +if [[ $? -ne 0 ]]; then + echo "$0: The nightly SonarQube scan FAILED in `basename ${RUN_DIR}`." >> ${LOGFILE} + cat ${LOGFILE} | mail -s "MET SonarQube scan Failed for ${1} in `basename ${RUN_DIR}` (autogen msg)" ${EMAIL_LIST} + exit 1 +fi + +# Convert SonarQube report from pdf to html + +exit 0 diff --git a/scripts/sonarqube/run_sonarqube.sh b/scripts/sonarqube/run_sonarqube.sh new file mode 100755 index 0000000000..98c4e15bbe --- /dev/null +++ b/scripts/sonarqube/run_sonarqube.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Run SonarQube Source Code Analyzer on a specified revision of MET +#======================================================================= +# +# This run_sonarqube.sh script will check out the specified version +# of MET and run the SonarQube Source Code Analyzer on it. First, +# go to the directory where you would like the SCA output written and +# then run: +# +# git clone https://github.com/dtcenter/MET +# MET/scripts/sonarqube/run_sonarqube_sca.sh name +# +# Usage: run_sonarqube_sca.sh name +# Test the specified branched version of MET: +# run_sonarqube_sca.sh {branch name} +# Test the specified tagged version of MET: +# run_sonarqube_sca.sh {tag name} +# +#======================================================================= + +# Constants +GIT_REPO="https://github.com/dtcenter/MET" + +function usage { + echo + echo "USAGE: $(basename $0) name" + echo " where \"name\" specifies a branch, tag, or hash." + echo +} + +# Check for arguments +if [[ $# -lt 1 ]]; then usage; exit; fi + +# Check that SONARQUBE_WRAPPER_BIN is defined +if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then + which build-wrapper-linux-x86-64 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_WRAPPER_BIN=$(which build-wrapper-linux-x86-64 2> /dev/null) + else + which build-wrapper 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_WRAPPER_BIN=$(which build-wrapper 2> /dev/null) + else + echo "ERROR: SONARQUBE_WRAPPER_BIN must be set" + exit 1 + fi + fi +fi +if [ ! -e ${SONARQUBE_WRAPPER_BIN} ]; then + echo "ERROR: SONARQUBE_WRAPPER_BIN (${SONARQUBE_WRAPPER_BIN}) does not exist" + exit 1 +fi + +# Check that SONARQUBE_SCANNER_BIN is defined +if [ -z ${SONARQUBE_SCANNER_BIN} ]; then + which sonar-scanner 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_SCANNER_BIN=$(which sonar-scanner 2> /dev/null) + else + echo "ERROR: SONARQUBE_SCANNER_BIN must be set" + exit 1 + fi +fi +if [ ! -e ${SONARQUBE_SCANNER_BIN} ]; then + echo "ERROR: SONARQUBE_SCANNER_BIN (${SONARQUBE_SCANNER_BIN}) does not exist" + exit 1 +fi + +if [ -z ${SONARQUBE_OUT_DIR} ]; then + export SONARQUBE_OUT_DIR=bw-outputs +fi + +# Sub-routine for running a command and checking return status +function run_command() { + + # Print the command being called + echo "CALLING: $1" + + # Run the command and store the return status + $1 + STATUS=$? + + # Check return status + if [[ ${STATUS} -ne 0 ]]; then + echo "ERROR: Command returned with non-zero status ($STATUS): $1" + exit ${STATUS} + fi + + return ${STATUS} +} + + +# Store the full path to the scripts directory +SCRIPT_DIR=`dirname $0` +if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi + +# Clone repo into a sub-directory and checkout the requested version +REPO_DIR="MET-${1}" + +if [ -e ${REPO_DIR} ]; then + run_command "rm -rf ${REPO_DIR}" +fi +run_command "git clone ${GIT_REPO} ${REPO_DIR}" +run_command "cd ${REPO_DIR}" +run_command "git checkout ${1}" + +# Build the MET instance +run_command "cd met" + +# Run bootstrap +run_command "./bootstrap" + +# Do no manually set the CXX and F77 compilers. +# Let the configure script pick them. +# Otherwise, the SonarQube logic does not work. +export MET_DEVELOPMENT=true + +# Run the configure script +run_command "./configure --prefix=`pwd` \ + --enable-grib2 \ + --enable-modis \ + --enable-mode_graphics \ + --enable-lidar2nc \ + --enable-python" + +# Set the build id +#BUILD_ID="MET-${1}" + +# Copy sonar-project.properties +[ ! -e "sonar-project.properties" ] && cp -p $SCRIPT_DIR/sonar-project.properties . + +# Run SonarQube clean +run_command "make clean" + + +# Run SonarQube make +run_command "${SONARQUBE_WRAPPER_BIN}/build-wrapper-linux-x86-64 --out-dir $SONARQUBE_OUT_DIR make" + +# Run SonarQube scan +run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" + +# Run SonarQube report generator to make a PDF file +#TODAY=`date +%Y%m%d` diff --git a/scripts/sonarqube/sonar-project.properties b/scripts/sonarqube/sonar-project.properties new file mode 100644 index 0000000000..387a9a3d74 --- /dev/null +++ b/scripts/sonarqube/sonar-project.properties @@ -0,0 +1,18 @@ +sonar.projectKey=org.sonarqube:MET_develop_NB +sonar.projectName=MET Nightly build +sonar.projectVersion=1.0 + +sonar.sources=src + +# The build-wrapper output dir +sonar.cfamily.build-wrapper-output=bw-outputs + +# Encoding of the source files +sonar.sourceEncoding=UTF-8 + +#----- Default SonarQube server +#sonar.host.url=http://localhost:9000 +sonar.host.url=http://mandan:9000 + +sonar.login=met +sonar.password=met@sonar.ucar diff --git a/test/bin/unit_test.sh b/test/bin/unit_test.sh index fd1c7149cb..0c95bba7ac 100755 --- a/test/bin/unit_test.sh +++ b/test/bin/unit_test.sh @@ -35,20 +35,25 @@ PERL_UNIT=${MET_TEST_BASE}/perl/unit.pl # Unit test XML UNIT_XML="unit_ascii2nc.xml \ + unit_ascii2nc_indy.xml \ unit_madis2nc.xml \ unit_trmm2nc.xml \ unit_pb2nc.xml \ + unit_pb2nc_indy.xml \ unit_gen_vx_mask.xml \ unit_gen_ens_prod.xml \ unit_pcp_combine.xml \ unit_wwmca_regrid.xml \ unit_point_stat.xml \ + unit_stat_analysis_ps.xml \ unit_duplicate_flag.xml \ unit_obs_summary.xml \ unit_grid_stat.xml \ + unit_stat_analysis_gs.xml \ unit_wavelet_stat.xml \ + unit_stat_analysis_ws.xml \ unit_ensemble_stat.xml \ - unit_stat_analysis.xml \ + unit_stat_analysis_es.xml \ unit_mode.xml \ unit_mode_analysis.xml \ unit_plot_point_obs.xml \ @@ -64,6 +69,11 @@ UNIT_XML="unit_ascii2nc.xml \ unit_tc_gen.xml \ unit_met_test_scripts.xml \ unit_modis.xml \ + unit_ref_config_lead_00.xml \ + unit_ref_config_lead_12.xml \ + unit_ref_config_lead_24.xml \ + unit_ref_config_lead_36.xml \ + unit_ref_config_lead_48.xml \ unit_ref_config.xml \ unit_mode_graphics.xml \ unit_regrid.xml \ @@ -71,7 +81,9 @@ UNIT_XML="unit_ascii2nc.xml \ unit_aeronet.xml \ unit_shift_data_plane.xml \ unit_mtd.xml \ - unit_climatology.xml \ + unit_climatology_1.0deg.xml \ + unit_climatology_1.5deg.xml \ + unit_climatology_2.5deg.xml \ unit_grib_tables.xml \ unit_grid_weight.xml \ unit_netcdf.xml \ diff --git a/test/xml/unit_ascii2nc.xml b/test/xml/unit_ascii2nc.xml index 11c46a3ea5..feef57c65d 100644 --- a/test/xml/unit_ascii2nc.xml +++ b/test/xml/unit_ascii2nc.xml @@ -13,6 +13,8 @@ + + &TEST_DIR; @@ -30,18 +32,6 @@ - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.22:30Z07Aug2012-10:30Z08Aug2012.G3.output.mtxt \ - &OUTPUT_DIR;/ascii2nc/trmm_2008080812_12hr.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/trmm_2008080812_12hr.nc - - - &MET_BIN;/ascii2nc \ @@ -66,44 +56,6 @@ - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/ascii/qc_out_2012-09-07_00:00:00.GRM_P+FCST \ - &OUTPUT_DIR;/ascii2nc/qc_out_2012-09-07_00:00:00.GRM_P_FCST.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/qc_out_2012-09-07_00:00:00.GRM_P_FCST.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/ascii/OBS:2015080700_bad_record \ - &OUTPUT_DIR;/ascii2nc/OBS_2015080700_bad_record.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/OBS_2015080700_bad_record.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/ascii/surfrad_tbl12136.txt \ - &DATA_DIR_OBS;/ascii/surfrad_tbl12137.txt \ - &DATA_DIR_OBS;/ascii/surfrad_tbl12138.txt \ - &OUTPUT_DIR;/ascii2nc/surfrad_tbl12.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/surfrad_tbl12.nc - - - &MET_BIN;/ascii2nc \ @@ -118,92 +70,6 @@ - - &MET_BIN;/ascii2nc - - BEG_TS 000000 - END_TS 235959 - STEP_TS 300 - WIDTH_TS 300 - - \ - &DATA_DIR_OBS;/surfrad/tbl12001.dat \ - &DATA_DIR_OBS;/surfrad/tbl12002.dat \ - &DATA_DIR_OBS;/surfrad/tbl12003.dat \ - -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ - &OUTPUT_DIR;/ascii2nc/surfrad_summary1.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/surfrad_summary1.nc - - - - - &MET_BIN;/ascii2nc - - BEG_TS 03 - END_TS 20 - STEP_TS 300 - WIDTH_TS 600 - - \ - &DATA_DIR_OBS;/surfrad/tbl12001.dat \ - &DATA_DIR_OBS;/surfrad/tbl12002.dat \ - &DATA_DIR_OBS;/surfrad/tbl12003.dat \ - -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ - &OUTPUT_DIR;/ascii2nc/surfrad_summary2.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/surfrad_summary2.nc - - - - - &MET_BIN;/ascii2nc - - BEG_TS 17 - END_TS 03 - STEP_TS 420 - WIDTH_TS 420 - - \ - &DATA_DIR_OBS;/surfrad/tbl12001.dat \ - &DATA_DIR_OBS;/surfrad/tbl12002.dat \ - &DATA_DIR_OBS;/surfrad/tbl12003.dat \ - -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ - &OUTPUT_DIR;/ascii2nc/surfrad_summary3.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/surfrad_summary3.nc - - - - - - - &MET_BIN;/ascii2nc - - BEG_TS 17 - END_TS 03 - STEP_TS 420 - WIDTH_TS {beg=-420;end=0;} - - \ - &DATA_DIR_OBS;/surfrad/tbl12001.dat \ - &DATA_DIR_OBS;/surfrad/tbl12002.dat \ - &DATA_DIR_OBS;/surfrad/tbl12003.dat \ - -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ - &OUTPUT_DIR;/ascii2nc/surfrad_summary4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/surfrad_summary4.nc - - - &MET_BIN;/ascii2nc \ @@ -223,187 +89,6 @@ - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_sid.nc \ - -mask_sid "MY_STATIONS:N526UA,N567UA,N571UA,N594UA" \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_sid.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_grid_data.nc \ - -mask_grid &TEST_DIR;/data/mnc/test_grid_valid.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_grid_data.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ - &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_named_grid.nc \ - -mask_grid G212 -v 1 - - - &OUTPUT_DIR;/ascii2nc/edr_hourly.20130827.mask_named_grid.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.10:30Z09Apr2012.G3.output.mtxt \ - &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr_mask_grid_dtc165.nc \ - -mask_grid DTC165 \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr_mask_grid_dtc165.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.10:30Z09Apr2012.G3.output.mtxt \ - &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr_mask_poly_lmv.nc \ - -mask_poly &MET_BASE;/poly/LMV.poly \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr_mask_poly_lmv.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/wwsis/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.csv \ - &OUTPUT_DIR;/ascii2nc/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc - - - - - &MET_BIN;/ascii2nc - \ - &DATA_DIR_OBS;/ascii/obs_test_name.txt \ - &OUTPUT_DIR;/ascii2nc/obs_test_name.nc \ - -v 1 - - - &OUTPUT_DIR;/ascii2nc/obs_test_name.nc - - - &MET_BIN;/ascii2nc \ diff --git a/test/xml/unit_ascii2nc_indy.xml b/test/xml/unit_ascii2nc_indy.xml new file mode 100644 index 0000000000..11ba012820 --- /dev/null +++ b/test/xml/unit_ascii2nc_indy.xml @@ -0,0 +1,340 @@ + + + + + + + + + + +]> + + + + + + + + &TEST_DIR; + true + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.22:30Z07Aug2012-10:30Z08Aug2012.G3.output.mtxt \ + &OUTPUT_DIR;/ascii2nc_indy/trmm_2008080812_12hr.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/trmm_2008080812_12hr.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/ascii/qc_out_2012-09-07_00:00:00.GRM_P+FCST \ + &OUTPUT_DIR;/ascii2nc_indy/qc_out_2012-09-07_00_00_00.GRM_P_FCST.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/qc_out_2012-09-07_00_00_00.GRM_P_FCST.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/ascii/OBS:2015080700_bad_record \ + &OUTPUT_DIR;/ascii2nc_indy/OBS_2015080700_bad_record.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/OBS_2015080700_bad_record.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/ascii/surfrad_tbl12136.txt \ + &DATA_DIR_OBS;/ascii/surfrad_tbl12137.txt \ + &DATA_DIR_OBS;/ascii/surfrad_tbl12138.txt \ + &OUTPUT_DIR;/ascii2nc_indy/surfrad_tbl12.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/surfrad_tbl12.nc + + + + + &MET_BIN;/ascii2nc + + BEG_TS 000000 + END_TS 235959 + STEP_TS 300 + WIDTH_TS 300 + + \ + &DATA_DIR_OBS;/surfrad/tbl12001.dat \ + &DATA_DIR_OBS;/surfrad/tbl12002.dat \ + &DATA_DIR_OBS;/surfrad/tbl12003.dat \ + -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary1.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary1.nc + + + + + &MET_BIN;/ascii2nc + + BEG_TS 03 + END_TS 20 + STEP_TS 300 + WIDTH_TS 600 + + \ + &DATA_DIR_OBS;/surfrad/tbl12001.dat \ + &DATA_DIR_OBS;/surfrad/tbl12002.dat \ + &DATA_DIR_OBS;/surfrad/tbl12003.dat \ + -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary2.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary2.nc + + + + + &MET_BIN;/ascii2nc + + BEG_TS 17 + END_TS 03 + STEP_TS 420 + WIDTH_TS 420 + + \ + &DATA_DIR_OBS;/surfrad/tbl12001.dat \ + &DATA_DIR_OBS;/surfrad/tbl12002.dat \ + &DATA_DIR_OBS;/surfrad/tbl12003.dat \ + -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary3.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary3.nc + + + + + + + &MET_BIN;/ascii2nc + + BEG_TS 17 + END_TS 03 + STEP_TS 420 + WIDTH_TS {beg=-420;end=0;} + + \ + &DATA_DIR_OBS;/surfrad/tbl12001.dat \ + &DATA_DIR_OBS;/surfrad/tbl12002.dat \ + &DATA_DIR_OBS;/surfrad/tbl12003.dat \ + -config &CONFIG_DIR;/Ascii2NcConfig.surfrad \ + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/surfrad_summary4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_sid.nc \ + -mask_sid "MY_STATIONS:N526UA,N567UA,N571UA,N594UA" \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_sid.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_grid_data.nc \ + -mask_grid &TEST_DIR;/data/mnc/test_grid_valid.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_grid_data.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.13.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.14.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.15.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.16.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.17.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.18.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.19.ascii \ + &DATA_DIR_OBS;/insitu_ascii/20130827/edr_hourly.20130827.20.ascii \ + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_named_grid.nc \ + -mask_grid G212 -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/edr_hourly.20130827.mask_named_grid.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.10:30Z09Apr2012.G3.output.mtxt \ + &OUTPUT_DIR;/ascii2nc_indy/trmm_2012040912_3hr_mask_grid_dtc165.nc \ + -mask_grid DTC165 \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/trmm_2012040912_3hr_mask_grid_dtc165.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/trmm/TRMM_3B42.007.accumulated_precipitation.10:30Z09Apr2012.G3.output.mtxt \ + &OUTPUT_DIR;/ascii2nc_indy/trmm_2012040912_3hr_mask_poly_lmv.nc \ + -mask_poly &MET_BASE;/poly/LMV.poly \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/trmm_2012040912_3hr_mask_poly_lmv.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_five_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_ten_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/clear_pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/HA_pvwatts_315510615_2006_30MW_sixty_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/pvwatts_315510615_2006_pv_30MW_one_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/wwsis/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.csv \ + &OUTPUT_DIR;/ascii2nc_indy/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/pvwatts_315510615_2006_pv_30MW_sixty_min_v3pt4.nc + + + + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/ascii/obs_test_name.txt \ + &OUTPUT_DIR;/ascii2nc_indy/obs_test_name.nc \ + -v 1 + + + &OUTPUT_DIR;/ascii2nc_indy/obs_test_name.nc + + + + diff --git a/test/xml/unit_climatology.xml b/test/xml/unit_climatology.xml deleted file mode 100644 index d50dc86ccf..0000000000 --- a/test/xml/unit_climatology.xml +++ /dev/null @@ -1,361 +0,0 @@ - - - - - - - - - - -]> - - - - - - &TEST_DIR; - true - - - &MET_BIN;/point_stat - - OUTPUT_PREFIX GFS_CLIMO_2.5DEG - DAY_INTERVAL 31 - HOUR_INTERVAL 6 - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_mean.19590315", - "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_mean.19590415" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_stdv.19590315", - "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_stdv.19590415" - - - - \ - &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ - &CONFIG_DIR;/PointStatConfig_climo \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_cnt.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_sl1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_sal1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_mpr.txt - - - - - &MET_BIN;/point_stat - - OUTPUT_PREFIX GFS_CLIMO_1.0DEG - DAY_INTERVAL 1 - HOUR_INTERVAL 6 - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" - - - - \ - &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ - &CONFIG_DIR;/PointStatConfig_climo \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_cnt.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_sl1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_sal1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_mpr.txt - - - - - &MET_BIN;/point_stat - - OUTPUT_PREFIX GFS_CLIMO_PREV_MONTH - DAY_INTERVAL NA - HOUR_INTERVAL 6 - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590309" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590309" - - - - \ - &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ - &CONFIG_DIR;/PointStatConfig_climo \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_cnt.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_sl1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_sal1l2.txt - &OUTPUT_DIR;/climatology/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_mpr.txt - - - - - &MET_BIN;/point_stat - - OUTPUT_PREFIX WMO_CLIMO_1.5DEG - CLIMO_DIR &DATA_DIR_CLIMO;/ERA_DAILY_1.5deg - - \ - &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ - &CONFIG_DIR;/PointStatConfig_climo_WMO \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_ctc.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_cnt.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_sl1l2.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_sal1l2.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_vl1l2.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_val1l2.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_pct.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_pstd.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_ecnt.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_rps.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_mpr.txt - &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_vcnt.txt - - - - - &MET_BIN;/stat_analysis - - OUTPUT_DIR &OUTPUT_DIR;/climatology - - \ - -lookin &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat \ - -job aggregate_stat -line_type MPR -out_line_type CTC -fcst_lev P850 -interp_mthd NEAREST -by FCST_VAR -out_thresh '>CDP90' \ - -out_stat &OUTPUT_DIR;/climatology/stat_analysis_WMO_1.5DEG_MPR_to_CTC_out.stat - - - &OUTPUT_DIR;/climatology/stat_analysis_WMO_1.5DEG_MPR_to_CTC_out.stat - - - - - &MET_BIN;/stat_analysis - - OUTPUT_DIR &OUTPUT_DIR;/climatology - - \ - -lookin &OUTPUT_DIR;/climatology/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat \ - -job filter -line_type MPR -column_thresh CLIMO_CDF 'lt0.1||gt0.9' \ - -dump_row &OUTPUT_DIR;/climatology/stat_analysis_WMO_1.5DEG_FILTER_CDF_dump.stat - - - &OUTPUT_DIR;/climatology/stat_analysis_WMO_1.5DEG_FILTER_CDF_dump.stat - - - - - &MET_BIN;/grid_stat - - OUTPUT_PREFIX WMO_CLIMO_1.5DEG - CLIMO_DIR &DATA_DIR_CLIMO;/ERA_DAILY_1.5deg - - \ - &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ - &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ - &CONFIG_DIR;/GridStatConfig_climo_WMO \ - -outdir &OUTPUT_DIR;/climatology -v 2 - - - &OUTPUT_DIR;/climatology/grid_stat_WMO_CLIMO_1.5DEG_240000L_20120410_000000V.stat - &OUTPUT_DIR;/climatology/grid_stat_WMO_CLIMO_1.5DEG_240000L_20120410_000000V_pairs.nc - - - - - &MET_BIN;/point_stat - - OUTPUT_PREFIX PROB_GFS_CLIMO_1.0DEG - DAY_INTERVAL 1 - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" - - - - \ - &DATA_DIR_MODEL;/grib2/sref_pr/sref_prob_2012040821_F015.grib2 \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ - &CONFIG_DIR;/PointStatConfig_climo_prob \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pct.txt - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pstd.txt - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pjc.txt - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_prc.txt - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_eclv.txt - &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_mpr.txt - - - - - &MET_BIN;/grid_stat - - OUTPUT_PREFIX PROB_GFS_CLIMO_1.0DEG - DAY_INTERVAL 1 - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" - - - - \ - &DATA_DIR_MODEL;/grib2/sref_pr/sref_prob_2012040821_F015.grib2 \ - &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \ - &CONFIG_DIR;/GridStatConfig_climo_prob \ - -outdir &OUTPUT_DIR;/climatology -v 4 - - - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pct.txt - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pstd.txt - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pjc.txt - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_prc.txt - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_eclv.txt - &OUTPUT_DIR;/climatology/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pairs.nc - - - - - &MET_BIN;/stat_analysis - - OUTPUT_DIR &OUTPUT_DIR;/climatology - - \ - -lookin &OUTPUT_DIR;/climatology/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat \ - -config &CONFIG_DIR;/STATAnalysisConfig_climo \ - -v 4 - - - &OUTPUT_DIR;/climatology/stat_analysis_MPR_to_PSTD.stat - - - - - &MET_BIN;/series_analysis - - CLIMO_MEAN_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409", - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410", - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590411" - - - CLIMO_STDEV_FILE_LIST - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409", - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590410", - "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590411" - - - - \ - -fcst &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F012.grib2 \ - &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ - &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F036.grib2 \ - &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F048.grib2 \ - -obs &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \ - &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ - &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_1200_000.grb2 \ - &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120411_0000_000.grb2 \ - -paired \ - -out &OUTPUT_DIR;/climatology/series_analysis_GFS_CLIMO_1.0DEG.nc \ - -config &CONFIG_DIR;/SeriesAnalysisConfig_climo \ - -v 2 - - - &OUTPUT_DIR;/climatology/series_analysis_GFS_CLIMO_1.0DEG.nc - - - - - echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib" \ - > &OUTPUT_DIR;/climatology/ensemble_stat_input_file_list; \ - &MET_BIN;/ensemble_stat - - OUTPUT_PREFIX NCEP_1.0DEG - CLIMO_MEAN_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410" - CLIMO_STDEV_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590410" - - \ - &OUTPUT_DIR;/climatology/ensemble_stat_input_file_list \ - &CONFIG_DIR;/EnsembleStatConfig_climo \ - -point_obs &OUTPUT_DIR;/pb2nc/ndas.20120410.t12z.prepbufr.tm00.nc \ - -grid_obs &DATA_DIR_OBS;/laps/laps_2012041012_F000.grib \ - -outdir &OUTPUT_DIR;/climatology - - - &OUTPUT_DIR;/climatology/ensemble_stat_NCEP_1.0DEG_20120410_120000V.stat - &OUTPUT_DIR;/climatology/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ecnt.txt - &OUTPUT_DIR;/climatology/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.txt - &OUTPUT_DIR;/climatology/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ens.nc - &OUTPUT_DIR;/climatology/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.nc - - - - - - - echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib" \ - > &OUTPUT_DIR;/climatology/ensemble_stat_input_file_list; \ - &MET_BIN;/ensemble_stat - - OUTPUT_PREFIX ONE_CDF_BIN - CLIMO_MEAN_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410" - - \ - &OUTPUT_DIR;/climatology/ensemble_stat_input_file_list \ - &CONFIG_DIR;/EnsembleStatConfig_one_cdf_bin \ - -point_obs &OUTPUT_DIR;/pb2nc/ndas.20120410.t12z.prepbufr.tm00.nc \ - -grid_obs &DATA_DIR_OBS;/laps/laps_2012041012_F000.grib \ - -outdir &OUTPUT_DIR;/climatology - - - &OUTPUT_DIR;/climatology/ensemble_stat_ONE_CDF_BIN_20120410_120000V.stat - &OUTPUT_DIR;/climatology/ensemble_stat_ONE_CDF_BIN_20120410_120000V_ecnt.txt - &OUTPUT_DIR;/climatology/ensemble_stat_ONE_CDF_BIN_20120410_120000V_ens.nc - - - - diff --git a/test/xml/unit_climatology_1.0deg.xml b/test/xml/unit_climatology_1.0deg.xml new file mode 100644 index 0000000000..8fad5f3be1 --- /dev/null +++ b/test/xml/unit_climatology_1.0deg.xml @@ -0,0 +1,252 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + &MET_BIN;/point_stat + + OUTPUT_PREFIX GFS_CLIMO_1.0DEG + DAY_INTERVAL 1 + HOUR_INTERVAL 6 + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" + + + + \ + &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &CONFIG_DIR;/PointStatConfig_climo \ + -outdir &OUTPUT_DIR;/climatology_1.0deg -v 4 + + + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_cnt.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_sl1l2.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_sal1l2.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_1.0DEG_120000L_20120409_120000V_mpr.txt + + + + + &MET_BIN;/point_stat + + OUTPUT_PREFIX GFS_CLIMO_PREV_MONTH + DAY_INTERVAL NA + HOUR_INTERVAL 6 + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590309" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590309" + + + + \ + &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &CONFIG_DIR;/PointStatConfig_climo \ + -outdir &OUTPUT_DIR;/climatology_1.0deg -v 4 + + + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_cnt.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_sl1l2.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_sal1l2.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_GFS_CLIMO_PREV_MONTH_120000L_20120409_120000V_mpr.txt + + + + + &MET_BIN;/point_stat + + OUTPUT_PREFIX PROB_GFS_CLIMO_1.0DEG + DAY_INTERVAL 1 + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" + + + + \ + &DATA_DIR_MODEL;/grib2/sref_pr/sref_prob_2012040821_F015.grib2 \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &CONFIG_DIR;/PointStatConfig_climo_prob \ + -outdir &OUTPUT_DIR;/climatology_1.0deg -v 4 + + + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pct.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pstd.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pjc.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_prc.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_eclv.txt + &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_mpr.txt + + + + + &MET_BIN;/grid_stat + + OUTPUT_PREFIX PROB_GFS_CLIMO_1.0DEG + DAY_INTERVAL 1 + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409" + + + + \ + &DATA_DIR_MODEL;/grib2/sref_pr/sref_prob_2012040821_F015.grib2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \ + &CONFIG_DIR;/GridStatConfig_climo_prob \ + -outdir &OUTPUT_DIR;/climatology_1.0deg -v 4 + + + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pct.txt + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pstd.txt + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pjc.txt + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_prc.txt + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_eclv.txt + &OUTPUT_DIR;/climatology_1.0deg/grid_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V_pairs.nc + + + + + &MET_BIN;/stat_analysis + + OUTPUT_DIR &OUTPUT_DIR;/climatology_1.0deg + + \ + -lookin &OUTPUT_DIR;/climatology_1.0deg/point_stat_PROB_GFS_CLIMO_1.0DEG_150000L_20120409_120000V.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_climo \ + -v 4 + + + &OUTPUT_DIR;/climatology_1.0deg/stat_analysis_MPR_to_PSTD.stat + + + + + &MET_BIN;/series_analysis + + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590409", + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410", + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590411" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590409", + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590410", + "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590411" + + + + \ + -fcst &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F012.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F036.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F048.grib2 \ + -obs &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_1200_000.grb2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120411_0000_000.grb2 \ + -paired \ + -out &OUTPUT_DIR;/climatology_1.0deg/series_analysis_GFS_CLIMO_1.0DEG.nc \ + -config &CONFIG_DIR;/SeriesAnalysisConfig_climo \ + -v 2 + + + &OUTPUT_DIR;/climatology_1.0deg/series_analysis_GFS_CLIMO_1.0DEG.nc + + + + + echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib" \ + > &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_input_file_list; \ + &MET_BIN;/ensemble_stat + + OUTPUT_PREFIX NCEP_1.0DEG + CLIMO_MEAN_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410" + CLIMO_STDEV_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cstdv_1d.19590410" + + \ + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_input_file_list \ + &CONFIG_DIR;/EnsembleStatConfig_climo \ + -point_obs &OUTPUT_DIR;/pb2nc/ndas.20120410.t12z.prepbufr.tm00.nc \ + -grid_obs &DATA_DIR_OBS;/laps/laps_2012041012_F000.grib \ + -outdir &OUTPUT_DIR;/climatology_1.0deg + + + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ecnt.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_ens.nc + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_NCEP_1.0DEG_20120410_120000V_orank.nc + + + + + + + echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib" \ + > &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_input_file_list; \ + &MET_BIN;/ensemble_stat + + OUTPUT_PREFIX ONE_CDF_BIN + CLIMO_MEAN_FILE_LIST "&DATA_DIR_CLIMO;/NCEP_NCAR_40YR_1.0deg/cmean_1d.19590410" + + \ + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_input_file_list \ + &CONFIG_DIR;/EnsembleStatConfig_one_cdf_bin \ + -point_obs &OUTPUT_DIR;/pb2nc/ndas.20120410.t12z.prepbufr.tm00.nc \ + -grid_obs &DATA_DIR_OBS;/laps/laps_2012041012_F000.grib \ + -outdir &OUTPUT_DIR;/climatology_1.0deg + + + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_ONE_CDF_BIN_20120410_120000V.stat + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_ONE_CDF_BIN_20120410_120000V_ecnt.txt + &OUTPUT_DIR;/climatology_1.0deg/ensemble_stat_ONE_CDF_BIN_20120410_120000V_ens.nc + + + + diff --git a/test/xml/unit_climatology_1.5deg.xml b/test/xml/unit_climatology_1.5deg.xml new file mode 100644 index 0000000000..52dd69897b --- /dev/null +++ b/test/xml/unit_climatology_1.5deg.xml @@ -0,0 +1,98 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + &MET_BIN;/point_stat + + OUTPUT_PREFIX WMO_CLIMO_1.5DEG + CLIMO_DIR &DATA_DIR_CLIMO;/ERA_DAILY_1.5deg + + \ + &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &CONFIG_DIR;/PointStatConfig_climo_WMO \ + -outdir &OUTPUT_DIR;/climatology_1.5deg -v 4 + + + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_ctc.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_cnt.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_sl1l2.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_sal1l2.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_vl1l2.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_val1l2.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_pct.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_pstd.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_ecnt.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_rps.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_mpr.txt + &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V_vcnt.txt + + + + + &MET_BIN;/stat_analysis + + OUTPUT_DIR &OUTPUT_DIR;/climatology_1.5deg + + \ + -lookin &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat \ + -job aggregate_stat -line_type MPR -out_line_type CTC -fcst_lev P850 -interp_mthd NEAREST -by FCST_VAR -out_thresh '>CDP90' \ + -out_stat &OUTPUT_DIR;/climatology_1.5deg/stat_analysis_WMO_1.5DEG_MPR_to_CTC_out.stat + + + &OUTPUT_DIR;/climatology_1.5deg/stat_analysis_WMO_1.5DEG_MPR_to_CTC_out.stat + + + + + &MET_BIN;/stat_analysis + + OUTPUT_DIR &OUTPUT_DIR;/climatology_1.5deg + + \ + -lookin &OUTPUT_DIR;/climatology_1.5deg/point_stat_WMO_CLIMO_1.5DEG_120000L_20120409_120000V.stat \ + -job filter -line_type MPR -column_thresh CLIMO_CDF 'lt0.1||gt0.9' \ + -dump_row &OUTPUT_DIR;/climatology_1.5deg/stat_analysis_WMO_1.5DEG_FILTER_CDF_dump.stat + + + &OUTPUT_DIR;/climatology_1.5deg/stat_analysis_WMO_1.5DEG_FILTER_CDF_dump.stat + + + + + &MET_BIN;/grid_stat + + OUTPUT_PREFIX WMO_CLIMO_1.5DEG + CLIMO_DIR &DATA_DIR_CLIMO;/ERA_DAILY_1.5deg + + \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ + &CONFIG_DIR;/GridStatConfig_climo_WMO \ + -outdir &OUTPUT_DIR;/climatology_1.5deg -v 2 + + + &OUTPUT_DIR;/climatology_1.5deg/grid_stat_WMO_CLIMO_1.5DEG_240000L_20120410_000000V.stat + &OUTPUT_DIR;/climatology_1.5deg/grid_stat_WMO_CLIMO_1.5DEG_240000L_20120410_000000V_pairs.nc + + + + diff --git a/test/xml/unit_climatology_2.5deg.xml b/test/xml/unit_climatology_2.5deg.xml new file mode 100644 index 0000000000..7c8c0b300a --- /dev/null +++ b/test/xml/unit_climatology_2.5deg.xml @@ -0,0 +1,53 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + &MET_BIN;/point_stat + + OUTPUT_PREFIX GFS_CLIMO_2.5DEG + DAY_INTERVAL 31 + HOUR_INTERVAL 6 + CLIMO_MEAN_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_mean.19590315", + "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_mean.19590415" + + + CLIMO_STDEV_FILE_LIST + "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_stdv.19590315", + "&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_stdv.19590415" + + + + \ + &DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &CONFIG_DIR;/PointStatConfig_climo \ + -outdir &OUTPUT_DIR;/climatology_2.5deg -v 4 + + + &OUTPUT_DIR;/climatology_2.5deg/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V.stat + &OUTPUT_DIR;/climatology_2.5deg/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_cnt.txt + &OUTPUT_DIR;/climatology_2.5deg/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_sl1l2.txt + &OUTPUT_DIR;/climatology_2.5deg/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_sal1l2.txt + &OUTPUT_DIR;/climatology_2.5deg/point_stat_GFS_CLIMO_2.5DEG_120000L_20120409_120000V_mpr.txt + + + + diff --git a/test/xml/unit_pb2nc.xml b/test/xml/unit_pb2nc.xml index d6d79def22..fc31e0e61b 100644 --- a/test/xml/unit_pb2nc.xml +++ b/test/xml/unit_pb2nc.xml @@ -14,6 +14,8 @@ + + &TEST_DIR; @@ -75,139 +77,5 @@ &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc - - - &MET_BIN;/pb2nc - - STATION_ID "72265","72274","72364","72426" - MASK_GRID - MASK_POLY - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_sid.nc \ - &CONFIG_DIR;/PB2NCConfig \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_sid.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID "&CONFIG_DIR;/SID_CO.txt" - MASK_GRID - MASK_POLY - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_sid_file.nc \ - &CONFIG_DIR;/PB2NCConfig \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_sid_file.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID - MASK_GRID &MET_DATA;/sample_fcst/2005080700/wrfprs_ruc13_00.tm00_G212 - MASK_POLY - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_grid_data.nc \ - &CONFIG_DIR;/PB2NCConfig \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.mask_grid_data.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID - MASK_GRID - MASK_POLY - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/nam.20210311.t00z.prepbufr.tm00 \ - &OUTPUT_DIR;/pb2nc/nam.20210311.t00z.prepbufr.tm00.pbl.nc \ - &CONFIG_DIR;/PB2NCConfig_pbl \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/nam.20210311.t00z.prepbufr.tm00.pbl.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID "72364","72265","72274","72426","72489","14008" - MASK_GRID - MASK_POLY - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.var_all.nc \ - &CONFIG_DIR;/PB2NCConfig_all \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.var_all.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID "SC-GSP","TX-HGX","KY-HPX" - MASK_GRID - MASK_POLY - QUALITY_MARK_THRESH 9 - - \ - &DATA_DIR_OBS;/bufr/nam_20170502.t00z.radwnd.tm02.bufr_d \ - &OUTPUT_DIR;/pb2nc/nam_20170502.t00z.radwnd.tm02.bufr_d.vlevel_500.nc \ - &CONFIG_DIR;/PB2NCConfig_vlevel -valid_end 20170501_22 \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/nam_20170502.t00z.radwnd.tm02.bufr_d.vlevel_500.nc - - - - - &MET_BIN;/pb2nc - - STATION_ID - MASK_GRID - MASK_POLY MET_BASE/poly/CONUS.poly - QUALITY_MARK_THRESH 2 - - \ - &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.summary.nc \ - &CONFIG_DIR;/PB2NCConfig_summary \ - -v 1 - - - &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.summary.nc - - - diff --git a/test/xml/unit_pb2nc_indy.xml b/test/xml/unit_pb2nc_indy.xml new file mode 100644 index 0000000000..d0483d2c6a --- /dev/null +++ b/test/xml/unit_pb2nc_indy.xml @@ -0,0 +1,157 @@ + + + + + + + + + + + +]> + + + + + + + + &TEST_DIR; + true + + + &MET_BIN;/pb2nc + + STATION_ID "72265","72274","72364","72426" + MASK_GRID + MASK_POLY + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_sid.nc \ + &CONFIG_DIR;/PB2NCConfig \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_sid.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID "&CONFIG_DIR;/SID_CO.txt" + MASK_GRID + MASK_POLY + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_sid_file.nc \ + &CONFIG_DIR;/PB2NCConfig \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_sid_file.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID + MASK_GRID &MET_DATA;/sample_fcst/2005080700/wrfprs_ruc13_00.tm00_G212 + MASK_POLY + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_grid_data.nc \ + &CONFIG_DIR;/PB2NCConfig \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.mask_grid_data.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID + MASK_GRID + MASK_POLY + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/nam.20210311.t00z.prepbufr.tm00 \ + &OUTPUT_DIR;/pb2nc_indy/nam.20210311.t00z.prepbufr.tm00.pbl.nc \ + &CONFIG_DIR;/PB2NCConfig_pbl \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/nam.20210311.t00z.prepbufr.tm00.pbl.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID "72364","72265","72274","72426","72489","14008" + MASK_GRID + MASK_POLY + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.var_all.nc \ + &CONFIG_DIR;/PB2NCConfig_all \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.var_all.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID "SC-GSP","TX-HGX","KY-HPX" + MASK_GRID + MASK_POLY + QUALITY_MARK_THRESH 9 + + \ + &DATA_DIR_OBS;/bufr/nam_20170502.t00z.radwnd.tm02.bufr_d \ + &OUTPUT_DIR;/pb2nc_indy/nam_20170502.t00z.radwnd.tm02.bufr_d.vlevel_500.nc \ + &CONFIG_DIR;/PB2NCConfig_vlevel -valid_end 20170501_22 \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/nam_20170502.t00z.radwnd.tm02.bufr_d.vlevel_500.nc + + + + + &MET_BIN;/pb2nc + + STATION_ID + MASK_GRID + MASK_POLY MET_BASE/poly/CONUS.poly + QUALITY_MARK_THRESH 2 + + \ + &DATA_DIR_OBS;/prepbufr/ndas/nam.20120409.t12z.prepbufr.tm00.nr \ + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.summary.nc \ + &CONFIG_DIR;/PB2NCConfig_summary \ + -v 1 + + + &OUTPUT_DIR;/pb2nc_indy/ndas.20120409.t12z.prepbufr.tm00.summary.nc + + + + diff --git a/test/xml/unit_pcp_combine.xml b/test/xml/unit_pcp_combine.xml index 4a1c39b76b..b2de557363 100644 --- a/test/xml/unit_pcp_combine.xml +++ b/test/xml/unit_pcp_combine.xml @@ -187,10 +187,10 @@ -subtract \ &DATA_DIR_MODEL;/p_interp/wrfout_d01_2008-08-08_12:00:00_PLEV 'name="RAINNC";level="(0,*,*)";' \ &DATA_DIR_MODEL;/p_interp/wrfout_d01_2008-08-08_06:00:00_PLEV 'name="RAINNC";level="(0,*,*)";' \ - &OUTPUT_DIR;/pcp_combine/wrfout_d01_2008-08-08_12:00:00_PLEV_APCP06.nc + &OUTPUT_DIR;/pcp_combine/wrfout_d01_2008-08-08_12_00_00_PLEV_APCP06.nc - &OUTPUT_DIR;/pcp_combine/wrfout_d01_2008-08-08_12:00:00_PLEV_APCP06.nc + &OUTPUT_DIR;/pcp_combine/wrfout_d01_2008-08-08_12_00_00_PLEV_APCP06.nc diff --git a/test/xml/unit_plot_data_plane.xml b/test/xml/unit_plot_data_plane.xml index 25d47c1397..c118078829 100644 --- a/test/xml/unit_plot_data_plane.xml +++ b/test/xml/unit_plot_data_plane.xml @@ -135,13 +135,13 @@ &MET_BIN;/plot_data_plane \ &DATA_DIR_MODEL;/p_interp/wrfout_d01_2008-08-08_12:00:00_PLEV \ - &OUTPUT_DIR;/plot_data_plane/wrfout_d01_2008-08-08_12:00:00_PLEV_NC_PINTERP_PRES_SFC.ps \ + &OUTPUT_DIR;/plot_data_plane/wrfout_d01_2008-08-08_12_00_00_PLEV_NC_PINTERP_PRES_SFC.ps \ 'name = "PSFC"; level = "(0,*,*)";' \ -title "NC PINTERP Pressure at the Surface" \ -v 1 - &OUTPUT_DIR;/plot_data_plane/wrfout_d01_2008-08-08_12:00:00_PLEV_NC_PINTERP_PRES_SFC.ps + &OUTPUT_DIR;/plot_data_plane/wrfout_d01_2008-08-08_12_00_00_PLEV_NC_PINTERP_PRES_SFC.ps diff --git a/test/xml/unit_python.xml b/test/xml/unit_python.xml index 698f550f69..b2b1dad44b 100644 --- a/test/xml/unit_python.xml +++ b/test/xml/unit_python.xml @@ -15,7 +15,7 @@ ]> - + @@ -378,7 +378,8 @@ - &MET_BIN;/plot_data_plane + echo "MET_PYTHON_EXE=&MET_PYTHON_EXE;"; \ + &MET_BIN;/plot_data_plane MET_PYTHON_EXE &MET_PYTHON_EXE; PYTHON_GRID G212 @@ -398,7 +399,8 @@ - &MET_BIN;/ascii2nc + echo "MET_PYTHON_EXE=&MET_PYTHON_EXE;"; \ + &MET_BIN;/ascii2nc MET_PYTHON_EXE &MET_PYTHON_EXE; @@ -415,6 +417,7 @@ export PATH='&ANACONDA_BIN;:${PATH}'; \ + echo "MET_PYTHON_EXE=&MET_PYTHON_EXE;"; \ &MET_BIN;/plot_data_plane MET_PYTHON_EXE &MET_PYTHON_EXE; @@ -434,7 +437,8 @@ - &MET_BIN;/stat_analysis + echo "MET_PYTHON_EXE=&MET_PYTHON_EXE;"; \ + &MET_BIN;/stat_analysis MET_PYTHON_EXE &MET_PYTHON_EXE; @@ -448,4 +452,111 @@ + + &MET_BIN;/point2grid + \ + 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \ + G212 \ + &OUTPUT_DIR;/python/pb2nc_TMP.nc \ + -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ <0 ]; censor_val=[0];' \ + -name TEMP \ + -v 1 + + + &OUTPUT_DIR;/python/pb2nc_TMP.nc + + + + + &MET_BIN;/plot_point_obs + + TO_GRID NONE + + \ + &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc \ + &OUTPUT_DIR;/python/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps \ + -point_obs &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr.nc \ + -plot_grid &DATA_DIR_MODEL;/grib2/nam/nam_2012040900_F012.grib2 \ + -config &CONFIG_DIR;/PlotPointObsConfig \ + -title "NAM 2012040900 F12 vs NDAS 500mb RH and TRMM 3h > 0" \ + -v 3 + + + &OUTPUT_DIR;/python/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps + + + + + &MET_BIN;/plot_point_obs + + TO_GRID NONE + + \ + 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \ + &OUTPUT_DIR;/python/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps \ + -point_obs 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/ascii2nc/trmm_2012040912_3hr.nc' \ + -plot_grid &DATA_DIR_MODEL;/grib2/nam/nam_2012040900_F012.grib2 \ + -config &CONFIG_DIR;/PlotPointObsConfig \ + -title "NAM 2012040900 F12 vs NDAS 500mb RH and TRMM 3h > 0" \ + -v 3 + + + &OUTPUT_DIR;/python/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps + + + + + echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + &DATA_DIR_MODEL;/grib1/arw-tom-gep7/arw-tom-gep7_2012040912_F024.grib" \ + > &OUTPUT_DIR;/python/ensemble_stat/input_file_list; \ + &MET_BIN;/ensemble_stat + + DESC NA + OBS_ERROR_FLAG FALSE + SKIP_CONST FALSE + OUTPUT_PREFIX FILE_LIST + + \ + &OUTPUT_DIR;/python/ensemble_stat/input_file_list \ + &CONFIG_DIR;/EnsembleStatConfig \ + -grid_obs &DATA_DIR_OBS;/laps/laps_2012041012_F000.grib \ + -point_obs 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/ascii2nc/gauge_2012041012_24hr.nc' \ + -outdir &OUTPUT_DIR;/python/ensemble_stat -v 1 + + + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V.stat + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_ecnt.txt + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_rhist.txt + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_phist.txt + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_orank.txt + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_ssvar.txt + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_ens.nc + &OUTPUT_DIR;/python/ensemble_stat/ensemble_stat_FILE_LIST_20120410_120000V_orank.nc + + + + + &MET_BIN;/point_stat + + BEG_DS -1800 + END_DS 1800 + OUTPUT_PREFIX GRIB1_NAM_GDAS_WINDS + CONFIG_DIR &CONFIG_DIR; + CLIMO_FILE "&DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012_gNam.grib" + + \ + &DATA_DIR_MODEL;/grib1/nam/nam_2012040900_F012.grib \ + 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/gdas1.20120409.t12z.prepbufr.nc' \ + &CONFIG_DIR;/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/python -v 1 + + + &OUTPUT_DIR;/python/point_stat_GRIB1_NAM_GDAS_WINDS_120000L_20120409_120000V.stat + + + diff --git a/test/xml/unit_ref_config.xml b/test/xml/unit_ref_config.xml index 5f9320ba5d..0cf9a8b8f3 100644 --- a/test/xml/unit_ref_config.xml +++ b/test/xml/unit_ref_config.xml @@ -11,930 +11,13 @@ ]> - + &TEST_DIR; true - - - - - - &MET_BIN;/gen_vx_mask - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ - &MET_BASE;/poly/CONUS.poly \ - &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc \ - -type poly -v 2 - - - &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - - - - - - - - --> - &MET_BIN;/pb2nc - - MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly - - \ - &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090212/ndas.t12z.prepbufr.tm12.nr \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PB2NCConfig \ - -valid_beg 20110901_223000\ - -valid_end 20110902_013000\ - -v 2 - - - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc - - - - --> - &MET_BIN;/pb2nc - - MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly - - \ - &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090300/ndas.t00z.prepbufr.tm12.nr \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PB2NCConfig \ - -valid_beg 20110902_103000\ - -valid_end 20110902_133000\ - -v 2 - - - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc - - - - --> - &MET_BIN;/pb2nc - - MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly - - \ - &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090312/ndas.t12z.prepbufr.tm12.nr \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PB2NCConfig \ - -valid_beg 20110902_223000\ - -valid_end 20110903_013000\ - -v 2 - - - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc - - - - --> - &MET_BIN;/pb2nc - - MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly - - \ - &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090400/ndas.t00z.prepbufr.tm12.nr \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PB2NCConfig \ - -valid_beg 20110903_103000\ - -valid_end 20110903_133000\ - -v 2 - - - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc - - - - --> - &MET_BIN;/pb2nc - - MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly - - \ - &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090412/ndas.t12z.prepbufr.tm12.nr \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PB2NCConfig \ - -valid_beg 20110903_223000\ - -valid_end 20110904_013000\ - -v 2 - - - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc - - - - - - - - - - &MET_BIN;/pcp_combine - \ - -sum \ - 00000000_000000 1 20110902_120000 3 \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110902 - - - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc - - - - - &MET_BIN;/pcp_combine - \ - -sum \ - 00000000_000000 1 20110903_000000 3 \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110902 \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 - - - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc - - - - - &MET_BIN;/pcp_combine - \ - -sum \ - 00000000_000000 1 20110903_120000 3 \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 - - - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc - - - - - &MET_BIN;/pcp_combine - \ - -sum \ - 00000000_000000 1 20110904_000000 3 \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 \ - -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110904 - - - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc - - - - - &MET_BIN;/pcp_combine - \ - -add \ - &DATA_DIR_OBS;/ref_config/pcp_combine/2011090324/ST2ml2011090312.24h.grb \ - 24 \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc - - - - - &MET_BIN;/pcp_combine - \ - -subtract \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 12 \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_009.tm00 9 \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_012.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_012.nc - - - - - &MET_BIN;/pcp_combine - \ - -subtract \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 24 \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_021.tm00 21 \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_024.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_024.nc - - - - - &MET_BIN;/pcp_combine - \ - -subtract \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 36 \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_033.tm00 33 \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_036.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_036.nc - - - - - &MET_BIN;/pcp_combine - \ - -subtract \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 48 \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_045.tm00 45 \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_048.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_048.nc - - - - - &MET_BIN;/pcp_combine - \ - -subtract \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 36 \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 12 \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp24_036.nc - - - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp24_036.nc - - - - - - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_ADPUPA_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_ONLYSF_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_WINDS_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_ADPUPA_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_ONLYSF_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_WINDS_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_ADPUPA_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_ONLYSF_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_WINDS_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_ADPUPA_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_ONLYSF_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_WINDS_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_ADPUPA_480000L_20110904_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_ONLYSF_480000L_20110904_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv2.7.1 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_WINDS_480000L_20110904_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_ADPUPA_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_ONLYSF_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 000 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_WINDS_000000L_20110902_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_ADPUPA_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_ONLYSF_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 012 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_WINDS_120000L_20110902_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_ADPUPA_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_ONLYSF_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 024 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_WINDS_240000L_20110903_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_ADPUPA_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_ONLYSF_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 036 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_WINDS_360000L_20110903_120000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_ADPUPA_480000L_20110904_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_ONLYSF_480000L_20110904_000000V.stat - - - - - &MET_BIN;/point_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 048 - - \ - &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ - &OUTPUT_DIR;/ref_config/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ - &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ - -outdir &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3 \ - -v 2 - - - &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_WINDS_480000L_20110904_000000V.stat - - - - - - - - - &MET_BIN;/grid_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 12 - - \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_012.nc \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc \ - &CONFIG_DIR;/ref_config/GridStatConfig_03h \ - -outdir &OUTPUT_DIR;/ref_config/grid_stat -v 1 - - - &OUTPUT_DIR;/ref_config/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F12_03h_120000L_20110902_120000V.stat - - - - - &MET_BIN;/grid_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 24 - - \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_024.nc \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc \ - &CONFIG_DIR;/ref_config/GridStatConfig_03h \ - -outdir &OUTPUT_DIR;/ref_config/grid_stat -v 1 - - - &OUTPUT_DIR;/ref_config/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F24_03h_240000L_20110903_000000V.stat - - - - - &MET_BIN;/grid_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 36 - - \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_036.nc \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc \ - &CONFIG_DIR;/ref_config/GridStatConfig_03h \ - -outdir &OUTPUT_DIR;/ref_config/grid_stat -v 1 - - - &OUTPUT_DIR;/ref_config/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F36_03h_360000L_20110903_120000V.stat - - - - - &MET_BIN;/grid_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 48 - - \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp03_048.nc \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc \ - &CONFIG_DIR;/ref_config/GridStatConfig_03h \ - -outdir &OUTPUT_DIR;/ref_config/grid_stat -v 1 - - - &OUTPUT_DIR;/ref_config/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F48_03h_480000L_20110904_000000V.stat - - - - - &MET_BIN;/grid_stat - - MASK_POLY_FILE &OUTPUT_DIR;/ref_config/gen_vx_mask/CONUS.nc - MODEL AFWAv3.4_Noahv3.3 - FCST_TIME 36 - - \ - &OUTPUT_DIR;/ref_config/pcp_combine/wrf/wrfpcp24_036.nc \ - &OUTPUT_DIR;/ref_config/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc \ - &CONFIG_DIR;/ref_config/GridStatConfig_24h \ - -outdir &OUTPUT_DIR;/ref_config/grid_stat -v 1 - - - &OUTPUT_DIR;/ref_config/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F36_24h_360000L_20110903_120000V.stat - - - @@ -942,8 +25,8 @@ &MET_BIN;/stat_analysis \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/ \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv3.3 \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv2.7.1 \ -job go_index -fcst_init_beg 2011090200 -fcst_init_end 2011090200 \ -model AFWAv3.4_Noahv3.3 -model AFWAv3.4_Noahv2.7.1 \ -vx_mask FULL -interp_mthd BILIN \ @@ -962,8 +45,8 @@ &MET_BIN;/stat_analysis \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/ \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv3.3 \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv2.7.1 \ -job go_index \ -by FCST_INIT_BEG,VX_MASK,OBTYPE -set_hdr DESC Noahv3.3_vs_v2.7.1 \ -model AFWAv3.4_Noahv3.3,AFWAv3.4_Noahv2.7.1 -ss_index_vld_thresh 0.5 \ @@ -981,8 +64,8 @@ &MET_BIN;/stat_analysis \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/ \ - -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv3.3 \ + -lookin &OUTPUT_DIR;/ref_config_lead_*/point_stat/AFWAv3.4_Noahv2.7.1 \ -job ss_index -config &CONFIG_DIR;/STATAnalysisConfig_SFC_SS_Index \ -by FCST_INIT_BEG,VX_MASK,OBTYPE \ -out_stat &OUTPUT_DIR;/ref_config/stat_analysis/sfc_ss_index_by_option.stat diff --git a/test/xml/unit_ref_config_lead_00.xml b/test/xml/unit_ref_config_lead_00.xml new file mode 100644 index 0000000000..ac65be1546 --- /dev/null +++ b/test/xml/unit_ref_config_lead_00.xml @@ -0,0 +1,178 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &MET_BASE;/poly/CONUS.poly \ + &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc \ + -type poly -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + + + + + + + + + &MET_BIN;/pb2nc + + MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly + + \ + &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090212/ndas.t12z.prepbufr.tm12.nr \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PB2NCConfig \ + -valid_beg 20110901_223000\ + -valid_end 20110902_013000\ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc + + + + + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_ADPUPA_000000L_20110902_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_ONLYSF_000000L_20110902_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F000_WINDS_000000L_20110902_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_ADPUPA_000000L_20110902_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_ONLYSF_000000L_20110902_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_00/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 000 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_000.tm00 \ + &OUTPUT_DIR;/ref_config_lead_00/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_00/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F000_WINDS_000000L_20110902_000000V.stat + + + + diff --git a/test/xml/unit_ref_config_lead_12.xml b/test/xml/unit_ref_config_lead_12.xml new file mode 100644 index 0000000000..989e548da5 --- /dev/null +++ b/test/xml/unit_ref_config_lead_12.xml @@ -0,0 +1,230 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &MET_BASE;/poly/CONUS.poly \ + &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc \ + -type poly -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + + + + + + + + + &MET_BIN;/pb2nc + + MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly + + \ + &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090300/ndas.t00z.prepbufr.tm12.nr \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PB2NCConfig \ + -valid_beg 20110902_103000\ + -valid_end 20110902_133000\ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc + + + + + + + + + &MET_BIN;/pcp_combine + \ + -sum \ + 00000000_000000 1 20110902_120000 3 \ + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110902 + + + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc + + + + + &MET_BIN;/pcp_combine + \ + -subtract \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 12 \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_009.tm00 9 \ + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/wrf/wrfpcp03_012.nc + + + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/wrf/wrfpcp03_012.nc + + + + + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_ADPUPA_120000L_20110902_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_ONLYSF_120000L_20110902_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F012_WINDS_120000L_20110902_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_ADPUPA_120000L_20110902_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_ONLYSF_120000L_20110902_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 012 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &OUTPUT_DIR;/ref_config_lead_12/pb2nc/NDAS_03h/20110902/prepbufr.ndas.20110902.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_12/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F012_WINDS_120000L_20110902_120000V.stat + + + + + + + + + &MET_BIN;/grid_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_12/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 12 + + \ + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/wrf/wrfpcp03_012.nc \ + &OUTPUT_DIR;/ref_config_lead_12/pcp_combine/ST2_03h/20110902/ST2ml2011090212.03h.nc \ + &CONFIG_DIR;/ref_config/GridStatConfig_03h \ + -outdir &OUTPUT_DIR;/ref_config_lead_12/grid_stat -v 1 + + + &OUTPUT_DIR;/ref_config_lead_12/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F12_03h_120000L_20110902_120000V.stat + + + + diff --git a/test/xml/unit_ref_config_lead_24.xml b/test/xml/unit_ref_config_lead_24.xml new file mode 100644 index 0000000000..78cff2ee56 --- /dev/null +++ b/test/xml/unit_ref_config_lead_24.xml @@ -0,0 +1,231 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &MET_BASE;/poly/CONUS.poly \ + &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc \ + -type poly -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + + + + + + + + + &MET_BIN;/pb2nc + + MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly + + \ + &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090312/ndas.t12z.prepbufr.tm12.nr \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PB2NCConfig \ + -valid_beg 20110902_223000\ + -valid_end 20110903_013000\ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc + + + + + + + + + &MET_BIN;/pcp_combine + \ + -sum \ + 00000000_000000 1 20110903_000000 3 \ + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110902 \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 + + + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc + + + + + &MET_BIN;/pcp_combine + \ + -subtract \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 24 \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_021.tm00 21 \ + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/wrf/wrfpcp03_024.nc + + + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/wrf/wrfpcp03_024.nc + + + + + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_ADPUPA_240000L_20110903_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_ONLYSF_240000L_20110903_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F024_WINDS_240000L_20110903_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_ADPUPA_240000L_20110903_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_ONLYSF_240000L_20110903_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 024 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_024.tm00 \ + &OUTPUT_DIR;/ref_config_lead_24/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_24/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F024_WINDS_240000L_20110903_000000V.stat + + + + + + + + + &MET_BIN;/grid_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_24/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 24 + + \ + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/wrf/wrfpcp03_024.nc \ + &OUTPUT_DIR;/ref_config_lead_24/pcp_combine/ST2_03h/20110903/ST2ml2011090300.03h.nc \ + &CONFIG_DIR;/ref_config/GridStatConfig_03h \ + -outdir &OUTPUT_DIR;/ref_config_lead_24/grid_stat -v 1 + + + &OUTPUT_DIR;/ref_config_lead_24/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F24_03h_240000L_20110903_000000V.stat + + + + diff --git a/test/xml/unit_ref_config_lead_36.xml b/test/xml/unit_ref_config_lead_36.xml new file mode 100644 index 0000000000..4832ba4ad1 --- /dev/null +++ b/test/xml/unit_ref_config_lead_36.xml @@ -0,0 +1,274 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &MET_BASE;/poly/CONUS.poly \ + &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc \ + -type poly -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + + + + + + + + + &MET_BIN;/pb2nc + + MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly + + \ + &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090400/ndas.t00z.prepbufr.tm12.nr \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PB2NCConfig \ + -valid_beg 20110903_103000\ + -valid_end 20110903_133000\ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc + + + + + + + + + &MET_BIN;/pcp_combine + \ + -sum \ + 00000000_000000 1 20110903_120000 3 \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 + + + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc + + + + + &MET_BIN;/pcp_combine + \ + -add \ + &DATA_DIR_OBS;/ref_config/pcp_combine/2011090324/ST2ml2011090312.24h.grb \ + 24 \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc + + + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc + + + + + &MET_BIN;/pcp_combine + \ + -subtract \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 36 \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_033.tm00 33 \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp03_036.nc + + + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp03_036.nc + + + + + &MET_BIN;/pcp_combine + \ + -subtract \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 36 \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 12 \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp24_036.nc + + + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp24_036.nc + + + + + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_ADPUPA_360000L_20110903_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_ONLYSF_360000L_20110903_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F036_WINDS_360000L_20110903_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_ADPUPA_360000L_20110903_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_ONLYSF_360000L_20110903_120000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 036 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_036.tm00 \ + &OUTPUT_DIR;/ref_config_lead_36/pb2nc/NDAS_03h/20110903/prepbufr.ndas.20110903.t12z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_36/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F036_WINDS_360000L_20110903_120000V.stat + + + + + + + + + &MET_BIN;/grid_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 36 + + \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp03_036.nc \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_03h/20110903/ST2ml2011090312.03h.nc \ + &CONFIG_DIR;/ref_config/GridStatConfig_03h \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/grid_stat -v 1 + + + &OUTPUT_DIR;/ref_config_lead_36/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F36_03h_360000L_20110903_120000V.stat + + + + + &MET_BIN;/grid_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_36/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 36 + + \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/wrf/wrfpcp24_036.nc \ + &OUTPUT_DIR;/ref_config_lead_36/pcp_combine/ST2_24h/20110903/ST2ml2011090312.24h.nc \ + &CONFIG_DIR;/ref_config/GridStatConfig_24h \ + -outdir &OUTPUT_DIR;/ref_config_lead_36/grid_stat -v 1 + + + &OUTPUT_DIR;/ref_config_lead_36/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F36_24h_360000L_20110903_120000V.stat + + + + diff --git a/test/xml/unit_ref_config_lead_48.xml b/test/xml/unit_ref_config_lead_48.xml new file mode 100644 index 0000000000..b096241afb --- /dev/null +++ b/test/xml/unit_ref_config_lead_48.xml @@ -0,0 +1,231 @@ + + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_012.tm00 \ + &MET_BASE;/poly/CONUS.poly \ + &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc \ + -type poly -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + + + + + + + + + &MET_BIN;/pb2nc + + MASK_POLY &CONFIG_DIR;/ref_config/RefConfig.poly + + \ + &DATA_DIR_OBS;/ref_config/prepbufr/ndas/2011090412/ndas.t12z.prepbufr.tm12.nr \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PB2NCConfig \ + -valid_beg 20110903_223000\ + -valid_end 20110904_013000\ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc + + + + + + + + + &MET_BIN;/pcp_combine + \ + -sum \ + 00000000_000000 1 20110904_000000 3 \ + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110903 \ + -pcpdir &DATA_DIR_OBS;/ref_config/pcp_combine/20110904 + + + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc + + + + + &MET_BIN;/pcp_combine + \ + -subtract \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 48 \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_045.tm00 45 \ + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/wrf/wrfpcp03_048.nc + + + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/wrf/wrfpcp03_048.nc + + + + + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_ADPUPA_480000L_20110904_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_ONLYSF_480000L_20110904_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv2.7.1 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv2.7.1/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv2.7.1/point_stat_AFWAv3.4_Noahv2.7.1_F048_WINDS_480000L_20110904_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ADPUPA \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_ADPUPA_480000L_20110904_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_ONLYSF \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_ONLYSF_480000L_20110904_000000V.stat + + + + + &MET_BIN;/point_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 048 + + \ + &DATA_DIR_MODEL;/grib1/ref_config/2011090200/AFWAv3.4_Noahv3.3/postprd/wrfprs_048.tm00 \ + &OUTPUT_DIR;/ref_config_lead_48/pb2nc/NDAS_03h/20110904/prepbufr.ndas.20110904.t00z.tm12.nc \ + &CONFIG_DIR;/ref_config/PointStatConfig_WINDS \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3 \ + -v 2 + + + &OUTPUT_DIR;/ref_config_lead_48/point_stat/AFWAv3.4_Noahv3.3/point_stat_AFWAv3.4_Noahv3.3_F048_WINDS_480000L_20110904_000000V.stat + + + + + + + + + &MET_BIN;/grid_stat + + MASK_POLY_FILE &OUTPUT_DIR;/ref_config_lead_48/gen_vx_mask/CONUS.nc + MODEL AFWAv3.4_Noahv3.3 + FCST_TIME 48 + + \ + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/wrf/wrfpcp03_048.nc \ + &OUTPUT_DIR;/ref_config_lead_48/pcp_combine/ST2_03h/20110904/ST2ml2011090400.03h.nc \ + &CONFIG_DIR;/ref_config/GridStatConfig_03h \ + -outdir &OUTPUT_DIR;/ref_config_lead_48/grid_stat -v 1 + + + &OUTPUT_DIR;/ref_config_lead_48/grid_stat/grid_stat_AFWAv3.4_Noahv3.3_F48_03h_480000L_20110904_000000V.stat + + + + diff --git a/test/xml/unit_stat_analysis.xml b/test/xml/unit_stat_analysis.xml deleted file mode 100644 index 078b76e072..0000000000 --- a/test/xml/unit_stat_analysis.xml +++ /dev/null @@ -1,331 +0,0 @@ - - - - - - - - - -]> - - - - - - &TEST_DIR; - true - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ - -job aggregate -line_type RHIST \ - -fcst_var TMP -vx_mask NWC -vx_mask GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_RHIST_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_RHIST_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_RHIST.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_RHIST_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_RHIST_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_RHIST.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ - -job aggregate -line_type PHIST \ - -fcst_var TMP -vx_mask NWC -vx_mask GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_PHIST_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_PHIST_out.stat \ - -set_hdr VX_MASK NWC_AND_GRB \ - -out &OUTPUT_DIR;/stat_analysis/AGG_PHIST.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_PHIST_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_PHIST_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_PHIST.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ - -job aggregate -line_type RELP -by FCST_VAR -vx_mask NWC,GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_RELP_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_RELP_out.stat \ - -set_hdr VX_MASK NWC_AND_GRB \ - -out &OUTPUT_DIR;/stat_analysis/AGG_RELP.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_RELP_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_RELP_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_RELP.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ - -job aggregate -line_type ECNT -by FCST_VAR -obs_thresh NA -vx_mask NWC,GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_ECNT_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_ECNT_out.stat \ - -set_hdr VX_MASK NWC_AND_GRB \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_ECNT_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_ECNT_out.stat - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ - -job aggregate_stat -line_type ORANK -out_line_type RHIST,PHIST \ - -fcst_var APCP_24 -vx_mask NWC -vx_mask GRB -out_bin_size 0.10 \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RHIST_PHIST.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ - -job aggregate_stat -line_type ORANK -out_line_type RELP \ - -fcst_var APCP_24 -vx_mask NWC,GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_RELP.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ - -job aggregate_stat -line_type ORANK -out_line_type SSVAR \ - -fcst_var APCP_24 -vx_mask NWC -vx_mask GRB -out_bin_size 0.25 \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_SSVAR.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_OBSERR_20120410_120000V.stat \ - -job aggregate_stat -line_type ORANK -out_line_type ECNT \ - -fcst_var APCP_24 -by VX_MASK \ - -set_hdr DESC VX_MASK \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_STAT_ORANK_ECNT.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ - -job aggregate -line_type SSVAR \ - -fcst_var APCP_24 -obtype ANALYS -vx_mask NWC -vx_mask GRB \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_SSVAR_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_SSVAR_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_SSVAR.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_SSVAR_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_SSVAR_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_SSVAR.out - - - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/wavelet_stat/wavelet_stat_GRIB1_NAM_STAGE4_120000L_20120409_120000V.stat \ - -job aggregate -line_type ISC \ - -fcst_var APCP_12 -fcst_thresh '>0.0' -vx_mask TILE1 -vx_mask TILE2 \ - -dump_row &OUTPUT_DIR;/stat_analysis/AGG_ISC_dump.stat \ - -out_stat &OUTPUT_DIR;/stat_analysis/AGG_ISC_out.stat \ - -out &OUTPUT_DIR;/stat_analysis/AGG_ISC.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/AGG_ISC_dump.stat - &OUTPUT_DIR;/stat_analysis/AGG_ISC_out.stat - &OUTPUT_DIR;/stat_analysis/AGG_ISC.out - - - - - - - - OUTPUT_DIR &OUTPUT_DIR;/stat_analysis - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_120000L_20120409_120000V.stat \ - -config &CONFIG_DIR;/STATAnalysisConfig_grid_stat \ - -out &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT.out - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_filter.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_stat_sl1l2_dump.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_stat_sl1l2_out.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_ctc_dump.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_ctc_out.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_stat_ctc_to_eclv_dump.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_GRID_STAT_agg_stat_ctc_to_eclv_out.stat - - - - - - - - CONFIG_DIR &CONFIG_DIR; - OUTPUT_DIR &OUTPUT_DIR;/stat_analysis - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ - -config &CONFIG_DIR;/STATAnalysisConfig_point_stat \ - -out &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT.out - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_dump.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_out.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_by_vx_mask_out.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_agg_ctc_by_fcst_thresh_out.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_agg_stat_mpr_to_wdir_dump.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_filter_mpr_sid.stat - &OUTPUT_DIR;/stat_analysis/CONFIG_POINT_STAT_filter_mpr_fcst_minus_obs.stat - - - - - - OUTPUT_DIR &OUTPUT_DIR;/stat_analysis - - &MET_BIN;/stat_analysis - \ - -lookin &DATA_DIR_MODEL;/time_series_met_6.0/*.stat \ - -config &CONFIG_DIR;/STATAnalysisConfig_ramps \ - -out &OUTPUT_DIR;/stat_analysis/RAMPS.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/RAMPS.out - &OUTPUT_DIR;/stat_analysis/RAMPS_100_100.stat - &OUTPUT_DIR;/stat_analysis/RAMPS_25_100.stat - &OUTPUT_DIR;/stat_analysis/RAMPS_25_100_30min.stat - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ - -job summary -line_type MPR -by FCST_VAR,FCST_LEV -column 'FCST,OBS,FCST-OBS,ABS(FCST-OBS)' \ - -boot_seed 1 -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_SUMMARY.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/POINT_STAT_SUMMARY.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ - -job summary -line_type MPR -by FCST_VAR,FCST_LEV -column 'FCST,OBS' -column_union true \ - -boot_seed 1 -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_SUMMARY_UNION.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/POINT_STAT_SUMMARY_UNION.out - - - - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ - -job filter -line_type MPR -fcst_var TMP -fcst_lev Z2 -vx_mask DTC165 \ - -column_str OBS_SID KDLN,KDHT,KDEN,KDLS,KDMA,KDMN,KDVT,KDEW \ - -column_str_exc OBS_SID KDLN,KDHT \ - -dump_row &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_OBS_SID.stat \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_OBS_SID.stat - - - - - - OUTPUT_DIR &OUTPUT_DIR;/stat_analysis - - &MET_BIN;/stat_analysis - \ - -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB2_NAM_NDAS_120000L_20120409_120000V.stat \ - -config &CONFIG_DIR;/STATAnalysisConfig_filter_times \ - -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES.out \ - -v 1 - - - &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES.out - - - - diff --git a/test/xml/unit_stat_analysis_es.xml b/test/xml/unit_stat_analysis_es.xml new file mode 100644 index 0000000000..a7b446c1c6 --- /dev/null +++ b/test/xml/unit_stat_analysis_es.xml @@ -0,0 +1,182 @@ + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ + -job aggregate -line_type RHIST \ + -fcst_var TMP -vx_mask NWC -vx_mask GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_RHIST.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ + -job aggregate -line_type PHIST \ + -fcst_var TMP -vx_mask NWC -vx_mask GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST_out.stat \ + -set_hdr VX_MASK NWC_AND_GRB \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_PHIST.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ + -job aggregate -line_type RELP -by FCST_VAR -vx_mask NWC,GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_RELP_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_RELP_out.stat \ + -set_hdr VX_MASK NWC_AND_GRB \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_RELP.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_RELP_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_RELP_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_RELP.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ + -job aggregate -line_type ECNT -by FCST_VAR -obs_thresh NA -vx_mask NWC,GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_ECNT_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_ECNT_out.stat \ + -set_hdr VX_MASK NWC_AND_GRB \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_ECNT_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_ECNT_out.stat + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ + -job aggregate_stat -line_type ORANK -out_line_type RHIST,PHIST \ + -fcst_var APCP_24 -vx_mask NWC -vx_mask GRB -out_bin_size 0.10 \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RHIST_PHIST.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_SKIP_CONST_20120410_120000V.stat \ + -job aggregate_stat -line_type ORANK -out_line_type RELP \ + -fcst_var APCP_24 -vx_mask NWC,GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_RELP.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ + -job aggregate_stat -line_type ORANK -out_line_type SSVAR \ + -fcst_var APCP_24 -vx_mask NWC -vx_mask GRB -out_bin_size 0.25 \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_SSVAR.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_OBSERR_20120410_120000V.stat \ + -job aggregate_stat -line_type ORANK -out_line_type ECNT \ + -fcst_var APCP_24 -by VX_MASK \ + -set_hdr DESC VX_MASK \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_STAT_ORANK_ECNT.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ensemble_stat/ensemble_stat_CMD_LINE_20120410_120000V.stat \ + -job aggregate -line_type SSVAR \ + -fcst_var APCP_24 -obtype ANALYS -vx_mask NWC -vx_mask GRB \ + -dump_row &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR_dump.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR_out.stat + &OUTPUT_DIR;/stat_analysis_es/AGG_SSVAR.out + + + + diff --git a/test/xml/unit_stat_analysis_gs.xml b/test/xml/unit_stat_analysis_gs.xml new file mode 100644 index 0000000000..6309f57aef --- /dev/null +++ b/test/xml/unit_stat_analysis_gs.xml @@ -0,0 +1,45 @@ + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + + + OUTPUT_DIR &OUTPUT_DIR;/stat_analysis_gs + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/grid_stat/grid_stat_GRIB2_NAM_RTMA_120000L_20120409_120000V.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_grid_stat \ + -out &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT.out + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_filter.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_stat_sl1l2_dump.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_stat_sl1l2_out.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_ctc_dump.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_ctc_out.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_stat_ctc_to_eclv_dump.stat + &OUTPUT_DIR;/stat_analysis_gs/CONFIG_GRID_STAT_agg_stat_ctc_to_eclv_out.stat + + + + diff --git a/test/xml/unit_stat_analysis_ps.xml b/test/xml/unit_stat_analysis_ps.xml new file mode 100644 index 0000000000..d32fe963f7 --- /dev/null +++ b/test/xml/unit_stat_analysis_ps.xml @@ -0,0 +1,120 @@ + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + + CONFIG_DIR &CONFIG_DIR; + OUTPUT_DIR &OUTPUT_DIR;/stat_analysis_ps + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_point_stat \ + -out &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT.out + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_dump.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_out.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_agg_stat_mpr_to_cnt_by_vx_mask_out.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_agg_ctc_by_fcst_thresh_out.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_agg_stat_mpr_to_wdir_dump.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_filter_mpr_sid.stat + &OUTPUT_DIR;/stat_analysis_ps/CONFIG_POINT_STAT_filter_mpr_fcst_minus_obs.stat + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ + -job summary -line_type MPR -by FCST_VAR,FCST_LEV -column 'FCST,OBS,FCST-OBS,ABS(FCST-OBS)' \ + -boot_seed 1 -out &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_SUMMARY.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_SUMMARY.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ + -job summary -line_type MPR -by FCST_VAR,FCST_LEV -column 'FCST,OBS' -column_union true \ + -boot_seed 1 -out &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_SUMMARY_UNION.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_SUMMARY_UNION.out + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ + -job filter -line_type MPR -fcst_var TMP -fcst_lev Z2 -vx_mask DTC165 \ + -column_str OBS_SID KDLN,KDHT,KDEN,KDLS,KDMA,KDMN,KDVT,KDEW \ + -column_str_exc OBS_SID KDLN,KDHT \ + -dump_row &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_FILTER_OBS_SID.stat \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_FILTER_OBS_SID.stat + + + + + + OUTPUT_DIR &OUTPUT_DIR;/stat_analysis_ps + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB2_NAM_NDAS_120000L_20120409_120000V.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_filter_times \ + -out &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_FILTER_TIMES.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_FILTER_TIMES.out + + + + + + OUTPUT_DIR &OUTPUT_DIR;/stat_analysis_ps + + &MET_BIN;/stat_analysis + \ + -lookin &DATA_DIR_MODEL;/time_series_met_6.0/*.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_ramps \ + -out &OUTPUT_DIR;/stat_analysis_ps/RAMPS.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ps/RAMPS.out + &OUTPUT_DIR;/stat_analysis_ps/RAMPS_100_100.stat + &OUTPUT_DIR;/stat_analysis_ps/RAMPS_25_100.stat + &OUTPUT_DIR;/stat_analysis_ps/RAMPS_25_100_30min.stat + + + + diff --git a/test/xml/unit_stat_analysis_ws.xml b/test/xml/unit_stat_analysis_ws.xml new file mode 100644 index 0000000000..4bb01eda9e --- /dev/null +++ b/test/xml/unit_stat_analysis_ws.xml @@ -0,0 +1,38 @@ + + + + + + + + + +]> + + + + + + &TEST_DIR; + true + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/wavelet_stat/wavelet_stat_GRIB1_NAM_STAGE4_120000L_20120409_120000V.stat \ + -job aggregate -line_type ISC \ + -fcst_var APCP_12 -fcst_thresh '>0.0' -vx_mask TILE1 -vx_mask TILE2 \ + -dump_row &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC_dump.stat \ + -out_stat &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC_out.stat \ + -out &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC_dump.stat + &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC_out.stat + &OUTPUT_DIR;/stat_analysis_ws/AGG_ISC.out + + + +