Merge pull request #287 from dtcenter/feature_240_reformat_tcdiag

Feature 240 reformat tcdiag
dtcenter · Apr 2, 2024 · 09c997d · 09c997d
2 parents a43bd89 + 565a82c
commit 09c997d
Show file tree

Hide file tree

Showing 11 changed files with 1,714 additions and 173 deletions.
diff --git a/.github/workflows/benchmark_pandas1_5.yml b/.github/workflows/benchmark_pandas1_5.yml
@@ -0,0 +1,83 @@
+# This workflow will install Python dependencies, run tests the specified Python version
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Benchmark for  pandas 1.5x
+
+on:
+  push:
+    branches:
+      - develop
+      - develop-ref
+      - feature_*
+      - main_*
+      - bugfix_*
+      - issue_*
+      - gha_*
+    paths-ignore:
+      - 'docs/**'
+      - '.github/pull_request_template.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '**/README.md'
+      - '**/LICENSE.md'
+
+
+  pull_request:
+    types: [ opened, reopened, synchronize ]
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10" ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }} for pandas1.5x benchmarking
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Retrieve METcalcpy repository develop branch
+        run: |
+          /usr/bin/git clone https://github.com/dtcenter/METcalcpy
+          cd METcalcpy
+          /usr/bin/git checkout develop
+          python -m pip install -e .
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pytest>=7.1.1
+          python -m pip install pytest_benchmark
+          python -m pip install netcdf4==1.6.2
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      
+
+      #   Checking the branch name, not necessary but useful when setting things up.
+      #     - name: Extract branch name
+      #       shell: bash
+      #       run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      #       id: extract_branch
+
+
+      - name: Test with pytest with pandas 1.5x
+        run: |
+          echo "GITHUB wspace: $GITHUB_WORKSPACE"
+          export PYTHONPATH=$GITHUB_WORKSPACE/:$GITHUB_WORKSPACE/METdbLoad:$GITHUB_WORKSPACE/METdbLoad/ush:$GITHUB_WORKSPACE/METreformat:$GITHUB_WORKSPACE/METreadnc
+          echo "PYTHONPATH is $PYTHONPATH"
+          cd $GITHUB_WORKSPACE/METreformat
+          cd test
+          py.test run_benchmark.py -v -s --benchmark-autosave
+          
+          echo "Finished benchmarking for pandas 1.5x "
+
+#      - name: Archive benchmark results
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: benchmark-report
+#          path: /home/runner/work/METdataio/METdatio/METreformat/test/**
diff --git a/.github/workflows/benchmark_pandas2_2.yml b/.github/workflows/benchmark_pandas2_2.yml
@@ -0,0 +1,85 @@
+# This workflow will install Python dependencies, run tests the specified Python version
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Benchmark for pandas 2.2
+
+on:
+  push:
+    branches:
+      - develop
+      - develop-ref
+      - feature_*
+      - main_*
+      - bugfix_*
+      - issue_*
+      - gha_*
+    paths-ignore:
+      - 'docs/**'
+      - '.github/pull_request_template.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '**/README.md'
+      - '**/LICENSE.md'
+
+
+  pull_request:
+    types: [ opened, reopened, synchronize ]
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10" ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }} for pandas2.2 testing
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Retrieve METcalcpy repository develop branch
+        run: |
+          /usr/bin/git clone https://github.com/dtcenter/METcalcpy
+          cd METcalcpy
+          /usr/bin/git checkout develop
+          python -m pip install -e .
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pytest>=7.1.1
+          python -m pip install pytest_benchmark
+          python -m pip install netcdf4==1.6.2
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          python -m pip install pandas==2.2
+      
+
+      #   Checking the branch name, not necessary but useful when setting things up.
+      #     - name: Extract branch name
+      #       shell: bash
+      #       run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      #       id: extract_branch
+
+
+      - name: Test with pytest with pandas 2.2
+        run: |
+          echo "GITHUB wspace: $GITHUB_WORKSPACE"
+          export PYTHONPATH=$GITHUB_WORKSPACE/:$GITHUB_WORKSPACE/METdbLoad:$GITHUB_WORKSPACE/METdbLoad/ush:$GITHUB_WORKSPACE/METreformat:$GITHUB_WORKSPACE/METreadnc
+          echo "PYTHONPATH is $PYTHONPATH"
+          cd $GITHUB_WORKSPACE/METreformat
+          cd test
+          py.test run_benchmark.py -v -s --benchmark-autosave
+          
+          
+          echo "Finished benchmark for pandas 2.2"
+
+#      - name: Archive benchmark results
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: benchmark-report
+#          path: /home/runner/work/METdataio/METdatio/METreformat/test/**
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -22,7 +22,7 @@ on:
 
 
   pull_request:
-    types: [opened, reopened, synchronize]
+    types: [ opened, reopened, synchronize ]
 
 
 jobs:
@@ -32,49 +32,56 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-         python-version: ["3.10"]
+        python-version: [ "3.10" ]
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Retrieve METcalcpy repository develop branch
-      run: |
-         /usr/bin/git clone https://github.com/dtcenter/METcalcpy
-         cd METcalcpy
-         /usr/bin/git checkout develop
-         python -m pip install -e .
-         
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install pytest>=7.1.1
-        python -m pip install netcdf4==1.6.2
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        
-
-#   Checking the branch name, not necessary but useful when setting things up.
-#     - name: Extract branch name
-#       shell: bash
-#       run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-#       id: extract_branch
-
-
-    - name: Test with pytest
-      run: |
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Retrieve METcalcpy repository develop branch
+        run: |
+          /usr/bin/git clone https://github.com/dtcenter/METcalcpy
+          python -m pip install --upgrade pip
+          cd METcalcpy
+          /usr/bin/git checkout develop
+          python -m pip install -e .
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pytest>=7.1.1
+          python -m pip install netcdf4==1.6.2
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          python -m pip install pytest-cov
+
+      #   Checking the branch name, not necessary but useful when setting things up.
+      #     - name: Extract branch name
+      #       shell: bash
+      #       run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      #       id: extract_branch
+
+
+      - name: Test with pytest
+        run: |
           echo "GITHUB wspace: $GITHUB_WORKSPACE"
           export PYTHONPATH=$GITHUB_WORKSPACE/:$GITHUB_WORKSPACE/METdbLoad:$GITHUB_WORKSPACE/METdbLoad/ush:$GITHUB_WORKSPACE/METreformat:$GITHUB_WORKSPACE/METreadnc
           echo "PYTHONPATH is $PYTHONPATH"
+          
+          # test reformatter
           cd $GITHUB_WORKSPACE/METreformat
           cd test
-
-          pytest test_reformatting.py
+          pytest test_reformatting.py 
+          pytest --cov 
+          
+          # test NetCDF reader 
           cd $GITHUB_WORKSPACE/METreadnc
           cd test
           pytest test_readnc.py
+          pytest --cov 
+
+          echo "Finished unit tests and coverage"
 
-          echo "Finished unit tests"
diff --git a/METdbLoad/ush/constants.py b/METdbLoad/ush/constants.py
@@ -111,7 +111,7 @@
 PJC = "PJC"  # Joint and Conditional Factorization for Probabilistic Forecasts
 PRC = "PRC"  # Receiver Operating Characteristic for Probabilistic Forecasts
 ECLV = "ECLV"  # Economic Cost/Loss Value derived from CTC and PCT lines
-MPR = "MPR"  # Matched Pair Data
+MPR = "MPR"  # Matched Pair TCDiag_Data
 NBRCTC = "NBRCTC"  # Neighborhood Contingency Table Counts
 NBRCTS = "NBRCTS"  # Neighborhood Contingency Table Statistics
 NBRCNT = "NBRCNT"  # Neighborhood Continuous Statistics
@@ -130,7 +130,7 @@
 RPS = "RPS"  # Ranked Probability Score
 SSIDX = "SSIDX"  # SKILL SCORE INDEX
 SEEPS = "SEEPS"  # Stable Equitable Error in Probability Space (SEEPS) score
-SEEPS_MPR = "SEEPS_MPR"  # SEEPS Matched Pair Data
+SEEPS_MPR = "SEEPS_MPR"  # SEEPS Matched Pair TCDiag_Data
 
 # VSDB line types
 BSS = "BSS"  # same as PSTD
@@ -1337,6 +1337,12 @@
 # N_THRESH value.
 NUM_STATIC_PCT_COLS = 26
 
+# Number of columns BEFORE the "variable" fields (i.e. the VERSION, MODEL, ..., LINETYPE, TOTAL,
+# INDEX precede the variable columns DIAG_i and VALUE_i).
+# The "variable" fields are DIAG_i and VALUE_i, the number of these is dictated by the N_DIAG
+# value.
+NUM_STATIC_TCDIAG_COLS = 20
+
 # Number of columns before the RANK_i variable columns (i.e. VERSION, MODEL,...,LINETYPE, TOTAL, N_RANK), including
 # the FCST_INIT_BEG and TOTAL columns.
 NUM_STATIC_RHIST_COLS = 26
@@ -1642,5 +1648,20 @@
 #### RHIST line type ####
 LC_RHIST_VARIABLE_HEADERS = ['rank']
 
+#### TCDIAG line type ####
+# These are the header names assigned by the METdbLoad
+# read_data_files module
+TC_DIAG_TOTAL_COLNAME = '0'
+TC_DIAG_INDEX_COLNAME = '1'
+TCDIAG_DIAG_SOURCE_COLNAME = '2'
+TCDIAG_TRACK_SOURCE_COLNAME = '3'
+TCDIAG_FIELD_SOURCE_COLNAME = '4'
+TCDIAG_N_DIAG_COLNAME = LINE_VAR_COUNTER[TCDIAG]
+# Common name for identical fields with different identifiers (due to different DIAG_SOURCE)
+# Used for cleaning up reformatted output
+SHMAG = 'SHEAR_MAGNITUDE'
+DLAND = 'DIST_TO_LAND'
+SSPEED = 'STORM_SPEED'
+TCDIAG_COMMON_NAMES = {'SHR_MAG':SHMAG, 'SHRD':SHMAG, 'LAND':DLAND, 'DTL':DLAND, 'STM_SPD':SSPEED }
 
 
diff --git a/METreformat/test/RHIST.yaml b/METreformat/test/RHIST.yaml
@@ -6,7 +6,7 @@ output_filename: reformatted_rhist_output.txt
 input_data_dir: ./data/rhist_phist_relp_orank
 
 # Set log_filename to STDOUT (or stdout, case-insensitive) if no log file is to be saved
-log_directory: /path/to/log-dir
+log_directory: ./output
 log_filename: stdout
 # most verbose is info, less verbose is error
 log_level: info