From f9160077f588dd40b39d0d5f182c0f62f793a5c7 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:32:46 -0400 Subject: [PATCH 01/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index a9f8748..1177c15 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -24,7 +24,7 @@ jobs: cd figure_tools; pip install .; cd .. cd freanalysis; pip install .; cd .. cd freanalysis_clouds; pip install .; cd .. - git clone https://github.com/aradhakrishnanGFDL/CatalogBuilder catalogbuilder + git clone https://github.com/noaa-gfdl/CatalogBuilder catalogbuilder cd catalogbuilder; pip install .; cd .. pip install pytest - name: Test freanalysis_clouds From 896eea4bf29c7e2339cdfa5e5753b7d5ea02c1b9 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:40:17 -0400 Subject: [PATCH 02/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 1177c15..4e17840 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -24,8 +24,7 @@ jobs: cd figure_tools; pip install .; cd .. cd freanalysis; pip install .; cd .. cd freanalysis_clouds; pip install .; cd .. - git clone https://github.com/noaa-gfdl/CatalogBuilder catalogbuilder - cd catalogbuilder; pip install .; cd .. + conda install noaa-gfdl::catalogbuilder pip install pytest - name: Test freanalysis_clouds run: | From 95d57be2cb29999751a453ac22f27d2b742c5676 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:41:08 -0400 Subject: [PATCH 03/16] Update test_freanalysis_clouds.py --- tests/test_freanalysis_clouds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py index eb7b889..1f46ba5 100644 --- a/tests/test_freanalysis_clouds.py +++ b/tests/test_freanalysis_clouds.py @@ -5,8 +5,8 @@ from tempfile import TemporaryDirectory from freanalysis.plugins import list_plugins, plugin_requirements, run_plugin -from scripts import gen_intake_gfdl - +import catalogbuilder +from catalogbuilder.scripts import gen_intake_gfdl def download_test_data(stem): """Downloads test datasets from a FTP server. From f2bc6e55dea9128a0d40e05aab7af3a633ed9512 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:45:54 -0400 Subject: [PATCH 04/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 4e17840..1226759 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -24,6 +24,8 @@ jobs: cd figure_tools; pip install .; cd .. cd freanalysis; pip install .; cd .. cd freanalysis_clouds; pip install .; cd .. + conda config --add channels noaa-gfdl + conda config --append channels conda-forge conda install noaa-gfdl::catalogbuilder pip install pytest - name: Test freanalysis_clouds From b430ae056771ee7964f132d7c3a1252e8a1b6586 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 19:49:15 -0400 Subject: [PATCH 05/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 1226759..98f0c37 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -17,17 +17,21 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | - python3 -m pip install --upgrade pip + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + conda config --add channels noaa-gfdl + conda config --append channels conda-forge + conda install noaa-gfdl::catalogbuilder -n analysis-scripts + conda activate analysis-scripts + pip install --upgrade pip cd analysis-scripts; pip install .; cd .. cd figure_tools; pip install .; cd .. cd freanalysis; pip install .; cd .. cd freanalysis_clouds; pip install .; cd .. - conda config --add channels noaa-gfdl - conda config --append channels conda-forge - conda install noaa-gfdl::catalogbuilder - pip install pytest + - name: Test freanalysis_clouds run: | - python3 -m pytest tests + $CONDA/bin/python -m pytest tests From 554b1dae89691a6f0badc7aff06af584208c5cd7 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 19:58:28 -0400 Subject: [PATCH 06/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 98f0c37..b97cc26 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -26,11 +26,11 @@ jobs: conda config --append channels conda-forge conda install noaa-gfdl::catalogbuilder -n analysis-scripts conda activate analysis-scripts - pip install --upgrade pip - cd analysis-scripts; pip install .; cd .. - cd figure_tools; pip install .; cd .. - cd freanalysis; pip install .; cd .. - cd freanalysis_clouds; pip install .; cd .. + conda install pip + $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/analysis-scripts + $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/figure_tools + $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis + $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis_clouds - name: Test freanalysis_clouds run: | From 6f7fcf63bac58a78e63cfdce9d0e29ed7caeffd9 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:02:21 -0400 Subject: [PATCH 07/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index b97cc26..4b37e2f 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -24,8 +24,9 @@ jobs: echo $CONDA/bin >> $GITHUB_PATH conda config --add channels noaa-gfdl conda config --append channels conda-forge - conda install noaa-gfdl::catalogbuilder -n analysis-scripts - conda activate analysis-scripts + conda create --name analysis-script-testing + conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl + conda activate analysis-script-testing conda install pip $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/analysis-scripts $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/figure_tools From 30479875b3e3ea5c02746651845771aa98d375c6 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:06:28 -0400 Subject: [PATCH 08/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 4b37e2f..6597002 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -26,12 +26,12 @@ jobs: conda config --append channels conda-forge conda create --name analysis-script-testing conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl - conda activate analysis-script-testing + #conda activate analysis-script-testing conda install pip - $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/analysis-scripts - $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/figure_tools - $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis - $CONDA/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis_clouds + $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/analysis-scripts + $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/figure_tools + $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis + $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis_clouds - name: Test freanalysis_clouds run: | From 7bf29abb0b026cec604d884efbd7e2c68a682b98 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:13:22 -0400 Subject: [PATCH 09/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 6597002..2fc6677 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -28,10 +28,14 @@ jobs: conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl #conda activate analysis-script-testing conda install pip - $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/analysis-scripts - $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/figure_tools - $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis - $CONDA/envs/analysis-script-testing/bin/python -m pip install /home/runner/work/analysis-scripts/freanalysis_clouds + cd analysis-scripts + $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. + cd figure_tools + $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. + cd freanalysis + $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. + cd freanalysis_clouds + $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - name: Test freanalysis_clouds run: | From 1186331fea4f54d91adef750fca9333dcc0ac4f1 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:18:31 -0400 Subject: [PATCH 10/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 2fc6677..dc9033b 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -39,4 +39,4 @@ jobs: - name: Test freanalysis_clouds run: | - $CONDA/bin/python -m pytest tests + $CONDA/envs/analysis-script-testing/bin/pytest tests From 3b463d003e577e44c16a64a263f29fc16579954b Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:25:05 -0400 Subject: [PATCH 11/16] Update test_freanalysis_clouds.py --- tests/test_freanalysis_clouds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py index 1f46ba5..4d13212 100644 --- a/tests/test_freanalysis_clouds.py +++ b/tests/test_freanalysis_clouds.py @@ -47,7 +47,7 @@ def create_data_catalog(path, output="data-catalog"): yaml_path = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" # Hack to stop click from exiting. - command = ["python3", "-m", "scripts.gen_intake_gfdl", str(path), output, + command = ["python", "-m", "gen_intake_gfdl", str(path), output, "--config", str(yaml_path)] run(command, check=True) return Path(f"{output}.json").resolve(), Path(f"{output}.csv").resolve() From 47dc923e72c6fe2fc6a256061d5b39520153b6a2 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 20:30:38 -0400 Subject: [PATCH 12/16] Update test_freanalysis_clouds.py --- tests/test_freanalysis_clouds.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py index 4d13212..a3db6c9 100644 --- a/tests/test_freanalysis_clouds.py +++ b/tests/test_freanalysis_clouds.py @@ -47,7 +47,8 @@ def create_data_catalog(path, output="data-catalog"): yaml_path = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" # Hack to stop click from exiting. - command = ["python", "-m", "gen_intake_gfdl", str(path), output, + # we have to move create catalog as its own pytest in the conda environment perhaps + command = ["$CONDA/envs/analysis-script-testing/bin/python", "-m", "gen_intake_gfdl", str(path), output, "--config", str(yaml_path)] run(command, check=True) return Path(f"{output}.json").resolve(), Path(f"{output}.csv").resolve() From 5308dff9f1e3f15031cd7ffbae7e41f8be59e1da Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 21:12:28 -0400 Subject: [PATCH 13/16] Update test_freanalysis_clouds.py --- tests/test_freanalysis_clouds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py index a3db6c9..3e71db2 100644 --- a/tests/test_freanalysis_clouds.py +++ b/tests/test_freanalysis_clouds.py @@ -48,7 +48,7 @@ def create_data_catalog(path, output="data-catalog"): # Hack to stop click from exiting. # we have to move create catalog as its own pytest in the conda environment perhaps - command = ["$CONDA/envs/analysis-script-testing/bin/python", "-m", "gen_intake_gfdl", str(path), output, + command = ["/usr/share/miniconda/envs/analysis-script-testing/bin/python", "-m", "gen_intake_gfdl", str(path), output, "--config", str(yaml_path)] run(command, check=True) return Path(f"{output}.json").resolve(), Path(f"{output}.csv").resolve() From 1f6d8cc240e8218da0e229fd2319dcd254d40f5e Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 21:19:05 -0400 Subject: [PATCH 14/16] Update test_freanalysis_clouds.py --- tests/test_freanalysis_clouds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py index 3e71db2..b80ff86 100644 --- a/tests/test_freanalysis_clouds.py +++ b/tests/test_freanalysis_clouds.py @@ -47,8 +47,8 @@ def create_data_catalog(path, output="data-catalog"): yaml_path = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" # Hack to stop click from exiting. - # we have to move create catalog as its own pytest in the conda environment perhaps - command = ["/usr/share/miniconda/envs/analysis-script-testing/bin/python", "-m", "gen_intake_gfdl", str(path), output, + # TODO we have to move create catalog as its own pytest in the conda environment perhaps, and avoid this hardcoding + command = ["/usr/share/miniconda/envs/analysis-script-testing/bin/python", "-m", "catalogbuilder.scripts.gen_intake_gfdl", str(path), output, "--config", str(yaml_path)] run(command, check=True) return Path(f"{output}.json").resolve(), Path(f"{output}.csv").resolve() From 1e21e5d8feb5726d21fdbba1b7a43eac87bfcdde Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Thu, 25 Jul 2024 12:38:01 -0400 Subject: [PATCH 15/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index dc9033b..b8e7985 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9']#, '3.10', '3.11'] steps: - name: Checkout repository uses: actions/checkout@v4 From 02639dd90426af7a1849676120916806624f826d Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Thu, 25 Jul 2024 15:14:32 -0400 Subject: [PATCH 16/16] Update ci-analysis.yml --- .github/workflows/ci-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index b8e7985..c506342 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: ['3.9']#, '3.10', '3.11'] + python-version: ['3.9'] steps: - name: Checkout repository uses: actions/checkout@v4