Add NEEM processing

TODO.md

@@ -205,18 +205,12 @@ Off the table improvements for now:
   - may also be relevant: https://stackoverflow.com/questions/32418045/running-python-code-by-clicking-a-button-in-bokeh?rq=4
   - probably also google 'bokeh update plot on click'
 
-### Data sources
-
-- NOAA: done
-- AGAGE: done
-
-- NEEM methane
-  - https://doi.pangaea.de/10.1594/PANGAEA.899040
-  - start with outliers removed (https://doi.pangaea.de/10.1594/PANGAEA.899037), then ask providers
-    - https://doi.pangaea.de/10.1594/PANGAEA.899037?format=textfile
+### Data sources to follow up on
 
 - UCI network? (I emailed them on 2024-03-27 asking about data access)
 
+- All the data sources for gases other than CO2, CH4 and N2O (that aren't AGAGAE)
+
 - Full list of data sources is in Table 12 of M17
   - https://gmd.copernicus.org/articles/10/2057/2017/gmd-10-2057-2017.pdf
 

notebooks/006y_process-neem-data/0051_process-epica.py → notebooks/006y_process-neem-data/0061_process-neem.py

@@ -13,9 +13,9 @@
 # ---
 
 # %% [markdown] editable=true slideshow={"slide_type": ""}
-# # EPICA - process
+# # NEEM - process
 #
-# Process data from the EPICA dataset.
+# Process data from the NEEM dataset.
 
 # %% [markdown]
 # ## Imports
@@ -36,7 +36,7 @@
 # ## Define branch this notebook belongs to
 
 # %% editable=true slideshow={"slide_type": ""}
-step: str = "retrieve_and_process_epica_data"
+step: str = "retrieve_and_process_neem_data"
 
 # %% [markdown] editable=true slideshow={"slide_type": ""}
 # ## Parameters
@@ -66,31 +66,30 @@
 
 assert "Methane [ppbv] (CH4)" in raw
 units = "ppb"
-assert "DATE/TIME END: 2006-01-17T00:00:00" in raw
-year_now = 2006
-assert "LATITUDE: -75.002500 * LONGITUDE: 0.068400" in raw
-lat = -75.002500
-lon = 0.068400
+assert "LATITUDE: 77.450000 * LONGITUDE: -51.060000" in raw
+lat = 77.450000
+lon = -51.060000
 
 
 read_df = pd.read_csv(
     config_step.raw_dir / config_step.download_url.url.split("/")[-1],
-    skiprows=17,
+    skiprows=37,
     header=0,
     delimiter="\t",
 )
 read_df = read_df.rename(
     {
-        "CH4 [ppbv]": "value",
+        "CH4 [ppbv] (5 yr medians, see abstract)": "value",
+        "Age [a AD/CE] (Ice age (yr CE) (Sigl et al. ...)": "year",
     },
     axis="columns",
 )
 read_df["unit"] = units
-read_df["year"] = year_now - (read_df["Age [ka BP]"] * 1000)
+# read_df["year"] = year_now - (read_df["Age [ka BP]"] * 1000)
 read_df["latitude"] = lat
 read_df["longitude"] = lon
 read_df["gas"] = "ch4"
-read_df = read_df.drop(["Depth ice/snow [m]", "Age [ka BP]"], axis="columns")
+read_df = read_df[["year", "value", "unit", "latitude", "longitude", "gas"]]
 read_df
 
 # %% [markdown]

notebooks/006y_process-neem-data/0059_plot-overview-scripps.py → notebooks/006y_process-neem-data/0069_plot-overview-neem.py

@@ -13,9 +13,9 @@
 # ---
 
 # %% [markdown] editable=true slideshow={"slide_type": ""}
-# # EPICA - overview
+# # NEEM - overview
 #
-# Overview of the EPICA data.
+# Overview of the NEEM data.
 
 # %% [markdown] editable=true slideshow={"slide_type": ""}
 # ## Imports
@@ -60,7 +60,7 @@
 )
 
 config_process = get_config_for_step_id(
-    config=config, step="retrieve_and_process_epica_data", step_config_id="only"
+    config=config, step="retrieve_and_process_neem_data", step_config_id="only"
 )
 
 # %% [markdown] editable=true slideshow={"slide_type": ""}
@@ -107,7 +107,7 @@
 )
 
 axes[2].set_xlim([0, 2020])
-axes[2].set_ylim([600, 750])
+axes[2].set_ylim([600, 1300])
 
 plt.tight_layout()
 plt.show()

src/local/notebook_steps/retrieve_and_process_epica_data.py

@@ -1,5 +1,5 @@
 """
-Retrieve and extract EPICA data notebook steps
+Retrieve and extract NEEM data notebook steps
 """
 
 from __future__ import annotations
@@ -55,7 +55,7 @@ def configure_notebooks(
     configured_notebooks = [
         ConfiguredNotebook(
             unconfigured_notebook=uc_nbs_dict[
-                Path("005y_process-epica-data") / "0050_download-epica"
+                Path("006y_process-neem-data") / "0060_download-neem"
             ],
             configuration=(config_step.download_url,),
             dependencies=(),
@@ -65,7 +65,7 @@ def configure_notebooks(
         ),
         ConfiguredNotebook(
             unconfigured_notebook=uc_nbs_dict[
-                Path("005y_process-epica-data") / "0051_process-epica"
+                Path("006y_process-neem-data") / "0061_process-neem"
             ],
             configuration=(),
             dependencies=(get_checklist_file(config_step.raw_dir),),
@@ -81,19 +81,19 @@ def configure_notebooks(
 step: UnconfiguredNotebookBasedStep[
     Config, ConfigBundle
 ] = UnconfiguredNotebookBasedStep(
-    step_name="retrieve_and_process_epica_data",
+    step_name="retrieve_and_process_neem_data",
     unconfigured_notebooks=[
         UnconfiguredNotebook(
-            notebook_path=Path("005y_process-epica-data") / "0050_download-epica",
+            notebook_path=Path("006y_process-neem-data") / "0060_download-neem",
             raw_notebook_ext=".py",
-            summary="process EPICA data - download",
-            doc="Download EPICA data",
+            summary="process NEEM data - download",
+            doc="Download NEEM data",
         ),
         UnconfiguredNotebook(
-            notebook_path=Path("005y_process-epica-data") / "0051_process-epica",
+            notebook_path=Path("006y_process-neem-data") / "0061_process-neem",
             raw_notebook_ext=".py",
-            summary="process EPICA data - process",
-            doc="Process EPICA data into a single file with monthly information etc.",
+            summary="process NEEM data - process",
+            doc="Process NEEM data into a single file with monthly information etc.",
         ),
     ],
     configure_notebooks=configure_notebooks,

src/local/notebook_steps/retrieve_and_process_neem_data.py

@@ -0,0 +1,106 @@
+"""
+Retrieve and extract NEEM data notebook steps
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pydoit_nb.checklist import get_checklist_file
+from pydoit_nb.config_handling import get_config_for_step_id
+from pydoit_nb.notebook import ConfiguredNotebook, UnconfiguredNotebook
+from pydoit_nb.notebook_step import UnconfiguredNotebookBasedStep
+
+if TYPE_CHECKING:
+    from ..config.base import Config, ConfigBundle
+
+
+def configure_notebooks(
+    unconfigured_notebooks: Iterable[UnconfiguredNotebook],
+    config_bundle: ConfigBundle,
+    step_name: str,
+    step_config_id: str,
+) -> list[ConfiguredNotebook]:
+    """
+    Configure notebooks
+
+    Parameters
+    ----------
+    unconfigured_notebooks
+        Unconfigured notebooks
+
+    config_bundle
+        Configuration bundle from which to take configuration values
+
+    step_name
+        Name of the step
+
+    step_config_id
+        Step config ID to use when configuring the notebook
+
+    Returns
+    -------
+        Configured notebooks
+    """
+    uc_nbs_dict = {nb.notebook_path: nb for nb in unconfigured_notebooks}
+
+    config = config_bundle.config_hydrated
+
+    config_step = get_config_for_step_id(
+        config=config, step=step_name, step_config_id=step_config_id
+    )
+
+    configured_notebooks = [
+        ConfiguredNotebook(
+            unconfigured_notebook=uc_nbs_dict[
+                Path("006y_process-neem-data") / "0060_download-neem"
+            ],
+            configuration=(
+                config_step.merged_ice_core_data,
+                config_step.station_data,
+            ),
+            dependencies=(),
+            targets=(
+                config_step.merged_ice_core_data_processed_data_file,
+                get_checklist_file(config_step.raw_dir),
+            ),
+            config_file=config_bundle.config_hydrated_path,
+            step_config_id=step_config_id,
+        ),
+        ConfiguredNotebook(
+            unconfigured_notebook=uc_nbs_dict[
+                Path("006y_process-neem-data") / "0061_process-neem"
+            ],
+            configuration=(),
+            dependencies=(get_checklist_file(config_step.raw_dir),),
+            targets=(config_step.processed_data_with_loc_file,),
+            config_file=config_bundle.config_hydrated_path,
+            step_config_id=step_config_id,
+        ),
+    ]
+
+    return configured_notebooks
+
+
+step: UnconfiguredNotebookBasedStep[
+    Config, ConfigBundle
+] = UnconfiguredNotebookBasedStep(
+    step_name="retrieve_and_process_neem_data",
+    unconfigured_notebooks=[
+        UnconfiguredNotebook(
+            notebook_path=Path("006y_process-neem-data") / "0060_download-neem",
+            raw_notebook_ext=".py",
+            summary="process NEEM data - download",
+            doc="Download NEEM data",
+        ),
+        UnconfiguredNotebook(
+            notebook_path=Path("006y_process-neem-data") / "0061_process-neem",
+            raw_notebook_ext=".py",
+            summary="process NEEM data - process",
+            doc="Process NEEM data into a single file with monthly information etc.",
+        ),
+    ],
+    configure_notebooks=configure_notebooks,
+)