Allow passing filters when reading from an ixmp4 platform (#838)

Co-authored-by: Philip Hackstock <[email protected]>

RELEASE_NOTES.md

@@ -6,6 +6,7 @@ Bumped minimum version of pandas and numpy to fit **ixmp4**'s requirement.
 
 ## Individual updates
 
+- [#838](https://github.com/IAMconsortium/pyam/pull/838) Support filters when reading from an ixmp4 platform
 - [#837](https://github.com/IAMconsortium/pyam/pull/837) Support filters as direct keyword arguments for `categorize()`
   similar to `validate()` signature (see [#804](https://github.com/IAMconsortium/pyam/pull/804))
 - [#832](https://github.com/IAMconsortium/pyam/pull/832) Improve the test-suite for the ixmp4 integration

docs/api/database.rst

@@ -3,22 +3,30 @@
 Data resources integration
 ==========================
 
-Connecting to an IIASA Scenario Explorer instance
--------------------------------------------------
+Connecting to an IIASA database instance
+----------------------------------------
 
 IIASA's ixmp Scenario Explorer infrastructure implements a RestAPI
 to directly query the database server connected to an explorer instance.
-See https://software.ece.iiasa.ac.at/ixmp-server for more information.
+See https://docs.ece.iiasa.ac.at/ for more information.
 
 The |pyam| package uses this interface to read timeseries data as well as
-categorization and quantitative indicators.
+categorization and quantitative meta indicators.
 The data is returned as an :class:`IamDataFrame`.
 See `this tutorial <../tutorials/iiasa.html>`_ for more information.
 
 .. autofunction:: read_iiasa
 
 .. autofunction:: lazy_read_iiasa
 
+Reading from an |ixmp4| platform
+--------------------------------
+
+The |pyam| package provides a simple interface to read timeseries data and meta
+indicators from local or remote |ixmp4| platform instancs.
+
+.. autofunction:: read_ixmp4
+
 Reading UNFCCC inventory data
 -----------------------------
 

docs/api/iiasa.rst

@@ -39,9 +39,9 @@ You will be prompted to enter your password.
 -------------------------
 
 The *Scenario Apps* use the |ixmp4| package as a database backend.
-You can list all available ixmp4 platforms hosted by IIASA using the following:
+You can list all available ixmp4 platforms hosted by IIASA using the following function:
 
-.. autofunctions:: platforms
+.. autofunction:: platforms
 
 *Scenario Explorer* instances (legacy service)
 ----------------------------------------------

pyam/iiasa.py

@@ -18,6 +18,7 @@
 from requests.auth import AuthBase
 
 from pyam.core import IamDataFrame
+from pyam.ixmp4 import read_ixmp4
 from pyam.logging import deprecation_warning
 from pyam.str import is_str
 from pyam.utils import (
@@ -53,7 +54,7 @@ def platforms() -> None:
     tabulate_manager_platforms(ixmp4.conf.settings.manager.list_platforms())
 
 
-def set_config(user, password, file=None):
+def set_config(*args, **kwargs):
     raise DeprecationWarning(f"This method is deprecated. {IXMP4_LOGIN}.")
 
 
@@ -155,8 +156,8 @@ class Connection(object):
 
     Notes
     -----
-    Credentials (username & password) are not required to access any public
-    Scenario Explorer instances (i.e., with Guest login).
+    Credentials (username & password) are not required to access public |ixmp4|
+    or Scenario Explorer databases (i.e., with Guest login).
     """
 
     def __init__(self, name=None, creds=None, auth_url=_AUTH_URL):
@@ -586,44 +587,47 @@ def _new_default_api(kwargs):
     )
 
 
-def read_iiasa(
-    name, default_only=True, meta=True, creds=None, base_url=_AUTH_URL, **kwargs
-):
-    """Query an IIASA Scenario Explorer database API and return as IamDataFrame
+def read_iiasa(name, default_only=True, meta=True, creds=None, **kwargs):
+    """Read data from an |ixmp4| platform or an IIASA Scenario Explorer database.
 
     Parameters
     ----------
     name : str
-        | Name of an IIASA Scenario Explorer database instance.
+        | Name of an |ixmp4| platform or an IIASA Scenario Explorer database instance.
+        | Use :attr:`platforms <pyam.iiasa.platforms>` for a list of |ixmp4| platforms
+          hosted by IIASA.
         | Use :attr:`valid_connections <pyam.iiasa.Connection.valid_connections>`
-          for a list of available instances.
+          for a list of available Scenario Explorer database instances.
     default_only : bool, optional
         If `True`, return *only* the default version of a model/scenario.
         If `False`, return all versions.
     meta : bool or list of strings, optional
         If `True`, include all meta categories & quantitative indicators
         (or subset if list is given).
     creds : str or :class:`pathlib.Path`, optional
-        | Credentials (username & password) are not required to access
-          any public Scenario Explorer instances (i.e., with Guest login).
-        | See :class:`pyam.iiasa.Connection` for details.
-        | Use :meth:`pyam.iiasa.set_config` to set credentials
-          for accessing private/restricted Scenario Explorer instances.
-    base_url : str
-        Authentication server URL
-    kwargs
-        Arguments for :meth:`pyam.iiasa.Connection.query`
+        Path to a file with authentication credentials. This feature is deprecated,
+        please run ``ixmp4 login <username>`` in a console instead.
+    **kwargs
+        Arguments for :meth:`pyam.read_ixmp4` or :meth:`pyam.iiasa.Connection.query`.
+
+    Notes
+    -----
+    Credentials (username & password) are not required to access any public |ixmp4|
+    or Scenario Explorer database (i.e., with Guest login).
     """
-    return Connection(name, creds, base_url).query(
-        default_only=default_only, meta=meta, **kwargs
-    )
+    if name in [i.name for i in ixmp4.conf.settings.manager.list_platforms()]:
+        if meta is not True:
+            raise NotImplementedError(
+                "Reading from ixmp4 platforms requires `meta=True`"
+            )
+        return read_ixmp4(name, default_only=default_only, **kwargs)
 
+    return Connection(name, creds).query(default_only=default_only, meta=meta, **kwargs)
 
-def lazy_read_iiasa(
-    file, name, default_only=True, meta=True, creds=None, base_url=_AUTH_URL, **kwargs
-):
+
+def lazy_read_iiasa(file, name, default_only=True, meta=True, creds=None, **kwargs):
     """
-    Try to load data from a local cache, failing that, loads it from the internet.
+    Try to load data from a local cache, failing that, loads it from an IIASA database.
 
     Check if the file in a given location is an up-to-date version of an IIASA
     database. If so, load it. If not, load  data from the IIASA scenario explorer
@@ -648,16 +652,24 @@ def lazy_read_iiasa(
         If `True`, include all meta categories & quantitative indicators
         (or subset if list is given).
     creds : str or :class:`pathlib.Path`, optional
-        | Credentials (username & password) are not required to access
-          any public Scenario Explorer instances (i.e., with Guest login).
-        | See :class:`pyam.iiasa.Connection` for details.
-        | Use :meth:`pyam.iiasa.set_config` to set credentials
-          for accessing private/restricted Scenario Explorer instances.
-    base_url : str
-        Authentication server URL
-    kwargs
-        Arguments for :meth:`pyam.iiasa.Connection.query`
+        Path to a file with authentication credentials. This feature is deprecated,
+        please run ``ixmp4 login <username>`` in a console instead.
+    **kwargs
+        Arguments for :meth:`pyam.read_ixmp4` or :meth:`pyam.iiasa.Connection.query`.
+
+    Notes
+    -----
+    This feature does currently not support reading data from |ixmp4| platforms.
+
+    Credentials (username & password) are not required to access any public |ixmp4|
+    or Scenario Explorer database (i.e., with Guest login).
     """
+    if name in [
+        platform.name for platform in ixmp4.conf.settings.manager.list_platforms()
+    ]:
+        raise NotImplementedError(
+            "The function `lazy_read_iiasa()` does not support ixmp4 platforms."
+        )
 
     file = Path(file)
     assert file.suffix in [
@@ -666,7 +678,7 @@ def lazy_read_iiasa(
     ], "We will only read and write to csv and xlsx format."
     if os.path.exists(file):
         date_set = pd.to_datetime(os.path.getmtime(file), unit="s")
-        version_info = Connection(name, creds, base_url).properties()
+        version_info = Connection(name, creds).properties()
         latest_new = np.nanmax(pd.to_datetime(version_info["create_date"]))
         latest_update = np.nanmax(pd.to_datetime(version_info["update_date"]))
         latest = pd.Series([latest_new, latest_update]).max()
@@ -684,7 +696,6 @@ def lazy_read_iiasa(
         meta=meta,
         default_only=default_only,
         creds=creds,
-        base_url=base_url,
         **kwargs,
     )
     Path(file).parent.mkdir(parents=True, exist_ok=True)

pyam/ixmp4.py

@@ -8,35 +8,61 @@
 logger = logging.getLogger(__name__)
 
 
-def read_ixmp4(platform: ixmp4.Platform | str, default_only: bool = True):
+def read_ixmp4(
+    platform: ixmp4.Platform | str,
+    default_only: bool = True,
+    model: str | list[str] | None = None,
+    scenario: str | list[str] | None = None,
+    region: str | list[str] | None = None,
+    variable: str | list[str] | None = None,
+    unit: str | list[str] | None = None,
+    year: int | list[int] | None = None,
+):
     """Read scenario runs from an ixmp4 platform database instance
 
     Parameters
     ----------
     platform : :class:`ixmp4.Platform` or str
-        The ixmp4 platform database instance to which the scenario data is saved
+        The ixmp4 platform database instance to which the scenario data is saved.
     default_only : :class:`bool`, optional
-        Read only default runs
+        Read only default runs.
+    model, scenario, region, variable, unit : str or list of str, optional
+        Filter by these dimensions.
+    year : int or list of int, optional
+        Filter by time domain.
     """
     from pyam import IamDataFrame
 
     if not isinstance(platform, ixmp4.Platform):
         platform = ixmp4.Platform(platform)
 
-    data = platform.iamc.tabulate(run={"default_only": default_only})
-    meta = platform.meta.tabulate(run={"default_only": default_only})
+    # TODO This may have to be revised, see https://github.com/iiasa/ixmp4/issues/72
+    meta_filters = dict(
+        run=dict(default_only=default_only, model=model, scenario=scenario)
+    )
+    iamc_filters = dict(
+        run=dict(default_only=default_only),
+        model=model,
+        scenario=scenario,
+        region=region,
+        variable=variable,
+        unit=unit,
+        year=year,
+    )
+    data = platform.iamc.tabulate(**iamc_filters)
+    meta = platform.meta.tabulate(**meta_filters)
 
     # if default-only, simplify to standard IAMC index, add `version` as meta indicator
     if default_only:
         index = ["model", "scenario"]
-        data.drop(columns="version", inplace=True)
         meta_version = (
-            meta[["model", "scenario", "version"]]
+            data[index + ["version"]]
             .drop_duplicates()
             .rename(columns={"version": "value"})
         )
         meta_version["key"] = "version"
         meta = pd.concat([meta.drop(columns="version"), meta_version])
+        data.drop(columns="version", inplace=True)
     else:
         index = ["model", "scenario", "version"]
 
@@ -70,8 +96,7 @@ def write_to_ixmp4(platform: ixmp4.Platform | str, df):
         if missing := set(values).difference(platform_values):
             raise model.NotFound(
                 ", ".join(missing)
-                + f". Use `Platform.{dimension}.create()` to add the missing "
-                f"{dimension}."
+                + f". Use `Platform.{dimension}.create()` to add missing elements."
             )
 
     # The "version" meta-indicator, added when reading from an ixmp4 platform,

pyproject.toml

@@ -44,7 +44,7 @@ classifiers = [
 [tool.poetry.dependencies]
 python = ">=3.10, <3.13"
 iam-units = ">=2020.4.21"
-ixmp4 = ">=0.7.3"
+ixmp4 = ">=0.8.0"
 matplotlib = ">=3.6.0"
 numpy = ">=1.26.0"
 openpyxl = ">=3.1.2"

tests/test_ixmp4.py

@@ -4,6 +4,7 @@
 
 import pyam
 from pyam import read_ixmp4
+from pyam.testing import assert_iamframe_equal
 
 
 def test_to_ixmp4_missing_region_raises(test_platform, test_df_year):
@@ -37,12 +38,12 @@ def test_ixmp4_integration(test_platform, test_df_year):
     obs = read_ixmp4(platform=test_platform)
     exp = test_df_year.copy()
     exp.set_meta(1, "version")  # add version number added from ixmp4
-    pyam.assert_iamframe_equal(exp, obs)
+    assert_iamframe_equal(exp, obs)
 
     # make one scenario a non-default scenario, make sure that it is not included
     test_platform.runs.get("model_a", "scen_b").unset_as_default()
     obs = read_ixmp4(platform=test_platform)
-    pyam.assert_iamframe_equal(exp.filter(scenario="scen_a"), obs)
+    assert_iamframe_equal(exp.filter(scenario="scen_a"), obs)
 
     # read all scenarios (runs) - version number used as additional index dimension
     obs = read_ixmp4(platform=test_platform, default_only=False)
@@ -54,18 +55,48 @@ def test_ixmp4_integration(test_platform, test_df_year):
     pyam.assert_iamframe_equal(exp, obs)
 
 
+@pytest.mark.parametrize(
+    "filters",
+    (
+        dict(model="model_a"),
+        dict(scenario="scen_a"),
+        dict(scenario="*n_a"),
+        dict(model="model_a", scenario="scen_a", region="World", variable="* Energy"),
+        dict(scenario="scen_a", region="World", variable="Primary Energy", year=2010),
+    ),
+)
+def test_ixmp4_filters(test_platform, test_df_year, filters):
+    """Write an IamDataFrame to the platform and read it back with filters"""
+
+    # test writing to platform
+    test_df_year.to_ixmp4(platform=test_platform)
+
+    # add 'version' meta indicator (indicator during ixmp4 roundtrip)
+    test_df_year.set_meta(1, "version")
+
+    # read with filters
+    assert_iamframe_equal(
+        read_ixmp4(test_platform, **filters),
+        test_df_year.filter(**filters),
+    )
+
+
 @pytest.mark.parametrize("drop_meta", (True, False))
 def test_ixmp4_reserved_columns(test_platform, test_df_year, drop_meta):
     """Make sure that a 'version' column in `meta` is not written to the platform"""
 
     if drop_meta:
         test_df_year = pyam.IamDataFrame(test_df_year.data)
 
-    # test writing to platform with a version-number as meta indicator
+    # write to platform with a version-number as meta indicator
     test_df_year.set_meta(1, "version")  # add version number added from ixmp4
     test_df_year.to_ixmp4(platform=test_platform)
 
+    # version is not saved to the platform
     if drop_meta:
         assert len(test_platform.runs.get("model_a", "scen_a").meta) == 0
     else:
         assert "version" not in test_platform.runs.get("model_a", "scen_a").meta
+
+    # version is included when reading again from the platform
+    assert_iamframe_equal(test_df_year, pyam.read_ixmp4(test_platform))