Merge current state of ixmp4-dev into ixmp4

* These files have been selected based on a diff of the two repos
* Where changes were detected, ixmp4-dev version was used

doc/source/modules.rst

@@ -0,0 +1,70 @@
+Developer Documentation
+=======================
+
+.. toctree::
+   :maxdepth: 1
+
+   ixmp4.core/modules
+   ixmp4.data/modules
+   ixmp4.server/modules
+   ixmp4.cli
+   ixmp4.db
+   ixmp4.db.utils
+   ixmp4.conf
+   tests
+
+
+Package/Folder Structure
+------------------------
+
+.. code:: bash
+
+   .
+   ├── ixmp4
+   │   ├── cli                 # cli
+   │   ├── conf                # configuration module, loads settings etc.
+   │   ├── core                # contains the facade layer for the core python API
+   │   ├── data
+   │   │   ├── abstract        # ABCs for data source models and repositories
+   │   │   ├── api             # data source implementation for the web api
+   │   │   ├── backend         # data source backends
+   │   │   └── db              # data source implementation for databases (sqlalchemy)
+   │   ├── db                  # database management
+   │   ├── server              # web application server
+   │       └── rest            # REST endpoints
+   ├── run                     # runtime artifacts
+   └── tests                   # tests
+
+Architecture
+------------
+
+ixmp4 provides a Python API, a REST API and a compatibility layer for Postgres and SQLite Databases.
+The Python API can interact with databases directly or use the REST API of a compatible ixmp4 server instance.
+
+::
+
+   -> calls -> 
+                            Web or SQL
+            Platform         Backend                    Server       SQL Backend
+      │  ┌────────────┐   ┌───────────┐    ┌─    │   ┌──────────┐   ┌───────────┐  ─┐      │  ┌─┐
+    P │  │            │   │           │    │     │   │          │   │           │   │    S │  │ │
+    y │  │ ┌────────┐ │   │ ┌───────┐ │    │   R │   │ ┌──────┐ │   │ ┌───────┐ │   │    Q │  │D│
+    t │  │ │        │ │   │ │       │ │  ┌─┘   E │   │ │Endp. │ │   │ │       │ │   └─┐  L │  │a│
+    h │  │ │Facade  │ │   │ │Model  │ │  │     S │   │ └──────┘ │   │ │Model  │ │     │  A │  │t│
+    o │  │ └────────┘ │   │ ├───────┤ │  │     T │   │          │   │ ├───────┤ │     │  l │  │a│
+    n │  │            │   │ ├───────┤ │  │       │   │ ┌──────┐ │   │ ├───────┤ │     │  c │  │b│
+      │  │    ...     │   │ │       │ │  │     A │   │ │Endp. │ │   │ │       │ │     │  h │  │a│
+    A │  │            │   │ │Repo.  │ │  └─┐   P │   │ └──────┘ │   │ │Repo.  │ │   ┌─┘  e │  │s│
+    P │  │            │   │ └───────┘ │    │   I │   │          │   │ └───────┘ │   │    m │  │e│
+    I │  │            │   │    ...    │    │     │   │   ...    │   │    ...    │   │    y │  │ │
+      │  └────────────┘   └───────────┘    └─    │   └──────────┘   └───────────┘  ─┘      │  └─┘
+
+           ixmp4.core        ixmp4.data                ixmp4.server      ixmp4.data
+
+Note that a REST SDK in another programming language would have to implement only the
+components before the bracketed part of the diagram (``ixmp4.data.api`` + optionally a facade layer).
+
+Overall both the “facade” layer and the “data source” layer are split
+into “models” (representing a row in a database or a json object) and
+“repositories” (representing a database table or a collection of REST
+endpoints) which manage these models.

doc/source/tests.rst

@@ -0,0 +1,82 @@
+Tests
+=====
+
+Run tests with the CLI for a default configuration:
+
+.. code:: bash
+
+   ixmp4 test [--with-backend] [--with-benchmarks]
+
+Unfortunately, since you are using ixmp4 to execute the tests, global statements are not
+included in the coverage calculations. To circumvent this, use the ``--dry`` parameter.
+
+.. code:: bash
+
+   ixmp4 test --with-backend --dry
+   # -> pytest --cov-report xml:.coverage.xml --cov-report term --cov=ixmp4 -rsx --benchmark-skip
+
+   eval $(ixmp4 test --with-backend --dry)
+   # -> executes pytest
+
+Alternatively, use ``pytest`` directly:
+
+.. code:: bash
+
+   py.test
+
+Running tests with PostgreSQL 
+-----------------------------
+
+In order to run the local tests with PostgreSQL you'll need to have a local
+instance of this database running. The easiest way to do this is using a docker
+container. 
+
+The docker container of the database needs to be started first and then the tests can be
+run normally using pytest. If everything is working correctly, the tests for
+PostgreSQL should not be skipped.
+
+
+For PostgreSQL using the official `postgres <https://hub.docker.com/_/postgres>`_ image
+is recommended. Get the latest version on your local machine using (having docker
+installed):
+
+.. code:: bash
+
+   docker pull postgres
+
+and run the container with:
+
+.. code:: bash
+
+   docker run -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=test -p 5432:5432 -d postgres
+
+please note that you'll have to wait for a few seconds for the databases to be up and
+running.
+
+In case there are any error messages during the start up of the container along those lines:
+
+.. code:: bash
+
+   ... Error response from daemon: driver failed programming external connectivity on
+   endpoint ...
+   Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in
+   use.
+
+you have to find the process running on the port in question (in the above case 5432)
+and kill it:
+
+.. code:: bash
+
+   sudo ss -lptn 'sport = :5432'
+   sudo kill <pid>
+
+Profiling
+---------
+
+Some tests will output profiler information to the ``.profiles/``
+directory (using the ``profiled`` fixture). You can analyze these using
+``snakeviz``. For example:
+
+.. code:: bash
+
+   snakeviz .profiles/test_add_datapoints_full_benchmark.prof

ixmp4/cli/platforms.py

@@ -1,6 +1,6 @@
 import re
-from typing import Optional
 from pathlib import Path
+from typing import Optional
 
 import typer
 
@@ -12,7 +12,6 @@
 
 from . import utils
 
-
 app = typer.Typer()
 
 
@@ -27,7 +26,7 @@ def validate_name(name: str):
 def validate_dsn(dsn: str | None):
     if dsn is None:
         return None
-    match = re.match(r"^(sqlite|postgresql|oracle|https|http)(\:\/\/)", dsn)
+    match = re.match(r"^(sqlite|postgresql|https|http)(\:\/\/)", dsn)
     if match is None:
         raise typer.BadParameter(
             "Platform dsn must be a valid URl or database connection string."

ixmp4/conf/credentials.py

@@ -1,6 +1,6 @@
 from contextlib import suppress
 from pathlib import Path
-import rtoml as toml
+import toml
 
 
 class Credentials(object):

ixmp4/conf/manager.py

@@ -101,7 +101,7 @@ def _request(
             params = hashabledict(params)
 
         if json is not None:
-            if type(json) == dict:
+            if type(json) is dict:
                 json = hashabledict(json)
             else:
                 json = tuple(json)

ixmp4/conf/settings.py

@@ -1,17 +1,18 @@
-import os
 import logging
 import logging.config
-from typing import Literal
+import os
 from pathlib import Path
-from httpx import ConnectError
+from typing import Literal
 
-from pydantic import BaseSettings, Field, validator, HttpUrl, Extra
+from httpx import ConnectError
+from pydantic import BaseSettings, Extra, Field, HttpUrl, validator
 
 from ixmp4.core.exceptions import InvalidCredentials
+
+from .auth import AnonymousAuth, ManagerAuth
 from .credentials import Credentials
-from .toml import TomlConfig
 from .manager import ManagerConfig
-from .auth import ManagerAuth, AnonymousAuth
+from .toml import TomlConfig
 from .user import local_user
 
 logger = logging.getLogger(__name__)
@@ -105,9 +106,7 @@ def get_auth(self):
             self._default_auth = AnonymousAuth()
 
     def load_manager_config(self):
-        self._manager = ManagerConfig(
-            self.manager_url, self.default_auth, remote=True
-        )
+        self._manager = ManagerConfig(self.manager_url, self.default_auth, remote=True)
 
     def load_toml_config(self):
         if self.default_auth is not None:

ixmp4/conf/toml.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 from typing import Any
-import rtoml as toml
+import toml
 import json
 
 from ixmp4.core.exceptions import PlatformNotFound, PlatformNotUnique
@@ -29,7 +29,6 @@ def load(self):
     def dump(self):
         obj = {}
         for c in self.platforms.values():
-            # needed for rtoml to serialize `Path` objects
             dict_ = json.loads(c.json())
             dict_.pop("user", None)
             name = dict_.pop("name")

ixmp4/core/__init__.py

@@ -1,8 +1,9 @@
 # flake8: noqa
+from .iamc.variable import Variable as Variable
 from .model import Model as Model
-from .scenario import Scenario as Scenario
-from .run import Run as Run
+from .optimization.indexset import IndexSet as IndexSet
+from .platform import Platform as Platform
 from .region import Region as Region
+from .run import Run as Run
+from .scenario import Scenario as Scenario
 from .unit import Unit as Unit
-from .platform import Platform as Platform
-from .iamc.variable import Variable as Variable

ixmp4/core/iamc/data.py

@@ -5,11 +5,11 @@
 from pandera.typing import Series
 
 from ixmp4.data.abstract import DataPoint as DataPointModel
+from ixmp4.data.abstract import Run
 
 from ..base import BaseFacade
-from .repository import IamcRepository
-from ..run import RunModel as Run
 from ..utils import substitute_type
+from .repository import IamcRepository
 
 
 def to_dimensionless(df: pd.DataFrame) -> pd.DataFrame:

ixmp4/core/optimization/__init__.py

@@ -0,0 +1 @@
+from .data import OptimizationData

ixmp4/core/optimization/data.py

@@ -0,0 +1,21 @@
+from functools import partial
+
+from ixmp4.data.abstract import Run
+
+from ..base import BaseFacade
+from .indexset import IndexSet as IndexSetModel
+from .indexset import IndexSetRepository
+
+
+class OptimizationData(BaseFacade):
+    """An optimization data instance, which provides access to optimization data such as
+    IndexSet, Table, Variable, etc."""
+
+    IndexSet: partial[IndexSetModel]
+
+    indexsets: IndexSetRepository
+
+    def __init__(self, *args, run: Run, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.IndexSet = partial(IndexSetModel, _backend=self.backend, _run=run)
+        self.indexsets = IndexSetRepository(_backend=self.backend, _run=run)