Refactor/logging

.tools/envs/testenv-linux.yml

@@ -33,4 +33,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

.tools/envs/testenv-others.yml

@@ -32,4 +32,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

.tools/envs/testenv-pandas.yml

@@ -30,4 +30,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

docs/source/how_to/how_to_logging.ipynb

@@ -53,6 +53,32 @@
     ")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In case the SQLite file already exists, this will raise a `FileExistsError` to prevent from accidentally polluting an existing database. If you want to reuse\n",
+    "an existing database on purpose, you must explicitly provide the corresponding option for `if_database_exists`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from optimagic.logging import ExistenceStrategy, SQLiteLogger\n",
+    "\n",
+    "logger = SQLiteLogger(\"my_log.db\", if_database_exists=ExistenceStrategy.EXTEND)\n",
+    "\n",
+    "res = om.minimize(\n",
+    "    fun=sphere,\n",
+    "    params=np.arange(5),\n",
+    "    algorithm=\"scipy_lbfgsb\",\n",
+    "    logging=logger,\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -63,7 +89,7 @@
     "\n",
     "However, this makes writing logs rather slow, which becomes notable when the criterion function is very fast. \n",
     "\n",
-    "In that case, you can enable ``\"fast_logging\"``, which is still quite safe!"
+    "In that case, you can enable `fast_logging`, which is still quite safe!"
    ]
   },
   {
@@ -72,12 +98,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "logger = SQLiteLogger(\"my_log.db\", fast_logging=True)\n",
+    "\n",
     "res = om.minimize(\n",
     "    fun=sphere,\n",
     "    params=np.arange(5),\n",
     "    algorithm=\"scipy_lbfgsb\",\n",
-    "    logging=\"my_log.db\",\n",
-    "    log_options={\"fast_logging\": True},\n",
+    "    logging=logger,\n",
     ")"
    ]
   },
@@ -87,7 +114,8 @@
    "source": [
     "## Handling existing tables\n",
     "\n",
-    "By default, we only append to databases and do not overwrite data in them. You have a few options to change this:"
+    "By default, we also raise an exception, when a table is present a database. Again, if you want to reuse or replace an existing table, provide the corresponding option\n",
+    "for `if_table_exists`:"
    ]
   },
   {
@@ -96,23 +124,24 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "logger = SQLiteLogger(\"my_log.db\", if_table_exists=ExistenceStrategy.EXTEND)\n",
+    "# or\n",
+    "logger = SQLiteLogger(\"my_log.db\", if_table_exists=ExistenceStrategy.REPLACE)\n",
+    "\n",
     "res = om.minimize(\n",
     "    fun=sphere,\n",
     "    params=np.arange(5),\n",
     "    algorithm=\"scipy_lbfgsb\",\n",
-    "    logging=\"my_log.db\",\n",
-    "    log_options={\n",
-    "        \"if_database_exists\": \"replace\",  # one of \"raise\", \"replace\", \"extend\",\n",
-    "        \"if_table_exists\": \"replace\",  # one of \"raise\", \"replace\", \"extend\"\n",
-    "    },\n",
+    "    logging=logger,\n",
     ")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reading the log"
+    "## Reading the log\n",
+    "To read the log after an optimization, extract the logger from the optimization result:"
    ]
   },
   {
@@ -121,7 +150,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "reader = om.OptimizeLogReader(\"my_log.db\")"
+    "res = om.minimize(\n",
+    "    fun=sphere,\n",
+    "    params=np.arange(5),\n",
+    "    algorithm=\"scipy_lbfgsb\",\n",
+    "    logging=\"my_log.db\",\n",
+    ")\n",
+    "\n",
+    "reader = res.logger"
    ]
   },
   {

environment.yml

@@ -47,3 +47,4 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests

src/estimagic/estimate_ml.py

@@ -60,8 +60,7 @@ def estimate_ml(
     *,
     bounds=None,
     constraints=None,
-    logging=False,
-    log_options=None,
+    logging=None,
     loglike_kwargs=None,
     jacobian=None,
     jacobian_kwargs=None,
@@ -71,6 +70,7 @@ def estimate_ml(
     hessian_numdiff_options=None,
     design_info=None,
     # deprecated
+    log_options=None,
     lower_bounds=None,
     upper_bounds=None,
     numdiff_options=None,
@@ -232,7 +232,6 @@ def estimate_ml(
             bounds=bounds,
             constraints=constraints,
             logging=logging,
-            log_options=log_options,
             **optimize_options,
         )
         estimates = opt_res.params

src/estimagic/estimate_msm.py

@@ -67,14 +67,14 @@ def estimate_msm(
     *,
     bounds=None,
     constraints=None,
-    logging=False,
-    log_options=None,
+    logging=None,
     simulate_moments_kwargs=None,
     weights="diagonal",
     jacobian=None,
     jacobian_kwargs=None,
     jacobian_numdiff_options=None,
     # deprecated
+    log_options=None,
     lower_bounds=None,
     upper_bounds=None,
     numdiff_options=None,
@@ -251,7 +251,6 @@ def estimate_msm(
             bounds=bounds,
             constraints=constraints,
             logging=logging,
-            log_options=log_options,
             params=params,
             **funcs,  # contains the criterion func and possibly more
             **optimize_options,

src/optimagic/deprecations.py

@@ -1,10 +1,13 @@
+import logging
 import warnings
 from dataclasses import replace
 from functools import wraps
-from typing import Any, Callable, ParamSpec
+from pathlib import Path
+from typing import Any, Callable, ParamSpec, cast
 
 from optimagic import mark
 from optimagic.constraints import Constraint, InvalidConstraintError
+from optimagic.logging.logger import Logger, SQLiteLogger
 from optimagic.optimization.fun_value import (
     LeastSquaresFunctionValue,
     LikelihoodFunctionValue,
@@ -13,6 +16,8 @@
 from optimagic.parameters.bounds import Bounds
 from optimagic.typing import AggregationLevel
 
+_logger = logging.getLogger(__name__)
+
 
 def throw_criterion_future_warning():
     msg = (
@@ -461,6 +466,39 @@ def replace_and_warn_about_deprecated_derivatives(candidate, name):
     return out
 
 
+def handle_log_options_throw_deprecated_warning(
+    log_options: dict[str, Any], logger: str | Path | Logger | None
+) -> str | Path | Logger | None:
+    msg = (
+        "Usage of the parameter log_options is deprecated "
+        "and will be removed in a future version. "
+        "Provide a Logger instance for the parameter `logging`, if you need to "
+        "configure the logging."
+    )
+    warnings.warn(msg, FutureWarning)
+
+    logging_is_path_or_string = isinstance(logger, str) or isinstance(logger, Path)
+    log_options_is_dict = isinstance(log_options, dict)
+    compatible_keys = {"fast_logging", "if_table_exists", "if_database_exists"}
+    log_options_is_compatible = set(log_options.keys()).issubset(compatible_keys)
+
+    if logging_is_path_or_string:
+        if log_options_is_dict and log_options_is_compatible:
+            warnings.warn(
+                f"\nUsing {log_options=} to create an instance of SQLiteLogger. "
+                f"This mechanism will be removed in the future."
+            )
+            return SQLiteLogger(cast(str | Path, logger), **log_options)
+        elif not log_options_is_compatible:
+            warnings.warn(
+                f"Found string or path for logger argument, but parameter"
+                f" {log_options=} is not compatible to {compatible_keys=}."
+                f"Explicitly create a Logger instance for configuration."
+            )
+
+    return logger
+
+
 def pre_process_constraints(
     constraints: list[Constraint | dict[str, Any]] | Constraint | dict[str, Any] | None,
 ) -> list[dict[str, Any]]:

src/optimagic/logging/__init__.py

@@ -0,0 +1,2 @@
+from .logger import SQLiteLogger as SQLiteLogger
+from .types import ExistenceStrategy as ExistenceStrategy