Warn users to migrate when old schema is detected and enable migratio…

…n CLI (#816)

This PR ensures that upgrading to signac 2.0 will not lead to silent failures, but will instead provide a meaningful error and a clear path to migration. Migration is now possible using a CLI, making it easy for users.

* Add note to enable migration CLI.

* Some cleanup of project comments and add TODO to support older config versions.

* Add support for older config versions in _locate_config_dir.

* Provide users a warning when they need to migrate.

* Add test of requesting migration.

* Enable migration CLI and add a test.

* Address PR comment.

signac/__main__.py

@@ -634,35 +634,36 @@ def main_update_cache(args):
         _print_err(f"Updated cache (size={n}).")
 
 
-# UNCOMMENT THE FOLLOWING BLOCK WHEN THE FIRST MIGRATION IS INTRODUCED.
-# def main_migrate(args):
-#     "Migrate the project's schema to the current schema version."
-#     from .contrib.migration import apply_migrations, _get_config_schema_version
-#     from packaging import version
-#     from .version import SCHEMA_VERSION
-#     project = get_project(_ignore_schema_version=True)
-#
-#     root = args.root_directory if args.root_directory else os.getcwd()
-#
-#     schema_version = version.parse(SCHEMA_VERSION)
-#     config_schema_version = _get_config_schema_version(root, schema_version)
-#
-#     if config_schema_version > schema_version:
-#         _print_err(
-#             "The schema version of the project ({}) is newer than the schema "
-#             "version supported by signac version {}: {}. Try updating signac.".format(
-#                 config_schema_version, __version__, schema_version))
-#     elif config_schema_version == schema_version:
-#         _print_err(
-#             "The schema version of the project ({}) is up to date. "
-#             "Nothing to do.".format(config_schema_version))
-#     elif args.yes or _query_yes_no(
-#         "Do you want to migrate this project's schema version from '{}' to '{}'? "
-#         "WARNING: THIS PROCESS IS IRREVERSIBLE!".format(
-#             config_schema_version, schema_version), 'no'):
-#         apply_migrations(root)
-#
-#
+def main_migrate(args):
+    """Migrate the project's schema to the current schema version."""
+    from .contrib.migration import _get_config_schema_version, apply_migrations
+    from .version import SCHEMA_VERSION
+
+    root = args.root_directory if args.root_directory else os.getcwd()
+
+    schema_version = int(SCHEMA_VERSION)
+    config_schema_version = _get_config_schema_version(root, schema_version)
+
+    if config_schema_version > schema_version:
+        _print_err(
+            f"The schema version of the project ({config_schema_version}) is "
+            "newer than the schema version {schema_version} supported by "
+            f"signac version {__version__}. Try updating signac."
+        )
+    elif config_schema_version == schema_version:
+        _print_err(
+            f"The schema version of the project ({config_schema_version}) is "
+            "up to date. Nothing to do."
+        )
+    elif args.yes or _query_yes_no(
+        "Do you want to migrate this project's schema version from "
+        f"'{config_schema_version}' to '{schema_version}'? WARNING: THIS "
+        "PROCESS IS IRREVERSIBLE!",
+        "no",
+    ):
+        apply_migrations(root)
+
+
 def verify_config(cfg, preserve_errors=True):
     """Verify provided configuration."""
     verification = cfg.verify(preserve_errors=preserve_errors)
@@ -1573,19 +1574,25 @@ def main():
     parser_verify = config_subparsers.add_parser("verify")
     parser_verify.set_defaults(func=main_config_verify)
 
-    # UNCOMMENT THE FOLLOWING BLOCK WHEN THE FIRST MIGRATION IS INTRODUCED.
-    # parser_migrate = subparsers.add_parser(
-    #     'migrate',
-    #     description="Irreversibly migrate this project's schema version to the "
-    #                 "supported version.")
-    # parser_migrate.add_argument(
-    #     "-r",
-    #     "--root-directory",
-    #     type=str,
-    #     default='',
-    #     help="The path to the project.",
-    # )
-    # parser_migrate.set_defaults(func=main_migrate)
+    parser_migrate = subparsers.add_parser(
+        "migrate",
+        description="Irreversibly migrate this project's schema version to the "
+        "supported version.",
+    )
+    parser_migrate.add_argument(
+        "-r",
+        "--root-directory",
+        type=str,
+        default="",
+        help="The path to the project.",
+    )
+    parser_migrate.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        help="Do not ask for confirmation.",
+    )
+    parser_migrate.set_defaults(func=main_migrate)
 
     # This is a hack, as argparse itself does not
     # allow to parse only --version without any

signac/common/config.py

@@ -17,14 +17,47 @@
 PROJECT_CONFIG_FN = os.path.join(".signac", "config")
 USER_CONFIG_FN = os.path.expanduser(os.path.join("~", ".signacrc"))
 
-# TODO: Consider making this entire module internal and removing all its
-# functions from the public API.
-
 
 def _get_project_config_fn(path):
     return os.path.abspath(os.path.join(path, PROJECT_CONFIG_FN))
 
 
+def _raise_if_older_schema(root):
+    """Raise if an older schema version is detected at the search path.
+
+    Parameters
+    ----------
+    root : str
+        Directory to check schema for.
+
+    Raises
+    ------
+    IncompatibleSchemaVersion
+        If the project uses an older schema version that requires migration.
+    """
+    from ..contrib.errors import IncompatibleSchemaVersion
+    from ..contrib.migration import _get_config_schema_version
+    from ..version import SCHEMA_VERSION, __version__
+
+    schema_version = int(SCHEMA_VERSION)
+
+    try:
+        schema_version = _get_config_schema_version(root, schema_version)
+        assert schema_version != int(SCHEMA_VERSION), (
+            "Migration schema loader succeeded in loading a config file "
+            "where normal loader failed. This indicates an internal "
+            "error, please contact the signac developers."
+        )
+        raise IncompatibleSchemaVersion(
+            "The signac schema version used by this project is "
+            f"{schema_version}, but signac {__version__} requires "
+            f"schema version {SCHEMA_VERSION}. Try running python -m "
+            "signac migrate."
+        )
+    except RuntimeError:
+        pass
+
+
 def _locate_config_dir(search_path):
     """Locates directory containing a signac configuration file in a directory hierarchy.
 
@@ -38,10 +71,24 @@ def _locate_config_dir(search_path):
     str or None
         The directory containing the configuration file if one is found, otherwise None.
     """
+    orig_search_path = search_path
     search_path = os.path.abspath(search_path)
     while True:
         if os.path.isfile(_get_project_config_fn(search_path)):
             return search_path
+        if (up := os.path.dirname(search_path)) == search_path:
+            break
+        else:
+            search_path = up
+
+    logger.debug(
+        "Reached filesystem root, no config found. Checking whether a "
+        "project created with an older signac schema may be found."
+    )
+
+    search_path = os.path.abspath(orig_search_path)
+    while True:
+        _raise_if_older_schema(search_path)
         if (up := os.path.dirname(search_path)) == search_path:
             logger.debug("Reached filesystem root, no config found.")
             return None

signac/contrib/migration/v0_to_v1.py

@@ -19,9 +19,12 @@
 
 
 def _load_config_v1(root_directory):
-    cfg = configobj.ConfigObj(
-        os.path.join(root_directory, "signac.rc"), configspec=_cfg.split("\n")
-    )
+    config_fn = os.path.join(root_directory, "signac.rc")
+    if not os.path.isfile(config_fn):
+        raise RuntimeError(
+            f"The directory {root_directory} does not contain a config file."
+        )
+    cfg = configobj.ConfigObj(config_fn, configspec=_cfg.split("\n"))
     validator = configobj.validate.Validator()
     if cfg.validate(validator) is not True:
         raise RuntimeError(

signac/contrib/migration/v1_to_v2.py

@@ -29,9 +29,12 @@
 
 
 def _load_config_v2(root_directory):
-    cfg = configobj.ConfigObj(
-        os.path.join(root_directory, ".signac", "config"), configspec=_cfg.split("\n")
-    )
+    config_fn = os.path.join(root_directory, ".signac", "config")
+    if not os.path.isfile(config_fn):
+        raise RuntimeError(
+            f"The directory {root_directory} does not contain a config file."
+        )
+    cfg = configobj.ConfigObj(config_fn, configspec=_cfg.split("\n"))
     validator = configobj.validate.Validator()
     if cfg.validate(validator) is not True:
         raise RuntimeError(

signac/contrib/project.py

@@ -25,6 +25,7 @@
     _get_project_config_fn,
     _load_config,
     _locate_config_dir,
+    _raise_if_older_schema,
     _read_config_file,
 )
 from ..core.h5store import H5StoreManager
@@ -104,6 +105,7 @@ def __init__(self, path=None):
         if path is None:
             path = os.getcwd()
         if not os.path.isfile(_get_project_config_fn(path)):
+            _raise_if_older_schema(path)
             raise LookupError(
                 f"Unable to find project at path '{os.path.abspath(path)}'."
             )
@@ -1435,13 +1437,6 @@ def init_project(cls, path=None):
         -------
         :class:`~signac.Project`
             Initialized project, an instance of :class:`~signac.Project`.
-
-        Raises
-        ------
-        RuntimeError
-            If the project path already contains a conflicting project
-            configuration.
-
         """
         if path is None:
             path = os.getcwd()

tests/test_project.py

@@ -2326,43 +2326,49 @@ def test_no_migration(self):
         assert len(migrations) == 0
 
 
+def _initialize_v1_project(dirname, with_workspace=True):
+    # Create v1 config file.
+    cfg_fn = os.path.join(dirname, "signac.rc")
+    workspace_dir = "workspace_dir"
+    with open(cfg_fn, "w") as f:
+        f.write(
+            textwrap.dedent(
+                f"""\
+                project = project
+                workspace_dir = {workspace_dir}
+                schema_version = 0"""
+            )
+        )
+
+    # Create a custom workspace
+    os.makedirs(os.path.join(dirname, workspace_dir))
+    if with_workspace:
+        os.makedirs(os.path.join(dirname, "workspace"))
+
+    # Create a shell history file.
+    history_fn = os.path.join(dirname, ".signac_shell_history")
+    with open(history_fn, "w") as f:
+        f.write("print(project)")
+
+    # Create a statepoint cache. Note that this cache does not
+    # correspond to actual statepoints since we don't currently have
+    # any in this project, but that's fine for migration testing.
+    sp_cache = os.path.join(dirname, ".signac_sp_cache.json.gz")
+    sp = {"a": 1}
+    with gzip.open(sp_cache, "wb") as f:
+        f.write(json.dumps({calc_id(sp): sp}).encode())
+
+    return cfg_fn
+
+
 class TestSchemaMigration:
     @pytest.mark.parametrize("implicit_version", [True, False])
     @pytest.mark.parametrize("workspace_exists", [True, False])
     def test_project_schema_version_migration(self, implicit_version, workspace_exists):
         from signac.contrib.migration import apply_migrations
 
         with TemporaryDirectory() as dirname:
-            # Create v1 config file.
-            cfg_fn = os.path.join(dirname, "signac.rc")
-            workspace_dir = "workspace_dir"
-            with open(cfg_fn, "w") as f:
-                f.write(
-                    textwrap.dedent(
-                        f"""\
-                        project = project
-                        workspace_dir = {workspace_dir}
-                        schema_version = 0"""
-                    )
-                )
-
-            # Create a custom workspace
-            os.makedirs(os.path.join(dirname, workspace_dir))
-            if workspace_exists:
-                os.makedirs(os.path.join(dirname, "workspace"))
-
-            # Create a shell history file.
-            history_fn = os.path.join(dirname, ".signac_shell_history")
-            with open(history_fn, "w") as f:
-                f.write("print(project)")
-
-            # Create a statepoint cache. Note that this cache does not
-            # correspond to actual statepoints since we don't currently have
-            # any in this project, but that's fine for migration testing.
-            history_fn = os.path.join(dirname, ".signac_sp_cache.json.gz")
-            sp = {"a": 1}
-            with gzip.open(history_fn, "wb") as f:
-                f.write(json.dumps({calc_id(sp): sp}).encode())
+            cfg_fn = _initialize_v1_project(dirname, workspace_exists)
 
             # If no schema version is present in the config it is equivalent to
             # version 0, so we test both explicit and implicit versions.
@@ -2394,6 +2400,17 @@ def test_project_schema_version_migration(self, implicit_version, workspace_exis
             assert os.path.isfile(project.fn(os.sep.join((".signac", "shell_history"))))
             assert os.path.isfile(project.fn(Project.FN_CACHE))
 
+    def test_project_init_old_schema(self):
+        with TemporaryDirectory() as dirname:
+            _initialize_v1_project(dirname)
+
+            # Initializing a project should detect the incompatible schema.
+            with pytest.raises(IncompatibleSchemaVersion):
+                signac.get_project(dirname)
+
+            with pytest.raises(IncompatibleSchemaVersion):
+                signac.Project(dirname)
+
 
 class TestProjectPickling(TestProjectBase):
     def test_pickle_project_empty(self):

tests/test_shell.py

@@ -9,6 +9,7 @@
 from tempfile import TemporaryDirectory
 
 import pytest
+from test_project import _initialize_v1_project
 
 import signac
 from signac.common import config
@@ -810,3 +811,12 @@ def test_update_cache(self):
 
         err = self.call("python -m signac update-cache".split(), error=True)
         assert "Cache is up to date" in err
+
+    def test_migrate_v1_to_v2(self):
+        dirname = self.tmpdir.name
+        _initialize_v1_project(dirname, False)
+        self.call("python -m signac migrate --yes".split())
+        assert not os.path.isfile(os.path.join(dirname, "signac.rc"))
+        assert not os.path.isdir(os.path.join(dirname, "workspace_dir"))
+        assert os.path.isdir(os.path.join(dirname, ".signac"))
+        assert os.path.isdir(os.path.join(dirname, "workspace"))