Remove index arguments and state point backups (#599)

* Remove index argument from _find_job_ids and CLI.

* Remove index argument from linked views.

* Remove access modules.

* Remove index argument from detect_schema.

* Improve docstrings.

* Remove unused _read_index function.

* Use generator in the same line it is constructed; remove unclear variable name jsi.

* Rename sp_index to statepoint_index.

* Remove index argument from repair method.

* Clarify docstrings.

* Remove index from docstring.

* Remove deprecated state point backup functions.

* Remove docs for removed methods.

* Fix tests for repair function to use only the persistent cache.

* Remove public get_statepoint method.

* Remove error.

* Improve rigor of state point validation after repair.

* Clarify docstring notes.

changelog.txt

@@ -16,14 +16,15 @@ Removed
  - The pre-SyncedCollection synchronized dictionary classes, including SyncedDict, SyncedAttrDict, and JSONDict (#577).
  - The old custom JSON encoder and dumps wrapper (#577).
  - The MPIPool and the filesystems.py module (#575).
- - The following Project methods: ``get_id``, ``build_job_search_index``, ``build_job_statepoint_index``, ``find_job_ids``, ``reset_statepoint``, ``update_statepoint``, ``create_access_module``, ``index`` (#574, #593).
+ - The following Project methods: ``get_id``, ``build_job_search_index``, ``build_job_statepoint_index``, ``find_job_ids``, ``reset_statepoint``, ``update_statepoint``, ``create_access_module``, ``index``, ``dump_statepoints``, ``get_statepoint``, ``read_statepoints``, ``write_statepoints`` (#574, #593, #599).
  - The following Job methods: ``get_id`` (#578).
  - The ``syncutil.copytree`` method (#581).
  - All Crawlers, including ``RegexFileCrawler``, ``MainCrawler``, ``MasterCrawler``, ``SignacProjectCrawler``, and ``BaseCrawler``, in addition to all associated functionality in indexing.py (#580).
  - The cite.py module (#594).
  - The following types of exceptions: ``AuthenticationError``, ``ExportError`, ``FetchError``, ``PermissionsError`` (#595, #596).
  - All MongoDB database functionality, including the ``db`` subpackage, the ``connection``, ``crypt``, and ``host`` submodules in the ``common`` subpackage, and all associated configuration information (#576, #596).
  - The ``JobSearchIndex`` class (#587).
+ - The ability to pass indexes to various ``Project`` methods (#599).
 
 Version 1
 =========

doc/api.rst

@@ -25,11 +25,9 @@ The Project
     Project.data
     Project.doc
     Project.document
-    Project.dump_statepoints
     Project.export_to
     Project.find_jobs
     Project.fn
-    Project.get_statepoint
     Project.groupby
     Project.groupbydoc
     Project.import_from
@@ -38,15 +36,13 @@ The Project
     Project.min_len_unique_id
     Project.num_jobs
     Project.open_job
-    Project.read_statepoints
     Project.repair
     Project.root_directory
     Project.stores
     Project.sync
     Project.update_cache
     Project.update_statepoint
     Project.workspace
-    Project.write_statepoints
 
 .. autoclass:: Project
     :members:

signac/__main__.py

@@ -113,13 +113,6 @@ def _fmt_bytes(nbytes, suffix="B"):
     return "{:.1f} {}{}".format(nbytes, "Yi", suffix)
 
 
-def _read_index(project, fn_index=None):
-    if fn_index is not None:
-        _print_err(f"Reading index from file '{fn_index}'...")
-        with open(fn_index) as file_descriptor:
-            return [json.loads(line) for line in file_descriptor]
-
-
 def _open_job_by_id(project, job_id):
     """Attempt to open a job by id and provide user feedback on error."""
     try:
@@ -160,14 +153,9 @@ def find_with_filter(args):
             return args.job_id
 
     project = get_project()
-    if hasattr(args, "index"):
-        index = _read_index(project, args.index)
-    else:
-        index = None
-
     f = parse_filter_arg(args.filter)
     df = parse_filter_arg(args.doc_filter)
-    return get_project()._find_job_ids(index=index, filter=f, doc_filter=df)
+    return project._find_job_ids(filter=f, doc_filter=df)
 
 
 def main_project(args):
@@ -178,10 +166,6 @@ def main_project(args):
         FutureWarning,
     )
     project = get_project()
-    if args.index:
-        for doc in project._index():
-            print(json.dumps(doc))
-        return
     if args.workspace:
         print(project.workspace)
     else:
@@ -362,7 +346,6 @@ def main_view(args):
         prefix=args.prefix,
         path=args.path,
         job_ids=find_with_filter(args),
-        index=_read_index(args.index),
     )
 
 
@@ -987,12 +970,6 @@ def main():
         action="store_true",
         help="Print the project's workspace path instead of the project id.",
     )
-    parser_project.add_argument(
-        "-i",
-        "--index",
-        action="store_true",
-        help="Generate and print an index for the project.",
-    )
     parser_project.set_defaults(func=main_project)
 
     parser_job = subparsers.add_parser("job")
@@ -1150,9 +1127,6 @@ def main():
         nargs="+",
         help="Show documents of job matching this document filter.",
     )
-    parser_document.add_argument(
-        "--index", type=str, help="The filename of an index file."
-    )
     parser_document.set_defaults(func=main_document)
 
     parser_remove = subparsers.add_parser("rm")
@@ -1227,9 +1201,6 @@ def main():
     parser_find.add_argument(
         "-d", "--doc-filter", type=str, nargs="+", help="A document filter."
     )
-    parser_find.add_argument(
-        "-i", "--index", type=str, help="The filename of an index file."
-    )
     parser_find.add_argument(
         "-s",
         "--show",
@@ -1326,9 +1297,6 @@ def main():
         nargs="+",
         help="Limit the view to jobs with these job ids.",
     )
-    selection_group.add_argument(
-        "-i", "--index", type=str, help="The filename of an index file."
-    )
     parser_view.set_defaults(func=main_view)
 
     parser_schema = subparsers.add_parser("schema")

signac/contrib/filterparse.py

@@ -40,28 +40,6 @@ def _with_message(query, file):
     return query
 
 
-def _read_index(project, fn_index=None):
-    """Read index from the file passed.
-
-    Parameters
-    ----------
-    project : :class:`~signac.Project`
-        Project handle.
-    fn_index : str
-        File name of the index (Default value = None).
-
-    Returns
-    -------
-    generator
-        Returns the file contents, parsed as JSON-encoded lines.
-
-    """
-    if fn_index is not None:
-        _print_err(f"Reading index from file '{fn_index}'...")
-        fd = open(fn_index)
-        return (json.loads(line) for line in fd)
-
-
 def _is_json_like(q):
     """Check if q is JSON like.
 

signac/contrib/import_export.py

@@ -63,11 +63,12 @@ def _make_schema_based_path_function(jobs, exclude_keys=None, delimiter_nested="
         return lambda job, sep=None: ""
 
     index = [{"_id": job.id, "sp": job.sp()} for job in jobs]
-    jsi = _build_job_statepoint_index(exclude_const=True, index=index)
-    sp_index = OrderedDict(jsi)
+    statepoint_index = OrderedDict(
+        _build_job_statepoint_index(exclude_const=True, index=index)
+    )
 
     paths = {}
-    for key_tokens, values in sp_index.items():
+    for key_tokens, values in statepoint_index.items():
         key = key_tokens.replace(".", delimiter_nested)
         if exclude_keys and key in exclude_keys:
             continue

signac/contrib/linked_view.py

@@ -14,7 +14,7 @@
 logger = logging.getLogger(__name__)
 
 
-def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None):
+def create_linked_view(project, prefix=None, job_ids=None, path=None):
     """Create or update a persistent linked view of the selected data space.
 
     Parameters
@@ -26,8 +26,6 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
     job_ids : iterable
         If None (the default), create the view for the complete data space,
         otherwise only for this iterable of job ids.
-    index :
-        A document index (Default value = None).
     path :
         The path (function) used to structure the linked data space (Default value = None).
 
@@ -42,8 +40,6 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
     OSError
         Linked views cannot be created on Windows because
         symbolic links are not supported by the platform.
-    ValueError
-        When the selected data space is provided with an insufficient index.
     RuntimeError
         When state points contain one of ``[os.sep, " ", "*"]``.
 
@@ -60,23 +56,10 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
     if prefix is None:
         prefix = "view"
 
-    if index is None:
-        if job_ids is None:
-            index = [{"_id": job.id, "sp": job.sp()} for job in project]
-            jobs = list(project)
-        else:
-            index = [
-                {"_id": job_id, "sp": project.open_job(id=job_id).sp()}
-                for job_id in job_ids
-            ]
-            jobs = list(project.open_job(id=job_id) for job_id in job_ids)
-    elif job_ids is not None:
-        if not isinstance(job_ids, set):
-            job_ids = set(job_ids)
-        index = [doc for doc in index if doc["_id"] in job_ids]
+    if job_ids is None:
+        jobs = list(project)
+    else:
         jobs = list(project.open_job(id=job_id) for job_id in job_ids)
-        if not job_ids.issubset({doc["_id"] for doc in index}):
-            raise ValueError("Insufficient index for selected data space.")
 
     key_list = [k for job in jobs for k in job.statepoint().keys()]
     value_list = [v for job in jobs for v in job.statepoint().values()]
@@ -92,7 +75,7 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
     if any(bad_items):
         err_msg = " ".join(
             [
-                f"In order to use view, statepoints should not contain {bad_chars}:",
+                f"In order to use view, state points should not contain {bad_chars}:",
                 *bad_items,
             ]
         )