Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove index arguments and state point backups #599

Merged
merged 18 commits into from
Aug 10, 2021
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions doc/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,9 @@ The Project
Project.data
Project.doc
Project.document
Project.dump_statepoints
Project.export_to
Project.find_jobs
Project.fn
Project.get_statepoint
Project.groupby
Project.groupbydoc
Project.import_from
Expand All @@ -38,15 +36,13 @@ The Project
Project.min_len_unique_id
Project.num_jobs
Project.open_job
Project.read_statepoints
Project.repair
Project.root_directory
Project.stores
Project.sync
Project.update_cache
Project.update_statepoint
Project.workspace
Project.write_statepoints

.. autoclass:: Project
:members:
Expand Down
2 changes: 1 addition & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ omit =

[tool:pytest]
filterwarnings =
ignore: .*[get_statepoint | Use of.+as key] is deprecated.*: DeprecationWarning
ignore: .*[Use of.+as key] is deprecated.*: DeprecationWarning

[bumpversion:file:setup.py]

Expand Down
34 changes: 1 addition & 33 deletions signac/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,6 @@ def _fmt_bytes(nbytes, suffix="B"):
return "{:.1f} {}{}".format(nbytes, "Yi", suffix)


def _read_index(project, fn_index=None):
if fn_index is not None:
_print_err(f"Reading index from file '{fn_index}'...")
with open(fn_index) as file_descriptor:
return [json.loads(line) for line in file_descriptor]


def _open_job_by_id(project, job_id):
"""Attempt to open a job by id and provide user feedback on error."""
try:
Expand Down Expand Up @@ -160,23 +153,14 @@ def find_with_filter(args):
return args.job_id

project = get_project()
if hasattr(args, "index"):
index = _read_index(project, args.index)
else:
index = None

f = parse_filter_arg(args.filter)
df = parse_filter_arg(args.doc_filter)
return get_project()._find_job_ids(index=index, filter=f, doc_filter=df)
return project._find_job_ids(filter=f, doc_filter=df)


def main_project(args):
"""Handle project subcommand."""
project = get_project()
if args.index:
for doc in project._index():
print(json.dumps(doc))
return
if args.workspace:
print(project.workspace())
else:
Expand Down Expand Up @@ -350,7 +334,6 @@ def main_view(args):
prefix=args.prefix,
path=args.path,
job_ids=find_with_filter(args),
index=_read_index(args.index),
)


Expand Down Expand Up @@ -973,12 +956,6 @@ def main():
action="store_true",
help="Print the project's workspace path instead of the project id.",
)
parser_project.add_argument(
"-i",
"--index",
action="store_true",
help="Generate and print an index for the project.",
)
parser_project.set_defaults(func=main_project)

parser_job = subparsers.add_parser("job")
Expand Down Expand Up @@ -1130,9 +1107,6 @@ def main():
nargs="+",
help="Show documents of job matching this document filter.",
)
parser_document.add_argument(
"--index", type=str, help="The filename of an index file."
)
parser_document.set_defaults(func=main_document)

parser_remove = subparsers.add_parser("rm")
Expand Down Expand Up @@ -1202,9 +1176,6 @@ def main():
parser_find.add_argument(
"-d", "--doc-filter", type=str, nargs="+", help="A document filter."
)
parser_find.add_argument(
"-i", "--index", type=str, help="The filename of an index file."
)
parser_find.add_argument(
"-s",
"--show",
Expand Down Expand Up @@ -1298,9 +1269,6 @@ def main():
nargs="+",
help="Limit the view to jobs with these job ids.",
)
selection_group.add_argument(
"-i", "--index", type=str, help="The filename of an index file."
)
parser_view.set_defaults(func=main_view)

parser_schema = subparsers.add_parser("schema")
Expand Down
22 changes: 0 additions & 22 deletions signac/contrib/filterparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,28 +40,6 @@ def _with_message(query, file):
return query


def _read_index(project, fn_index=None):
"""Read index from the file passed.

Parameters
----------
project : :class:`~signac.Project`
Project handle.
fn_index : str
File name of the index (Default value = None).

Returns
-------
generator
Returns the file contents, parsed as JSON-encoded lines.

"""
if fn_index is not None:
_print_err(f"Reading index from file '{fn_index}'...")
fd = open(fn_index)
return (json.loads(line) for line in fd)


def _is_json(q):
"""Check if q is JSON.

Expand Down
7 changes: 4 additions & 3 deletions signac/contrib/import_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,12 @@ def _make_schema_based_path_function(jobs, exclude_keys=None, delimiter_nested="
return lambda job, sep=None: ""

index = [{"_id": job.id, "sp": job.sp()} for job in jobs]
jsi = _build_job_statepoint_index(exclude_const=True, index=index)
sp_index = OrderedDict(jsi)
statepoint_index = OrderedDict(
_build_job_statepoint_index(exclude_const=True, index=index)
)

paths = {}
for key_tokens, values in sp_index.items():
for key_tokens, values in statepoint_index.items():
key = key_tokens.replace(".", delimiter_nested)
if exclude_keys and key in exclude_keys:
continue
Expand Down
27 changes: 5 additions & 22 deletions signac/contrib/linked_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
logger = logging.getLogger(__name__)


def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None):
def create_linked_view(project, prefix=None, job_ids=None, path=None):
"""Create or update a persistent linked view of the selected data space.

Parameters
Expand All @@ -26,8 +26,6 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
job_ids : iterable
If None (the default), create the view for the complete data space,
otherwise only for this iterable of job ids.
index :
A document index (Default value = None).
path :
The path (function) used to structure the linked data space (Default value = None).

Expand All @@ -42,8 +40,6 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
OSError
Linked views cannot be created on Windows because
symbolic links are not supported by the platform.
ValueError
When the selected data space is provided with an insufficient index.
RuntimeError
When state points contain one of ``[os.sep, " ", "*"]``.

Expand All @@ -60,23 +56,10 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
if prefix is None:
prefix = "view"

if index is None:
if job_ids is None:
index = [{"_id": job.id, "sp": job.sp()} for job in project]
jobs = list(project)
else:
index = [
{"_id": job_id, "sp": project.open_job(id=job_id).sp()}
for job_id in job_ids
]
jobs = list(project.open_job(id=job_id) for job_id in job_ids)
elif job_ids is not None:
if not isinstance(job_ids, set):
job_ids = set(job_ids)
index = [doc for doc in index if doc["_id"] in job_ids]
if job_ids is None:
jobs = list(project)
else:
jobs = list(project.open_job(id=job_id) for job_id in job_ids)
if not job_ids.issubset({doc["_id"] for doc in index}):
raise ValueError("Insufficient index for selected data space.")

key_list = [k for job in jobs for k in job.statepoint().keys()]
value_list = [v for job in jobs for v in job.statepoint().values()]
Expand All @@ -92,7 +75,7 @@ def create_linked_view(project, prefix=None, job_ids=None, index=None, path=None
if any(bad_items):
err_msg = " ".join(
[
f"In order to use view, statepoints should not contain {bad_chars}:",
f"In order to use view, state points should not contain {bad_chars}:",
*bad_items,
]
)
Expand Down
Loading