Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove the use of arguments scheduled to be removed in 0.12 #424

Merged
merged 6 commits into from
Jan 16, 2021
Merged

Remove the use of arguments scheduled to be removed in 0.12 #424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cec7b11
Remove the use of argument
kidrahahjo Jan 13, 2021
db1ca86
Remove the use of env attribute while submit
kidrahahjo Jan 13, 2021
4e0ae52
Pass environment as keyword argument.
bdice Jan 15, 2021
c51648e
Remove env argument in a few missed places.
bdice Jan 15, 2021
69f8754
Fix template reference data. It was using mpiexec (default) and shoul…
bdice Jan 16, 2021
8d8ec81
Remove extra comment about status parallelization.
bdice Jan 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changelog.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ Removed
+++++++

- Removed the deprecated method ``flow.util.misc.write_human_readable_statepoints`` (#397).
- Removed the deprecated argument ``--no-parallelize`` (#424).
- Removed the deprecated ``env`` argument from submission methods (#424).

Version 0.11
============
Expand Down
81 changes: 11 additions & 70 deletions flow/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2421,13 +2421,6 @@ def _fetch_status(
operations, labels, and any errors caught.

"""
# The argument status_parallelization is used so that _fetch_status method
# gets to know whether the deprecated argument no_parallelization passed
# while calling print_status is True or False. This can also be done by
# setting self.config['flow']['status_parallelization']='none' if the argument
# is True. But the later functionality will last the rest of the session but in order
# to do proper deprecation, it is not required for now.

# Update the project's status cache
self._fetch_scheduler_status(aggregates, err, ignore_errors)
# Get project status cache
Expand Down Expand Up @@ -2643,7 +2636,6 @@ def print_status(
file=None,
err=None,
ignore_errors=False,
no_parallelize=False,
template=None,
profile=False,
eligible_jobs_max_lines=None,
Expand Down Expand Up @@ -2697,8 +2689,6 @@ def print_status(
output_format : str
Status output format, supports:
'terminal' (default), 'markdown' or 'html'.
no_parallelize : bool
Disable parallelization. (Default value = False)

Returns
-------
Expand All @@ -2720,19 +2710,7 @@ def print_status(
"eligible_jobs_max_lines"
)

if no_parallelize:
print(
"WARNING: "
"The no_parallelize argument is deprecated as of 0.10 "
"and will be removed in 0.12. "
"Instead, set the status_parallelization configuration value to 'none'. "
"In order to do this from the CLI, execute "
"`signac config set flow.status_parallelization 'none'`\n",
file=sys.stderr,
)
status_parallelization = "none"
else:
status_parallelization = self.config["flow"]["status_parallelization"]
status_parallelization = self.config["flow"]["status_parallelization"]

# initialize jinja2 template environment and necessary filters
template_environment = self._template_environment()
Expand Down Expand Up @@ -3706,14 +3684,14 @@ def script(
return self._script(operations, parallel, template, show_template_help)

def _generate_submit_script(
self, _id, operations, template, show_template_help, env, **kwargs
self, _id, operations, template, show_template_help, **kwargs
):
"""Generate submission script to submit the execution of operations to a scheduler."""
if template is None:
template = env.template
template = self._environment.template
assert _id is not None

template_environment = self._template_environment(env)
template_environment = self._template_environment(self._environment)
template = template_environment.get_template(template)
context = self._get_standard_template_context()
# The flow 'script.sh' file simply extends the base script
Expand All @@ -3722,10 +3700,10 @@ def _generate_submit_script(
# with signac-flow unless additional environment information is
# detected.

logger.info("Use environment '%s'.", env)
logger.info("Set 'base_script=%s'.", env.template)
context["base_script"] = env.template
context["environment"] = env
logger.info("Set 'base_script=%s'.", self._environment.template)
logger.info("Use environment '%s'.", self._environment)
context["base_script"] = self._environment.template
context["environment"] = self._environment
context["id"] = _id
context["operations"] = list(operations)
context.update(kwargs)
Expand All @@ -3737,7 +3715,6 @@ def _submit_operations(
self,
operations,
_id=None,
env=None,
parallel=False,
flags=None,
force=False,
Expand All @@ -3754,9 +3731,6 @@ def _submit_operations(
The operations to submit.
_id : str
The _id to be used for this submission. (Default value = None)
env : :class:`~.ComputeEnvironment`
The environment to use for submission. Uses the environment defined
by the :class:`~.FlowProject` if None (Default value = None).
parallel : bool
Execute all bundled operations in parallel. (Default value = False)
flags : list
Expand All @@ -3783,14 +3757,6 @@ def _submit_operations(
"""
if _id is None:
_id = self._store_bundled(operations)
if env is None:
env = self._environment
else:
warnings.warn(
"The env argument is deprecated as of 0.10 and will be removed in 0.12. "
"Instead, set the environment when constructing a FlowProject.",
DeprecationWarning,
)

print(f"Submitting cluster job '{_id}':", file=sys.stderr)

Expand All @@ -3804,7 +3770,6 @@ def _msg(group):
operations=map(_msg, operations),
template=template,
show_template_help=show_template_help,
env=env,
parallel=parallel,
force=force,
**kwargs,
Expand Down Expand Up @@ -3841,14 +3806,15 @@ def _msg(group):
print(script)

else:
return env.submit(_id=_id, script=script, flags=flags, **kwargs)
return self._environment.submit(
_id=_id, script=script, flags=flags, **kwargs
)

@deprecated(deprecated_in="0.11", removed_in="0.13", current_version=__version__)
def submit_operations(
self,
operations,
_id=None,
env=None,
parallel=False,
flags=None,
force=False,
Expand All @@ -3865,9 +3831,6 @@ def submit_operations(
The operations to submit.
_id : str
The _id to be used for this submission. (Default value = None)
env : :class:`~.ComputeEnvironment`
The environment to use for submission. Uses the environment defined
by the :class:`~.FlowProject` if None (Default value = None).
parallel : bool
Execute all bundled operations in parallel. (Default value = False)
flags : list
Expand Down Expand Up @@ -3895,7 +3858,6 @@ def submit_operations(
return self._submit_operations(
operations,
_id,
env,
parallel,
flags,
force,
Expand All @@ -3914,7 +3876,6 @@ def submit(
parallel=False,
force=False,
walltime=None,
env=None,
ignore_conditions=IgnoreConditions.NONE,
ignore_conditions_on_execution=IgnoreConditions.NONE,
**kwargs,
Expand Down Expand Up @@ -3949,9 +3910,6 @@ def submit(
Specify if preconditions and/or postconditions are to be ignored
when determining eligibility after submitting. The default is
:class:`IgnoreConditions.NONE`.
env : :class:`~.ComputeEnvironment`
The environment to use for submission. Uses the environment defined
by the :class:`~.FlowProject` if None (Default value = None).
\*\*kwargs
Additional keyword arguments forwarded to :meth:`~.ComputeEnvironment.submit`.

Expand All @@ -3964,14 +3922,6 @@ def submit(
"The 'names' argument must be a sequence of strings, however "
f"a single string was provided: {names}."
)
if env is None:
env = self._environment
else:
warnings.warn(
"The env argument is deprecated as of 0.10 and will be removed in 0.12. "
"Instead, set the environment when constructing a FlowProject.",
DeprecationWarning,
)
if walltime is not None:
try:
walltime = datetime.timedelta(hours=walltime)
Expand Down Expand Up @@ -4010,7 +3960,6 @@ def submit(
for bundle in _make_bundles(operations, bundle_size):
status = self._submit_operations(
operations=bundle,
env=env,
parallel=parallel,
force=force,
walltime=walltime,
Expand Down Expand Up @@ -4282,14 +4231,6 @@ def _add_print_status_args(cls, parser):
action="store_true",
help="Ignore errors that might occur when querying the scheduler.",
)
parser.add_argument(
"--no-parallelize",
action="store_true",
help="Do not parallelize the status determination. "
"The '--no-parallelize' argument is deprecated. "
"Please use the status_parallelization configuration "
"instead (see above).",
)
view_group.add_argument(
"-o",
"--output-format",
Expand Down
7 changes: 3 additions & 4 deletions tests/generate_template_reference_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,12 @@ def _store_bundled(self, operations):
return bid


def get_masked_flowproject(p):
def get_masked_flowproject(p, **kwargs):
"""Mock environment-dependent attributes and functions. Need to mock
sys.executable before the FlowProject is instantiated, and then modify the
root_directory and project_dir elements after creation."""
sys.executable = "/usr/local/bin/python"
fp = TestProject.get_project(root=p.root_directory())
fp = TestProject.get_project(root=p.root_directory(), **kwargs)
fp._entrypoint.setdefault("path", "generate_template_reference_data.py")
fp.root_directory = lambda: PROJECT_DIRECTORY
fp.config.project_dir = PROJECT_DIRECTORY
Expand Down Expand Up @@ -183,6 +183,7 @@ def main(args):
with job:
kwargs = job.statepoint()
env = get_nested_attr(flow, kwargs["environment"])
fp._environment = env
parameters = kwargs["parameters"]
if "bundle" in parameters:
bundle = parameters.pop("bundle")
Expand All @@ -191,7 +192,6 @@ def main(args):
with redirect_stdout(tmp_out):
try:
fp.submit(
env=env,
jobs=[job],
names=bundle,
pretend=True,
Expand Down Expand Up @@ -224,7 +224,6 @@ def main(args):
with redirect_stdout(tmp_out):
try:
fp.submit(
env=env,
jobs=[job],
names=[op],
pretend=True,
Expand Down
Binary file modified tests/template_reference_data.tar.gz
View file
Open in desktop
Binary file not shown.
4 changes: 1 addition & 3 deletions tests/test_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def test_env(env, monkeypatch):

# Must import the data into the project.
with signac.TemporaryProject(name=gen.PROJECT_NAME) as p:
fp = gen.get_masked_flowproject(p)
fp = gen.get_masked_flowproject(p, environment=env)
# Here we set the appropriate executable for all the operations. This
# is necessary as otherwise the default executable between submitting
# and running could look different depending on the environment.
Expand All @@ -62,7 +62,6 @@ def test_env(env, monkeypatch):
with redirect_stderr(devnull):
with redirect_stdout(tmp_out):
fp.submit(
env=env,
jobs=[job],
names=bundle,
pretend=True,
Expand Down Expand Up @@ -93,7 +92,6 @@ def test_env(env, monkeypatch):
with redirect_stderr(devnull):
with redirect_stdout(tmp_out):
fp.submit(
env=env,
jobs=[job],
names=[op],
pretend=True,
Expand Down