From 70509481299edce1b38fb5cf7cf10393b6bd7392 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 13 Mar 2020 13:24:58 -0500 Subject: [PATCH] v0.5.1 (#189) * Provide htcondor module version info as a constant instead of a function * Test infrastructure update (#188) * resolve #186 --- .codecov.yml | 7 -- .coveragerc | 2 + .gitignore | 2 + .travis.yml | 10 +- codecov.yml | 11 ++ docs/source/devs/release.rst | 26 ++-- docs/source/versions/v0_5_1.rst | 29 +++++ dr | 4 +- htmap/maps.py | 17 ++- htmap/state.py | 4 +- htmap/utils.py | 4 +- htmap/version.py | 2 +- pytest.ini | 9 ++ requirements-dev.txt | 2 + requirements.txt | 6 +- tests/_inf/.htmaprc | 4 - tests/_inf/Dockerfile | 65 ++++------ tests/_inf/condor_config.local | 17 +-- tests/_inf/entrypoint.sh | 8 +- tests/_inf/travis.sh | 13 +- tests/conftest.py | 38 +++--- tests/integration/test_checkpointing.py | 12 +- .../integration/test_late_materialization.py | 16 +-- tests/integration/test_maps.py | 14 +-- tests/integration/test_remove_map.py | 23 ++++ tests/unit/test_delivery_method_setup.py | 4 +- tests/unit/test_options.py | 113 +++++++----------- 27 files changed, 247 insertions(+), 215 deletions(-) delete mode 100644 .codecov.yml create mode 100644 codecov.yml create mode 100644 docs/source/versions/v0_5_1.rst mode change 100644 => 100755 dr delete mode 100644 tests/_inf/.htmaprc diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 7674d4be..00000000 --- a/.codecov.yml +++ /dev/null @@ -1,7 +0,0 @@ -coverage: - range: 50..90 - round: down - precision: 0 - -comment: - layout: "diff, files" diff --git a/.coveragerc b/.coveragerc index 9eec3c18..0ca8a1fa 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,8 @@ [run] branch = True +data_file = /tmp/htmap-test-coverage + include = htmap/* diff --git a/.gitignore b/.gitignore index de5eb444..e0409807 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,5 @@ venv.bak/ .vscode/ !htmap-exec/singularity.d/* + +prof/ diff --git a/.travis.yml b/.travis.yml index 9c22631d..a37a9cb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ +os: linux dist: xenial -sudo: required services: - docker @@ -7,15 +7,17 @@ language: python python: - "3.6" - "3.7" +# - "3.8" env: - HTCONDOR_VERSION=8.8 - HTCONDOR_VERSION=8.9 -matrix: +jobs: fast_finish: true install: - - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=$TRAVIS_PYTHON_VERSION . + - pip install codecov + - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} . script: - - docker run htmap-test tests/_inf/travis.sh + - docker run --mount type=bind,src="$PWD",dst=/home/mapper/htmap,readonly htmap-test bash tests/_inf/travis.sh diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..df72f7f2 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,11 @@ +coverage: + range: 50..90 + round: down + precision: 0 + status: + project: + default: + threshold: 1% # allow coverage to drop by up to 1% in a PR before marking it failed + +comment: + layout: "diff, files" diff --git a/docs/source/devs/release.rst b/docs/source/devs/release.rst index ef486774..bb51ac42 100644 --- a/docs/source/devs/release.rst +++ b/docs/source/devs/release.rst @@ -6,16 +6,16 @@ How to Release a New HTMap Version To release a new version of HTMap: -1. Merge the version PR into ``master`` via GitHub. -1. Make a GitHub release from https://github.com/htcondor/htmap/releases . -Name it exactly ``vX.Y.Z``, and link to the release notes for that version -(like https://htmap.readthedocs.io/en/latest/versions/vX_Y_Z.html ) -in the description (the page will not actually exist yet). -1. Delete anything in the ``dist/`` directory in your copy of the repository. -1. On your machine, make sure ``master`` is up-to-date, then run -``python3 setup.py sdist bdist_wheel`` to create the source distribution -and the wheel. (This is where the files in ``dist/`` are created.) -1. Install Twine: ``pip install twine``. -1. Upload to PyPI: -``python3 -m twine upload dist/*``. -You will be prompted for your PyPI login. +#. Merge the version PR into ``master`` via GitHub. +#. Make a GitHub release from https://github.com/htcondor/htmap/releases . + Name it exactly ``vX.Y.Z``, and link to the release notes for that version + (like https://htmap.readthedocs.io/en/latest/versions/vX_Y_Z.html ) + in the description (the page will not actually exist yet). +#. Delete anything in the ``dist/`` directory in your copy of the repository. +#. On your machine, make sure ``master`` is up-to-date, then run + ``python3 setup.py sdist bdist_wheel`` to create the source distribution + and the wheel. (This is where the files in ``dist/`` are created.) +#. Install Twine: ``pip install twine``. +#. Upload to PyPI: + ``python3 -m twine upload dist/*``. + You will be prompted for your PyPI login. diff --git a/docs/source/versions/v0_5_1.rst b/docs/source/versions/v0_5_1.rst new file mode 100644 index 00000000..aae8e6a0 --- /dev/null +++ b/docs/source/versions/v0_5_1.rst @@ -0,0 +1,29 @@ +v0.5.1 +====== + +New Features +------------ + + +Deprecated Features +------------------- + + +Bug Fixes +--------- + +* Maps can now be force-removed even if the schedd cannot be contacted. + Graceful removal still requires contact with the schedd. + Issue: https://github.com/htcondor/htmap/issues/186 + + +Known Issues +------------ + +* Execution errors that result in the job being terminated but no output being + produced are still not handled entirely gracefully. Right now, the component + state will just show as ``ERRORED``, but there won't be an actual error report. +* Map component state may become corrupted when a map is manually vacated. + Force-removal may be needed to clean up maps if HTCondor and HTMap disagree + about the state of their components. + Issue: https://github.com/htcondor/htmap/issues/129 diff --git a/dr b/dr old mode 100644 new mode 100755 index 28085fbc..cceec85b --- a/dr +++ b/dr @@ -1,8 +1,8 @@ #!/usr/bin/env bash -CONTAINER_TAG=htmap-tests +CONTAINER_TAG='htmap-tests' set -e echo "Building HTMap testing container..." docker build --quiet -t ${CONTAINER_TAG} --file tests/_inf/Dockerfile . -docker run -it --rm ${CONTAINER_TAG} $@ +docker run -it --rm --mount type=bind,src="$PWD",dst=/home/mapper/htmap ${CONTAINER_TAG} $@ diff --git a/htmap/maps.py b/htmap/maps.py index 08303ca6..022b4c74 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -262,7 +262,7 @@ def wait( previous_pbar_len = 0 - ok_statuses = set([state.ComponentStatus.COMPLETED]) + ok_statuses = {state.ComponentStatus.COMPLETED} if holds_ok: ok_statuses.add(state.ComponentStatus.HELD) if errors_ok: @@ -707,7 +707,14 @@ def remove(self, force: bool = False) -> None: force If ``True``, do not wait for HTCondor to confirm that all map components have been removed. """ - self._remove_from_queue() + try: + self._remove_from_queue() + except Exception as e: + if not force: + raise e + + logger.exception(f"Encountered error while force-removing map {self.tag}; ignoring and moving to cleanup step") + self._cleanup_local_data(force = force) MAPS.remove(self) @@ -736,7 +743,7 @@ def _cleanup_local_data(self, force: bool = False) -> None: ) for cs in self.component_statuses ): - time.sleep(.01) + time.sleep(settings["WAIT_TIME"]) # move the tagfile to the removed tags dir # renamed by uid to prevent duplicates @@ -757,8 +764,8 @@ def _cleanup_local_data(self, force: bool = False) -> None: logger.debug(f'Removed tag file for map {self.tag}') return # break out of the loop except OSError: - logger.exception(f'Failed to remove map directory for map {self.tag}, retrying in .1 seconds') - time.sleep(.1) + logger.exception(f'Failed to remove map directory for map {self.tag}, retrying in {settings["WAIT_TIME"]} seconds') + time.sleep(settings["WAIT_TIME"]) logger.error(f'Failed to remove map directory for map {self.tag}, run htmap.clean() to try to remove later') @property diff --git a/htmap/state.py b/htmap/state.py index bbe3a003..1439476e 100644 --- a/htmap/state.py +++ b/htmap/state.py @@ -170,7 +170,7 @@ def _read_events(self): if handled_events: self.map._local_data = None # invalidate cache if any events were received - if utils.htcondor_version_info() >= (8, 9, 3): + if utils.HTCONDOR_VERSION_INFO >= (8, 9, 3): self.save() def save(self) -> Path: @@ -188,7 +188,7 @@ def save(self) -> Path: @staticmethod def load(map): - if utils.htcondor_version_info() < (8, 9, 3): + if utils.HTCONDOR_VERSION_INFO < (8, 9, 3): raise exceptions.InsufficientHTCondorVersion("Map state can only be saved with HTCondor 8.9.3 or greater") with (map._map_dir / names.MAP_STATE).open(mode = 'rb') as f: diff --git a/htmap/utils.py b/htmap/utils.py index a521d7b2..faf680cc 100644 --- a/htmap/utils.py +++ b/htmap/utils.py @@ -253,6 +253,4 @@ def parse_version(v: str) -> Tuple[int, int, int, str, int]: EXTRACT_HTCONDOR_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)", flags = re.ASCII) - -def htcondor_version_info() -> Tuple[int, int, int, str, int]: - return parse_version(EXTRACT_HTCONDOR_VERSION_RE.search(htcondor.version()).group(0)) +HTCONDOR_VERSION_INFO = parse_version(EXTRACT_HTCONDOR_VERSION_RE.search(htcondor.version()).group(0)) diff --git a/htmap/version.py b/htmap/version.py index 4e062c4f..8797b6cc 100644 --- a/htmap/version.py +++ b/htmap/version.py @@ -17,7 +17,7 @@ from . import utils -__version__ = '0.5.0' +__version__ = '0.5.1' def version() -> str: diff --git a/pytest.ini b/pytest.ini index d9b3f1ab..830bccfe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,12 @@ testspaths = tests console_output_style = count + +log_format = %(levelname)-8s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S + +cache_dir = /tmp/htmap-pytest-cache + +markers = + issue: marks a test as corresponding to a particular GitHub issue + diff --git a/requirements-dev.txt b/requirements-dev.txt index 92c1aa96..6b83b0b2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,8 @@ pytest >= 4.1.1 pytest-mock pytest-xdist pytest-timeout +pytest-watch +pytest-profiling coverage pytest-cov codecov diff --git a/requirements.txt b/requirements.txt index 949245a9..257e6b8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ htcondor >= 8.8.0 cloudpickle ~= 1.2 -toml ~= 0.10.0 +toml ~= 0.10 tqdm ~= 4.36 click ~= 7.0 -click-didyoumean == 0.0.3 -halo == 0.0.28 +click-didyoumean ~= 0.0.3 +halo ~= 0.0.28 diff --git a/tests/_inf/.htmaprc b/tests/_inf/.htmaprc deleted file mode 100644 index da8b5bfa..00000000 --- a/tests/_inf/.htmaprc +++ /dev/null @@ -1,4 +0,0 @@ -DELIVERY_METHOD = "assume" - -[MAP_OPTIONS] -request_disk = "100MB" diff --git a/tests/_inf/Dockerfile b/tests/_inf/Dockerfile index be7faa79..695cd787 100644 --- a/tests/_inf/Dockerfile +++ b/tests/_inf/Dockerfile @@ -15,22 +15,21 @@ # this dockerfile builds a test environment for HTMap -FROM ubuntu:bionic +ARG PYTHON_VERSION=3.6 +FROM python:${PYTHON_VERSION} + +# build config +ARG HTCONDOR_VERSION=8.8 # switch to root to do root-level config USER root -# build config -ARG PYTHON_VERSION=3.7 -ARG HTCONDOR_VERSION=8.9 -ARG MINICONDA_VERSION=latest - # environment setup ENV DEBIAN_FRONTEND=noninteractive # install utils and dependencies RUN apt-get update \ - && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales \ + && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz \ && apt-get -y clean \ && rm -rf /var/lib/apt/lists/* \ && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ @@ -41,50 +40,38 @@ ENV LC_ALL=en_US.UTF-8 \ LANGUAGE=en_US.UTF-8 # install HTCondor version specified in config -RUN wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | apt-key add - \ - && echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/${HTCONDOR_VERSION}/bionic bionic contrib" >> /etc/apt/sources.list \ +RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ + && echo "deb http://research.cs.wisc.edu/htcondor/debian/${HTCONDOR_VERSION}/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list \ && apt-get -y update \ && apt-get -y install --no-install-recommends htcondor \ && apt-get -y clean \ && rm -rf /var/lib/apt/lists/* -# create a user to be our submitter and set conda install location -ENV SUBMIT_USER=mapper -ENV CONDA_DIR=/home/${SUBMIT_USER}/conda -ENV PATH=${CONDA_DIR}/bin:${PATH} +# copy entrypoint into place and make executable +COPY tests/_inf/entrypoint.sh /.entrypoint.sh +RUN chmod +x /.entrypoint.sh + +# create a user, set their PATH and PYTHONPATH +ENV SUBMIT_USER=mapper \ + PATH="/home/mapper/.local/bin:${PATH}" \ + PYTHONPATH="/home/mapper/htmap:${PYTHONPATH}" RUN groupadd ${SUBMIT_USER} \ && useradd -m -g ${SUBMIT_USER} ${SUBMIT_USER} -# switch to submit user, don't need root anymore +# switch to the user, don't need root anymore USER ${SUBMIT_USER} -# install miniconda and python version specified in config -# (and ipython, which is nice for debugging inside the container) -RUN cd /tmp \ - && wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh \ - && bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR \ - && rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh \ - && conda install python=${PYTHON_VERSION} ipython \ - && conda clean -y --all - -# install htmap dependencies early for docker build caching -COPY requirements* /home/${SUBMIT_USER}/ -RUN pip install --no-cache-dir -r /home/${SUBMIT_USER}/requirements-dev.txt \ - && pip install --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.* \ - && rm /home/${SUBMIT_USER}/requirements* - -# set default entrypoint and command -# the entrypoint is critical: it starts HTCondor in the container -ENTRYPOINT ["tests/_inf/entrypoint.sh"] -CMD ["pytest"] +# install htmap dependencies and debugging tools early for docker build caching +COPY requirements* /tmp/ +RUN pip install --user --no-cache-dir -r /tmp/requirements-dev.txt \ + && pip install --user --no-cache-dir ipython \ + && pip install --user --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.* # copy HTCondor and HTMap testing configs into place COPY tests/_inf/condor_config.local /etc/condor/condor_config.local -COPY tests/_inf/.htmaprc /home/${SUBMIT_USER}/.htmaprc -# copy htmap package into container and install it -# this is the only part that can't be cached against editing the package -COPY --chown=mapper:mapper . /home/${SUBMIT_USER}/htmap -RUN chmod +x /home/${SUBMIT_USER}/htmap/tests/_inf/entrypoint.sh /home/${SUBMIT_USER}/htmap/tests/_inf/travis.sh \ - && pip install --no-cache-dir --no-deps -e /home/${SUBMIT_USER}/htmap +# set default entrypoint and command +# the entrypoint is critical: it starts HTCondor in the container WORKDIR /home/${SUBMIT_USER}/htmap +ENTRYPOINT ["/.entrypoint.sh"] +CMD ["pytest"] diff --git a/tests/_inf/condor_config.local b/tests/_inf/condor_config.local index 260cfca4..a4ebe437 100644 --- a/tests/_inf/condor_config.local +++ b/tests/_inf/condor_config.local @@ -22,18 +22,21 @@ SHADOW_QUEUE_UPDATE_INTERVAL=10 UPDATE_INTERVAL=5 RUNBENCHMARKS=0 -# Don't use all the machine resources -RESERVED_MEMORY = ( $(DETECTED_MEMORY) / 2 ) +# Put all of the machine resources under a single partitionable slot +NUM_SLOTS = 1 +NUM_SLOTS_TYPE_1 = 1 +SLOT_TYPE_1 = 100% +SLOT_TYPE_1_PARTITIONABLE = TRUE + JOB_RENICE_INCREMENT=5 SCHED_UNIV_RENICE_INCREMENT=5 SHADOW_RENICE_INCREMENT=5 -# If the job does not explicitly set an environment, define -# some default environment variables that put Conda in the path. -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SetCondaVars -JOB_TRANSFORM_SetCondaVars @=end +# Get the HTMap source into the Python path +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SetPyVars +JOB_TRANSFORM_SetPyVars @=end [ Requirements = ((Env?:"") == "") && ((Environment?:"") == ""); - set_Environment = "PATH=/home/mapper/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CONDA_DIR=/home/mapper/conda"; + set_Environment = "PATH=/home/mapper/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PYTHONPATH=/home/mapper/htmap"; ] @end diff --git a/tests/_inf/entrypoint.sh b/tests/_inf/entrypoint.sh index 591d2f25..e4847d38 100644 --- a/tests/_inf/entrypoint.sh +++ b/tests/_inf/entrypoint.sh @@ -8,13 +8,15 @@ mkdir -p "$_condor_local_dir/lock" "$_condor_local_dir/log" "$_condor_local_dir/ # start condor condor_master -echo "HTCondor is starting..." +echo "Starting HTCondor..." -# wait until the scheduler is awake -while [[ ! -f ${_condor_local_dir}/spool/job_queue.log ]] +# once the shared port daemon wakes up, use condor_who to wait for condor to stand up +while [[ ! -s "${_condor_local_dir}/log/SharedPortLog" ]] do sleep .01 done +sleep 1 # fudge factor to let shared port *actually* wake up +condor_who -wait:60 'IsReady && STARTD_State =?= "Ready"' > /dev/null if [[ -n $@ ]]; then diff --git a/tests/_inf/travis.sh b/tests/_inf/travis.sh index 02b7bfbf..cd09bed5 100644 --- a/tests/_inf/travis.sh +++ b/tests/_inf/travis.sh @@ -2,7 +2,14 @@ set -e -echo condor_version -pytest -n 6 --cov +echo "python bindings version" +python -c "import htcondor; print(htcondor.version())" -codecov -t 492519e2-1bcf-4e8a-8a3e-e28be5d9de8d +echo "pytest version" +pytest --version + +pytest -n 4 --cov --durations=20 + +coverage xml -o /tmp/coverage.xml + +codecov -t 492519e2-1bcf-4e8a-8a3e-e28be5d9de8d -f /tmp/coverage.xml diff --git a/tests/conftest.py b/tests/conftest.py index 725db321..57309da7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,8 @@ # start with base settings (ignore user settings for tests) htmap.settings.replace(BASE_SETTINGS) htmap.settings['DELIVERY_METHOD'] = 'assume' # assume is the default for testing +htmap.settings['WAIT_TIME'] = 0.01 +htmap.settings['MAP_OPTIONS.request_memory'] = '10MB' @pytest.fixture(scope = 'session', autouse = True) @@ -37,8 +39,8 @@ def set_transplant_dir(tmpdir_factory): def pytest_addoption(parser): parser.addoption( "--delivery", - action = "store", - default = 'assume', + nargs = "+", + default = ['assume'], ) @@ -46,29 +48,22 @@ def pytest_generate_tests(metafunc): if 'delivery_methods' in metafunc.fixturenames: metafunc.parametrize( 'delivery_method', - metafunc.config.getoption('delivery').split(), + metafunc.config.getoption('delivery'), ) -@pytest.fixture() +@pytest.fixture(scope = 'function') def delivery_methods(delivery_method): htmap.settings['DELIVERY_METHOD'] = delivery_method -@pytest.fixture(scope = 'session', autouse = True) +@pytest.fixture(scope = 'function', autouse = True) def set_htmap_dir(tmpdir_factory): path = Path(tmpdir_factory.mktemp('htmap_dir')) htmap.settings['HTMAP_DIR'] = path ensure_htmap_dir_exists() -@pytest.fixture(scope = 'function', autouse = True) -def clean_after_test(): - yield - - htmap.clean(all = True) - - @pytest.fixture(scope = 'session') def doubler(): def doubler(x): @@ -98,18 +93,19 @@ def mapped_power(power): @pytest.fixture(scope = 'session') -def sleepy_double(): - def sleepy_double(x): - time.sleep(30) - return 2 * x +def never_returns(): + def never(_): + while True: + time.sleep(1) - return sleepy_double + return never -@pytest.fixture(scope = 'session') -def mapped_sleepy_double(sleepy_double): - mapper = htmap.mapped(sleepy_double) - return mapper +@pytest.fixture(scope = 'function') +def map_that_never_finishes(never_returns): + m = htmap.map(never_returns, [None]) + yield m + m.remove() @pytest.fixture(scope = 'session') diff --git a/tests/integration/test_checkpointing.py b/tests/integration/test_checkpointing.py index 4545ea14..da77ffbb 100644 --- a/tests/integration/test_checkpointing.py +++ b/tests/integration/test_checkpointing.py @@ -21,6 +21,7 @@ import htmap +@pytest.mark.timeout(120) def test_checkpoint_file_exists_after_restart(): @htmap.mapped def test(_): @@ -44,12 +45,10 @@ def test(_): time.sleep(5) m.vacate() - while m.component_statuses[0] is not htmap.ComponentStatus.COMPLETED: - time.sleep(.1) - - assert m[0] is True + assert m.get(0, timeout = 60) is True +@pytest.mark.timeout(120) def test_checkpoint_file_has_expected_contents_after_restart(): @htmap.mapped def test(_): @@ -73,7 +72,4 @@ def test(_): time.sleep(5) m.vacate() - while m.component_statuses[0] is not htmap.ComponentStatus.COMPLETED: - time.sleep(.1) - - assert m[0] is True + assert m.get(0, timeout = 60) is True diff --git a/tests/integration/test_late_materialization.py b/tests/integration/test_late_materialization.py index de6df437..ac51c894 100644 --- a/tests/integration/test_late_materialization.py +++ b/tests/integration/test_late_materialization.py @@ -21,25 +21,25 @@ @pytest.fixture(scope = 'function') -def late_sleep(): +def late_noop(): @htmap.mapped(map_options = htmap.MapOptions(max_idle = "1")) - def sleep(_): - return time.sleep(1) + def noop(_): + return True - return sleep + return noop @pytest.mark.timeout(10) -def test_can_be_removed_immediately(late_sleep): - m = late_sleep.map(range(1000)) +def test_can_be_removed_immediately(late_noop): + m = late_noop.map(range(1000)) m.remove() assert m.is_removed -def test_wait_with_late_materialization(late_sleep): - m = late_sleep.map(range(3)) +def test_wait_with_late_materialization(late_noop): + m = late_noop.map(range(3)) m.wait(timeout = 180) diff --git a/tests/integration/test_maps.py b/tests/integration/test_maps.py index 729eb75c..9d65d08d 100644 --- a/tests/integration/test_maps.py +++ b/tests/integration/test_maps.py @@ -39,12 +39,9 @@ def test_starmap_produces_correct_output(mapped_power): assert list(m) == [x ** p for x, p in zip(range(n), range(n))] -def test_getitem_too_soon_raises_output_not_found(mapped_sleepy_double): - n = 3 - m = mapped_sleepy_double.map(range(n)) - +def test_getitem_too_soon_raises_output_not_found(map_that_never_finishes): with pytest.raises(htmap.exceptions.OutputNotFound): - print(m[0]) + print(map_that_never_finishes[0]) @pytest.mark.parametrize( @@ -54,12 +51,9 @@ def test_getitem_too_soon_raises_output_not_found(mapped_sleepy_double): datetime.timedelta(seconds = 0.01), ] ) -def test_get_with_too_short_timeout_raises_timeout_error(mapped_sleepy_double, timeout): - n = 3 - m = mapped_sleepy_double.map(range(n)) - +def test_get_with_too_short_timeout_raises_timeout_error(map_that_never_finishes, timeout): with pytest.raises(htmap.exceptions.TimeoutError): - print(m.get(n - 1, timeout = timeout)) + print(map_that_never_finishes.get(0, timeout = timeout)) def test_get_waits_until_ready(mapped_doubler): diff --git a/tests/integration/test_remove_map.py b/tests/integration/test_remove_map.py index 311a6ab9..20eb0415 100644 --- a/tests/integration/test_remove_map.py +++ b/tests/integration/test_remove_map.py @@ -76,3 +76,26 @@ def test_calling_public_methods_after_remove_raises(method, mapped_doubler): # some of the methods take arguments # but this will actually fail before that check happens getattr(m, method)() + + +@pytest.mark.issue(186) +def test_failure_to_contact_schedd_when_removing_map_reraises_underlying_exception(mapped_doubler, mocker): + m = mapped_doubler.map(range(1)) + + # simulate failing to get the schedd for some reason + mocker.patch('htmap.mapping.get_schedd', side_effect = FileNotFoundError("poison")) + + with pytest.raises(FileNotFoundError): + m.remove() + + +@pytest.mark.issue(186) +def test_can_force_remove_map_without_contacting_schedd(mapped_doubler, mocker): + m = mapped_doubler.map(range(1)) + + # simulate failing to get the schedd for some reason + mocker.patch('htmap.mapping.get_schedd', side_effect = FileNotFoundError("poison")) + + m.remove(force = True) + + assert m.is_removed diff --git a/tests/unit/test_delivery_method_setup.py b/tests/unit/test_delivery_method_setup.py index d980d59f..157c1cb7 100644 --- a/tests/unit/test_delivery_method_setup.py +++ b/tests/unit/test_delivery_method_setup.py @@ -21,6 +21,6 @@ from htmap.options import run_delivery_setup -def test_unknown_delivery_method_raises(): +def test_unknown_delivery_method_for_delivery_setup_raises(tmp_path): with pytest.raises(htmap.exceptions.UnknownPythonDeliveryMethod): - run_delivery_setup('foo', Path.cwd(), 'definitely-not-real') + run_delivery_setup('foo', tmp_path, 'definitely-not-real') diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index 1d44f501..abc5cc38 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -64,9 +64,8 @@ def test_request_disk_for_str(): assert opts['request_disk'] == '1.2GB' -def test_single_shared_input_file(): +def test_single_shared_input_file(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( fixed_input_files = ['foo.txt'], @@ -74,7 +73,7 @@ def test_single_shared_input_file(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -82,9 +81,8 @@ def test_single_shared_input_file(): assert 'foo.txt' in sub['transfer_input_files'] -def test_single_shared_input_file_can_be_single_str(): +def test_single_shared_input_file_can_be_single_str(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( fixed_input_files = 'foo.txt', @@ -92,7 +90,7 @@ def test_single_shared_input_file_can_be_single_str(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -100,9 +98,8 @@ def test_single_shared_input_file_can_be_single_str(): assert 'foo.txt' in sub['transfer_input_files'] -def test_two_shared_input_files(): +def test_two_shared_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( fixed_input_files = ['foo.txt', 'bar.txt'], @@ -110,7 +107,7 @@ def test_two_shared_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -119,9 +116,8 @@ def test_two_shared_input_files(): assert 'bar.txt' in sub['transfer_input_files'] -def test_list_of_list_of_str_input_files(): +def test_list_of_list_of_str_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( input_files = [['foo.txt'], ['bar.txt'], ['buz.txt']], @@ -129,7 +125,7 @@ def test_list_of_list_of_str_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -143,9 +139,8 @@ def test_list_of_list_of_str_input_files(): assert itemdata == expected -def test_list_of_str_input_files(): +def test_list_of_str_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( input_files = ['foo.txt', 'bar.txt', 'buz.txt'], @@ -153,7 +148,7 @@ def test_list_of_str_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -167,9 +162,8 @@ def test_list_of_str_input_files(): assert itemdata == expected -def test_list_of_path_input_files(): +def test_list_of_path_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( input_files = [Path('foo.txt'), Path('bar.txt'), Path('buz.txt')], @@ -177,7 +171,7 @@ def test_list_of_path_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -191,9 +185,8 @@ def test_list_of_path_input_files(): assert itemdata == expected -def test_list_of_list_of_path_input_files(): +def test_list_of_list_of_path_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( input_files = [ @@ -205,7 +198,7 @@ def test_list_of_list_of_path_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -219,9 +212,8 @@ def test_list_of_list_of_path_input_files(): assert itemdata == expected -def test_fewer_components_than_input_files(): +def test_fewer_components_than_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( input_files = [['foo.txt'], ['bar.txt'], ['buz.txt']], @@ -230,7 +222,7 @@ def test_fewer_components_than_input_files(): with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -238,9 +230,8 @@ def test_fewer_components_than_input_files(): assert 'input_files' in exception_msg(exc_info) -def test_fewer_input_files_than_components(): +def test_fewer_input_files_than_components(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 5 map_options = htmap.MapOptions( input_files = [['foo.txt']], @@ -249,7 +240,7 @@ def test_fewer_input_files_than_components(): with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -257,65 +248,52 @@ def test_fewer_input_files_than_components(): assert 'input_files' in exception_msg(exc_info) -@pytest.mark.parametrize( - 'rm', - [ - ['239MB'], - ] -) -def test_list_request_memory(rm): +def test_list_request_memory(tmp_path): tag = 'foo' - map_dir = Path().cwd() - num_components = 1 + num_components = 2 map_options = htmap.MapOptions( - request_memory = rm, + request_memory = ['239MB', '136MB'], ) sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) expected = [ {'component': '0', 'itemdata_for_request_memory': '239MB'}, + {'component': '1', 'itemdata_for_request_memory': '136MB'}, ] assert itemdata == expected -@pytest.mark.parametrize( - 'rd', - [ - ['239GB'], - ] -) -def test_list_request_disk(rd): +def test_list_request_disk(tmp_path): tag = 'foo' - map_dir = Path().cwd() - num_components = 1 + num_components = 2 map_options = htmap.MapOptions( - request_disk = rd, + request_disk = ['239MB', '136MB'], ) sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) expected = [ - {'component': '0', 'itemdata_for_request_disk': '239GB'}, + {'component': '0', 'itemdata_for_request_disk': '239MB'}, + {'component': '1', 'itemdata_for_request_disk': '136MB'}, ] assert itemdata == expected -def test_generic_itemdata(): +def test_generic_itemdata(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( stooge = ['larry', 'moe', 'curly'], @@ -323,7 +301,7 @@ def test_generic_itemdata(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -337,9 +315,8 @@ def test_generic_itemdata(): assert itemdata == expected -def test_generic_itemdata_too_few(): +def test_generic_itemdata_too_few(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( stooge = ['larry', 'moe'], @@ -348,7 +325,7 @@ def test_generic_itemdata_too_few(): with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -404,9 +381,8 @@ def test_option_from_settings_is_visible_in_base_options(): assert opts['zing'] == 'hit' -def test_url_in_fixed_input_files(): +def test_url_in_fixed_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 url = 'http://www.baz.test' map_options = htmap.MapOptions( @@ -415,7 +391,7 @@ def test_url_in_fixed_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -423,9 +399,8 @@ def test_url_in_fixed_input_files(): assert url in sub['transfer_input_files'] -def test_url_in_input_files(): +def test_url_in_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 url = 'http://www.baz.test' map_options = htmap.MapOptions( @@ -434,7 +409,7 @@ def test_url_in_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -442,9 +417,8 @@ def test_url_in_input_files(): assert url in itemdata[0]['extra_input_files'] -def test_two_urls_in_input_files(): +def test_two_urls_in_input_files(tmp_path): tag = 'foo' - map_dir = Path().cwd() num_components = 1 url_1 = 'http://www.baz.test' url_2 = 'http://www.bong.test' @@ -454,7 +428,7 @@ def test_two_urls_in_input_files(): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, ) @@ -463,9 +437,9 @@ def test_two_urls_in_input_files(): assert url_2 in itemdata[0]['extra_input_files'] -def test_unknown_delivery_mechanism(): +def test_unknown_delivery_mechanism_for_base_descriptors_raises(tmp_path): with pytest.raises(htmap.exceptions.UnknownPythonDeliveryMethod): - get_base_descriptors('foo', Path.cwd(), delivery = 'unknown') + get_base_descriptors('foo', tmp_path, delivery = 'unknown') @pytest.mark.parametrize( @@ -479,9 +453,8 @@ def test_unknown_delivery_mechanism(): 'My.foo', ] ) -def test_custom_options(key): +def test_custom_options(key, tmp_path): tag = 'test' - map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( custom_options = {key: 'bar'}, @@ -489,7 +462,7 @@ def test_custom_options(key): sub, itemdata = create_submit_object_and_itemdata( tag, - map_dir, + tmp_path, num_components, map_options, )