diff --git a/.ci/test.sh b/.ci/test.sh index 13b4ceb..0e62bf8 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -5,6 +5,6 @@ set -e pytest -flake8 creevey tests -black creevey tests --skip-string-normalization --check +flake8 wildebeest tests +black wildebeest tests --skip-string-normalization --check sphinx-build docs docs/_html diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6914e12..7ece5e5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,22 +1,25 @@ -Contributors from outside ShopRunner should feel free to submit a PR without having completed all of the items below. See [CONTRIBUTING.md](https://github.com/ShopRunner/creevey/blob/master/CONTRIBUTING.md) for additional information. +# Pull Request Process -**General Pull Request Checklist** +Contributors from outside ShopRunner should feel free to submit a PR without having completed all of the items below. See [CONTRIBUTING.md](https://github.com/ShopRunner/wildebeest/blob/master/CONTRIBUTING.md) for additional information. - - [ ] Pull request has been made against the `wip` branch (or from `wip` into `master`). - - [ ] Pull request includes a description of the change and the reason behind it. - - [ ] Pull request [uses keywords](https://help.github.com/en/articles/closing-issues-using-keywords) to close relevant [issues](https://github.com/ShopRunner/creevey/issues). - - [ ] Pull request includes unit tests for any bug fixes and new functionality. - - [ ] `./.ci/test.sh` passes locally. - - [ ] Docs have been updated as needed. (`./.ci/test.sh` rebuilds the docs. Open `docs/_html/index.html` to check them.) - -The maintainer will complete the following steps for external contributions. +You can run `./.ci/test.sh` locally to check for style issues, run tests, rebuild docs, etc. before submitting changes. -**Additional Items for ShopRunner Contributors** +## Description - - [ ] `CHANGELOG.md` has been updated. - - [ ] `_version.py` has been updated. - - [ ] Version number has been updated in `docs/conf.py`. +DESCRIBE THE CHANGE AND EXPLAIN THE REASON BEHIND IT -**For merges from `wip` to `master`**:: +## Checklist - - [ ] Confirm that docs build correctly on [readthedocs](https://readthedocs.org/projects/creevey/builds/). +### General + +- [ ] Pull request [uses keywords](https://help.github.com/en/articles/closing-issues-using-keywords) to close relevant [issues](https://github.com/ShopRunner/wildebeest/issues). +- [ ] Pull request includes unit tests for any bug fixes and new functionality. +- [ ] Docs have been updated as needed. (`sphinx-build docs docs/_html` rebuilds the docs. Open `docs/_html/index.html` to check them.) + +### ShopRunner Contributors + +The maintainer will complete the following steps for contributions from outside ShopRunner. + +- [ ] `CHANGELOG.md` has been updated. +- [ ] `_version.py` has been updated. +- [ ] Version number has been updated in `docs/conf.py`. diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..ff7ce85 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,33 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + pull_request: + branches: [ master ] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . + python -m pip install -r requirements-dev.txt + - name: Run checks and build docs + run: .ci/test.sh + - name: Verify build + run: | + pip install --upgrade pip wheel twine + python setup.py sdist bdist_wheel diff --git a/.github/workflows/prod.yaml b/.github/workflows/prod.yaml new file mode 100644 index 0000000..2118024 --- /dev/null +++ b/.github/workflows/prod.yaml @@ -0,0 +1,52 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ master ] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . + python -m pip install -r requirements-dev.txt + - name: Run checks and build docs + run: | + .ci/test.sh + + publish: + + runs-on: ubuntu-latest + needs: [test] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index a396648..8d133df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +#[3.0.0] - 2020-7-28 +### Changed + - Rename library "wildebeest" + - Use GitHub Actions for CI/CD. + +==== Below is Creevey development ==== + +#[3.0.0] - 2020-7-28 +### Added + - Warning about upcoming name change + #[2.3.0] - 2020-7-24 ### Added - Use `pip-tools`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d399b79..a88cf91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ We welcome contributions in the form of issues or pull requests! We want this to be a place where all are welcome to discuss and contribute, so please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. Find the code of conduct in the ``CONDUCT.md`` file on GitHub. -If you have a problem using Creevey or see a possible improvement, open an issue in the GitHub issue tracker. Please be as specific as you can. +If you have a problem using Wildebeest or see a possible improvement, open an issue in the GitHub issue tracker. Please be as specific as you can. If you see an open issue you'd like to be fixed, take a stab at it and open a PR! @@ -16,7 +16,7 @@ If you see an open issue you'd like to be fixed, take a stab at it and open a PR 2. Clone the forked repo to your local disk. ```bash -git clone https://github.com//creevey.git +git clone https://github.com//wildebeest.git ``` 3. `cd` into the directory. @@ -33,11 +33,11 @@ pip install -e . ``` 6. Make your changes. -7. Complete all items in the [pull request checklist](https://github.com/ShopRunner/creevey/blob/master/.github/pull_request_template.md) that you feel comfortable with. Feel free to submit a PR with some loose ends, particularly if you are not sure about how to complete a checklist item. +7. Complete all items in the [pull request checklist](https://github.com/ShopRunner/wildebeest/blob/master/.github/pull_request_template.md) that you feel comfortable with. Feel free to submit a PR with some loose ends, particularly if you are not sure about how to complete a checklist item. 8. Submit your PR against the `wip` branch. ## Additional Notes - We prefer single quotes for one-libe strings unless using double quotes allows us to avoid escaping internal single quotes. - We use [numpy style](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_numpy.html) for docstrings. -- We want Creevey to be compatible with Python 3.6+. +- We want Wildebeest to be compatible with Python 3.6+. diff --git a/README.md b/README.md index 24a26fb..30210ea 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ -# Creevey +# Wildebeest -> **Note 6/15/20:** Our team previously had a tradition of naming projects with terms or characters from the Harry Potter series, but we are disappointed by J.K. Rowling’s persistent transphobic comments. In response, we will be renaming this repository, and are working to develop an inclusive solution that minimizes disruption to our users. +![wildebeest stampede](docs/images/wildebeest_stampede.jpg) -Creevey is a file processing framework. It is designed for IO-bound workflows that involve reading files into memory, doing some processing on their contents, and writing out the results. It makes running those workflows faster and more reliable by parallelizing them across files, optional skipping files that have already been processed, handling errors, and keeping organized records of what happened. +Wildebeest is a file processing framework. It is designed for IO-bound workflows that involve reading files into memory, doing some processing on their contents, and writing out the results. It makes running those workflows faster and more reliable by parallelizing them across files, optional skipping files that have already been processed, handling errors, and keeping organized records of what happened. -Creevey was developed for deep learning computer vision projects, so in addition to the general framework it also provides predefined components for image processing. However, it can be used for any project that involves processing data from many sources. +Wildebeest was developed for deep learning computer vision projects, so in addition to the general framework it also provides predefined components for image processing. However, it can be used for any project that involves processing data from many sources. -See [the docs](https://creevey.readthedocs.io/) for more details. +See [the docs](https://wildebeest.readthedocs.io/) for more details. + +Wildebeest was known as Creevey until version 3.0.0. diff --git a/creevey/__init__.py b/creevey/__init__.py deleted file mode 100644 index 4e19ad7..0000000 --- a/creevey/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from creevey._version import __version__ -from creevey.pipelines import * diff --git a/creevey/_version.py b/creevey/_version.py deleted file mode 100644 index 8219039..0000000 --- a/creevey/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '2.3.0' diff --git a/creevey/load_funcs/__init__.py b/creevey/load_funcs/__init__.py deleted file mode 100644 index 27aa20b..0000000 --- a/creevey/load_funcs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.load_funcs.load_funcs import * diff --git a/creevey/ops/__init__.py b/creevey/ops/__init__.py deleted file mode 100644 index ce379b2..0000000 --- a/creevey/ops/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.ops.helpers import * diff --git a/creevey/ops/helpers/__init__.py b/creevey/ops/helpers/__init__.py deleted file mode 100644 index 9b68dfb..0000000 --- a/creevey/ops/helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.ops.helpers.report import * diff --git a/creevey/ops/image/__init__.py b/creevey/ops/image/__init__.py deleted file mode 100644 index 5ff1fce..0000000 --- a/creevey/ops/image/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from creevey.ops.image.stats import * -from creevey.ops.image.transforms import * diff --git a/creevey/path_funcs/__init__.py b/creevey/path_funcs/__init__.py deleted file mode 100644 index 919c89b..0000000 --- a/creevey/path_funcs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.path_funcs.path_funcs import * diff --git a/creevey/pipelines/__init__.py b/creevey/pipelines/__init__.py deleted file mode 100644 index 8f65ef8..0000000 --- a/creevey/pipelines/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.pipelines.pipelines import * diff --git a/creevey/util/__init__.py b/creevey/util/__init__.py deleted file mode 100644 index de8cc4c..0000000 --- a/creevey/util/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from creevey.util.util import * diff --git a/docs/api.rst b/docs/api.rst index 6b8a376..1b77d06 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,7 +1,7 @@ API === -Generally, each module has a submodule which shares its name that defines generic components and an ``image`` submodule that defines components for working with images. Items in the former are imported into the module namespace, so that you can write e.g. from ``creevey.path_funcs import combine_outdir_dirname_extension`` rather than ``from creevey.path_funcs.path_funcs import combine_outdir_dirname_extension``. +Generally, each module has a submodule which shares its name that defines generic components and an ``image`` submodule that defines components for working with images. Items in the former are imported into the module namespace, so that you can write e.g. from ``wildebeest.path_funcs import combine_outdir_dirname_extension`` rather than ``from wildebeest.path_funcs.path_funcs import combine_outdir_dirname_extension``. pipelines --------- @@ -9,14 +9,14 @@ pipelines pipelines ^^^^^^^^^ -.. automodule:: creevey.pipelines.pipelines +.. automodule:: wildebeest.pipelines.pipelines :special-members: __call__ :members: image ^^^^^ -.. automodule:: creevey.pipelines.image +.. automodule:: wildebeest.pipelines.image :members: load_funcs @@ -25,13 +25,13 @@ load_funcs load_funcs ^^^^^^^^^^ -.. automodule:: creevey.load_funcs.load_funcs +.. automodule:: wildebeest.load_funcs.load_funcs :members: image ^^^^^ -.. automodule:: creevey.load_funcs.image +.. automodule:: wildebeest.load_funcs.image :members: ops @@ -43,7 +43,7 @@ helpers report """""" -.. automodule:: creevey.ops.helpers.report +.. automodule:: wildebeest.ops.helpers.report :members: image @@ -52,13 +52,13 @@ image stats """"" -.. automodule:: creevey.ops.image.stats +.. automodule:: wildebeest.ops.image.stats :members: transforms """""""""" -.. automodule:: creevey.ops.image.transforms +.. automodule:: wildebeest.ops.image.transforms :members: path_funcs @@ -67,7 +67,7 @@ path_funcs path_funcs ^^^^^^^^^^ -.. automodule:: creevey.path_funcs.path_funcs +.. automodule:: wildebeest.path_funcs.path_funcs :members: write_funcs @@ -76,7 +76,7 @@ write_funcs image ^^^^^ -.. automodule:: creevey.write_funcs.image +.. automodule:: wildebeest.write_funcs.image :members: util @@ -85,17 +85,17 @@ util util ^^^^ -.. automodule:: creevey.util.util +.. automodule:: wildebeest.util.util :members: image ^^^^^ -.. automodule:: creevey.util.image +.. automodule:: wildebeest.util.image :members: constants --------- -.. automodule:: creevey.constants +.. automodule:: wildebeest.constants :members: diff --git a/docs/conf.py b/docs/conf.py index 465b925..30102de 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,15 +15,15 @@ import os import sys -sys.path.insert(0, os.path.abspath('../creevey')) +sys.path.insert(0, os.path.abspath('../wildebeest')) # -- Project information ----------------------------------------------------- -project = 'creevey' +project = 'wildebeest' copyright = '2019 Shoprunner' author = 'gsganden, sugi-chan' -release = version = '2.3.0' +release = version = '3.0.0' # -- General configuration --------------------------------------------------- @@ -115,7 +115,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'creeveydoc' +htmlhelp_basename = 'wildebeestdoc' # -- Options for LaTeX output ------------------------------------------------ @@ -141,8 +141,8 @@ latex_documents = [ ( master_doc, - 'creevey.tex', - 'creevey Documentation', + 'wildebeest.tex', + 'wildebeest Documentation', 'gsganden, sugi-chan', 'manual', ) @@ -153,7 +153,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, 'creevey', 'creevey Documentation', [author], 1)] +man_pages = [(master_doc, 'wildebeest', 'wildebeest Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -164,10 +164,10 @@ texinfo_documents = [ ( master_doc, - 'creevey', - 'creevey Documentation', + 'wildebeest', + 'wildebeest Documentation', author, - 'creevey', + 'wildebeest', 'One line description of project.', 'Miscellaneous', ) diff --git a/docs/images/wildebeest_stampede.jpg b/docs/images/wildebeest_stampede.jpg new file mode 100644 index 0000000..329a905 Binary files /dev/null and b/docs/images/wildebeest_stampede.jpg differ diff --git a/docs/index.rst b/docs/index.rst index b95d683..fb82d44 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1,18 @@ -.. creevey documentation master file, created by +.. wildebeest documentation master file, created by sphinx-quickstart on Wed Feb 13 13:07:01 2019. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Creevey -======= +Wildebeest +========== -Creevey is a file processing framework. It is designed for IO-bound workflows that involve reading files into memory, processing their contents, and writing out the results. It makes running those workflows faster and more reliable by parallelizing across files, handling errors, making it easy to skip files that have already been processed, and keeping organized records of what was done. +.. image:: images/wildebeest_stampede.jpg -Creevey was developed for deep learning computer vision projects, so in addition to the general framework it also provides predefined components for image processing. However, it can be used for any project that involves processing data from many sources. +Wildebeest is a file processing framework. It is designed for IO-bound workflows that involve reading files into memory, processing their contents, and writing out the results. It makes running those workflows faster and more reliable by parallelizing across files, handling errors, making it easy to skip files that have already been processed, and keeping organized records of what was done. + +Wildebeest was developed for deep learning computer vision projects, so in addition to the general framework it also provides predefined components for image processing. However, it can be used for any project that involves processing data from many sources. + +Wildebeest was known as Creevey until version 3.0.0. .. toctree:: diff --git a/docs/limitations.rst b/docs/limitations.rst index df7d8e9..e3b045f 100644 --- a/docs/limitations.rst +++ b/docs/limitations.rst @@ -2,6 +2,6 @@ Limitations ----------- -Creevey provides concurrency through threading rather than multiprocessing, which is appropriate for **IO-bound rather than CPU-bound** workflows. +Wildebeest provides concurrency through threading rather than multiprocessing, which is appropriate for **IO-bound rather than CPU-bound** workflows. -**Creevey does not force you to write threadsafe code.** It is intended to be used for workflows in which multiple files are all processed separately and written out to separate locations. This pattern is generally threadsafe, but it has some pitfalls. For instance, if your ``write_func`` checks whether an output directory exists and creates it if it does not, then it can happen that the process switches threads between checking whether the directory exists and attempting to create it, so that the attempt to create it raises an error. (One solution to this problem is to wrap the directory creation step in a ``try/except`` block, as in ``creevey.write_funcs.image.write_image``.) +**Wildebeest does not force you to write threadsafe code.** It is intended to be used for workflows in which multiple files are all processed separately and written out to separate locations. This pattern is generally threadsafe, but it has some pitfalls. For instance, if your ``write_func`` checks whether an output directory exists and creates it if it does not, then it can happen that the process switches threads between checking whether the directory exists and attempting to create it, so that the attempt to create it raises an error. (One solution to this problem is to wrap the directory creation step in a ``try/except`` block, as in ``wildebeest.write_funcs.image.write_image``.) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 330fff2..dc38da4 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -5,16 +5,16 @@ Quickstart Basic Image Download Example ---------------------------- -The following code uses a fairly minimal Creevey pipeline to download a list of images to the current working directory as PNGs, parallelizing across up to ten threads. +The following code uses a fairly minimal Wildebeest pipeline to download a list of images to the current working directory as PNGs, parallelizing across up to ten threads. .. code-block:: python from functools import partial - from creevey import Pipeline - from creevey.load_funcs.image import load_image_from_url - from creevey.path_funcs import join_outdir_filename_extension - from creevey.write_funcs.image import write_image + from wildebeest import Pipeline + from wildebeest.load_funcs.image import load_image_from_url + from wildebeest.path_funcs import join_outdir_filename_extension + from wildebeest.write_funcs.image import write_image image_urls = [ @@ -47,14 +47,14 @@ The trailing underscore in ``run_report_`` indicates that the attribute exists o If ``n_jobs`` is greater than 1, then the order of the input files in the run report typically will not match the order in ``inpaths``\ ; a command like ``run_report.loc[inpaths, :]`` can be used to restore the original order if desired. -Because downloading images is a common use case for Creevey, we have provided a ``download_image_pipeline`` pipeline that you can simply import, resulting in the following simplified version of the code above: +Because downloading images is a common use case for Wildebeest, we have provided a ``download_image_pipeline`` pipeline that you can simply import, resulting in the following simplified version of the code above: .. code-block:: python from functools import partial - from creevey.path_funcs import join_outdir_filename_extension - from creevey.pipelines.image import download_image_pipeline + from wildebeest.path_funcs import join_outdir_filename_extension + from wildebeest.pipelines.image import download_image_pipeline image_urls = [ @@ -81,11 +81,11 @@ The following example processes a few more files with three additional wrinkles: from pathlib import Path from functools import partial - from creevey import Pipeline - from creevey.load_funcs.image import load_image_from_url - from creevey.ops.image import resize - from creevey.write_funcs.image import write_image - from creevey.path_funcs import join_outdir_filename_extension + from wildebeest import Pipeline + from wildebeest.load_funcs.image import load_image_from_url + from wildebeest.ops.image import resize + from wildebeest.write_funcs.image import write_image + from wildebeest.path_funcs import join_outdir_filename_extension image_urls += [ @@ -129,7 +129,7 @@ We could simplify the code above by using the provided ``download_image_pipeline .. code-block:: python - from creevey.pipelines.image import download_image_pipeline + from wildebeest.pipelines.image import download_image_pipeline trim_resize_pipeline = download_image_pipeline trim_resize_pipeline.ops = [ @@ -143,7 +143,7 @@ More generally, you can modify attributes of an existing ``Pipeline`` object. Another Use for ``skip_func`` ----------------------------- -``skip_func=lambda inpath, outpath: Path(outpath).is_file()`` often makes sense when you are writing files locally and don't want to repeat work. However, suppose you are using Creevey to copy files from one S3 bucket to another. You could write a function to check whether a file exists at the output URL (e.g. ``requests.head(outpath).status_code < 400``), but in my testing running that function was slower than copying the file. An alternative approach is to make a set of all of the files in the output bucket before running the pipeline and pass ``skip_func`` a function that checks whether ``outpath`` is in that set. +``skip_func=lambda inpath, outpath: Path(outpath).is_file()`` often makes sense when you are writing files locally and don't want to repeat work. However, suppose you are using Wildebeest to copy files from one S3 bucket to another. You could write a function to check whether a file exists at the output URL (e.g. ``requests.head(outpath).status_code < 400``), but in my testing running that function was slower than copying the file. An alternative approach is to make a set of all of the files in the output bucket before running the pipeline and pass ``skip_func`` a function that checks whether ``outpath`` is in that set. More generally, (1) ``skip_func`` is not limited to checking whether the output file already exists locally, and (2) generating a set of paths to skip up front and checking against that set with ``skip_func`` may be faster than performing some operation on each path on the fly to decide whether to skip it. @@ -161,12 +161,12 @@ The ``CustomReportingPipeline`` class allows you to add additional information t from functools import partial - from creevey import CustomReportingPipeline - from creevey.load_funcs.image import load_image_from_url - from creevey.ops import get_report_output_decorator - from creevey.ops.image import report_mean_brightness - from creevey.path_funcs import join_outdir_filename_extension - from creevey.write_funcs.image import write_image + from wildebeest import CustomReportingPipeline + from wildebeest.load_funcs.image import load_image_from_url + from wildebeest.ops import get_report_output_decorator + from wildebeest.ops.image import report_mean_brightness + from wildebeest.path_funcs import join_outdir_filename_extension + from wildebeest.write_funcs.image import write_image @get_report_output_decorator(key="is_grayscale") @@ -197,15 +197,15 @@ Note that if we use ``skip_func`` to skip files in a ``CustomReportingPipeline`` Text Scraping Example --------------------- -Creevey is not limited to images! It applies anywhere you want to process data from many sources. For instance, we can use it to scrape online text. The example below uses it to get titles and crude word counts for four blog posts. +Wildebeest is not limited to images! It applies anywhere you want to process data from many sources. For instance, we can use it to scrape online text. The example below uses it to get titles and crude word counts for four blog posts. .. code-block:: python import re import urllib.request - from creevey import CustomReportingPipeline - from creevey.ops import get_report_output_decorator + from wildebeest import CustomReportingPipeline + from wildebeest.ops import get_report_output_decorator URLS = [ "http://gandenberger.org/2019/10/29/evaluating-classification-models-part-1-weighing-false-positives-against-false-negatives/", diff --git a/requirements-dev.in b/requirements-dev.in index 42f3544..56aa4d5 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -4,8 +4,12 @@ black flake8 flake8-docstrings flake8-import-order +isort +git+https://github.com/crossnox/m2r@dev#egg=m2r # See https://github.com/sphinx-doc/sphinx/issues/7420 matplotlib +pip-tools pre-commit pytest pytest-cov responses +sphinx_autodoc_typehints diff --git a/requirements-dev.txt b/requirements-dev.txt index d6d1eef..d6c5299 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,48 +4,68 @@ # # pip-compile requirements-dev.in # +alabaster==0.7.12 # via sphinx appdirs==1.4.4 # via black, virtualenv attrs==19.3.0 # via black, pytest +babel==2.8.0 # via sphinx black==19.10b0 # via -r requirements-dev.in certifi==2020.6.20 # via -c requirements.txt, requests cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via -c requirements.txt, requests -click==7.1.2 # via black +click==7.1.2 # via black, pip-tools coverage==5.2.1 # via pytest-cov cycler==0.10.0 # via matplotlib distlib==0.3.1 # via virtualenv +docutils==0.15.2 # via -c requirements.txt, m2r, sphinx filelock==3.0.12 # via virtualenv flake8-docstrings==1.5.0 # via -r requirements-dev.in flake8-import-order==0.18.1 # via -r requirements-dev.in flake8==3.8.3 # via -r requirements-dev.in, flake8-docstrings identify==1.4.25 # via pre-commit idna==2.10 # via -c requirements.txt, requests +imagesize==1.2.0 # via sphinx importlib-metadata==1.7.0 # via flake8, pluggy, pre-commit, pytest, virtualenv +isort==5.2.0 # via -r requirements-dev.in +jinja2==2.11.2 # via sphinx kiwisolver==1.2.0 # via matplotlib +git+https://github.com/crossnox/m2r@dev#egg=m2r # via -r requirements-dev.in +markupsafe==1.1.1 # via jinja2 matplotlib==3.3.0 # via -r requirements-dev.in mccabe==0.6.1 # via flake8 +mistune==0.8.4 # via m2r more-itertools==8.4.0 # via pytest nodeenv==1.4.0 # via pre-commit numpy==1.19.1 # via -c requirements.txt, matplotlib -packaging==20.4 # via pytest +packaging==20.4 # via pytest, sphinx pathspec==0.8.0 # via black pillow==7.2.0 # via -c requirements.txt, matplotlib +pip-tools==5.3.0 # via -r requirements-dev.in pluggy==0.13.1 # via pytest pre-commit==2.6.0 # via -r requirements-dev.in py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8, flake8-import-order pydocstyle==5.0.2 # via flake8-docstrings pyflakes==2.2.0 # via flake8 +pygments==2.6.1 # via sphinx pyparsing==2.4.7 # via matplotlib, packaging pytest-cov==2.10.0 # via -r requirements-dev.in pytest==5.4.3 # via -r requirements-dev.in, pytest-cov python-dateutil==2.8.1 # via -c requirements.txt, matplotlib +pytz==2020.1 # via -c requirements.txt, babel pyyaml==5.3.1 # via pre-commit regex==2020.7.14 # via black -requests==2.24.0 # via -c requirements.txt, responses +requests==2.24.0 # via -c requirements.txt, responses, sphinx responses==0.10.15 # via -r requirements-dev.in -six==1.15.0 # via -c requirements.txt, cycler, packaging, python-dateutil, responses, virtualenv -snowballstemmer==2.0.0 # via pydocstyle +six==1.15.0 # via -c requirements.txt, cycler, packaging, pip-tools, python-dateutil, responses, virtualenv +snowballstemmer==2.0.0 # via pydocstyle, sphinx +sphinx-autodoc-typehints==1.11.0 # via -r requirements-dev.in +sphinx==3.1.2 # via sphinx-autodoc-typehints +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx toml==0.10.1 # via black, pre-commit typed-ast==1.4.1 # via black urllib3==1.25.10 # via -c requirements.txt, requests @@ -54,4 +74,5 @@ wcwidth==0.2.5 # via pytest zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: +# pip # setuptools diff --git a/requirements-read-the-docs.txt b/requirements-read-the-docs.txt deleted file mode 100644 index 4ac5b1c..0000000 --- a/requirements-read-the-docs.txt +++ /dev/null @@ -1,3 +0,0 @@ -git+https://github.com/crossnox/m2r@dev#egg=m2r -sphinx_autodoc_typehints -. diff --git a/requirements.txt b/requirements.txt index 7b16eda..73da970 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,28 +2,28 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile +# pip-compile --output-file=requirements.txt # -boto3==1.14.27 # via creevey (setup.py) +boto3==1.14.27 # via wildebeest (setup.py) botocore==1.17.27 # via boto3, s3transfer certifi==2020.6.20 # via requests chardet==3.0.4 # via requests docutils==0.15.2 # via botocore idna==2.10 # via requests jmespath==0.10.0 # via boto3, botocore -joblib==0.16.0 # via creevey (setup.py), scikit-learn +joblib==0.16.0 # via scikit-learn, wildebeest (setup.py) numpy==1.19.1 # via opencv-contrib-python, pandas, scikit-learn, scipy -opencv-contrib-python==4.3.0.36 # via creevey (setup.py) -pandas==1.0.5 # via creevey (setup.py) -pillow==7.2.0 # via creevey (setup.py) +opencv-contrib-python==4.3.0.36 # via wildebeest (setup.py) +pandas==1.0.5 # via wildebeest (setup.py) +pillow==7.2.0 # via wildebeest (setup.py) python-dateutil==2.8.1 # via botocore, pandas pytz==2020.1 # via pandas -requests==2.24.0 # via creevey (setup.py) -retrying==1.3.3 # via creevey (setup.py) +requests==2.24.0 # via wildebeest (setup.py) +retrying==1.3.3 # via wildebeest (setup.py) s3transfer==0.3.3 # via boto3 -scikit-learn==0.23.1 # via creevey (setup.py) +scikit-learn==0.23.1 # via wildebeest (setup.py) scipy==1.5.2 # via scikit-learn six==1.15.0 # via python-dateutil, retrying threadpoolctl==2.1.0 # via scikit-learn -tqdm==4.48.0 # via creevey (setup.py) +tqdm==4.48.0 # via wildebeest (setup.py) urllib3==1.25.10 # via botocore, requests diff --git a/setup.cfg b/setup.cfg index 6d9f008..9ca0449 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ description-file = README.md [flake8] max-line-length = 110 -application-import-names = creevey, tests +application-import-names = wildebeest, tests import-order-style = appnexus exclude = tests/conftest.py, @@ -31,4 +31,7 @@ ignore = per-file-ignores = # not requiring docstrings for tests tests/*: D100, D101, D102, D103 - creevey/_version.py: D100 + wildebeest/_version.py: D100 + +[isort] +known_first_party=wildebeest, tests diff --git a/setup.py b/setup.py index 890d74f..8165a1f 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from pathlib import Path from setuptools import find_packages, setup -version_file_path = Path(__file__).parent / 'creevey' / '_version.py' +version_file_path = Path(__file__).parent / 'wildebeest' / '_version.py' with open(version_file_path, 'r') as version_file: exec(version_file.read()) @@ -33,7 +33,7 @@ setup( - name='creevey', + name='wildebeest', version=__version__, description='Bulk image processing', long_description=readme, @@ -42,7 +42,7 @@ extras_require={'dev': regular_packages + dev_packages}, author='Greg Gandenberger', author_email='gsganden@gmail.com', - url='https://github.com/ShopRunner/creevey', - download_url=f'https://github.com/ShopRunner/creevey/tarball/{__version__}', + url='https://github.com/ShopRunner/wildebeest', + download_url=f'https://github.com/ShopRunner/wildebeest/tarball/{__version__}', long_description_content_type='text/markdown', ) diff --git a/tests/__init__.py b/tests/__init__.py index fc61fe5..ca0e42e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,3 @@ import logging - logging.basicConfig(level=logging.INFO) diff --git a/tests/conftest.py b/tests/conftest.py index da7f256..0e848b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,8 @@ from functools import partial -import sys from pathlib import Path +import sys -from creevey.path_funcs import join_outdir_filename_extension +from wildebeest.path_funcs import join_outdir_filename_extension TEST_DIR = Path(__file__).parent SAMPLE_DATA_DIR = Path(TEST_DIR) / 'sample_data' @@ -12,7 +12,6 @@ from tests.fixtures import * - sys.path.append(TEST_DIR / 'fixtures') keep_filename_save_png_in_tempdir = partial( diff --git a/tests/fixtures/image.py b/tests/fixtures/image.py index 4030cf6..4c9e064 100644 --- a/tests/fixtures/image.py +++ b/tests/fixtures/image.py @@ -1,130 +1,130 @@ import pytest -from creevey.load_funcs.image import load_image_from_disk from tests.conftest import SAMPLE_DATA_DIR +from wildebeest.load_funcs.image import load_image_from_disk @pytest.fixture def sample_image_square_rgb(): - filename = 'creevey_rgb.jpg' + filename = 'wildebeest_rgb.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_tall_grayscale(): - filename = 'creevey_gray.jpg' + filename = 'wildebeest_gray.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba(): - filename = 'creevey_rgba.png' + filename = 'wildebeest_rgba.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba_flipped_horiz(): - filename = 'creevey_rgba_flipped_horiz.png' + filename = 'wildebeest_rgba_flipped_horiz.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba_flipped_vert(): - filename = 'creevey_rgba_flipped_vert.png' + filename = 'wildebeest_rgba_flipped_vert.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgb_flipped_horiz(): - filename = 'creevey_rgb_flipped_horiz.jpg' + filename = 'wildebeest_rgb_flipped_horiz.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgb_flipped_vert(): - filename = 'creevey_rgb_flipped_vert.jpg' + filename = 'wildebeest_rgb_flipped_vert.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_grayscale_flipped_horiz(): - filename = 'creevey_gray_flipped_horiz.jpg' + filename = 'wildebeest_gray_flipped_horiz.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_grayscale_flipped_vert(): - filename = 'creevey_gray_flipped_vert.jpg' + filename = 'wildebeest_gray_flipped_vert.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_tall_grayscale_rotated_90(): - filename = 'creevey_gray_rotated_90.jpg' + filename = 'wildebeest_gray_rotated_90.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgb_rotated_90(): - filename = 'creevey_rgb_rotated_90.jpg' + filename = 'wildebeest_rgb_rotated_90.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba_rotated_90(): - filename = 'creevey_rgba_rotated_90.png' + filename = 'wildebeest_rgba_rotated_90.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_tall_grayscale_rotated_180(): - filename = 'creevey_gray_rotated_180.jpg' + filename = 'wildebeest_gray_rotated_180.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgb_rotated_180(): - filename = 'creevey_rgb_rotated_180.jpg' + filename = 'wildebeest_rgb_rotated_180.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba_rotated_180(): - filename = 'creevey_rgba_rotated_180.png' + filename = 'wildebeest_rgba_rotated_180.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_tall_grayscale_rotated_270(): - filename = 'creevey_gray_rotated_270.jpg' + filename = 'wildebeest_gray_rotated_270.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgb_rotated_270(): - filename = 'creevey_rgb_rotated_270.jpg' + filename = 'wildebeest_rgb_rotated_270.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture def sample_image_square_rgba_rotated_270(): - filename = 'creevey_rgba_rotated_270.png' + filename = 'wildebeest_rgba_rotated_270.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) diff --git a/tests/sample_data/creevey_gray.jpg b/tests/sample_data/creevey_gray.jpg deleted file mode 100644 index dc43156..0000000 Binary files a/tests/sample_data/creevey_gray.jpg and /dev/null differ diff --git a/tests/sample_data/creevey_rgb_no_extension b/tests/sample_data/creevey_rgb_no_extension deleted file mode 100644 index 9b642e9..0000000 Binary files a/tests/sample_data/creevey_rgb_no_extension and /dev/null differ diff --git a/tests/sample_data/creevey_rgba.png b/tests/sample_data/creevey_rgba.png deleted file mode 100644 index dfab4c4..0000000 Binary files a/tests/sample_data/creevey_rgba.png and /dev/null differ diff --git a/tests/sample_data/creevey_gray_flipped_horiz.jpg b/tests/sample_data/wildebeest_gray_flipped_horiz.jpg similarity index 100% rename from tests/sample_data/creevey_gray_flipped_horiz.jpg rename to tests/sample_data/wildebeest_gray_flipped_horiz.jpg diff --git a/tests/sample_data/creevey_gray_flipped_vert.jpg b/tests/sample_data/wildebeest_gray_flipped_vert.jpg similarity index 100% rename from tests/sample_data/creevey_gray_flipped_vert.jpg rename to tests/sample_data/wildebeest_gray_flipped_vert.jpg diff --git a/tests/sample_data/creevey_gray_rotated_180.jpg b/tests/sample_data/wildebeest_gray_rotated_180.jpg similarity index 100% rename from tests/sample_data/creevey_gray_rotated_180.jpg rename to tests/sample_data/wildebeest_gray_rotated_180.jpg diff --git a/tests/sample_data/creevey_gray_rotated_270.jpg b/tests/sample_data/wildebeest_gray_rotated_270.jpg similarity index 100% rename from tests/sample_data/creevey_gray_rotated_270.jpg rename to tests/sample_data/wildebeest_gray_rotated_270.jpg diff --git a/tests/sample_data/creevey_gray_rotated_90.jpg b/tests/sample_data/wildebeest_gray_rotated_90.jpg similarity index 100% rename from tests/sample_data/creevey_gray_rotated_90.jpg rename to tests/sample_data/wildebeest_gray_rotated_90.jpg diff --git a/tests/sample_data/creevey_rgb_flipped_horiz.jpg b/tests/sample_data/wildebeest_rgb_flipped_horiz.jpg similarity index 100% rename from tests/sample_data/creevey_rgb_flipped_horiz.jpg rename to tests/sample_data/wildebeest_rgb_flipped_horiz.jpg diff --git a/tests/sample_data/creevey_rgb_flipped_vert.jpg b/tests/sample_data/wildebeest_rgb_flipped_vert.jpg similarity index 100% rename from tests/sample_data/creevey_rgb_flipped_vert.jpg rename to tests/sample_data/wildebeest_rgb_flipped_vert.jpg diff --git a/tests/sample_data/creevey_rgb.jpg b/tests/sample_data/wildebeest_rgb_no_extension similarity index 100% rename from tests/sample_data/creevey_rgb.jpg rename to tests/sample_data/wildebeest_rgb_no_extension diff --git a/tests/sample_data/creevey_rgb_rotated_180.jpg b/tests/sample_data/wildebeest_rgb_rotated_180.jpg similarity index 100% rename from tests/sample_data/creevey_rgb_rotated_180.jpg rename to tests/sample_data/wildebeest_rgb_rotated_180.jpg diff --git a/tests/sample_data/creevey_rgb_rotated_270.jpg b/tests/sample_data/wildebeest_rgb_rotated_270.jpg similarity index 100% rename from tests/sample_data/creevey_rgb_rotated_270.jpg rename to tests/sample_data/wildebeest_rgb_rotated_270.jpg diff --git a/tests/sample_data/creevey_rgb_rotated_90.jpg b/tests/sample_data/wildebeest_rgb_rotated_90.jpg similarity index 100% rename from tests/sample_data/creevey_rgb_rotated_90.jpg rename to tests/sample_data/wildebeest_rgb_rotated_90.jpg diff --git a/tests/sample_data/creevey_rgba_flipped_horiz.png b/tests/sample_data/wildebeest_rgba_flipped_horiz.png similarity index 100% rename from tests/sample_data/creevey_rgba_flipped_horiz.png rename to tests/sample_data/wildebeest_rgba_flipped_horiz.png diff --git a/tests/sample_data/creevey_rgba_flipped_vert.png b/tests/sample_data/wildebeest_rgba_flipped_vert.png similarity index 100% rename from tests/sample_data/creevey_rgba_flipped_vert.png rename to tests/sample_data/wildebeest_rgba_flipped_vert.png diff --git a/tests/sample_data/creevey_rgba_rotated_180.png b/tests/sample_data/wildebeest_rgba_rotated_180.png similarity index 100% rename from tests/sample_data/creevey_rgba_rotated_180.png rename to tests/sample_data/wildebeest_rgba_rotated_180.png diff --git a/tests/sample_data/creevey_rgba_rotated_270.png b/tests/sample_data/wildebeest_rgba_rotated_270.png similarity index 100% rename from tests/sample_data/creevey_rgba_rotated_270.png rename to tests/sample_data/wildebeest_rgba_rotated_270.png diff --git a/tests/sample_data/creevey_rgba_rotated_90.png b/tests/sample_data/wildebeest_rgba_rotated_90.png similarity index 100% rename from tests/sample_data/creevey_rgba_rotated_90.png rename to tests/sample_data/wildebeest_rgba_rotated_90.png diff --git a/tests/test_load_funcs/test_load_funcs_image.py b/tests/test_load_funcs/test_load_funcs_image.py index fc3093e..c5b1f28 100644 --- a/tests/test_load_funcs/test_load_funcs_image.py +++ b/tests/test_load_funcs/test_load_funcs_image.py @@ -5,17 +5,17 @@ from requests.exceptions import HTTPError import responses -from creevey.load_funcs.image import load_image_from_disk, load_image_from_url from tests.conftest import SAMPLE_DATA_DIR +from wildebeest.load_funcs.image import load_image_from_disk, load_image_from_url SAMPLE_DATA_BASE_URL = ( - 'https://github.com/ShopRunner/creevey/raw/master/tests/sample_data' + 'https://github.com/ShopRunner/wildebeest/raw/master/tests/sample_data' ) @responses.activate def test_load_error_406(): - filename = 'creevey_gray.jpg' + filename = 'wildebeest_gray.jpg' url = f'{SAMPLE_DATA_BASE_URL}/{filename}' responses.add(responses.GET, url, status=406) @@ -28,49 +28,49 @@ def test_load_error_406(): @pytest.fixture -def creevey_gray_local(): - filename = 'creevey_gray.jpg' +def wildebeest_gray_local(): + filename = 'wildebeest_gray.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture -def creevey_gray_remote(): - filename = 'creevey_gray.jpg' +def wildebeest_gray_remote(): + filename = 'wildebeest_gray.jpg' url = f'{SAMPLE_DATA_BASE_URL}/{filename}' return load_image_from_url(url) @pytest.fixture -def creevey_rgb_local(): - filename = 'creevey_rgb.jpg' +def wildebeest_rgb_local(): + filename = 'wildebeest_rgb.jpg' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture -def creevey_rgb_remote(): - filename = 'creevey_rgb.jpg' +def wildebeest_rgb_remote(): + filename = 'wildebeest_rgb.jpg' url = f'{SAMPLE_DATA_BASE_URL}/{filename}' return load_image_from_url(url) @pytest.fixture -def creevey_rgba_local(): - filename = 'creevey_rgba.png' +def wildebeest_rgba_local(): + filename = 'wildebeest_rgba.png' path = SAMPLE_DATA_DIR / filename return load_image_from_disk(path) @pytest.fixture -def creevey_rgba_remote(): - filename = 'creevey_rgba.png' +def wildebeest_rgba_remote(): + filename = 'wildebeest_rgba.png' url = f'{SAMPLE_DATA_BASE_URL}/{filename}' return load_image_from_url(url) -def test_load_image_from_disk_grayscale_shape(creevey_gray_local): - image = creevey_gray_local +def test_load_image_from_disk_grayscale_shape(wildebeest_gray_local): + image = wildebeest_gray_local expected_shape = (34, 25) actual_shape = image.shape @@ -78,8 +78,8 @@ def test_load_image_from_disk_grayscale_shape(creevey_gray_local): assert actual_shape == expected_shape -def test_load_image_from_disk_rgb_shape(creevey_rgb_local): - image = creevey_rgb_local +def test_load_image_from_disk_rgb_shape(wildebeest_rgb_local): + image = wildebeest_rgb_local expected_shape = (32, 32, 3) actual_shape = image.shape @@ -87,8 +87,8 @@ def test_load_image_from_disk_rgb_shape(creevey_rgb_local): assert actual_shape == expected_shape -def test_load_image_from_disk_rgba_shape(creevey_rgba_local): - image = creevey_rgba_local +def test_load_image_from_disk_rgba_shape(wildebeest_rgba_local): + image = wildebeest_rgba_local expected_shape = (32, 32, 4) actual_shape = image.shape @@ -129,13 +129,13 @@ def test_load_image_from_disk_blue(): np.testing.assert_equal(actual_image, expected_image) -def test_load_image_from_url_gray(creevey_gray_local, creevey_gray_remote): - np.testing.assert_equal(creevey_gray_remote, creevey_gray_local) +def test_load_image_from_url_gray(wildebeest_gray_local, wildebeest_gray_remote): + np.testing.assert_equal(wildebeest_gray_remote, wildebeest_gray_local) -def test_load_image_from_url_rgb(creevey_rgb_local, creevey_rgb_remote): - np.testing.assert_equal(creevey_rgb_local, creevey_rgb_remote) +def test_load_image_from_url_rgb(wildebeest_rgb_local, wildebeest_rgb_remote): + np.testing.assert_equal(wildebeest_rgb_local, wildebeest_rgb_remote) -def test_load_image_from_url_rgba(creevey_rgba_local, creevey_rgba_remote): - np.testing.assert_equal(creevey_rgba_local, creevey_rgba_remote) +def test_load_image_from_url_rgba(wildebeest_rgba_local, wildebeest_rgba_remote): + np.testing.assert_equal(wildebeest_rgba_local, wildebeest_rgba_remote) diff --git a/tests/test_ops/test_helpers/test_report.py b/tests/test_ops/test_helpers/test_report.py index 0ae912a..441384d 100644 --- a/tests/test_ops/test_helpers/test_report.py +++ b/tests/test_ops/test_helpers/test_report.py @@ -1,6 +1,6 @@ from collections import defaultdict -from creevey.ops import get_report_output_decorator, report_output +from wildebeest.ops import get_report_output_decorator, report_output def test_report_output(): diff --git a/tests/test_ops/test_image/test_stats.py b/tests/test_ops/test_image/test_stats.py index 2bc3070..dcad076 100644 --- a/tests/test_ops/test_image/test_stats.py +++ b/tests/test_ops/test_image/test_stats.py @@ -2,14 +2,14 @@ import numpy as np -from creevey.ops.image.stats import ( +from tests.conftest import SAMPLE_DATA_DIR +from wildebeest.ops.image.stats import ( calculate_dhash, calculate_mean_brightness, report_dhash, report_mean_brightness, ) -from creevey.ops.image.transforms import resize -from tests.conftest import SAMPLE_DATA_DIR +from wildebeest.ops.image.transforms import resize class TestCalculateMeanBrightness: @@ -29,7 +29,7 @@ def test_calculate_mean_brightness_rgba(self, sample_image_square_rgba): class TestReportMeanBrightness: def test_report_mean_brightness(self, sample_image_square_rgb): log_dict = defaultdict(dict) - image_path = SAMPLE_DATA_DIR / 'creevey_rgb.jpg' + image_path = SAMPLE_DATA_DIR / 'wildebeest_rgb.jpg' result = report_mean_brightness( sample_image_square_rgb, log_dict=log_dict, inpath=image_path ) @@ -92,7 +92,7 @@ def _hashlen(hash): class TestReportDHash: def test_report_dhash(self, sample_image_square_rgb): log_dict = defaultdict(dict) - image_path = SAMPLE_DATA_DIR / 'creevey_rgb.jpg' + image_path = SAMPLE_DATA_DIR / 'wildebeest_rgb.jpg' result = report_dhash( sample_image_square_rgb, inpath=image_path, log_dict=log_dict ) diff --git a/tests/test_ops/test_image/test_transforms.py b/tests/test_ops/test_image/test_transforms.py index 43e148a..ee2c678 100644 --- a/tests/test_ops/test_image/test_transforms.py +++ b/tests/test_ops/test_image/test_transforms.py @@ -4,7 +4,7 @@ import numpy as np import pytest -from creevey.ops.image import ( +from wildebeest.ops.image import ( centercrop, flip_horiz, flip_vert, diff --git a/tests/test_path_funcs/test_path_funcs.py b/tests/test_path_funcs/test_path_funcs.py index 7c4238f..fe152cf 100644 --- a/tests/test_path_funcs/test_path_funcs.py +++ b/tests/test_path_funcs/test_path_funcs.py @@ -1,6 +1,6 @@ from pathlib import Path -from creevey.path_funcs import ( +from wildebeest.path_funcs import ( join_outdir_filename_extension, join_outdir_hashed_path_extension, replace_dir, diff --git a/tests/test_pipelines/test_pipelines.py b/tests/test_pipelines/test_pipelines.py index a353409..e9f346b 100644 --- a/tests/test_pipelines/test_pipelines.py +++ b/tests/test_pipelines/test_pipelines.py @@ -5,11 +5,6 @@ import pytest import requests -from creevey import CustomReportingPipeline, Pipeline -from creevey.load_funcs.image import load_image_from_url -from creevey.ops import get_report_output_decorator -from creevey.ops.image import calculate_mean_brightness -from creevey.write_funcs.image import write_image from tests.conftest import ( delete_file_if_exists, IMAGE_FILENAMES, @@ -17,6 +12,11 @@ keep_filename_save_png_in_tempdir, TEMP_DATA_DIR, ) +from wildebeest import CustomReportingPipeline, Pipeline +from wildebeest.load_funcs.image import load_image_from_url +from wildebeest.ops import get_report_output_decorator +from wildebeest.ops.image import calculate_mean_brightness +from wildebeest.write_funcs.image import write_image @get_report_output_decorator(key='mean_brightness') diff --git a/tests/test_pipelines/test_pipelines_image.py b/tests/test_pipelines/test_pipelines_image.py index 8a0a15c..04e1e03 100644 --- a/tests/test_pipelines/test_pipelines_image.py +++ b/tests/test_pipelines/test_pipelines_image.py @@ -5,10 +5,6 @@ import pandas as pd import pytest -from creevey import Pipeline -from creevey.load_funcs.image import load_image_from_url -from creevey.ops.image import resize -from creevey.write_funcs.image import write_image from tests.conftest import ( delete_file_if_exists, IMAGE_FILENAMES, @@ -16,6 +12,10 @@ keep_filename_save_png_in_tempdir, TEMP_DATA_DIR, ) +from wildebeest import Pipeline +from wildebeest.load_funcs.image import load_image_from_url +from wildebeest.ops.image import resize +from wildebeest.write_funcs.image import write_image IMAGE_RESIZE_SHAPE = (224, 224) diff --git a/tests/test_util/test_util.py b/tests/test_util/test_util.py index cd49bee..8af74a7 100644 --- a/tests/test_util/test_util.py +++ b/tests/test_util/test_util.py @@ -1,5 +1,5 @@ -from creevey.util import find_files_with_extensions from tests.conftest import TEMP_DATA_DIR +from wildebeest.util import find_files_with_extensions def test_find_files_with_extensions_without_dot(generate_file_tree): # noqa: F811 diff --git a/tests/test_util/test_util_image.py b/tests/test_util/test_util_image.py index fb23cf4..acba2d5 100644 --- a/tests/test_util/test_util_image.py +++ b/tests/test_util/test_util_image.py @@ -1,5 +1,5 @@ -from creevey.util.image import find_image_files from tests.conftest import TEMP_DATA_DIR +from wildebeest.util.image import find_image_files def test_find_images(generate_file_tree): # noqa: F811 diff --git a/wildebeest/__init__.py b/wildebeest/__init__.py new file mode 100644 index 0000000..14707c2 --- /dev/null +++ b/wildebeest/__init__.py @@ -0,0 +1,2 @@ +from wildebeest._version import __version__ +from wildebeest.pipelines import * diff --git a/wildebeest/_version.py b/wildebeest/_version.py new file mode 100644 index 0000000..4eb28e3 --- /dev/null +++ b/wildebeest/_version.py @@ -0,0 +1 @@ +__version__ = '3.0.0' diff --git a/creevey/constants.py b/wildebeest/constants.py similarity index 100% rename from creevey/constants.py rename to wildebeest/constants.py diff --git a/wildebeest/load_funcs/__init__.py b/wildebeest/load_funcs/__init__.py new file mode 100644 index 0000000..c28e317 --- /dev/null +++ b/wildebeest/load_funcs/__init__.py @@ -0,0 +1 @@ +from wildebeest.load_funcs.load_funcs import * diff --git a/creevey/load_funcs/image.py b/wildebeest/load_funcs/image.py similarity index 95% rename from creevey/load_funcs/image.py rename to wildebeest/load_funcs/image.py index 4cc0ce2..8bf849c 100644 --- a/creevey/load_funcs/image.py +++ b/wildebeest/load_funcs/image.py @@ -2,8 +2,8 @@ import cv2 as cv import numpy as np -from creevey.constants import PathOrStr -from creevey.load_funcs import get_response +from wildebeest.constants import PathOrStr +from wildebeest.load_funcs import get_response def load_image_from_url(inpath: str, **kwargs) -> np.array: diff --git a/creevey/load_funcs/load_funcs.py b/wildebeest/load_funcs/load_funcs.py similarity index 100% rename from creevey/load_funcs/load_funcs.py rename to wildebeest/load_funcs/load_funcs.py diff --git a/wildebeest/ops/__init__.py b/wildebeest/ops/__init__.py new file mode 100644 index 0000000..ed571ba --- /dev/null +++ b/wildebeest/ops/__init__.py @@ -0,0 +1 @@ +from wildebeest.ops.helpers import * diff --git a/wildebeest/ops/helpers/__init__.py b/wildebeest/ops/helpers/__init__.py new file mode 100644 index 0000000..4a7a0be --- /dev/null +++ b/wildebeest/ops/helpers/__init__.py @@ -0,0 +1 @@ +from wildebeest.ops.helpers.report import * diff --git a/creevey/ops/helpers/report.py b/wildebeest/ops/helpers/report.py similarity index 85% rename from creevey/ops/helpers/report.py rename to wildebeest/ops/helpers/report.py index 018619f..8944279 100644 --- a/creevey/ops/helpers/report.py +++ b/wildebeest/ops/helpers/report.py @@ -2,7 +2,7 @@ from functools import partial from typing import Any, Callable, DefaultDict, Hashable -from creevey.constants import PathOrStr +from wildebeest.constants import PathOrStr def report_output( @@ -25,12 +25,12 @@ def report_output( -------- >>> from functools import partial >>> - >>> from creevey import CustomReportingPipeline - >>> from creevey.load_funcs.image import load_image_from_url - >>> from creevey.ops import report_output - >>> from creevey.ops.image import calculate_mean_brightness - >>> from creevey.path_funcs import join_outdir_filename_extension - >>> from creevey.write_funcs.image import write_image + >>> from wildebeest import CustomReportingPipeline + >>> from wildebeest.load_funcs.image import load_image_from_url + >>> from wildebeest.ops import report_output + >>> from wildebeest.ops.image import calculate_mean_brightness + >>> from wildebeest.path_funcs import join_outdir_filename_extension + >>> from wildebeest.write_funcs.image import write_image >>> >>> report_mean_brightness = partial( >>> report_output, func=calculate_mean_brightness, key='mean_brightness' @@ -102,12 +102,12 @@ def get_report_output_decorator(key: Hashable) -> Callable: -------- >>> from functools import partial >>> - >>> from creevey import CustomReportingPipeline - >>> from creevey.load_funcs.image import load_image_from_url - >>> from creevey.ops import get_report_output_decorator - >>> from creevey.ops.image import calculate_mean_brightness - >>> from creevey.path_funcs import join_outdir_filename_extension - >>> from creevey.write_funcs.image import write_image + >>> from wildebeest import CustomReportingPipeline + >>> from wildebeest.load_funcs.image import load_image_from_url + >>> from wildebeest.ops import get_report_output_decorator + >>> from wildebeest.ops.image import calculate_mean_brightness + >>> from wildebeest.path_funcs import join_outdir_filename_extension + >>> from wildebeest.write_funcs.image import write_image >>> >>> >>> @get_report_output_decorator(key='mean_brightness') diff --git a/wildebeest/ops/image/__init__.py b/wildebeest/ops/image/__init__.py new file mode 100644 index 0000000..02b73c9 --- /dev/null +++ b/wildebeest/ops/image/__init__.py @@ -0,0 +1,2 @@ +from wildebeest.ops.image.stats import * +from wildebeest.ops.image.transforms import * diff --git a/creevey/ops/image/stats.py b/wildebeest/ops/image/stats.py similarity index 93% rename from creevey/ops/image/stats.py rename to wildebeest/ops/image/stats.py index 066fa55..a47fca1 100644 --- a/creevey/ops/image/stats.py +++ b/wildebeest/ops/image/stats.py @@ -2,8 +2,8 @@ import cv2 as cv import numpy as np -from creevey.ops.helpers.report import get_report_output_decorator -from creevey.ops.image.transforms import convert_to_grayscale +from wildebeest.ops.helpers.report import get_report_output_decorator +from wildebeest.ops.image.transforms import convert_to_grayscale def calculate_mean_brightness(image: np.array) -> float: diff --git a/creevey/ops/image/transforms.py b/wildebeest/ops/image/transforms.py similarity index 99% rename from creevey/ops/image/transforms.py rename to wildebeest/ops/image/transforms.py index 2c7a1fc..e9484aa 100644 --- a/creevey/ops/image/transforms.py +++ b/wildebeest/ops/image/transforms.py @@ -15,7 +15,7 @@ def resize( Resize input image `shape` or `min_dim` needs to be specified with `partial` before - this function can be used in a Creevey pipeline. + this function can be used in a Wildebeest pipeline. `kwargs` is included only for compatibility with the `CustomReportingPipeline` class. diff --git a/wildebeest/path_funcs/__init__.py b/wildebeest/path_funcs/__init__.py new file mode 100644 index 0000000..07dff73 --- /dev/null +++ b/wildebeest/path_funcs/__init__.py @@ -0,0 +1 @@ +from wildebeest.path_funcs.path_funcs import * diff --git a/creevey/path_funcs/path_funcs.py b/wildebeest/path_funcs/path_funcs.py similarity index 97% rename from creevey/path_funcs/path_funcs.py rename to wildebeest/path_funcs/path_funcs.py index c12e7c1..3162d6d 100644 --- a/creevey/path_funcs/path_funcs.py +++ b/wildebeest/path_funcs/path_funcs.py @@ -3,7 +3,7 @@ from typing import Optional import uuid -from creevey.constants import PathOrStr +from wildebeest.constants import PathOrStr def join_outdir_filename_extension( diff --git a/wildebeest/pipelines/__init__.py b/wildebeest/pipelines/__init__.py new file mode 100644 index 0000000..1344992 --- /dev/null +++ b/wildebeest/pipelines/__init__.py @@ -0,0 +1 @@ +from wildebeest.pipelines.pipelines import * diff --git a/creevey/pipelines/image.py b/wildebeest/pipelines/image.py similarity index 55% rename from creevey/pipelines/image.py rename to wildebeest/pipelines/image.py index 0ff6712..5f6ff1b 100644 --- a/creevey/pipelines/image.py +++ b/wildebeest/pipelines/image.py @@ -1,7 +1,7 @@ """Image-processing pipelines""" -from creevey import Pipeline -from creevey.load_funcs.image import load_image_from_url -from creevey.write_funcs.image import write_image +from wildebeest import Pipeline +from wildebeest.load_funcs.image import load_image_from_url +from wildebeest.write_funcs.image import write_image download_image_pipeline = Pipeline( diff --git a/creevey/pipelines/pipelines.py b/wildebeest/pipelines/pipelines.py similarity index 98% rename from creevey/pipelines/pipelines.py rename to wildebeest/pipelines/pipelines.py index 8b0352a..a568f75 100644 --- a/creevey/pipelines/pipelines.py +++ b/wildebeest/pipelines/pipelines.py @@ -9,7 +9,7 @@ import pandas as pd from tqdm import tqdm -from creevey.constants import PathOrStr +from wildebeest.constants import PathOrStr RUN_REPORT_COLS = ['outpath', 'skipped', 'error', 'time_finished'] @@ -248,7 +248,7 @@ class CustomReportingPipeline(Pipeline): Each element of `ops` and `write_func` must each accept a `defaultdict(dict)` object as an additional positional argument. - Functions defined in Creevey call this item `log_dict`. + Functions defined in Wildebeest call this item `log_dict`. Inside those functions, adding items to `log_dict[inpath]` causes them to be added to the "run record" DataFrame that the pipeline diff --git a/wildebeest/util/__init__.py b/wildebeest/util/__init__.py new file mode 100644 index 0000000..5c32a3d --- /dev/null +++ b/wildebeest/util/__init__.py @@ -0,0 +1 @@ +from wildebeest.util.util import * diff --git a/creevey/util/image.py b/wildebeest/util/image.py similarity index 83% rename from creevey/util/image.py rename to wildebeest/util/image.py index 6290cc0..1a38300 100644 --- a/creevey/util/image.py +++ b/wildebeest/util/image.py @@ -2,7 +2,7 @@ from functools import partial import mimetypes -from creevey.util import find_files_with_extensions +from wildebeest.util import find_files_with_extensions find_image_files = partial( find_files_with_extensions, diff --git a/creevey/util/util.py b/wildebeest/util/util.py similarity index 95% rename from creevey/util/util.py rename to wildebeest/util/util.py index 20b0ce8..2a3ac50 100644 --- a/creevey/util/util.py +++ b/wildebeest/util/util.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Iterable, List -from creevey.constants import PathOrStr +from wildebeest.constants import PathOrStr def find_files_with_extensions( diff --git a/creevey/write_funcs/__init__.py b/wildebeest/write_funcs/__init__.py similarity index 100% rename from creevey/write_funcs/__init__.py rename to wildebeest/write_funcs/__init__.py diff --git a/creevey/write_funcs/image.py b/wildebeest/write_funcs/image.py similarity index 97% rename from creevey/write_funcs/image.py rename to wildebeest/write_funcs/image.py index 600b9f0..06b77b2 100644 --- a/creevey/write_funcs/image.py +++ b/wildebeest/write_funcs/image.py @@ -5,7 +5,7 @@ import cv2 as cv import numpy as np -from creevey.constants import PathOrStr +from wildebeest.constants import PathOrStr def write_image(image: np.array, path: PathOrStr, **kwargs) -> None: