[MRG] use tox for running tests locally

.github/workflows/asv.yml

@@ -14,7 +14,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: make latest branch available
+      run: |
+        git checkout latest
+        git checkout -
 
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
@@ -29,9 +36,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install asv virtualenv==16.7.9
+        pip install tox
 
     - name: Runs benchmarks against latest
-      run: |
-        asv machine --yes
-        asv continuous origin/latest $(git rev-parse HEAD)
+      run: tox -e asv

.github/workflows/hypothesis.yml

@@ -22,14 +22,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[test]
+          pip install tox
 
       - name: Run Hypothesis tests
-        run: |
-          python -m pytest --cov=. --cov-report=xml --run-hypothesis --hypothesis-show-statistics --hypothesis-profile ci
-
-      - name: Upload coverage to Codecov  
-        uses: codecov/codecov-action@v1
-        with:
-            file: ./coverage.xml
-            fail_ci_if_error: true
+        run: tox -e hypothesis,coverage

.github/workflows/khmer.yml

@@ -22,20 +22,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[test]
+          pip install tox
 
       - name: Run tests with latest released khmer
-        run: |
-          python -m pip install khmer
-          python -m pytest -k test_nodegraph --cov=. --cov-report=xml
-
-      - name: Run tests with khmer master tests
-        run: |
-          python -m pip install -U git+https://github.com/dib-lab/khmer.git#egg=khmer
-          python -m pytest -k test_nodegraph --cov=. --cov-report=xml
+        run: tox -e khmer,coverage
 
-      - name: Upload coverage to Codecov  
-        uses: codecov/codecov-action@v1
-        with:
-            file: ./coverage.xml
-            fail_ci_if_error: true
+      - name: Run tests with khmer master branch
+        run: tox -e khmer_master,coverage

.github/workflows/python.yml

@@ -73,8 +73,3 @@ jobs:
         run: tox
         env:
           PYTHONDEVMODE: 1
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true

.github/workflows/rust.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Upload Rust coverage to codecov
         uses: codecov/codecov-action@v1
         with:
-          flags: rusttests
+          flags: rust
           fail_ci_if_error: true
 
   lints:

.gitignore

@@ -18,10 +18,10 @@ sourmash.egg-info
 .coverage
 .pytest_cache
 .python-version
-sourmash/version.py
+src/sourmash/version.py
 *.DS_Store
 .tox
-sourmash/_lowlevel*.py
+src/sourmash/_lowlevel*.py
 .env
 Pipfile
 Pipfile.lock

.pre-commit-config.yaml

@@ -0,0 +1,71 @@
+default_language_version:
+  python: python3
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+  - id: check-ast
+#  - id: check-builtin-literals
+  - id: check-docstring-first
+  - id: check-merge-conflict
+  - id: check-yaml
+  - id: check-toml
+  - id: debug-statements
+#  - id: end-of-file-fixer
+#    exclude: 'tests/test-data'
+#  - id: trailing-whitespace
+#    exclude: 'tests/test-data'
+#- repo: https://github.com/asottile/pyupgrade
+#  rev: v2.7.2
+#  hooks:
+#  - id: pyupgrade
+#- repo: https://github.com/pre-commit/mirrors-isort
+#  rev: v5.4.2
+#  hooks:
+#  - id: isort
+#    additional_dependencies: [toml]
+
+# format using black
+# when the full codebase is black, use it directly;
+#  while it isn't, let's use darker to format new/changed code
+-   repo: https://github.com/akaihola/darker
+    rev: 1.2.1
+    hooks:
+    -   id: darker
+#- repo: https://github.com/psf/black
+#  rev: 20.8b1
+#  hooks:
+#  - id: black
+#    args:
+#    - --safe
+#    language_version: python3.8
+#- repo: https://github.com/asottile/blacken-docs
+#  rev: v1.8.0
+#  hooks:
+#  - id: blacken-docs
+#    additional_dependencies:
+#    - black==19.10b0
+#    language_version: python3.8
+
+#- repo: https://github.com/asottile/add-trailing-comma
+#  rev: v2.0.1
+#  hooks:
+#  - id: add-trailing-comma
+#- repo: https://github.com/pre-commit/pygrep-hooks
+#  rev: v1.6.0
+#  hooks:
+#  - id: rst-backticks
+#- repo: https://github.com/asottile/setup-cfg-fmt
+#  rev: v1.11.0
+#  hooks:
+#  - id: setup-cfg-fmt
+#    args:
+#    - --min-py3-version
+#    - '3.7'
+#- repo: https://gitlab.com/pycqa/flake8
+#  rev: 3.8.3
+#  hooks:
+#  - id: flake8
+#    additional_dependencies:
+#    - flake8-bugbear == 20.1.2
+#    language_version: python3.8

MANIFEST.in

@@ -1,16 +1,10 @@
-include LICENSE Makefile Dockerfile LICENSE Makefile README.md requirements.txt
-include index.ipynb
-include sourmash VERSION
-recursive-include sourmash_lib *
-recursive-include sourmash *
-recursive-include src *.rs
-recursive-include benches *.rs
+include LICENSE Makefile README.md requirements.txt
 include Cargo.toml
 include include/sourmash.h
 prune .eggs
 global-exclude *.rlib
 global-exclude *.orig
 global-exclude *.pyc
 global-exclude *.so
-prune tests/test-data/
-global-exclude *.git/
+prune tests/test-data
+global-exclude *.git

Makefile

@@ -18,13 +18,12 @@ install: all
 dist: FORCE
 	$(PYTHON) setup.py sdist
 
-test: all
-	$(PYTHON) -m pip install -e '.[test]'
-	$(PYTHON) -m pytest
+test:
+	tox -e py38
 	cargo test
 
-doc: build .PHONY
-	cd doc && make html
+doc: .PHONY
+	tox -e docs
 
 include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/hyperloglog.rs \
@@ -36,11 +35,10 @@ include/sourmash.h: src/core/src/lib.rs \
 	RUSTUP_TOOLCHAIN=nightly cbindgen -c cbindgen.toml . -o ../../$@
 
 coverage: all
-	$(PYTHON) setup.py build_ext -i
-	$(PYTHON) -m pytest --cov=. --cov-report term-missing --cov-report=xml
+	tox -e coverage
 
 benchmark:
-	asv continuous latest `git rev-parse HEAD`
+	tox -e asv
 	cargo bench
 
 check:

asv.conf.json

@@ -12,7 +12,7 @@
     "html_dir": ".asv/html",
     "build_cache_size": 8,
     "build_command": [
-      "python -m pip install 'setuptools_scm[toml]<5' milksnake",
+      "python -m pip install 'setuptools_scm[toml]>=4,<6' milksnake",
       "python setup.py build",
       "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
     ]

doc/developer.md

@@ -2,7 +2,6 @@
 
 ## Development environment
 
-
 You can get the latest development branch with:
 ```
 git clone https://github.com/dib-lab/sourmash.git
@@ -13,24 +12,29 @@ extension code). We suggest using `rustup` to install the Rust environment:
 
     curl https://sh.rustup.rs -sSf | sh
 
-To install all of the necessary Python dependencies, do:
+We use [`tox`](https://tox.readthedocs.io) for managing dependencies and
+running tests and checks during development.
+To install it, do:
 ```
-pip install -r requirements.txt
+python -m pip install tox
 ```
-Briefly, we use `py.test` and `cargo test` for testing, and `coverage` for code
-coverage analysis.
+and use `tox -l` to list available tasks.
 
 We suggest working on sourmash in a virtualenv; e.g. from within the
 sourmash clone directory, you can do:
 ```
-python -m virtualenv dev
-. dev/bin/activate
-pip install -e .
+tox -e dev
+. .tox/dev/bin/activate
 ```
 
 You can run tests by invoking `make test` in the sourmash directory;
-`python -m pytest` will run the Python tests, and `cargo test` will
-run the Rust tests.
+`tox -e py39` will run the Python tests with Python 3.9,
+and `cargo test` will run the Rust tests.
+
+You can also explicitly install all the dependencies for sourmash by running
+```
+pip install -r requirements.txt
+```
 
 ### If you're having trouble installing or using the development environment
 
@@ -40,14 +44,22 @@ If you are getting an error that contains `ImportError: cannot import name 'to_b
 make clean
 ```
 
+## Adding new changes
+
+We use [`pre-commit`](https://pre-commit.com/) to run automatic checks and fixes
+when developing sourmash. You can run it with
+```
+tox -e fix_lint
+```
+which prints a "hint" at the end of the run with instructions to set it up to
+run automatically every time you run `git commit`.
 
 ## Automated tests and code coverage calculation
 
-We use [Travis][0] and [GitHub Actions][2] for continuous integration.
+We use [GitHub Actions][2] for continuous integration.
 
 Code coverage can be viewed interactively at [codecov.io][1].
 
-[0]: https://travis-ci.com/dib-lab/sourmash
 [1]: https://codecov.io/gh/dib-lab/sourmash/
 [2]: https://github.com/dib-lab/sourmash/actions
 

nix.shell

@@ -0,0 +1,51 @@
+let
+  moz_overlay = import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz);
+  nixpkgs = import <nixpkgs> { overlays = [ moz_overlay ]; };
+  ruststable = (nixpkgs.latest.rustChannels.stable.rust);
+
+  mach-nix = import (
+    builtins.fetchGit {
+      url = "https://github.com/DavHau/mach-nix/";
+      ref = "2.0.0";
+    }
+  );
+
+  customPython = mach-nix.mkPython {
+    python = nixpkgs.python38;
+    requirements = ''
+       screed>=0.9
+       cffi>=1.14.0
+       numpy
+       matplotlib
+       scipy
+       deprecation>=2.0.6
+       cachetools >=4,<5
+       setuptools>=38.6.0
+       milksnake
+       setuptools_scm>=3.2.0
+       setuptools_scm_git_archive
+       pytest
+       pytest-cov
+       hypothesis
+       tox
+    '';
+  };
+
+in
+  with nixpkgs;
+
+  nixpkgs.mkShell {
+    buildInputs = [
+      customPython
+      git
+      stdenv
+      ruststable
+      stdenv.cc.cc.lib
+   ];
+
+  shellHook = ''
+     # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
+     export SOURCE_DATE_EPOCH=315532800 # 1980
+     export LD_LIBRARY_PATH="${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH";
+  '';
+}

pyproject.toml

@@ -1,9 +1,21 @@
 [build-system]
 requires = [
     "setuptools >= 48",
-    "setuptools_scm[toml] >= 4, <5",
+    "setuptools_scm[toml] >= 4, <6",
     "setuptools_scm_git_archive",
     "milksnake",
     "wheel >= 0.29.0",
 ]
 build-backend = 'setuptools.build_meta'
+
+[tool.setuptools_scm]
+write_to = "src/sourmash/version.py"
+git_describe_command = "git describe --dirty --tags --long --match v* --first-parent"
+
+[tool.isort]
+known_third_party = ["deprecation", "hypothesis", "mmh3", "numpy", "pkg_resources", "pytest", "screed", "setuptools", "sourmash_tst_utils"]
+multi_line_output = 3 
+include_trailing_comma = true
+force_grid_wrap = 0
+line_length = 88
+known_first_party = ["sourmash"]