use tox for running tests

move most of setup.py to pyproject.toml and setup.cfg
use isolated build for package creation
isolate test running
add some pyscaffold-inspired changes
update precommit
isort config
tox.ini updates based on tox-dev/tox
fix benchmarks for CI, add pypy wheels
add nix.shell for env setup
move sourmash to src/sourmash
coverage fixes

.gitignore

@@ -18,10 +18,10 @@ sourmash.egg-info
 .coverage
 .pytest_cache
 .python-version
-sourmash/version.py
+src/sourmash/version.py
 *.DS_Store
 .tox
-sourmash/_lowlevel*.py
+src/sourmash/_lowlevel*.py
 .env
 Pipfile
 Pipfile.lock

.pre-commit-config.yaml

@@ -0,0 +1,62 @@
+default_language_version:
+  python: python3
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+  - id: check-ast
+#  - id: check-builtin-literals
+  - id: check-docstring-first
+  - id: check-merge-conflict
+  - id: check-yaml
+  - id: check-toml
+  - id: debug-statements
+#  - id: end-of-file-fixer
+#    exclude: 'tests/test-data'
+#  - id: trailing-whitespace
+#    exclude: 'tests/test-data'
+#- repo: https://github.com/asottile/pyupgrade
+#  rev: v2.7.2
+#  hooks:
+#  - id: pyupgrade
+#- repo: https://github.com/pre-commit/mirrors-isort
+#  rev: v5.4.2
+#  hooks:
+#  - id: isort
+#    additional_dependencies: [toml]
+#- repo: https://github.com/psf/black
+#  rev: 20.8b1
+#  hooks:
+#  - id: black
+#    args:
+#    - --safe
+#    language_version: python3.8
+#- repo: https://github.com/asottile/blacken-docs
+#  rev: v1.8.0
+#  hooks:
+#  - id: blacken-docs
+#    additional_dependencies:
+#    - black==19.10b0
+#    language_version: python3.8
+#- repo: https://github.com/asottile/add-trailing-comma
+#  rev: v2.0.1
+#  hooks:
+#  - id: add-trailing-comma
+#- repo: https://github.com/pre-commit/pygrep-hooks
+#  rev: v1.6.0
+#  hooks:
+#  - id: rst-backticks
+#- repo: https://github.com/asottile/setup-cfg-fmt
+#  rev: v1.11.0
+#  hooks:
+#  - id: setup-cfg-fmt
+#    args:
+#    - --min-py3-version
+#    - '3.7'
+#- repo: https://gitlab.com/pycqa/flake8
+#  rev: 3.8.3
+#  hooks:
+#  - id: flake8
+#    additional_dependencies:
+#    - flake8-bugbear == 20.1.2
+#    language_version: python3.8

MANIFEST.in

@@ -1,16 +1,10 @@
-include LICENSE Makefile Dockerfile LICENSE Makefile README.md requirements.txt
-include index.ipynb
-include sourmash VERSION
-recursive-include sourmash_lib *
-recursive-include sourmash *
-recursive-include src *.rs
-recursive-include benches *.rs
+include LICENSE Makefile README.md requirements.txt
 include Cargo.toml
 include include/sourmash.h
 prune .eggs
 global-exclude *.rlib
 global-exclude *.orig
 global-exclude *.pyc
 global-exclude *.so
-prune tests/test-data/
-global-exclude *.git/
+prune tests/test-data
+global-exclude *.git

Makefile

@@ -18,13 +18,12 @@ install: all
 dist: FORCE
 	$(PYTHON) setup.py sdist
 
-test: all
-	$(PYTHON) -m pip install -e '.[test]'
-	$(PYTHON) -m pytest
+test:
+	tox -e py38
 	cargo test
 
-doc: build .PHONY
-	cd doc && make html
+doc: .PHONY
+	tox -e docs
 
 include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/hyperloglog.rs \
@@ -36,8 +35,7 @@ include/sourmash.h: src/core/src/lib.rs \
 	RUSTUP_TOOLCHAIN=nightly cbindgen -c cbindgen.toml . -o ../../$@
 
 coverage: all
-	$(PYTHON) setup.py build_ext -i
-	$(PYTHON) -m pytest --cov=. --cov-report term-missing --cov-report=xml
+	tox -e coverage
 
 benchmark:
 	asv continuous latest `git rev-parse HEAD`

nix.shell

@@ -0,0 +1,51 @@
+let
+  moz_overlay = import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz);
+  nixpkgs = import <nixpkgs> { overlays = [ moz_overlay ]; };
+  ruststable = (nixpkgs.latest.rustChannels.stable.rust);
+
+  mach-nix = import (
+    builtins.fetchGit {
+      url = "https://github.com/DavHau/mach-nix/";
+      ref = "2.0.0";
+    }
+  );
+
+  customPython = mach-nix.mkPython {
+    python = nixpkgs.python38;
+    requirements = ''
+       screed>=0.9
+       cffi>=1.14.0
+       numpy
+       matplotlib
+       scipy
+       deprecation>=2.0.6
+       cachetools >=4,<5
+       setuptools>=38.6.0
+       milksnake
+       setuptools_scm>=3.2.0
+       setuptools_scm_git_archive
+       pytest
+       pytest-cov
+       hypothesis
+       tox
+    '';
+  };
+
+in
+  with nixpkgs;
+
+  nixpkgs.mkShell {
+    buildInputs = [
+      customPython
+      git
+      stdenv
+      ruststable
+      stdenv.cc.cc.lib
+   ];
+
+  shellHook = ''
+     # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
+     export SOURCE_DATE_EPOCH=315532800 # 1980
+     export LD_LIBRARY_PATH="${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH";
+  '';
+}

pyproject.toml

@@ -7,3 +7,15 @@ requires = [
     "wheel >= 0.29.0",
 ]
 build-backend = 'setuptools.build_meta'
+
+[tool.setuptools_scm]
+write_to = "src/sourmash/version.py"
+git_describe_command = "git describe --dirty --tags --long --match v* --first-parent"
+
+[tool.isort]
+known_third_party = ["deprecation", "hypothesis", "mmh3", "numpy", "pkg_resources", "pytest", "screed", "setuptools", "sourmash_tst_utils"]
+multi_line_output = 3 
+include_trailing_comma = true
+force_grid_wrap = 0
+line_length = 88
+known_first_party = ["sourmash"]

setup.cfg

@@ -0,0 +1,102 @@
+[metadata]
+name = sourmash
+description = tools for comparing DNA sequences with MinHash sketches
+long_description = file: README.md
+long_description_content_type = text/markdown; charset=UTF-8
+url = https://github.com/dib-lab/sourmash
+author = C. Titus Brown
+author_email = [email protected]
+license = BSD 3-clause
+license_file = LICENSE
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Environment :: Console
+    Environment :: MacOS X
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: BSD License
+    Natural Language :: English
+    Operating System :: POSIX :: Linux
+    Operating System :: MacOS :: MacOS X
+    Programming Language :: Rust
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Topic :: Scientific/Engineering :: Bio-Informatics
+project_urls = 
+    Documentation = https://sourmash.readthedocs.io
+    CI = https://travis-ci.com/dib-lab/sourmash
+    Source = https://github.com/dib-lab/sourmash
+    Tracker = https://github.com/dib-lab/sourmash/issues
+
+[options]
+zip_safe = False
+packages = find:
+platforms = any
+include_package_data = True
+install_requires =
+    screed>=1.0
+    cffi>=1.14.0
+    numpy
+    matplotlib
+    scipy
+    deprecation>=2.0.6
+    cachetools>=4,<5
+python_requires = >=3.7
+
+[bdist_wheel]
+universal = 1
+
+[options.packages.find]
+where = src
+
+# for explanation of %(extra)s syntax see:
+# https://github.com/pypa/setuptools/issues/1260#issuecomment-438187625
+# this syntax may change in the future
+[options.extras_require]
+test =
+    pytest>=6
+    pytest-cov<2.6
+    recommonmark
+    hypothesis
+demo =
+    jupyter
+    jupyter_client
+    ipython
+doc =
+    sphinx
+    myst-parser[sphinx]>=0.12.2
+    alabaster
+    sphinxcontrib-napoleon
+    nbsphinx
+    ipython
+10x =
+    bam2fasta==1.0.4
+storage =
+    ipfshttpclient>=0.4.13
+    redis
+all = 
+    %(test)s
+    %(demo)s
+    %(doc)s
+    %(10x)s
+    %(storage)s
+
+[options.entry_points]
+console_scripts =
+    sourmash = sourmash.__main__:main
+
+[tool:pytest]
+addopts =
+    --doctest-glob='doc/*.md'
+norecursedirs =
+    utils
+    build
+    buildenv
+    .tox
+    .asv
+    .eggs
+python_files =
+    src/sourmash/*.py
+    tests/*.py
+testpaths =
+    tests
+    doc

setup.py

@@ -1,7 +1,8 @@
 import os
-from setuptools import setup, find_packages
 import sys
 
+from setuptools import setup
+
 
 DEBUG_BUILD = os.environ.get("SOURMASH_DEBUG") == "1"
 
@@ -26,67 +27,7 @@ def build_native(spec):
         rtld_flags=rtld_flags,
     )
 
-
-CLASSIFIERS = [
-    "Environment :: Console",
-    "Environment :: MacOS X",
-    "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
-    "Natural Language :: English",
-    "Operating System :: POSIX :: Linux",
-    "Operating System :: MacOS :: MacOS X",
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
-    "Topic :: Scientific/Engineering :: Bio-Informatics",
-]
-
-CLASSIFIERS.append("Development Status :: 5 - Production/Stable")
-
-with open("README.md", "r") as readme:
-    LONG_DESCRIPTION = readme.read()
-
-SETUP_METADATA = {
-    "name": "sourmash",
-    "description": "tools for comparing DNA sequences with MinHash sketches",
-    "long_description": LONG_DESCRIPTION,
-    "long_description_content_type": "text/markdown",
-    "url": "https://github.com/dib-lab/sourmash",
-    "author": "C. Titus Brown",
-    "author_email": "[email protected]",
-    "license": "BSD 3-clause",
-    "packages": find_packages(exclude=["tests", "benchmarks"]),
-    "entry_points": {'console_scripts': [
-        'sourmash = sourmash.__main__:main'
-        ]
-    },
-    "install_requires": ['screed>=0.9', 'cffi>=1.14.0', 'numpy',
-                         'matplotlib', 'scipy', 'deprecation>=2.0.6',
-                         'cachetools >=4,<5'],
-    "setup_requires": [
-        "setuptools>=48",
-        "milksnake",
-        "setuptools_scm[toml] >= 4, <5",
-        "setuptools_scm_git_archive",
-        "wheel >= 0.29.0",
-    ],
-    "use_scm_version": {
-        "write_to": "sourmash/version.py",
-        "git_describe_command": "git describe --dirty --tags --long --match v* --first-parent"
-    },
-    "zip_safe": False,
-    "platforms": "any",
-    "extras_require": {
-        'test' : ['pytest', 'pytest-cov', 'recommonmark', 'hypothesis'],
-        'demo' : ['jupyter', 'jupyter_client', 'ipython'],
-        'doc' : ['sphinx', 'myst-parser[sphinx]>=0.12.2', 'alabaster',
-                 "sphinxcontrib-napoleon", "nbsphinx",
-                 "ipython"],
-        'storage': ["ipfshttpclient>=0.4.13", "redis"]
-    },
-    "include_package_data": True,
-    "classifiers": CLASSIFIERS,
-    "milksnake_tasks": [build_native],
-}
-
-setup(**SETUP_METADATA)
+setup(
+  milksnake_tasks=[build_native],
+  package_dir={"": "src"},
+)