Pre commit Hooks (black, flake8, mypy, etc) [CT-105]

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+select =
+    E
+    W
+    F
+ignore =
+    W503
+    W504
+    E203
+    E741
+max-line-length = 99
+exclude = test

.github/workflows/main.yml

@@ -37,19 +37,10 @@ defaults:
 
 jobs:
   code-quality:
-    name: ${{ matrix.toxenv }}
+    name: code-quality
 
     runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        toxenv: [flake8, mypy]
-
-    env:
-      TOXENV: ${{ matrix.toxenv }}
-      PYTEST_ADDOPTS: "-v --color=yes"
-
     steps:
       - name: Check out the repository
         uses: actions/checkout@v2
@@ -62,12 +53,12 @@ jobs:
       - name: Install python dependencies
         run: |
           pip install --user --upgrade pip
-          pip install tox
+          pip install pre-commit
           pip --version
-          tox --version
+          pre-commit --version
 
-      - name: Run tox
-        run: tox
+      - name: Run pre-commit hooks
+        run: pre-commit
 
   unit:
     name: unit test / python ${{ matrix.python-version }}

.github/workflows/release.yml

@@ -1,5 +1,5 @@
 # **what?**
-# Take the given commit, run unit tests specifically on that sha, build and 
+# Take the given commit, run unit tests specifically on that sha, build and
 # package it, and then release to GitHub and PyPi with that specific build
 
 # **why?**
@@ -142,9 +142,8 @@ jobs:
         run: |
           dbt --version
 
-
   github-release:
-    name: GitHub Release 
+    name: GitHub Release
 
     needs: test-build
 
@@ -155,7 +154,7 @@ jobs:
         with:
           name: dist
           path: '.'
-          
+
       # Need to set an output variable because env variables can't be taken as input
       # This is needed for the next step with releasing to GitHub
       - name: Find release type
@@ -179,7 +178,7 @@ jobs:
             dbt_core-${{github.event.inputs.version_number}}-py3-none-any.whl
             dbt-postgres-${{github.event.inputs.version_number}}.tar.gz
             dbt-core-${{github.event.inputs.version_number}}.tar.gz
-          
+
   pypi-release:
     name: Pypi release
 
@@ -188,12 +187,12 @@ jobs:
     needs: github-release
 
     environment: PypiProd
-    steps:    
+    steps:
       - uses: actions/download-artifact@v2
         with:
           name: dist
           path: 'dist'
-      
+
       - name: Publish distribution to PyPI
         uses: pypa/[email protected]
         with:

.gitignore

@@ -49,9 +49,8 @@ coverage.xml
 *,cover
 .hypothesis/
 test.env
+*.pytest_cache/
 
-# Mypy
-.mypy_cache/
 
 # Translations
 *.mo
@@ -66,10 +65,10 @@ docs/_build/
 # PyBuilder
 target/
 
-#Ipython Notebook
+# Ipython Notebook
 .ipynb_checkpoints
 
-#Emacs
+# Emacs
 *~
 
 # Sublime Text
@@ -78,6 +77,7 @@ target/
 # Vim
 *.sw*
 
+# Pyenv
 .python-version
 
 # Vim
@@ -90,6 +90,7 @@ venv/
 # AWS credentials
 .aws/
 
+# MacOS
 .DS_Store
 
 # vscode

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# Configuration for pre-commit hooks (see https://pre-commit.com/).
+# Eventually the hooks described here will be run as tests before merging each PR.
+
+# TODO: remove global exclusion of tests when testing overhaul is complete
+exclude: ^test/
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+  - id: check-yaml
+    args: [--unsafe]
+  - id: check-json
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-case-conflict
+- repo: https://github.com/psf/black
+  rev: 21.12b0
+  hooks:
+  - id: black
+    language_version: python3
+    args: ['--line-length=99']
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 4.0.1
+  hooks:
+  - id: flake8
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.782
+  hooks:
+  - id: mypy
+    args: [--show-error-codes]
+    files: ^core/dbt/
+    entry: mypy core/dbt/
+    pass_filenames: false
+    # N.B.: By using `language: system` we run this hook in the local
+    # environment instead of a pre-commit isolated one.  This is needed
+    # to ensure mypy correctly parses the project.  It may cause trouble
+    # in that it adds environmental variables out of our control to the
+    # mix.  Unfortunately, there's nothing we can do about per pre-commit's
+    # author.
+    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+    language: system

CONTRIBUTING.md

@@ -47,6 +47,7 @@ Here's a good workflow:
 - Outline your planned implementation. If you want help getting started, ask!
 - Follow the steps outlined below to develop locally. Once you have opened a PR, one of the `dbt-core` maintainers will work with you to review your code.
 - Add a test! Tests are crucial for both fixes and new features alike. We want to make sure that code works as intended, and that it avoids any bugs  previously encountered. Currently, the best resource for understanding `dbt-core`'s [unit](test/unit) and [integration](test/integration) tests is the tests themselves. One of the maintainers can help by pointing out relevant examples.
+- Check your formatting and linting with [Flake8](https://flake8.pycqa.org/en/latest/#), [Black](https://github.com/psf/black), and the rest of the hooks we have in our [pre-commit](https://pre-commit.com/) [config](https://github.com/dbt-labs/dbt-core/blob/75201be9db1cb2c6c01fa7e71a314f5e5beb060a/.pre-commit-config.yaml).
 
 In some cases, the right resolution to an open issue might be tangential to the `dbt-core` codebase. The right path forward might be a documentation update or a change that can be made in user-space. In other cases, the issue might describe functionality that the `dbt-core` maintainers are unwilling or unable to incorporate into the `dbt-core` codebase. When it is determined that an open issue describes functionality that will not translate to a code change in the `dbt-core` repository, the issue will be tagged with the `wontfix` label (see below) and closed.
 
@@ -106,6 +107,7 @@ A short list of tools used in `dbt-core` testing that will be helpful to your un
 - [`pytest`](https://docs.pytest.org/en/latest/) to discover/run tests
 - [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
 - [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
+- [`black`](https://github.com/psf/black) for code formatting
 - [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking
 - [Github Actions](https://github.com/features/actions)
 

Makefile

@@ -8,27 +8,44 @@ endif
 
 .PHONY: dev
 dev: ## Installs dbt-* packages in develop mode along with development dependencies.
-	pip install -r dev-requirements.txt -r editable-requirements.txt
+	@\
+	pip install -r dev-requirements.txt -r editable-requirements.txt && \
+	pre-commit install
 
 .PHONY: mypy
 mypy: .env ## Runs mypy for static type checking.
-	$(DOCKER_CMD) tox -e mypy
+	@\
+	$(DOCKER_CMD) pre-commit run mypy | grep -v "[INFO]"
 
 .PHONY: flake8
 flake8: .env ## Runs flake8 to enforce style guide.
-	$(DOCKER_CMD) tox -e flake8
+	@\
+	$(DOCKER_CMD) pre-commit run flake8 | grep -v "[INFO]"
+
+.PHONY: black
+black: .env ## Runs black to enforce style guide.
+	@\
+	$(DOCKER_CMD) pre-commit run black | grep -v "[INFO]"
 
 .PHONY: lint
 lint: .env ## Runs all code checks in parallel.
-	$(DOCKER_CMD) tox -p -e flake8,mypy
+	@\
+	$(DOCKER_CMD) pre-commit run flake8 | grep -v "[INFO]"; \
+	$(DOCKER_CMD) pre-commit run mypy | grep -v "[INFO]"; \
+	$(DOCKER_CMD) pre-commit run black | grep -v "[INFO]" \
 
 .PHONY: unit
 unit: .env ## Runs unit tests with py38.
+	@\
 	$(DOCKER_CMD) tox -e py38
 
 .PHONY: test
 test: .env ## Runs unit tests with py38 and code checks in parallel.
-	$(DOCKER_CMD) tox -p -e py38,flake8,mypy
+	@\
+	$(DOCKER_CMD) tox -p -e py38; \
+	$(DOCKER_CMD) pre-commit run flake8 | grep -v "[INFO]"; \
+	$(DOCKER_CMD) pre-commit run mypy | grep -v "[INFO]"; \
+	$(DOCKER_CMD) pre-commit run black | grep -v "[INFO]"
 
 .PHONY: integration
 integration: .env integration-postgres ## Alias for integration-postgres.
@@ -38,15 +55,18 @@ integration-fail-fast: .env integration-postgres-fail-fast ## Alias for integrat
 
 .PHONY: integration-postgres
 integration-postgres: .env ## Runs postgres integration tests with py38.
+	@\
 	$(DOCKER_CMD) tox -e py38-postgres -- -nauto
 
 .PHONY: integration-postgres-fail-fast
 integration-postgres-fail-fast: .env ## Runs postgres integration tests with py38 in "fail fast" mode.
+	@\
 	$(DOCKER_CMD) tox -e py38-postgres -- -x -nauto
 
 .PHONY: setup-db
 setup-db: ## Setup Postgres database with docker-compose for system testing.
-	docker-compose up -d database
+	@\
+	docker-compose up -d database && \
 	PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
 
 # This rule creates a file named .env that is used by docker-compose for passing
@@ -62,27 +82,29 @@ endif
 
 .PHONY: clean
 clean: ## Resets development environment.
-	rm -f .coverage
-	rm -rf .eggs/
-	rm -f .env
-	rm -rf .tox/
-	rm -rf build/
-	rm -rf dbt.egg-info/
-	rm -f dbt_project.yml
-	rm -rf dist/
-	rm -f htmlcov/*.{css,html,js,json,png}
-	rm -rf logs/
-	rm -rf target/
-	find . -type f -name '*.pyc' -delete
-	find . -type d -name '__pycache__' -depth -delete
+	@echo 'cleaning repo...'
+	@rm -f .coverage
+	@rm -rf .eggs/
+	@rm -f .env
+	@rm -rf .tox/
+	@rm -rf build/
+	@rm -rf dbt.egg-info/
+	@rm -f dbt_project.yml
+	@rm -rf dist/
+	@rm -f htmlcov/*.{css,html,js,json,png}
+	@rm -rf logs/
+	@rm -rf target/
+	@find . -type f -name '*.pyc' -delete
+	@find . -type d -name '__pycache__' -depth -delete
+	@echo 'done.'
+
 
 .PHONY: help
 help: ## Show this help message.
 	@echo 'usage: make [target] [USE_DOCKER=true]'
 	@echo
 	@echo 'targets:'
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 	@echo
 	@echo 'options:'
 	@echo 'use USE_DOCKER=true to run target in a docker container'
-

dev-requirements.txt

@@ -5,6 +5,7 @@ freezegun==0.3.12
 ipdb
 mypy==0.782
 pip-tools
+pre-commit
 pytest
 pytest-dotenv
 pytest-logbook

tox.ini

@@ -2,25 +2,6 @@
 skipsdist = True
 envlist = py37,py38,py39,flake8,mypy
 
-[testenv:flake8]
-description = flake8 code checks
-basepython = python3.8
-skip_install = true
-commands = flake8 --select=E,W,F --ignore=W504,E741 --max-line-length 99 \
-  core/dbt \
-  plugins/postgres/dbt
-deps =
-  -rdev-requirements.txt
-
-[testenv:mypy]
-description = mypy static type checking
-basepython = python3.8
-skip_install = true
-commands = mypy --show-error-codes core/dbt
-deps =
-  -rdev-requirements.txt
-  -reditable-requirements.txt
-
 [testenv:{unit,py37,py38,py39,py}]
 description = unit testing
 skip_install = true