Some modernization and tidying up

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+exclude =
+  venv,
+  **/migrations/*
+# So flake8 plays nicely with black
+# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html
+max-line-length = 120
+extend-ignore = E203

.github/workflows/python-package.yml

@@ -0,0 +1,54 @@
+name: Python package
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  release:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest black isort
+        pip install .[tests]
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Check black
+      run: black --check irs_reader tests
+    - name: isort
+      run: isort --profile=black --check-only irs_reader tests
+    - name: Test with pytest
+      run: |
+        pytest
+    - name: Build distribution
+      if: ${{ github.event_name == 'release' }}
+      run: |
+        pip install build
+        python -m build
+    - name: Upload source distribution
+      if: ${{ github.event_name == 'release' }}
+      uses: softprops/action-gh-release@v2
+      with:
+        files: dist/*

.gitmodules

@@ -1,8 +1,3 @@
 [submodule "irs_reader/metadata"]
 	path = irs_reader/metadata
-	url = https://github.com/jsfenfen/990-xml-metadata
-	branch = master
-[submodule "metadata"]
-	path = metadata
-	url = https://github.com/jsfenfen/990-xml-metadata
-	branch = master
+	url = https://github.com/datamade/990-xml-metadata.git

README.md

@@ -1,8 +1,5 @@
 # IRSx
 
-Update: 12/16. The IRS has announced it will no longer post xml 990 filings to AWS, thereby undermining irsx' ability to automatically retrieve filings. The IRS does appear to make the raw filings available in [bulk format on this page](https://www.irs.gov/charities-non-profits/form-990-series-downloads). It is possible to use IRSx by retrieving the files and placing them at the location that IRSX expects to find them. We are seeking additional information from IRS and plan to address this soon. 
-
-
 ## Table of Contents
 
 - [Installation](#installation)
@@ -289,21 +286,11 @@ For example:
 
 ### Legacy configuration ###
 
-You also can configure IRSx's cache location by setting the local_settings.py file. To figure out where that settings file is, log in to a terminal and type:
-
-	>>> from irsx.settings import IRSX_SETTINGS_LOCATION
-	>>> IRSX_SETTINGS_LOCATION
-	'/long/path/to/lib/python3.6/site-packages/irsx/settings.py'
-
-[ If you get an error, try upgrading irsx with `pip install irsx --upgrade` -- this feature was added in 0.1.1. ]
-
-
-Go to that directory. You can either modify the settings.py file or the local_settings.py file. To do the latter, first `cd` into the directory where the settings files live and run:
-
-	$ cp local_settings.py-example local_settings.py
-
-Then edit local_settings.py to set WORKING\_DIRECTORY to where the raw xml files are found. 
+You also can configure IRSx's cache location by setting an environmntal variable.
 
+```console
+> export IRSX_CACHE_DIRECTORY=/where/you/like
+```
 
 ## IRSx from python 
 
@@ -477,20 +464,16 @@ You can still add command line args, like this:
 
 ## Testing
 
-Nosetests - Test coverage is incomplete, improve it with coverage.py;  run 'pip install coverage' 
-then:
-
-	$ nosetests --with-coverage --cover-erase --cover-package=irs_reader
-
-or
-
-	$ coverage report -m 
-
-
-
-Tox -- see tox.ini; testing for: 2.7,3.4,3.5,3.6. You may need to run `pip install tox` in the testing environment. 
+Install dependencies
+```console
+> pip install .[tests]
+```
 
+And run tests
 
+```console
+> pytest
+```
 
 ## Acknowledgements
 

irs_reader/_version.py

@@ -1 +1 @@
-__version__ = '0.3.2'
+__version__ = "0.3.2"

irs_reader/dir_utils.py

@@ -3,7 +3,7 @@
 
 
 def mkdir_p(paths):
-    """ Makedirs, from http://stackoverflow.com/a/600612 """
+    """Makedirs, from http://stackoverflow.com/a/600612"""
     for path in paths:
         try:
             os.makedirs(path)

irs_reader/file_utils.py

@@ -1,11 +1,17 @@
-import re
 import os
+import re
+from datetime import datetime
+
 import requests
 
-from datetime import datetime
-from .settings import IRS_XML_HTTP_BASE, WORKING_DIRECTORY, INDEX_DIRECTORY, IRS_INDEX_BASE
+from .settings import (
+    INDEX_DIRECTORY,
+    IRS_INDEX_BASE,
+    IRS_XML_HTTP_BASE,
+    WORKING_DIRECTORY,
+)
 
-OBJECT_ID_RE = re.compile(r'20\d{16}')
+OBJECT_ID_RE = re.compile(r"20\d{16}")
 
 # Not sure how much detail we need to go into here
 OBJECT_ID_MSG = """
@@ -18,40 +24,37 @@
 
 
 def stream_download(url, target_path, verbose=False):
-    """ Download a large file without loading it into memory. """
+    """Download a large file without loading it into memory."""
     response = requests.get(url, stream=True)
     handle = open(target_path, "wb")
     if verbose:
         print("Beginning streaming download of %s" % url)
         start = datetime.now()
         try:
-            content_length = int(response.headers['Content-Length'])
-            content_MB = content_length/1048576.0
+            content_length = int(response.headers["Content-Length"])
+            content_MB = content_length / 1048576.0
             print("Total file size: %.2f MB" % content_MB)
         except KeyError:
-            pass      # allow Content-Length to be missing
+            pass  # allow Content-Length to be missing
     for chunk in response.iter_content(chunk_size=512):
-        if chunk:     # filter out keep-alive new chunks
+        if chunk:  # filter out keep-alive new chunks
             handle.write(chunk)
 
     if verbose:
-        print(
-            "Download completed to %s in %s" %
-            (target_path, datetime.now() - start))
+        print("Download completed to %s in %s" % (target_path, datetime.now() - start))
 
 
 def validate_object_id(object_id):
-    """ It's easy to make a mistake entering these, validate the format """
+    """It's easy to make a mistake entering these, validate the format"""
     result = re.match(OBJECT_ID_RE, str(object_id))
     if not result:
         print("'%s' appears not to be a valid 990 object_id" % object_id)
         raise RuntimeError(OBJECT_ID_MSG)
     return object_id
 
 
-# Files are no longer available on S3
-# def get_s3_URL(object_id):
-#     return ("%s/%s_public.xml" % (IRS_XML_HTTP_BASE, object_id))
+def get_s3_URL(object_id):
+    return "%s/%s_public.xml" % (IRS_XML_HTTP_BASE, object_id)
 
 
 def get_local_path(object_id):