support Python 3.10

.github/workflows/pythonbuild.yml

@@ -13,14 +13,16 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: 3.6
+          - python-version: "3.6"
             tox-env: py36-core
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-core
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-core
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-core
+          - python-version: "3.10"
+            tox-env: py310-core
 
     steps:
       - uses: actions/checkout@v2
@@ -37,7 +39,7 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip 'tox<3.0'
+          python -m pip install --upgrade pip 'tox<4.0'
       - name: Setup MySQL DB
         run: |
           sudo /etc/init.d/mysql start
@@ -49,7 +51,7 @@ jobs:
           TOXENV: ${{ matrix.tox-env }}
         run: tox
       - name: Codecov
-        env: 
+        env:
           COVERAGE_PROCESS_START: .coveragerc
         run: |
           pip install codecov
@@ -76,14 +78,16 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: 3.6
+          - python-version: "3.6"
             tox-env: py36-postgres
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-postgres
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-postgres
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-postgres
+          - python-version: "3.10"
+            tox-env: py310-postgres
 
     steps:
       - uses: actions/checkout@v2
@@ -100,7 +104,7 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip 'tox<3.0'
+          python -m pip install --upgrade pip 'tox<4.0'
       - name: Create PSQL database
         run: |
           PGPASSWORD=postgres psql -h localhost -p 5432 -c 'create database spotify;' -U postgres
@@ -109,7 +113,7 @@ jobs:
           TOXENV: ${{ matrix.tox-env }}
         run: tox
       - name: Codecov
-        env: 
+        env:
           COVERAGE_PROCESS_START: .coveragerc
         run: |
           pip install codecov
@@ -125,45 +129,54 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: 3.6
+          - python-version: "3.6"
             tox-env: py36-aws
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-aws
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-aws
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-aws
+          - python-version: "3.10"
+            tox-env: py310-aws
 
-          - python-version: 3.6
+          - python-version: "3.6"
             tox-env: py36-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
+          - python-version: "3.10"
+            tox-env: py310-unixsocket
+            OVERRIDE_SKIP_CI_TESTS: True
 
-          - python-version: 3.6
+          - python-version: "3.6"
             tox-env: py36-apache
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-apache
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-apache
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-apache
-
-          - python-version: 3.6
+          - python-version: "3.10"
+            tox-env: py310-apache
+
+          - python-version: "3.6"
             tox-env: py36-azureblob
-          - python-version: 3.7
+          - python-version: "3.7"
             tox-env: py37-azureblob
-          - python-version: 3.8
+          - python-version: "3.8"
             tox-env: py38-azureblob
-          - python-version: 3.9
+          - python-version: "3.9"
             tox-env: py39-azureblob
+          - python-version: "3.10"
+            tox-env: py310-azureblob
 
 
           - python-version: 3.9
@@ -194,7 +207,7 @@ jobs:
         run: tox
       - name: Codecov
         if: ${{ matrix.tox-env != 'flake8' && matrix.tox-env != 'docs' }}
-        env: 
+        env:
           COVERAGE_PROCESS_START: .coveragerc
         run: |
           pip install codecov

README.rst

@@ -14,7 +14,7 @@
 .. image:: https://img.shields.io/pypi/l/luigi.svg?style=flat
    :target: https://pypi.python.org/pypi/luigi
 
-Luigi is a Python (3.6, 3.7, 3.8, 3.9 tested) package that helps you build complex
+Luigi is a Python (3.6, 3.7, 3.8, 3.9, 3.10 tested) package that helps you build complex
 pipelines of batch jobs. It handles dependency resolution, workflow management,
 visualization, handling failures, command line integration, and much more.
 

setup.py

@@ -115,6 +115,7 @@ def get_static_files(path):
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'Topic :: System :: Monitoring',
     ],
 )

test/lock_test.py

@@ -21,6 +21,7 @@
 import mock
 from helpers import unittest
 
+from tenacity import retry, retry_if_result, stop_after_attempt, wait_exponential
 import luigi
 import luigi.lock
 import luigi.notifications
@@ -31,13 +32,21 @@
 class TestCmd(unittest.TestCase):
 
     def test_getpcmd(self):
+        def _is_empty(cmd):
+            return cmd == ""
+
+        # for CI stability, add retring
+        @retry(retry=retry_if_result(_is_empty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3))
+        def _getpcmd(pid):
+            return luigi.lock.getpcmd(pid)
+
         if os.name == 'nt':
             command = ["ping", "1.1.1.1", "-w", "1000"]
         else:
             command = ["sleep", "1"]
 
         external_process = subprocess.Popen(command)
-        result = luigi.lock.getpcmd(external_process.pid)
+        result = _getpcmd(external_process.pid)
 
         self.assertTrue(
             result.strip() in ["sleep 1", '[sleep]', 'ping 1.1.1.1 -w 1000']
@@ -57,9 +66,17 @@ def tearDown(self):
         os.rmdir(self.pid_dir)
 
     def test_get_info(self):
+        def _is_empty(result):
+            return result[1] == ""  # cmd is empty
+
+        # for CI stability, add retring
+        @retry(retry=retry_if_result(_is_empty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3))
+        def _get_info(pid_dir, pid):
+            return luigi.lock.get_info(pid_dir, pid)
+
         try:
             p = subprocess.Popen(["yes", u"à我ф"], stdout=subprocess.PIPE)
-            pid, cmd, pid_file = luigi.lock.get_info(self.pid_dir, p.pid)
+            pid, cmd, pid_file = _get_info(self.pid_dir, p.pid)
         finally:
             p.kill()
         self.assertEqual(cmd, u'yes à我ф')

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{35,36,37,38,39}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8
+envlist = py{35,36,37,38,39,310}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8
 skipsdist = True
 
 [pytest]