From 4103d10740cae300a8633f5a6e0e411569688873 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Sat, 22 Jan 2022 07:10:49 +0900 Subject: [PATCH 1/8] support python 3.10 --- .github/workflows/pythonbuild.yml | 73 ++++++++++++++++++------------- README.rst | 2 +- setup.py | 1 + tox.ini | 2 +- 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index f28342ca3c..388b0569cd 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -13,14 +13,16 @@ jobs: strategy: matrix: include: - - python-version: 3.6 + - python-version: "3.6" tox-env: py36-core - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-core - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-core - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-core + - python-version: "3.10" + tox-env: py310-core steps: - uses: actions/checkout@v2 @@ -37,7 +39,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }} - name: Install dependencies run: | - python -m pip install --upgrade pip 'tox<3.0' + python -m pip install --upgrade pip 'tox<4.0' - name: Setup MySQL DB run: | sudo /etc/init.d/mysql start @@ -49,7 +51,7 @@ jobs: TOXENV: ${{ matrix.tox-env }} run: tox - name: Codecov - env: + env: COVERAGE_PROCESS_START: .coveragerc run: | pip install codecov @@ -76,14 +78,16 @@ jobs: strategy: matrix: include: - - python-version: 3.6 + - python-version: "3.6" tox-env: py36-postgres - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-postgres - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-postgres - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-postgres + - python-version: "3.10" + tox-env: py310-postgres steps: - uses: actions/checkout@v2 @@ -100,7 +104,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }} - name: Install dependencies run: | - python -m pip install --upgrade pip 'tox<3.0' + python -m pip install --upgrade pip 'tox<4.0' - name: Create PSQL database run: | PGPASSWORD=postgres psql -h localhost -p 5432 -c 'create database spotify;' -U postgres @@ -109,7 +113,7 @@ jobs: TOXENV: ${{ matrix.tox-env }} run: tox - name: Codecov - env: + env: COVERAGE_PROCESS_START: .coveragerc run: | pip install codecov @@ -125,45 +129,54 @@ jobs: strategy: matrix: include: - - python-version: 3.6 + - python-version: "3.6" tox-env: py36-aws - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-aws - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-aws - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-aws + - python-version: "3.10" + tox-env: py310-aws - - python-version: 3.6 + - python-version: "3.6" tox-env: py36-unixsocket OVERRIDE_SKIP_CI_TESTS: True - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-unixsocket OVERRIDE_SKIP_CI_TESTS: True - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-unixsocket OVERRIDE_SKIP_CI_TESTS: True - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-unixsocket OVERRIDE_SKIP_CI_TESTS: True + - python-version: "3.10" + tox-env: py310-unixsocket + OVERRIDE_SKIP_CI_TESTS: True - - python-version: 3.6 + - python-version: "3.6" tox-env: py36-apache - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-apache - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-apache - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-apache - - - python-version: 3.6 + - python-version: "3.10" + tox-env: py310-apache + + - python-version: "3.6" tox-env: py36-azureblob - - python-version: 3.7 + - python-version: "3.7" tox-env: py37-azureblob - - python-version: 3.8 + - python-version: "3.8" tox-env: py38-azureblob - - python-version: 3.9 + - python-version: "3.9" tox-env: py39-azureblob + - python-version: "3.10" + tox-env: py310-azureblob - python-version: 3.9 @@ -194,7 +207,7 @@ jobs: run: tox - name: Codecov if: ${{ matrix.tox-env != 'flake8' && matrix.tox-env != 'docs' }} - env: + env: COVERAGE_PROCESS_START: .coveragerc run: | pip install codecov diff --git a/README.rst b/README.rst index c6946fc62e..dd2b5db623 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ .. image:: https://img.shields.io/pypi/l/luigi.svg?style=flat :target: https://pypi.python.org/pypi/luigi -Luigi is a Python (3.6, 3.7, 3.8, 3.9 tested) package that helps you build complex +Luigi is a Python (3.6, 3.7, 3.8, 3.9, 3.10 tested) package that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization, handling failures, command line integration, and much more. diff --git a/setup.py b/setup.py index 6ea5ac0612..08a942c679 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,7 @@ def get_static_files(path): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: System :: Monitoring', ], ) diff --git a/tox.ini b/tox.ini index f66a6a79ff..85894dc8d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{35,36,37,38,39}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8 +envlist = py{35,36,37,38,39,310}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8 skipsdist = True [pytest] From c5ff9439a19e6e71d8a5e64d326b50509971bee9 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Tue, 13 Dec 2022 06:37:31 +0900 Subject: [PATCH 2/8] add sleep --- test/lock_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/lock_test.py b/test/lock_test.py index 7ef8761cc1..d9ddb175f6 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -18,6 +18,7 @@ import os import subprocess import tempfile +import time import mock from helpers import unittest @@ -37,6 +38,7 @@ def test_getpcmd(self): command = ["sleep", "1"] external_process = subprocess.Popen(command) + time.sleep(0.42) result = luigi.lock.getpcmd(external_process.pid) self.assertTrue( From 4a1495900f9ff8796fbc2177fca6af3df9607386 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Tue, 13 Dec 2022 06:59:44 +0900 Subject: [PATCH 3/8] add more sleep --- test/lock_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/lock_test.py b/test/lock_test.py index d9ddb175f6..84f80c7126 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -61,6 +61,7 @@ def tearDown(self): def test_get_info(self): try: p = subprocess.Popen(["yes", u"à我ф"], stdout=subprocess.PIPE) + time.sleep(0.42) pid, cmd, pid_file = luigi.lock.get_info(self.pid_dir, p.pid) finally: p.kill() From cc02b3c34c5b7fc9337082091d55310b35a41e69 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Tue, 13 Dec 2022 07:22:47 +0900 Subject: [PATCH 4/8] add comment --- test/lock_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lock_test.py b/test/lock_test.py index 84f80c7126..172b8ae68b 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -38,7 +38,7 @@ def test_getpcmd(self): command = ["sleep", "1"] external_process = subprocess.Popen(command) - time.sleep(0.42) + time.sleep(0.42) # need this sleep for CI running on GitHub from Python 3.10 result = luigi.lock.getpcmd(external_process.pid) self.assertTrue( @@ -61,7 +61,7 @@ def tearDown(self): def test_get_info(self): try: p = subprocess.Popen(["yes", u"à我ф"], stdout=subprocess.PIPE) - time.sleep(0.42) + time.sleep(0.42) # need this sleep for CI running on GitHub from Python 3.10 pid, cmd, pid_file = luigi.lock.get_info(self.pid_dir, p.pid) finally: p.kill() From 276211ec3e512a233dd76a6f5bb52ad721c485f0 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Sun, 25 Dec 2022 09:06:39 +0900 Subject: [PATCH 5/8] add retry on getpcmd and get_info --- test/lock_test.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test/lock_test.py b/test/lock_test.py index 172b8ae68b..d0a6014258 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -22,24 +22,35 @@ import mock from helpers import unittest +from tenacity import retry, retry_if_result, stop_after_attempt, wait_exponential import luigi import luigi.lock import luigi.notifications luigi.notifications.DEBUG = True +pcmd_retry = retry(retry) + + +def _is_empyty(s): + return s == "" + class TestCmd(unittest.TestCase): def test_getpcmd(self): + # for CI stability, add retring + @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) + def _getpcmd(pid): + return luigi.lock.getpcmd(pid) + if os.name == 'nt': command = ["ping", "1.1.1.1", "-w", "1000"] else: command = ["sleep", "1"] external_process = subprocess.Popen(command) - time.sleep(0.42) # need this sleep for CI running on GitHub from Python 3.10 - result = luigi.lock.getpcmd(external_process.pid) + result = _getpcmd(external_process.pid) self.assertTrue( result.strip() in ["sleep 1", '[sleep]', 'ping 1.1.1.1 -w 1000'] @@ -59,10 +70,14 @@ def tearDown(self): os.rmdir(self.pid_dir) def test_get_info(self): + # for CI stability, add retring + @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) + def _get_info(pid_dir, pid): + return luigi.lock.get_info(pid_dir, pid) + try: p = subprocess.Popen(["yes", u"à我ф"], stdout=subprocess.PIPE) - time.sleep(0.42) # need this sleep for CI running on GitHub from Python 3.10 - pid, cmd, pid_file = luigi.lock.get_info(self.pid_dir, p.pid) + pid, cmd, pid_file = _get_info(self.pid_dir, p.pid) finally: p.kill() self.assertEqual(cmd, u'yes à我ф') From c7c95ed1acfe5aa73e4a1e03ae13987451028f29 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Sun, 25 Dec 2022 09:16:10 +0900 Subject: [PATCH 6/8] fix --- test/lock_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/lock_test.py b/test/lock_test.py index d0a6014258..405e398e29 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -18,7 +18,6 @@ import os import subprocess import tempfile -import time import mock from helpers import unittest @@ -32,13 +31,12 @@ pcmd_retry = retry(retry) -def _is_empyty(s): - return s == "" - - class TestCmd(unittest.TestCase): def test_getpcmd(self): + def _is_empyty(cmd): + return cmd == "" + # for CI stability, add retring @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) def _getpcmd(pid): @@ -70,6 +68,9 @@ def tearDown(self): os.rmdir(self.pid_dir) def test_get_info(self): + def _is_empyty(pid, cmd, pid_file): + return cmd == "" + # for CI stability, add retring @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) def _get_info(pid_dir, pid): From add010729f33d0c3d2d5587c166d2cba2e817226 Mon Sep 17 00:00:00 2001 From: hirosassa Date: Sun, 25 Dec 2022 10:07:23 +0900 Subject: [PATCH 7/8] fix --- test/lock_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/lock_test.py b/test/lock_test.py index 405e398e29..5de26b9d81 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -34,11 +34,11 @@ class TestCmd(unittest.TestCase): def test_getpcmd(self): - def _is_empyty(cmd): + def _is_empty(cmd): return cmd == "" # for CI stability, add retring - @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) + @retry(retry=retry_if_result(_is_empty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) def _getpcmd(pid): return luigi.lock.getpcmd(pid) @@ -68,11 +68,11 @@ def tearDown(self): os.rmdir(self.pid_dir) def test_get_info(self): - def _is_empyty(pid, cmd, pid_file): - return cmd == "" + def _is_empty(result): + return result[1] == "" # cmd is empty # for CI stability, add retring - @retry(retry=retry_if_result(_is_empyty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) + @retry(retry=retry_if_result(_is_empty), wait=wait_exponential(multiplier=0.2, min=0.1, max=3), stop=stop_after_attempt(3)) def _get_info(pid_dir, pid): return luigi.lock.get_info(pid_dir, pid) From e8a957cfa01f4f62f4df42359420e9f4815f437f Mon Sep 17 00:00:00 2001 From: hirosassa Date: Wed, 28 Dec 2022 06:16:30 +0900 Subject: [PATCH 8/8] remove unused --- test/lock_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/lock_test.py b/test/lock_test.py index 5de26b9d81..2701bd963f 100644 --- a/test/lock_test.py +++ b/test/lock_test.py @@ -28,8 +28,6 @@ luigi.notifications.DEBUG = True -pcmd_retry = retry(retry) - class TestCmd(unittest.TestCase):