From 62b0e5235cb245da8fb8e16b7bb4ef11aea9ef73 Mon Sep 17 00:00:00 2001 From: jumpei-ukita Date: Wed, 20 Dec 2023 21:28:06 +0900 Subject: [PATCH 1/7] add py312 --- .github/workflows/pythonbuild.yml | 13 +++++++++++++ README.rst | 2 +- setup.py | 1 + tox.ini | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 26d2980cc3..2f8e12f63b 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -25,6 +25,8 @@ jobs: tox-env: py310-core - python-version: "3.11" tox-env: py311-core + - python-version: "3.12" + tox-env: py312-core steps: - uses: actions/checkout@v2 @@ -92,6 +94,8 @@ jobs: tox-env: py310-postgres - python-version: "3.11" tox-env: py311-postgres + - python-version: "3.12" + tox-env: py312-postgres steps: - uses: actions/checkout@v2 @@ -145,6 +149,8 @@ jobs: tox-env: py310-aws - python-version: "3.11" tox-env: py311-aws + - python-version: "3.12" + tox-env: py312-aws - python-version: "3.6" tox-env: py36-unixsocket @@ -164,6 +170,9 @@ jobs: - python-version: "3.11" tox-env: py311-unixsocket OVERRIDE_SKIP_CI_TESTS: True + - python-version: "3.12" + tox-env: py312-unixsocket + OVERRIDE_SKIP_CI_TESTS: True - python-version: "3.6" tox-env: py36-apache @@ -177,6 +186,8 @@ jobs: tox-env: py310-apache - python-version: "3.11" tox-env: py311-apache + - python-version: "3.12" + tox-env: py312-apache - python-version: "3.6" tox-env: py36-azureblob @@ -190,6 +201,8 @@ jobs: tox-env: py310-azureblob - python-version: "3.11" tox-env: py311-azureblob + - python-version: "3.12" + tox-env: py312-azureblob - python-version: 3.9 diff --git a/README.rst b/README.rst index 102388157b..cdc236ffee 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ .. image:: https://img.shields.io/pypi/l/luigi.svg?style=flat :target: https://pypi.python.org/pypi/luigi -Luigi is a Python (3.6, 3.7, 3.8, 3.9, 3.10, 3.11 tested) package that helps you build complex +Luigi is a Python (3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 tested) package that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization, handling failures, command line integration, and much more. diff --git a/setup.py b/setup.py index c56e54811e..e88e8f3804 100644 --- a/setup.py +++ b/setup.py @@ -118,6 +118,7 @@ def get_static_files(path): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: System :: Monitoring', ], ) diff --git a/tox.ini b/tox.ini index 4208e98eca..c82d97b0aa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{35,36,37,38,39,310,311}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8 +envlist = py{35,36,37,38,39,310,311,312}-{cdh,hdp,core,contrib,apache,aws,gcloud,postgres,unixsocket,azureblob,dropbox}, visualiser, docs, flake8 skipsdist = True [pytest] From 0369cfba2ade6af1c50af4472eb46add9e1c9b17 Mon Sep 17 00:00:00 2001 From: jumpei-ukita Date: Wed, 20 Dec 2023 21:37:52 +0900 Subject: [PATCH 2/7] import boto when python<=311 --- test/contrib/s3_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/contrib/s3_test.py b/test/contrib/s3_test.py index 845806606b..d21b8240b1 100644 --- a/test/contrib/s3_test.py +++ b/test/contrib/s3_test.py @@ -20,7 +20,8 @@ import tempfile import boto3 -from boto.s3 import key +if sys.version_info[:2] <= (3, 11): + from boto.s3 import key from botocore.exceptions import ClientError from mock import patch From 2a9da0c1c9064140b42692cf4e4910577e928787 Mon Sep 17 00:00:00 2001 From: jumpei-ukita Date: Wed, 20 Dec 2023 21:53:55 +0900 Subject: [PATCH 3/7] fix removed assert --- test/cmdline_test.py | 2 +- test/contrib/hadoop_test.py | 2 +- test/contrib/s3_test.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/cmdline_test.py b/test/cmdline_test.py index 7db317ded2..f8d2b4ced8 100644 --- a/test/cmdline_test.py +++ b/test/cmdline_test.py @@ -362,7 +362,7 @@ def test_cmd_line_params_are_available_for_execution_summary(self): print(stdout) print(stderr) - self.assertNotEquals(returncode, 1) + self.assertNotEqual(returncode, 1) self.assertFalse(b'required_test_param' in stderr) diff --git a/test/contrib/hadoop_test.py b/test/contrib/hadoop_test.py index b3db99a0bc..8e1c2fe64d 100644 --- a/test/contrib/hadoop_test.py +++ b/test/contrib/hadoop_test.py @@ -200,7 +200,7 @@ def test_run_2(test_case): job = WordFreqJob(use_hdfs=test_case.use_hdfs) luigi.build([job], local_scheduler=True) c = read_wordcount_output(job.output()) - test_case.assertAlmostEquals(float(c['jk']), 6.0 / 33.0) + test_case.assertAlmostEqual(float(c['jk']), 6.0 / 33.0) @staticmethod def test_map_only(test_case): diff --git a/test/contrib/s3_test.py b/test/contrib/s3_test.py index d21b8240b1..a043beff40 100644 --- a/test/contrib/s3_test.py +++ b/test/contrib/s3_test.py @@ -343,7 +343,7 @@ def test_get(self): s3_client.get('s3://mybucket/putMe', tmp_file_path) with open(tmp_file_path, 'r') as f: content = f.read() - self.assertEquals(content, self.tempFileContents.decode("utf-8")) + self.assertEqual(content, self.tempFileContents.decode("utf-8")) tmp_file.close() def test_get_as_bytes(self): @@ -353,7 +353,7 @@ def test_get_as_bytes(self): contents = s3_client.get_as_bytes('s3://mybucket/putMe') - self.assertEquals(contents, self.tempFileContents) + self.assertEqual(contents, self.tempFileContents) def test_get_as_string(self): create_bucket() @@ -362,7 +362,7 @@ def test_get_as_string(self): contents = s3_client.get_as_string('s3://mybucket/putMe2') - self.assertEquals(contents, self.tempFileContents.decode('utf-8')) + self.assertEqual(contents, self.tempFileContents.decode('utf-8')) def test_get_as_string_latin1(self): create_bucket() @@ -371,7 +371,7 @@ def test_get_as_string_latin1(self): contents = s3_client.get_as_string('s3://mybucket/putMe3', encoding='ISO-8859-1') - self.assertEquals(contents, self.tempFileContents.decode('ISO-8859-1')) + self.assertEqual(contents, self.tempFileContents.decode('ISO-8859-1')) def test_get_key(self): create_bucket() From e7130f83a06a49a4ab550e7341b7cd496dc594ad Mon Sep 17 00:00:00 2001 From: jumpei-ukita Date: Wed, 20 Dec 2023 21:59:00 +0900 Subject: [PATCH 4/7] fix removed assert --- test/contrib/external_daily_snapshot_test.py | 8 ++++---- test/contrib/postgres_test.py | 4 ++-- test/interface_test.py | 4 ++-- test/task_test.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/contrib/external_daily_snapshot_test.py b/test/contrib/external_daily_snapshot_test.py index ff273dd751..53c9d8a0b5 100644 --- a/test/contrib/external_daily_snapshot_test.py +++ b/test/contrib/external_daily_snapshot_test.py @@ -32,18 +32,18 @@ class ExternalDailySnapshotTest(unittest.TestCase): def test_latest(self): MockTarget('data-xyz-zebra-Congo-2012-01-01').open('w').close() d = DataDump.latest(date=datetime.date(2012, 1, 10), param='xyz') - self.assertEquals(d.date, datetime.date(2012, 1, 1)) + self.assertEqual(d.date, datetime.date(2012, 1, 1)) def test_latest_not_exists(self): MockTarget('data-abc-zebra-Congo-2012-01-01').open('w').close() d = DataDump.latest(date=datetime.date(2012, 1, 11), param='abc', lookback=5) - self.assertEquals(d.date, datetime.date(2012, 1, 7)) + self.assertEqual(d.date, datetime.date(2012, 1, 7)) def test_deterministic(self): MockTarget('data-pqr-zebra-Congo-2012-01-01').open('w').close() d = DataDump.latest(date=datetime.date(2012, 1, 10), param='pqr', a='zebra', aa='Congo') - self.assertEquals(d.date, datetime.date(2012, 1, 1)) + self.assertEqual(d.date, datetime.date(2012, 1, 1)) MockTarget('data-pqr-zebra-Congo-2012-01-05').open('w').close() d = DataDump.latest(date=datetime.date(2012, 1, 10), param='pqr', aa='Congo', a='zebra') - self.assertEquals(d.date, datetime.date(2012, 1, 1)) # Should still be the same + self.assertEqual(d.date, datetime.date(2012, 1, 1)) # Should still be the same diff --git a/test/contrib/postgres_test.py b/test/contrib/postgres_test.py index f08a330f84..bf9c0b89ac 100644 --- a/test/contrib/postgres_test.py +++ b/test/contrib/postgres_test.py @@ -132,11 +132,11 @@ def test_bulk_complete(self, mock_connect): def test_override_port(self): output = DummyPostgresQueryWithPort(date=datetime.datetime(1991, 3, 24)).output() - self.assertEquals(output.port, 1234) + self.assertEqual(output.port, 1234) def test_port_encoded_in_host(self): output = DummyPostgresQueryWithPortEncodedInHost(date=datetime.datetime(1991, 3, 24)).output() - self.assertEquals(output.port, '1234') + self.assertEqual(output.port, '1234') @pytest.mark.postgres diff --git a/test/interface_test.py b/test/interface_test.py index 8418683a34..502a408640 100644 --- a/test/interface_test.py +++ b/test/interface_test.py @@ -204,9 +204,9 @@ class CoreConfigTest(LuigiTestCase): @with_config({}) def test_parallel_scheduling_processes_default(self): - self.assertEquals(0, core().parallel_scheduling_processes) + self.assertEqual(0, core().parallel_scheduling_processes) @with_config({'core': {'parallel-scheduling-processes': '1234'}}) def test_parallel_scheduling_processes(self): from luigi.interface import core - self.assertEquals(1234, core().parallel_scheduling_processes) + self.assertEqual(1234, core().parallel_scheduling_processes) diff --git a/test/task_test.py b/test/task_test.py index e2da34796c..926106ea54 100644 --- a/test/task_test.py +++ b/test/task_test.py @@ -577,4 +577,4 @@ def __init_subclass__(cls, x, **kwargs): class Receiver(ReceivesClassKwargs, x=1): pass - self.assertEquals(Receiver.x, 1) + self.assertEqual(Receiver.x, 1) From b8733d421beefa2bebef33113e4b48b505458188 Mon Sep 17 00:00:00 2001 From: jumpei-ukita Date: Wed, 20 Dec 2023 22:00:03 +0900 Subject: [PATCH 5/7] skip boto test for py312 --- test/contrib/s3_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/contrib/s3_test.py b/test/contrib/s3_test.py index a043beff40..c24e69899a 100644 --- a/test/contrib/s3_test.py +++ b/test/contrib/s3_test.py @@ -108,6 +108,7 @@ def test_read_no_file_sse(self): t = self.create_target(encrypt_key=True) self.assertRaises(FileNotFoundException, t.open) + @unittest.skipIf(tuple(sys.version_info) >= (3, 12), "boto is not supported on Python 3.12+") def test_read_iterator_long(self): # write a file that is 5X the boto buffersize # to test line buffering From 43239958c8ef3aff58a4790574b6f559e1bcbedb Mon Sep 17 00:00:00 2001 From: Jumpei Ukita Date: Wed, 8 May 2024 20:21:13 +0900 Subject: [PATCH 6/7] Update README.rst From 3a66f28cacb337d93951233c3003c383d0c4ba85 Mon Sep 17 00:00:00 2001 From: Jumpei Ukita Date: Wed, 8 May 2024 20:40:37 +0900 Subject: [PATCH 7/7] Update README.rst