diff --git a/doc/workflows.rst b/doc/workflows.rst index c243c343a6..6fe916877d 100644 --- a/doc/workflows.rst +++ b/doc/workflows.rst @@ -19,7 +19,7 @@ In practice, implementing Target subclasses is rarely needed. Luigi comes with a toolbox of several useful Targets. In particular, :class:`~luigi.file.LocalTarget` and :class:`~luigi.contrib.hdfs.target.HdfsTarget`, but there is also support for other file systems: -:class:`luigi.s3.S3Target`, +:class:`luigi.contrib.s3.S3Target`, :class:`luigi.contrib.ssh.RemoteTarget`, :class:`luigi.contrib.ftp.RemoteTarget`, :class:`luigi.contrib.mysqldb.MySqlTarget`, diff --git a/examples/pyspark_wc.py b/examples/pyspark_wc.py index 46e2b2fa3e..3194df40b6 100644 --- a/examples/pyspark_wc.py +++ b/examples/pyspark_wc.py @@ -15,7 +15,7 @@ # limitations under the License. # import luigi -from luigi.s3 import S3Target +from luigi.contrib.s3 import S3Target from luigi.contrib.spark import SparkSubmitTask, PySparkTask diff --git a/examples/top_artists.py b/examples/top_artists.py index c156e8c817..a80e683494 100755 --- a/examples/top_artists.py +++ b/examples/top_artists.py @@ -246,12 +246,13 @@ def _input_iterator(self): class ArtistToplistToDatabase(luigi.contrib.postgres.CopyToTable): """ - This task runs a :py:class:`luigi.postgres.CopyToTable` task + This task runs a :py:class:`luigi.contrib.postgres.CopyToTable` task over the target data returned by :py:meth:`~/.Top10Artists.output` and writes the result into its :py:meth:`~.ArtistToplistToDatabase.output` target which, - by default, is :py:class:`luigi.postgres.PostgresTarget` (a table in PostgreSQL). + by default, is :py:class:`luigi.contrib.postgres.PostgresTarget` (a table in PostgreSQL). - This class uses :py:meth:`luigi.postgres.CopyToTable.run` and :py:meth:`luigi.postgres.CopyToTable.output`. + This class uses :py:meth:`luigi.contrib.postgres.CopyToTable.run` + and :py:meth:`luigi.contrib.postgres.CopyToTable.output`. """ date_interval = luigi.DateIntervalParameter() diff --git a/luigi/contrib/opener.py b/luigi/contrib/opener.py index 3c25f4ee58..9640c55164 100644 --- a/luigi/contrib/opener.py +++ b/luigi/contrib/opener.py @@ -33,7 +33,7 @@ from luigi.file import LocalTarget from luigi.mock import MockTarget -from luigi.s3 import S3Target +from luigi.contrib.s3 import S3Target from luigi.target import FileSystemException from six.moves.urllib.parse import urlsplit, parse_qs diff --git a/luigi/contrib/redshift.py b/luigi/contrib/redshift.py index a3ac8fc3c9..fa32b4661a 100644 --- a/luigi/contrib/redshift.py +++ b/luigi/contrib/redshift.py @@ -22,9 +22,9 @@ import os import luigi -from luigi import postgres +from luigi.contrib import postgres from luigi.contrib import rdbms -from luigi.s3 import S3PathTask, S3Target +from luigi.contrib.s3 import S3PathTask, S3Target logger = logging.getLogger('luigi-interface') diff --git a/luigi/contrib/s3.py b/luigi/contrib/s3.py index 4d77766756..2d90e1960a 100644 --- a/luigi/contrib/s3.py +++ b/luigi/contrib/s3.py @@ -412,7 +412,7 @@ def __copy_multipart(self, pool, src_bucket, src_key, dst_bucket, dst_key, part_ """ Copy a single S3 object to another S3 object, falling back to multipart copy where necessary - NOTE: This is a private method and should only be called from within the `luigi.s3.copy` method + NOTE: This is a private method and should only be called from within the `s3.copy` method :param pool: The threadpool to put the s3 copy processes onto :param src_bucket: source bucket name diff --git a/luigi/postgres.py b/luigi/postgres.py deleted file mode 100644 index 561d27aac7..0000000000 --- a/luigi/postgres.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017 VNG Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -luigi.postgres has moved to :py:mod:`luigi.contrib.postgres` -""" -# Delete this file any time after 24 march 2017 - -import warnings - -from luigi.contrib.postgres import * # NOQA -warnings.warn("luigi.postgres module has been moved to luigi.contrib.postgres", - DeprecationWarning) diff --git a/luigi/s3.py b/luigi/s3.py deleted file mode 100644 index 4f710600db..0000000000 --- a/luigi/s3.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017 VNG Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -luigi.s3 has moved to :py:mod:`luigi.contrib.s3` -""" -# Delete this file any time after 24 march 2017 - -import warnings - -from luigi.contrib.s3 import * # NOQA -warnings.warn("luigi.s3 module has been moved to luigi.contrib.s3", - DeprecationWarning) diff --git a/luigi/tools/deps.py b/luigi/tools/deps.py index f2b827641b..0239d84cba 100755 --- a/luigi/tools/deps.py +++ b/luigi/tools/deps.py @@ -40,8 +40,8 @@ from __future__ import print_function import luigi.interface from luigi.contrib.ssh import RemoteTarget -from luigi.postgres import PostgresTarget -from luigi.s3 import S3Target +from luigi.contrib.postgres import PostgresTarget +from luigi.contrib.s3 import S3Target from luigi.target import FileSystemTarget from luigi.task import flatten from luigi import parameter diff --git a/test/contrib/opener_test.py b/test/contrib/opener_test.py index 34ac58dca2..33336a6718 100644 --- a/test/contrib/opener_test.py +++ b/test/contrib/opener_test.py @@ -89,7 +89,7 @@ def test_local_tmp_target(self, lt_del_patch, lt_init_patch): OpenerTarget(local_file) lt_init_patch.assert_called_with(self.local_file, is_tmp=True) - @mock.patch('luigi.s3.S3Target.__init__') + @mock.patch('luigi.contrib.s3.S3Target.__init__') def test_s3_parse(self, s3_init_patch): '''Verify basic s3 target url @@ -100,7 +100,7 @@ def test_s3_parse(self, s3_init_patch): OpenerTarget(local_file) s3_init_patch.assert_called_with("s3://zefr/foo/bar.txt") - @mock.patch('luigi.s3.S3Target.__init__') + @mock.patch('luigi.contrib.s3.S3Target.__init__') def test_s3_parse_param(self, s3_init_patch): '''Verify s3 target url with params diff --git a/test/contrib/postgres_test.py b/test/contrib/postgres_test.py index 6b0069be49..5df6888343 100644 --- a/test/contrib/postgres_test.py +++ b/test/contrib/postgres_test.py @@ -14,7 +14,7 @@ import datetime import luigi -import luigi.postgres +import luigi.contrib.postgres from luigi.tools.range import RangeDaily from helpers import unittest import mock @@ -44,7 +44,7 @@ def fetchone(self): return self.fetchone_result -class DummyPostgresImporter(luigi.postgres.CopyToTable): +class DummyPostgresImporter(luigi.contrib.postgres.CopyToTable): date = luigi.DateParameter() host = 'dummy_host' @@ -83,7 +83,7 @@ def test_bulk_complete(self, mock_connect): self.assertFalse(task.complete()) -class DummyPostgresQuery(luigi.postgres.PostgresQuery): +class DummyPostgresQuery(luigi.contrib.postgres.PostgresQuery): date = luigi.DateParameter() host = 'dummy_host' diff --git a/test/contrib/postgres_with_server_test.py b/test/contrib/postgres_with_server_test.py index 142b14d7c0..e0f6459d7e 100644 --- a/test/contrib/postgres_with_server_test.py +++ b/test/contrib/postgres_with_server_test.py @@ -21,7 +21,7 @@ import luigi import luigi.notifications -from luigi import postgres +from luigi.contrib import postgres """ Typical use cases that should be tested: diff --git a/test/contrib/s3_test.py b/test/contrib/s3_test.py index 1a0cce6ed8..96ae90646c 100644 --- a/test/contrib/s3_test.py +++ b/test/contrib/s3_test.py @@ -29,7 +29,7 @@ from moto import mock_sts from luigi import configuration -from luigi.s3 import FileNotFoundException, InvalidDeleteException, S3Client, S3Target +from luigi.contrib.s3 import FileNotFoundException, InvalidDeleteException, S3Client, S3Target from luigi.target import MissingParentDirectory if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3): diff --git a/test/redshift_test.py b/test/redshift_test.py index 8af3830afe..76b3df4730 100644 --- a/test/redshift_test.py +++ b/test/redshift_test.py @@ -26,7 +26,7 @@ from luigi.contrib import redshift from moto import mock_s3 from boto.s3.key import Key -from luigi.s3 import S3Client +from luigi.contrib.s3 import S3Client if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3):