From 1e0ca7a348543bd5279bc0796721513820e3aecd Mon Sep 17 00:00:00 2001 From: Scott Rohrer Date: Sat, 28 Oct 2023 12:05:30 -0400 Subject: [PATCH] Make Dropbox listdir optionally recursive --- luigi/contrib/dropbox.py | 4 ++-- test/contrib/dropbox_test.py | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/luigi/contrib/dropbox.py b/luigi/contrib/dropbox.py index 76760b8688..633ff5a20d 100644 --- a/luigi/contrib/dropbox.py +++ b/luigi/contrib/dropbox.py @@ -133,9 +133,9 @@ def isdir(self, path): raise e @accept_trailing_slash_in_existing_dirpaths - def listdir(self, path, **kwargs): + def listdir(self, path, recursive=True, **kwargs): dirs = [] - lister = self.conn.files_list_folder(path, recursive=True, **kwargs) + lister = self.conn.files_list_folder(path, recursive, **kwargs) dirs.extend(lister.entries) while lister.has_more: lister = self.conn.files_list_folder_continue(lister.cursor) diff --git a/test/contrib/dropbox_test.py b/test/contrib/dropbox_test.py index 38371d0a9c..7ce9792eca 100644 --- a/test/contrib/dropbox_test.py +++ b/test/contrib/dropbox_test.py @@ -87,19 +87,33 @@ def test_listdir_simple(self): list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH) self.assertTrue('/' not in list_of_dirs) self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs) - self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity + self.assertTrue(DROPBOX_TEST_SIMPLE_DIR in list_of_dirs) + self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) + self.assertTrue(DROPBOX_TEST_FILE_IN_DIR in list_of_dirs) # we verify recursivity def test_listdir_simple_with_one_slash(self): list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH + '/') self.assertTrue('/' not in list_of_dirs) self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs) - self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity + self.assertTrue(DROPBOX_TEST_SIMPLE_DIR in list_of_dirs) + self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) + self.assertTrue(DROPBOX_TEST_FILE_IN_DIR in list_of_dirs) # we verify recursivity def test_listdir_multiple(self): list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH, limit=2) self.assertTrue('/' not in list_of_dirs) self.assertTrue(DROPBOX_TEST_PATH in list_of_dirs) - self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) # we verify recursivity + self.assertTrue(DROPBOX_TEST_SIMPLE_DIR in list_of_dirs) + self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) + self.assertTrue(DROPBOX_TEST_FILE_IN_DIR in list_of_dirs) # we verify recursivity + + def test_listdir_non_recursive(self): + list_of_dirs = self.luigiconn.listdir(DROPBOX_TEST_PATH, recursive=False) + self.assertTrue('/' not in list_of_dirs) + self.assertTrue(DROPBOX_TEST_PATH not in list_of_dirs) + self.assertTrue(DROPBOX_TEST_SIMPLE_DIR in list_of_dirs) + self.assertTrue(DROPBOX_TEST_SIMPLE_FILE in list_of_dirs) + self.assertTrue(DROPBOX_TEST_FILE_IN_DIR not in list_of_dirs) def test_listdir_nonexisting(self): with self.assertRaises(dropbox.exceptions.ApiError):