Merge pull request #249 from allegro/aj-support-pytest

Add support for 'pytest'

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.pytest.ini_options]
+minversion = "6.0"
+#addopts = "-ra -q"
+testpaths = [
+    "test",
+]

test/__init__.py

@@ -1,4 +0,0 @@
-# Made 'example_project.main_package' importable as a module
-# It is need to complain with bigflow project structure - top-level package X corresponds to project name.
-import sys, os.path
-sys.path.append(os.path.join(os.path.dirname(__file__), "example_project"))

test/bf-projects/example_project/resources/requirements.txt

@@ -0,0 +1,23 @@
+# *** autogenerated: don't edit ***
+# $source-hash: sha256:690d1edf921b7215a3b1b3e29efc291db86b55ff5d543d64892b06e2874b06c7
+# $source-file: resources/requirements.in
+#
+# run 'bigflow build-requirements resources/requirements.in' to update this file
+
+--index-url https://artifactory.allegrogroup.com/artifactory/api/pypi/pypi.python.org/simple
+
+datetime_truncate==1.1.0
+    # via -r resources/requirements.in
+freezegun==0.3.14
+    # via -r resources/requirements_base.in
+python-dateutil==2.8.1
+    # via freezegun
+schedule==1.1.0
+    # via -r resources/requirements_base.in
+six==1.15.0
+    # via
+    #   datetime-truncate
+    #   freezegun
+    #   python-dateutil
+typing==3.7.4.3
+    # via datetime-truncate

test/example_project/setup.py → test/bf-projects/example_project/setup.py

@@ -7,16 +7,6 @@
 from setuptools import setup
 
 
-
-def add_bigflow_to_path():
-    # For Travis
-    bf_path_index = str(Path(__file__).absolute()).split(os.sep).index('bigflow')
-    bf_path_parts = str(Path(__file__).absolute()).split(os.sep)[:bf_path_index + 1]
-    bf_package = os.path.join(os.sep, *bf_path_parts)
-    #print(f'Adding to path: {bf_package}')
-    sys.path.insert(0, bf_package)
-
-
 PROJECT_DIR = Path(__file__).parent
 PROJECT_NAME = 'main_package'
 BUILD_PATH = Path(__file__).parent / 'build'
@@ -32,7 +22,6 @@ def add_bigflow_to_path():
 RESOURCES_PATH = Path(__file__).parent / 'resources'
 
 if __name__ == '__main__':
-    add_bigflow_to_path()
     from bigflow import build
 
     setup(**build.project_setup(

test/build/test_dist.py → test/buildd/test_dist.py

@@ -4,47 +4,30 @@
 import subprocess
 
 from datetime import timedelta
-
 from pathlib import Path
 from unittest import TestCase, mock
 
-import bigflow.build.spec as spec
+from test import mixins
 
+import bigflow.build.spec as spec
+from bigflow.commons import run_process
 from bigflow.cli import walk_module_files
 from bigflow.build.dist import SETUP_VALIDATION_MESSAGE
 
 from bigflow.commons import (
     get_docker_image_id,
     build_docker_image_tag,
 )
-from bigflow.build import (
-    auto_configuration,
-    project_setup,
-)
 from bigflow.build.operate import (
     clear_image_leftovers,
     clear_dags_leftovers,
     clear_package_leftovers,
     build_image,
 )
 
-
 PROJECT_NAME = 'main_package'
 DOCKER_REPOSITORY = 'test_docker_repository'
-
-TEST_PROJECT_PATH = Path(__file__).parent.parent / 'example_project'
-IMAGE_DIR_PATH = TEST_PROJECT_PATH / '.image'
-DAGS_DIR_PATH = TEST_PROJECT_PATH / '.dags'
-DIST_DIR_PATH = TEST_PROJECT_PATH / 'dist'
-EGGS_DIR_PATH = TEST_PROJECT_PATH / f'{PROJECT_NAME}.egg-info'
-BUILD_PATH = TEST_PROJECT_PATH / 'build'
-
-
-class TestProject:
-    def run_build(self, cmd: str):
-        output = subprocess.getoutput(f'cd {TEST_PROJECT_PATH};{cmd}')
-        print(output)
-        return output
+TEST_PROJECT_PATH = None
 
 
 def mkdir(dir_path: Path):
@@ -57,14 +40,18 @@ def rmdir(dir_path: Path):
         shutil.rmtree(dir_path)
 
 
-def create_image_leftovers(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def create_image_leftovers(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
+
     mkdir(test_project_dir_path / '.image')
     (test_project_dir_path / '.image' / 'leftover').touch()
 
 
 def create_package_leftovers(
-        test_project_dir_path: Path = TEST_PROJECT_PATH,
+        test_project_dir_path=None,
         project_name: str = PROJECT_NAME):
+
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     mkdir(test_project_dir_path / 'build')
     (test_project_dir_path / 'build' / 'leftover').touch()
     mkdir(test_project_dir_path / 'dist')
@@ -73,40 +60,47 @@ def create_package_leftovers(
     (test_project_dir_path / f'{project_name}.egg-info' / 'leftover').touch()
 
 
-def create_dags_leftovers(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def create_dags_leftovers(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     mkdir(test_project_dir_path / '.dags')
     (test_project_dir_path / '.dags' / 'leftover').touch()
 
 
-def dags_leftovers_exist(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def dags_leftovers_exist(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return os.path.exists(test_project_dir_path / '.dags' / 'leftover')
 
 
 def package_leftovers_exist(
-        test_project_dir_path: Path = TEST_PROJECT_PATH,
+        test_project_dir_path=None,
         project_name: str = PROJECT_NAME):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return os.path.exists(test_project_dir_path / 'build' / 'leftover') and\
            os.path.exists(test_project_dir_path / 'dist' / 'leftover') and\
            os.path.exists(test_project_dir_path / f'{project_name}.egg-info' / 'leftover')
 
 
-def image_leftovers_exist(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def image_leftovers_exist(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return os.path.exists(test_project_dir_path / '.image' / 'leftover')
 
 
 def dir_not_empty(dir_path: Path):
     return len(os.listdir(dir_path)) != 0
 
 
-def deployment_config_copied(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def deployment_config_copied(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return (test_project_dir_path / '.image' / 'deployment_config.py').exists()
 
 
-def python_package_built(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def python_package_built(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return (test_project_dir_path / 'dist' / 'main_package-0.1.0-py3-none-any.whl').exists()
 
 
-def test_run(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def is_test_run(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return (test_project_dir_path / 'build' / 'junit-reports').exists()
 
 
@@ -125,7 +119,8 @@ def dags_built(test_project_dir_path: Path, expected_workflow_count: int):
     return workflows_count == expected_workflow_count
 
 
-def docker_image_as_file_built(test_project_dir_path: Path = TEST_PROJECT_PATH):
+def docker_image_as_file_built(test_project_dir_path=None):
+    test_project_dir_path = test_project_dir_path or TEST_PROJECT_PATH
     return (test_project_dir_path / '.image' / 'image-0.1.0.tar').exists()
 
 
@@ -142,16 +137,28 @@ def dags_contain(test_project_dir_path: Path, substring: str):
     return True
 
 
-class SetupTestCase(TestCase):
+class SetupTestCase(
+    mixins.PrototypedDirMixin,
+    mixins.SubprocessMixin,
+    TestCase,
+):
+    proto_dir = "bf-projects/example_project"
+
     def setUp(self) -> None:
-        self.test_project = TestProject()
+        super().setUp()
         self.prj = spec.parse_project_spec(
             name=PROJECT_NAME,
-            project_dir=TEST_PROJECT_PATH,
+            project_dir=self.cwd,
             docker_repository=DOCKER_REPOSITORY,
             requries=[],
         )
 
+        global TEST_PROJECT_PATH
+        TEST_PROJECT_PATH = self.cwd
+
+    def run_build(self, cmd):
+        return self.subprocess_run(cmd, text=True).stdout
+
 
 class BuildProjectE2E(SetupTestCase):
 
@@ -160,11 +167,11 @@ def test_should_build_project_artifacts(self):
         create_package_leftovers()
         create_image_leftovers()
         create_dags_leftovers()
-        self.test_project.run_build('python setup.py build_project')
+        self.run_build('python setup.py build_project')
 
         # then
         self.assertTrue(python_package_built())
-        self.assertTrue(test_run())
+        self.assertTrue(is_test_run())
         self.assertTrue(dags_built(TEST_PROJECT_PATH, 2))
 
         self.assertTrue(docker_image_as_file_built())
@@ -179,10 +186,10 @@ def test_should_build_project_artifacts(self):
 
     def test_should_validate_project_setup(self):
         # expected
-        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.test_project.run_build('python setup.py build_project --validate-project-setup'))
-        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.test_project.run_build('python setup.py build_project --build-dags --validate-project-setup'))
-        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.test_project.run_build('python setup.py build_project --build-image --validate-project-setup'))
-        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.test_project.run_build('python setup.py build_project --build-package --validate-project-setup'))
+        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.run_build('python setup.py build_project --validate-project-setup'))
+        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.run_build('python setup.py build_project --build-dags --validate-project-setup'))
+        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.run_build('python setup.py build_project --build-image --validate-project-setup'))
+        self.assertTrue(SETUP_VALIDATION_MESSAGE in self.run_build('python setup.py build_project --build-package --validate-project-setup'))
 
 
 class BuildPackageCommandE2E(SetupTestCase):
@@ -193,11 +200,11 @@ def test_should_execute_build_package_command(self):
         clear_dags_leftovers(self.prj)
 
         # when
-        self.test_project.run_build('python setup.py build_project --build-package')
+        self.run_build('python setup.py build_project --build-package')
 
         # then
         self.assertTrue(python_package_built())
-        self.assertTrue(test_run())
+        self.assertTrue(is_test_run())
         self.assertFalse(package_leftovers_exist())
 
 
@@ -211,21 +218,21 @@ def test_should_execute_build_dags_command(self):
         clear_package_leftovers(self.prj)
 
         # when
-        self.test_project.run_build('python setup.py build_project --build-dags')
+        self.run_build('python setup.py build_project --build-dags')
 
         # then
        # self.assertTrue(dags_built(TEST_PROJECT_PATH, 2))
         self.assertFalse(dags_leftovers_exist(TEST_PROJECT_PATH))
         self.assertTrue(dags_contain(TEST_PROJECT_PATH, repr(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(hours=24))))
 
         # when
-        self.test_project.run_build("python setup.py build_project --build-dags --start-time '2020-01-02 00:00:00'")
+        self.run_build(["python", "setup.py", "build_project", "--build-dags", "--start-time", "2020-01-02 00:00:00"])
 
         # then
         self.assertTrue(dags_contain(TEST_PROJECT_PATH, 'datetime.datetime(2020, 1, 1, 0, 0)'))
 
         # when
-        self.test_project.run_build('python setup.py build_project --build-dags --workflow workflow1')
+        self.run_build('python setup.py build_project --build-dags --workflow workflow1')
 
         # then
         self.assertTrue(self.single_dag_for_workflow_exists('workflow1'))
@@ -248,10 +255,10 @@ def test_should_execute_build_image_command(self):
         clear_dags_leftovers(self.prj)
         clear_package_leftovers(self.prj)
 
-        self.test_project.run_build('python setup.py build_project --build-package')
+        self.run_build('python setup.py build_project --build-package')
 
         # when
-        self.test_project.run_build('python setup.py build_project --build-image')
+        self.run_build('python setup.py build_project --build-image')
 
         # then
         self.assertFalse(image_leftovers_exist())

test/build/test_reflect.py → test/buildd/test_reflect.py

@@ -31,7 +31,7 @@ class _BaseBuildReflectTest(
     mixins.PrototypedDirMixin,
     unittest.TestCase,
 ):
-    proto_dir = "build/bf-projects/bf_selfbuild_project"
+    proto_dir = "buildd/bf-projects/bf_selfbuild_project"
 
     def runpy_n_dump(self, func_name: str):
         mod, _ = func_name.rsplit(".", 1)

test/test_cli.py → test/cli/test_cli.py

@@ -9,17 +9,21 @@
 
 from bigflow.cli import *
 
+from test import mixins
+
 TESTS_DIR = Path(__file__).parent
-EXAMPLE_PROJECT_PATH = TESTS_DIR / "example_project"
-EXAMPLE_PROJECT_SETUPPY = EXAMPLE_PROJECT_PATH / "setup.py"
 
 
-class CliTestCase(TestCase):
+class CliTestCase(
+    mixins.PrototypedDirMixin,
+    TestCase,
+):
+    proto_dir = "bf-projects/example_project"
 
     def setUp(self) -> None:
-        cwd = os.getcwd()
-        self.addCleanup(os.chdir, cwd)
-        os.chdir(EXAMPLE_PROJECT_PATH)
+        super().setUp()
+
+        self.project_setuppy = self.cwd / "setup.py"
 
         global TEST_MODULE_PATH
         TEST_MODULE_PATH = Path(__file__).parent / 'test_module'
@@ -46,7 +50,7 @@ def test_should_walk_through_all_files_inside_package_tree(self):
         # and at the first position there are absolute paths
         for (path, name) in res_as_list:
             self.assertEqual('/', path[0], "Path should be absolute and start with /")
-            expected_ending = 'bigflow/test/test_module'
+            expected_ending = 'bigflow/test/cli/test_module'
             self.assertEqual(expected_ending, path[-len(expected_ending):])
 
     def test_should_walk_through_all_module_paths_inside_package_tree(self):
@@ -68,7 +72,7 @@ def test_should_build_module_path_for_example_file(self):
         res = build_module_path(root_path, file_path, module_file)
 
         # then
-        self.assertEqual("test.test_module.py_unused1", res)
+        self.assertEqual("cli.test_module.py_unused1", res)
 
     def test_should_walk_through_all_modules_inside_package_tree(self):
         # when
@@ -132,7 +136,7 @@ def test_should_not_find_non_existing_workflow(self):
         # then
         exception_message = cm.exception.args[0]
         expected_prefix = "Workflow with id NOT_EXISTING_ID not found in package "
-        expected_suffix = "bigflow/test/test_module"
+        expected_suffix = "bigflow/test/cli/test_module"
         self.assertEqual(exception_message[:len(expected_prefix)], expected_prefix)
         self.assertEqual(exception_message[-len(expected_suffix):], expected_suffix)
 
@@ -487,7 +491,8 @@ def test_should_call_cli_deploy_image_command__when_all_parameters_are_given_by_
     @mock.patch('bigflow.cli.deploy_docker_image')
     def test_should_find_tar_in_image_directory(self, deploy_docker_image_mock):
         # given
-        shutil.rmtree(Path.cwd() / ".image")
+
+        shutil.rmtree(Path.cwd() / ".image", ignore_errors=True)
         self._touch_file('image-123.tar', '', '.image')
 
         # when