From 1b5293d5843351843350505af1f1586a6131a904 Mon Sep 17 00:00:00 2001 From: Lukas Piatkowski Date: Fri, 16 Oct 2020 12:21:28 -0700 Subject: [PATCH] mononoke/integration: create a Makefile to run tests as part of getdeps.py build (#67) Summary: Pull Request resolved: https://github.com/facebookexperimental/eden/pull/67 With this change it will be possible to build dependencies of and run integration tests using getdeps.py. This is the first goal of Q4 as per https://fb.quip.com/v8YzAYNSYgot: "Get Open Source version of integration tests running on Legocastle". Before this diff: The OSS integration tests run now on GitHub by: - Building some test dependencies with getdeps.py - Building some test dependencies with homebrew/apt-get - Running tests via python script The OSS integration tests were not running on Sandcastle. After this diff: The OSS integration tests run on Github by: - Building and executing tests via getdeps.py (execution of tests happens by getdeps.py calling Make calling python script) The OSS integration tests run on Sandcastle using the same getdeps.py setup as Github. Reviewed By: krallin Differential Revision: D24253268 fbshipit-source-id: cae249b72d076222673b8bbe4ec21866dcdbb253 --- .../workflows/mononoke-integration_linux.yml | 79 ++++--- .../workflows/mononoke-integration_mac.yml | 75 ++++--- build/fbcode_builder/getdeps/builder.py | 24 ++- build/fbcode_builder/getdeps/manifest.py | 3 + build/fbcode_builder/manifests/mononoke | 1 + .../manifests/mononoke_integration | 6 + eden/mononoke/tests/integration/Makefile | 36 +++- .../integration/integration_runner_real.py | 4 +- .../tests/integration/run_tests_getdeps.py | 197 ++++++++++++------ 9 files changed, 296 insertions(+), 129 deletions(-) diff --git a/.github/workflows/mononoke-integration_linux.yml b/.github/workflows/mononoke-integration_linux.yml index a16e6d1400d8f..215c5e327eeb4 100644 --- a/.github/workflows/mononoke-integration_linux.yml +++ b/.github/workflows/mononoke-integration_linux.yml @@ -33,21 +33,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: '2.7' - - name: Install Python 2 dependencies - run: | - python -m pip install --upgrade pip - pip install "dulwich==0.18.6" - - name: Install Apt-get dependencies - run: | - sudo apt-get install nmap tree - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm + run: >- + sudo python3 build/fbcode_builder/getdeps.py + --allow-system-packages + install-system-deps + --recursive + mononoke_integration - name: Build eden_scm dependencies run: >- python3 build/fbcode_builder/getdeps.py build --allow-system-packages --scratch-path /tmp/build --only-deps + --no-tests --src-dir=. eden_scm - name: Build eden_scm @@ -56,6 +55,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --no-deps + --no-tests --src-dir=. eden_scm - name: Check space before cleanup @@ -69,7 +69,7 @@ jobs: python3 build/fbcode_builder/getdeps.py build --allow-system-packages --scratch-path /tmp/build - --no-deps + --no-tests --src-dir=. eden_scm_lib_edenapi_tools - name: Check space before cleanup @@ -84,6 +84,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --only-deps + --no-tests --src-dir=. mononoke - name: Build mononoke @@ -92,6 +93,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --no-deps + --no-tests --src-dir=. mononoke - name: Check space before cleanup @@ -104,27 +106,48 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install Python 3 dependencies - run: | - python -m pip install --upgrade pip - pip install click - name: Check space before running tests run: df -h - - name: Run Monononke integration tests + - name: Build mononoke_integration dependencies + # This is a way of getting all the dependencies of mononoke_integration + # which were not covered by the dependencies of the previous 3 projects. + # We have to do this unfortunately to avoid rebuilding the previous + # project as they have been built with "--src-dir=." option. + run: >- + export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; + for x in $( + sort + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration) + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke) + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools) + <(echo mononoke_integration) + | uniq -u + ) + ;do + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-tests + "$x" + ;done + - name: Build mononoke_integration run: >- - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed - /tmp/build/build/mononoke_integration_test - continue-on-error: true + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-deps + --src-dir=. + mononoke_integration + - name: Test mononoke_integration + run: >- + python3 build/fbcode_builder/getdeps.py test + --allow-system-packages + --scratch-path /tmp/build + --src-dir=. + mononoke_integration - name: Check space after running tests run: df -h - - name: Rerun failed Monononke integration tests (reduce flakiness) - run: >- - cat eden/mononoke/tests/integration/.test* || true; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed /tmp/build/build/mononoke_integration_test - --rerun-failed diff --git a/.github/workflows/mononoke-integration_mac.yml b/.github/workflows/mononoke-integration_mac.yml index 1aaa450c6ea8c..3fd56e2774e57 100644 --- a/.github/workflows/mononoke-integration_mac.yml +++ b/.github/workflows/mononoke-integration_mac.yml @@ -25,13 +25,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: '2.7' - - name: Install Python 2 dependencies + - name: Install curl-openssl run: | - python -m pip install --upgrade pip - pip install "dulwich==0.18.6" - - name: Install Brew dependencies - run: | - brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree + brew install curl-openssl - name: Install system deps run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; @@ -39,7 +35,7 @@ jobs: --allow-system-packages install-system-deps --recursive - eden_scm + mononoke_integration - name: Build eden_scm dependencies run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; @@ -47,6 +43,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --only-deps + --no-tests --src-dir=. eden_scm - name: Build eden_scm @@ -56,6 +53,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --no-deps + --no-tests --src-dir=. eden_scm - name: Build eden_scm_lib_edenapi_tools @@ -64,7 +62,7 @@ jobs: python3 build/fbcode_builder/getdeps.py build --allow-system-packages --scratch-path /tmp/build - --no-deps + --no-tests --src-dir=. eden_scm_lib_edenapi_tools - name: Build mononoke dependencies @@ -74,6 +72,7 @@ jobs: --allow-system-packages --scratch-path /tmp/build --only-deps + --no-tests --src-dir=. mononoke - name: Build mononoke @@ -83,33 +82,57 @@ jobs: --allow-system-packages --scratch-path /tmp/build --no-deps + --no-tests --src-dir=. mononoke - name: Install Python 3.7 uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install Python 3 dependencies - run: | - python -m pip install --upgrade pip - pip install click - name: Check space run: df -h - - name: Run Monononke integration tests + - name: Build mononoke_integration dependencies + # This is a way of getting all the dependencies of mononoke_integration + # which were not covered by the dependencies of the previous 3 projects. + # We have to do this unfortunately to avoid rebuilding the previous + # project as they have been built with "--src-dir=." option. + run: >- + export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; + for x in $( + sort + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration) + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke) + <(python3 build/fbcode_builder/getdeps.py list-deps mononoke) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools) + <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools) + <(echo mononoke_integration) + | uniq -u + ) + ;do + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-tests + "$x" + ;done + - name: Build mononoke_integration run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed - /tmp/build/build/mononoke_integration_test - continue-on-error: true - - name: Rerun failed Monononke integration tests (reduce flakiness) + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-deps + --src-dir=. + mononoke_integration + - name: Test mononoke_integration run: >- - cat eden/mononoke/tests/integration/.test* || true; export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed /tmp/build/build/mononoke_integration_test - --rerun-failed + python3 build/fbcode_builder/getdeps.py test + --allow-system-packages + --scratch-path /tmp/build + --src-dir=. + mononoke_integration + - name: Check space after running tests + run: df -h diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index e774eac9e12e7..237234a7fb33a 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -145,12 +145,17 @@ def __init__( inst_dir, build_args, install_args, + test_args, ): super(MakeBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) self.build_args = build_args or [] self.install_args = install_args or [] + self.test_args = test_args + + def _get_prefix(self): + return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] def _build(self, install_dirs, reconfigure): env = self._compute_env(install_dirs) @@ -161,17 +166,24 @@ def _build(self, install_dirs, reconfigure): cmd = ( ["make", "-j%s" % self.build_opts.num_jobs] + self.build_args - + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] + + self._get_prefix() ) self._run_cmd(cmd, env=env) - install_cmd = ( - ["make"] - + self.install_args - + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] - ) + install_cmd = ["make"] + self.install_args + self._get_prefix() self._run_cmd(install_cmd, env=env) + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + if not self.test_args: + return + + env = self._compute_env(install_dirs) + + cmd = ["make"] + self.test_args + self._get_prefix() + self._run_cmd(cmd, env=env) + class AutoconfBuilder(BuilderBase): def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 1094deb97da43..1e1526dcfaa14 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -88,6 +88,7 @@ "b2.args": {"optional_section": True}, "make.build_args": {"optional_section": True}, "make.install_args": {"optional_section": True}, + "make.test_args": {"optional_section": True}, "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, "shipit.pathmap": {"optional_section": True}, "shipit.strip": {"optional_section": True}, @@ -437,6 +438,7 @@ def create_builder( # noqa:C901 if builder == "make": build_args = self.get_section_as_args("make.build_args", ctx) install_args = self.get_section_as_args("make.install_args", ctx) + test_args = self.get_section_as_args("make.test_args", ctx) return MakeBuilder( build_options, ctx, @@ -446,6 +448,7 @@ def create_builder( # noqa:C901 inst_dir, build_args, install_args, + test_args, ) if builder == "autoconf": diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke index 9103a342898f7..7df92c77b3bcf 100644 --- a/build/fbcode_builder/manifests/mononoke +++ b/build/fbcode_builder/manifests/mononoke @@ -34,6 +34,7 @@ tools/rust/ossconfigs = . ^fbcode/eden/mononoke/Cargo\.toml$ ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$ ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$ +^.*/facebook/.*$ [dependencies] fbthrift-source diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration index 07a6d4a581fc2..a796e967e6aeb 100644 --- a/build/fbcode_builder/manifests/mononoke_integration +++ b/build/fbcode_builder/manifests/mononoke_integration @@ -18,6 +18,9 @@ build-getdeps [make.install_args] install-getdeps +[make.test_args] +test-getdeps + [shipit.pathmap] fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration @@ -25,7 +28,10 @@ fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration ^.*/facebook/.*$ [dependencies] +eden_scm +eden_scm_lib_edenapi_tools jq +mononoke nmap python-click python-dulwich diff --git a/eden/mononoke/tests/integration/Makefile b/eden/mononoke/tests/integration/Makefile index 60f46449c8a9e..55c8edfa7311c 100644 --- a/eden/mononoke/tests/integration/Makefile +++ b/eden/mononoke/tests/integration/Makefile @@ -13,12 +13,38 @@ all: help build-getdeps: mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration - @echo "building..." - touch $(GETDEPS_BUILD_DIR)/mononoke_integration/build + # In this step just generate the manifest.json file + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --generate_manifest install-getdeps: mkdir -p $(GETDEPS_INSTALL_DIR)/mononoke_integration - @echo "installing..." - touch $(GETDEPS_BUILD_DIR)/mononoke_integration/install + # In this step copy the integration/ folder and the manifest.json file + # to the installation directory + cp -r ../ $(GETDEPS_INSTALL_DIR)/mononoke_integration + cp $(GETDEPS_BUILD_DIR)/mononoke_integration/manifest.json $(GETDEPS_INSTALL_DIR)/mononoke_integration -.PHONY: help all build-getdeps install-getdeps +test-getdeps: + # Custom tmp folder inside getdeps scratch path, just to make sure it + # has all proper permissions + mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp + # Remove the .testfailed and .testerrored files so that after this next + # step they are written clean + rm -f $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test* + # Unsetting http_proxy and https_proxy, because all the traffic from + # tests go to localhost (and for some reason the no_proxy=localhost env + # variable is not respected). + unset http_proxy; \ + unset https_proxy; \ + export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \ + export GETDEPS_BUILD=1; \ + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) || true + # Rerunnig the failed test again, because with so many tests run + # concurrently there is a certain amount of flakiness involved. + cat $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test* || true + unset http_proxy; \ + unset https_proxy; \ + export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \ + export GETDEPS_BUILD=1; \ + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --rerun-failed + +.PHONY: help all build-getdeps install-getdeps test-getdeps diff --git a/eden/mononoke/tests/integration/integration_runner_real.py b/eden/mononoke/tests/integration/integration_runner_real.py index 716c08340361c..c827e249f3df6 100644 --- a/eden/mononoke/tests/integration/integration_runner_real.py +++ b/eden/mononoke/tests/integration/integration_runner_real.py @@ -152,9 +152,11 @@ def _hg_runner( interactive: bool = False, quiet: bool = False, ): - if "SANDCASTLE" in os.environ: + if "SANDCASTLE" in os.environ and "GETDEPS_BUILD" not in os.environ: # Sandcastle's /tmp might be mounted on a slow device # In that case let's move the test tmp dir to /dev/shm + # But if this is a getdeps build it might be running in environment + # without /dev/shm (like Legocastle's OSX), so leave it be. os.environ["TMPDIR"] = "/dev/shm" with tempfile.TemporaryDirectory() as output_dir: diff --git a/eden/mononoke/tests/integration/run_tests_getdeps.py b/eden/mononoke/tests/integration/run_tests_getdeps.py index 712a4dbb492b1..56445ba4407b3 100755 --- a/eden/mononoke/tests/integration/run_tests_getdeps.py +++ b/eden/mononoke/tests/integration/run_tests_getdeps.py @@ -28,51 +28,107 @@ def __str__(self): return self.value +def script_dir(): + return dirname(abspath(__file__)) + + def parse_args(): parser = argparse.ArgumentParser( description="Run Mononoke integration tests from getdeps.py build" ) - parser.add_argument( - "install_dir", - help="Location of getdeps.py install dir (With installed mononoke and eden_scm projects)", - ) - parser.add_argument( - "build_dir", help="Location where to put generated manifest.json file" - ) - parser.add_argument( - "tests", - nargs="*", - help="Optional list of tests to run. If provided the --tests default is None", + subs = parser.add_subparsers() + + local_parser = subs.add_parser( + "local", + help=( + "Command to run tests from the current checkout of repo assuming that" + " the required dependencies were already installed by getdeps" + " (getdeps.py build mononoke_integration)." + ), ) - parser.add_argument( - "-t", - "--test-groups", - type=TestGroup, - nargs="*", - choices=list(TestGroup), - help=f"Choose groups of tests to run, default: [{TestGroup.PASSING}]", + local_parser.set_defaults(func=local_cmd) + + getdeps_parser = subs.add_parser( + "getdeps", + help=( + "Command that is invoked by getdeps, you probably don't want to call it" + " directly." + ), ) - parser.add_argument( - "-r", - "--rerun-failed", + getdeps_parser.set_defaults(func=getdeps_cmd) + getdeps_parser.add_argument( + "--generate_manifest", + help="Generate manifest.json file in build directory and don't run tests", action="store_true", - help="Rerun failed tests based on '.testfailed' file", ) + + for p in (local_parser, getdeps_parser): + p.add_argument("getdeps_install_dir", help="Location of getdeps.py install dir") + p.add_argument( + "tests", + nargs="*", + help="Optional list of tests to run. If provided the --test-groups default is None", + ) + p.add_argument( + "-t", + "--test-groups", + type=TestGroup, + nargs="*", + choices=list(TestGroup), + help=f"Choose groups of tests to run, default: [{TestGroup.PASSING}]", + ) + p.add_argument( + "-r", + "--rerun-failed", + action="store_true", + help="Rerun failed tests based on '.testfailed' file", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Just print which tests will be run without running them", + ) + p.add_argument( + "--keep-tmpdir", + action="store_true", + help="Keep temporary directory after running tests", + ) + return parser.parse_args() -def prepare_manifest_deps(install_dir, build_dir, repo_root): +def local_cmd(args): + install_dir = args.getdeps_install_dir + repo_root = dirname(dirname(dirname(dirname(script_dir())))) + + if not args.dry_run: + prepare_manifest_deps(install_dir, repo_root) + run_tests(args, join(install_dir, "../build/mononoke_integration")) + + +def getdeps_cmd(args): + install_dir = args.getdeps_install_dir + mononoke_repo_root = join(install_dir, "mononoke/source") + + if args.generate_manifest: + prepare_manifest_deps(install_dir, mononoke_repo_root) + else: + run_tests(args, join(install_dir, "mononoke_integration")) + + +def prepare_manifest_deps(install_dir, mononoke_repo_root): + build_dir = join(install_dir, "../build/mononoke_integration") + manifest_deps_path = join(script_dir(), "manifest_deps") + exec( "global OSS_DEPS; global MONONOKE_BINS; global EDENSCM_BINS; global EDENSCMLIBEDENAPITOOLS_BINS; " - + open( - join(repo_root, "eden/mononoke/tests/integration/manifest_deps"), "r" - ).read() + + open(manifest_deps_path, "r").read() ) MANIFEST_DEPS = {} for k, v in OSS_DEPS.items(): # noqa: F821 if v.startswith("//"): - MANIFEST_DEPS[k] = join(repo_root, v[2:]) + MANIFEST_DEPS[k] = join(mononoke_repo_root, v[2:]) else: MANIFEST_DEPS[k] = v for k, v in MONONOKE_BINS.items(): # noqa: F821 @@ -87,7 +143,7 @@ def prepare_manifest_deps(install_dir, build_dir, repo_root): f.write(json.dumps(MANIFEST_DEPS, sort_keys=True, indent=4)) -def get_test_groups(repo_root): +def get_test_groups(): test_groups = { TestGroup.TIMING_OUT: { "test-blobimport-lfs.t", @@ -137,8 +193,7 @@ def get_test_groups(repo_root): assert not not_unique, f"The test groups contain not unique tests: {not_unique}" test_groups[TestGroup.ALL] = all_tests = { - basename(p) - for p in iglob(join(repo_root, "eden/mononoke/tests/integration/*.t")) + basename(p) for p in iglob(join(script_dir(), "*.t")) } not_existing = manual_groups - all_tests @@ -156,8 +211,8 @@ def get_test_groups(repo_root): return test_groups -def get_tests_to_run(repo_root, tests, groups_to_run, rerun_failed): - test_groups = get_test_groups(repo_root) +def get_tests_to_run(tests, groups_to_run, rerun_failed, test_root_public): + test_groups = get_test_groups() groups_to_run = set( groups_to_run or ([TestGroup.PASSING] if not (tests or rerun_failed) else []) @@ -173,51 +228,67 @@ def get_tests_to_run(repo_root, tests, groups_to_run, rerun_failed): if rerun_failed: # Based on eden/scm/tests/run-tests.py for title in ("failed", "errored"): - failed = Path(repo_root) / "eden/mononoke/tests/integration/.test{}".format( - title - ) + failed = Path(test_root_public) / ".test{}".format(title) if failed.is_file(): tests_to_run.update(t for t in failed.read_text().splitlines() if t) return tests_to_run -def main(): - args = parse_args() - install_dir = args.install_dir - build_dir = args.build_dir - repo_root = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) +def get_pythonpath(getdeps_install_dir): + paths = [join(getdeps_install_dir, "eden_scm/lib/python2.7/site-packages")] - prepare_manifest_deps(install_dir, build_dir, repo_root) + _, installed, _ = next(os.walk(getdeps_install_dir)) + + packages = ["click", "dulwich"] + + for package in packages: + candidates = [i for i in installed if i.startswith(f"python-{package}-")] + if len(candidates) == 0: + raise Exception( + f"Failed to find 'python-{package}' in installed directory," + " did you run getdeps?" + ) + if len(candidates) > 1: + raise Exception( + f"Found more than one 'python-{package}' package in installed" + "directory, try cleaning the install dir and rerunning getdeps" + ) + paths.append( + join(getdeps_install_dir, candidates[0], f"lib/fb-py-libs/python-{package}") + ) + + pythonpath = os.environ.get("PYTHONPATH") + return ":".join(paths) + (":{}".format(pythonpath) if pythonpath else "") + + +def run_tests(args, manifest_json_dir): + manifest_json_path = join(manifest_json_dir, "manifest.json") + with open(manifest_json_path) as json_file: + manifest = json.load(json_file) + test_root_public = manifest["TEST_ROOT_PUBLIC"] tests_to_run = get_tests_to_run( - repo_root, args.tests, args.test_groups, args.rerun_failed + args.tests, args.test_groups, args.rerun_failed, test_root_public ) env = dict(os.environ.items()) env["NO_LOCAL_PATHS"] = "1" - eden_scm_packages = join(install_dir, "eden_scm/lib/python2.7/site-packages") - pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = eden_scm_packages + ( - ":{}".format(pythonpath) if pythonpath else "" - ) + env["PYTHONPATH"] = get_pythonpath(args.getdeps_install_dir) - if tests_to_run: - sys.exit( - subprocess.run( - [ - sys.executable, - join( - repo_root, - "eden/mononoke/tests/integration/integration_runner_real.py", - ), - join(build_dir, "manifest.json"), - ] - + list(tests_to_run), - env=env, - ).returncode - ) + if args.dry_run: + print("\n".join(tests_to_run)) + elif tests_to_run: + cmd = [ + sys.executable, + join(script_dir(), "integration_runner_real.py"), + manifest_json_path, + ] + if args.keep_tmpdir: + cmd.append("--keep-tmpdir") + sys.exit(subprocess.run(cmd + list(tests_to_run), env=env).returncode) if __name__ == "__main__": - main() + args = parse_args() + args.func(args)