From efd448b1b748a4c58f47188e29db795b6a9aaf64 Mon Sep 17 00:00:00 2001 From: Lukas Piatkowski Date: Wed, 14 Oct 2020 11:40:52 -0700 Subject: [PATCH] mononoke/integration: create a Makefile to run tests as part of getdeps.py build (#67) Summary: Pull Request resolved: https://github.com/facebookexperimental/eden/pull/67 With this change it will be possible to build dependencies of and run integration tests using getdeps.py. Differential Revision: D24253268 fbshipit-source-id: 57d53933731fd421eb901a481a028d14e3ee365f --- .../workflows/mononoke-integration_linux.yml | 56 ++--- .../workflows/mononoke-integration_mac.yml | 53 ++--- build/fbcode_builder/getdeps/builder.py | 24 ++- build/fbcode_builder/getdeps/manifest.py | 3 + build/fbcode_builder/manifests/mononoke | 1 + .../manifests/mononoke_integration | 6 + eden/mononoke/tests/integration/Makefile | 36 +++- .../integration/integration_runner_real.py | 4 +- .../tests/integration/run_tests_getdeps.py | 197 ++++++++++++------ 9 files changed, 251 insertions(+), 129 deletions(-) diff --git a/.github/workflows/mononoke-integration_linux.yml b/.github/workflows/mononoke-integration_linux.yml index a16e6d1400d8f..4e5c73596f411 100644 --- a/.github/workflows/mononoke-integration_linux.yml +++ b/.github/workflows/mononoke-integration_linux.yml @@ -33,15 +33,13 @@ jobs: uses: actions/setup-python@v2 with: python-version: '2.7' - - name: Install Python 2 dependencies - run: | - python -m pip install --upgrade pip - pip install "dulwich==0.18.6" - - name: Install Apt-get dependencies - run: | - sudo apt-get install nmap tree - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm + run: >- + sudo python3 build/fbcode_builder/getdeps.py + --allow-system-packages + install-system-deps + --recursive + mononoke_integration - name: Build eden_scm dependencies run: >- python3 build/fbcode_builder/getdeps.py build @@ -69,7 +67,6 @@ jobs: python3 build/fbcode_builder/getdeps.py build --allow-system-packages --scratch-path /tmp/build - --no-deps --src-dir=. eden_scm_lib_edenapi_tools - name: Check space before cleanup @@ -104,27 +101,30 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install Python 3 dependencies - run: | - python -m pip install --upgrade pip - pip install click - name: Check space before running tests run: df -h - - name: Run Monononke integration tests + - name: Build mononoke_integration dependencies + run: >- + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --only-deps + --src-dir=. + mononoke_integration + - name: Build mononoke_integration run: >- - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed - /tmp/build/build/mononoke_integration_test - continue-on-error: true + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-deps + --src-dir=. + mononoke_integration + - name: Test mononoke_integration + run: >- + python3 build/fbcode_builder/getdeps.py test + --allow-system-packages + --scratch-path /tmp/build + --src-dir=. + mononoke_integration - name: Check space after running tests run: df -h - - name: Rerun failed Monononke integration tests (reduce flakiness) - run: >- - cat eden/mononoke/tests/integration/.test* || true; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed /tmp/build/build/mononoke_integration_test - --rerun-failed diff --git a/.github/workflows/mononoke-integration_mac.yml b/.github/workflows/mononoke-integration_mac.yml index 1aaa450c6ea8c..fcd65738c72a5 100644 --- a/.github/workflows/mononoke-integration_mac.yml +++ b/.github/workflows/mononoke-integration_mac.yml @@ -25,13 +25,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: '2.7' - - name: Install Python 2 dependencies + - name: Install curl-openssl run: | - python -m pip install --upgrade pip - pip install "dulwich==0.18.6" - - name: Install Brew dependencies - run: | - brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree + brew install curl-openssl - name: Install system deps run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; @@ -39,7 +35,7 @@ jobs: --allow-system-packages install-system-deps --recursive - eden_scm + mononoke_integration - name: Build eden_scm dependencies run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; @@ -64,7 +60,6 @@ jobs: python3 build/fbcode_builder/getdeps.py build --allow-system-packages --scratch-path /tmp/build - --no-deps --src-dir=. eden_scm_lib_edenapi_tools - name: Build mononoke dependencies @@ -89,27 +84,33 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install Python 3 dependencies - run: | - python -m pip install --upgrade pip - pip install click - name: Check space run: df -h - - name: Run Monononke integration tests + - name: Build mononoke_integration dependencies + run: >- + export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --only-deps + --src-dir=. + mononoke_integration + - name: Build mononoke_integration run: >- export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed - /tmp/build/build/mononoke_integration_test - continue-on-error: true - - name: Rerun failed Monononke integration tests (reduce flakiness) + python3 build/fbcode_builder/getdeps.py build + --allow-system-packages + --scratch-path /tmp/build + --no-deps + --src-dir=. + mononoke_integration + - name: Test mononoke_integration run: >- - cat eden/mononoke/tests/integration/.test* || true; export PATH="/usr/local/opt/curl-openssl/bin:$PATH"; - PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages" - python3 - eden/mononoke/tests/integration/run_tests_getdeps.py - /tmp/build/installed /tmp/build/build/mononoke_integration_test - --rerun-failed + python3 build/fbcode_builder/getdeps.py test + --allow-system-packages + --scratch-path /tmp/build + --src-dir=. + mononoke_integration + - name: Check space after running tests + run: df -h diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 3095d64397bf0..516dac62a2a0c 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -145,12 +145,17 @@ def __init__( inst_dir, build_args, install_args, + test_args, ): super(MakeBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) self.build_args = build_args or [] self.install_args = install_args or [] + self.test_args = test_args + + def _get_prefix(self): + return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] def _build(self, install_dirs, reconfigure): env = self._compute_env(install_dirs) @@ -161,17 +166,24 @@ def _build(self, install_dirs, reconfigure): cmd = ( ["make", "-j%s" % self.build_opts.num_jobs] + self.build_args - + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] + + self._get_prefix() ) self._run_cmd(cmd, env=env) - install_cmd = ( - ["make"] - + self.install_args - + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] - ) + install_cmd = ["make"] + self.install_args + self._get_prefix() self._run_cmd(install_cmd, env=env) + def run_tests( + self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot + ): + if not self.test_args: + return + + env = self._compute_env(install_dirs) + + cmd = ["make"] + self.test_args + self._get_prefix() + self._run_cmd(cmd, env=env) + class AutoconfBuilder(BuilderBase): def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 95e1c3b99e1bd..c24c273b14687 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -88,6 +88,7 @@ "b2.args": {"optional_section": True}, "make.build_args": {"optional_section": True}, "make.install_args": {"optional_section": True}, + "make.test_args": {"optional_section": True}, "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, "shipit.pathmap": {"optional_section": True}, "shipit.strip": {"optional_section": True}, @@ -437,6 +438,7 @@ def create_builder( # noqa:C901 if builder == "make": build_args = self.get_section_as_args("make.build_args", ctx) install_args = self.get_section_as_args("make.install_args", ctx) + test_args = self.get_section_as_args("make.test_args", ctx) return MakeBuilder( build_options, ctx, @@ -446,6 +448,7 @@ def create_builder( # noqa:C901 inst_dir, build_args, install_args, + test_args, ) if builder == "autoconf": diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke index 9103a342898f7..7df92c77b3bcf 100644 --- a/build/fbcode_builder/manifests/mononoke +++ b/build/fbcode_builder/manifests/mononoke @@ -34,6 +34,7 @@ tools/rust/ossconfigs = . ^fbcode/eden/mononoke/Cargo\.toml$ ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$ ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$ +^.*/facebook/.*$ [dependencies] fbthrift-source diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration index 07a6d4a581fc2..a796e967e6aeb 100644 --- a/build/fbcode_builder/manifests/mononoke_integration +++ b/build/fbcode_builder/manifests/mononoke_integration @@ -18,6 +18,9 @@ build-getdeps [make.install_args] install-getdeps +[make.test_args] +test-getdeps + [shipit.pathmap] fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration @@ -25,7 +28,10 @@ fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration ^.*/facebook/.*$ [dependencies] +eden_scm +eden_scm_lib_edenapi_tools jq +mononoke nmap python-click python-dulwich diff --git a/eden/mononoke/tests/integration/Makefile b/eden/mononoke/tests/integration/Makefile index 60f46449c8a9e..55c8edfa7311c 100644 --- a/eden/mononoke/tests/integration/Makefile +++ b/eden/mononoke/tests/integration/Makefile @@ -13,12 +13,38 @@ all: help build-getdeps: mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration - @echo "building..." - touch $(GETDEPS_BUILD_DIR)/mononoke_integration/build + # In this step just generate the manifest.json file + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --generate_manifest install-getdeps: mkdir -p $(GETDEPS_INSTALL_DIR)/mononoke_integration - @echo "installing..." - touch $(GETDEPS_BUILD_DIR)/mononoke_integration/install + # In this step copy the integration/ folder and the manifest.json file + # to the installation directory + cp -r ../ $(GETDEPS_INSTALL_DIR)/mononoke_integration + cp $(GETDEPS_BUILD_DIR)/mononoke_integration/manifest.json $(GETDEPS_INSTALL_DIR)/mononoke_integration -.PHONY: help all build-getdeps install-getdeps +test-getdeps: + # Custom tmp folder inside getdeps scratch path, just to make sure it + # has all proper permissions + mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp + # Remove the .testfailed and .testerrored files so that after this next + # step they are written clean + rm -f $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test* + # Unsetting http_proxy and https_proxy, because all the traffic from + # tests go to localhost (and for some reason the no_proxy=localhost env + # variable is not respected). + unset http_proxy; \ + unset https_proxy; \ + export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \ + export GETDEPS_BUILD=1; \ + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) || true + # Rerunnig the failed test again, because with so many tests run + # concurrently there is a certain amount of flakiness involved. + cat $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test* || true + unset http_proxy; \ + unset https_proxy; \ + export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \ + export GETDEPS_BUILD=1; \ + ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --rerun-failed + +.PHONY: help all build-getdeps install-getdeps test-getdeps diff --git a/eden/mononoke/tests/integration/integration_runner_real.py b/eden/mononoke/tests/integration/integration_runner_real.py index 716c08340361c..c827e249f3df6 100644 --- a/eden/mononoke/tests/integration/integration_runner_real.py +++ b/eden/mononoke/tests/integration/integration_runner_real.py @@ -152,9 +152,11 @@ def _hg_runner( interactive: bool = False, quiet: bool = False, ): - if "SANDCASTLE" in os.environ: + if "SANDCASTLE" in os.environ and "GETDEPS_BUILD" not in os.environ: # Sandcastle's /tmp might be mounted on a slow device # In that case let's move the test tmp dir to /dev/shm + # But if this is a getdeps build it might be running in environment + # without /dev/shm (like Legocastle's OSX), so leave it be. os.environ["TMPDIR"] = "/dev/shm" with tempfile.TemporaryDirectory() as output_dir: diff --git a/eden/mononoke/tests/integration/run_tests_getdeps.py b/eden/mononoke/tests/integration/run_tests_getdeps.py index 712a4dbb492b1..56445ba4407b3 100755 --- a/eden/mononoke/tests/integration/run_tests_getdeps.py +++ b/eden/mononoke/tests/integration/run_tests_getdeps.py @@ -28,51 +28,107 @@ def __str__(self): return self.value +def script_dir(): + return dirname(abspath(__file__)) + + def parse_args(): parser = argparse.ArgumentParser( description="Run Mononoke integration tests from getdeps.py build" ) - parser.add_argument( - "install_dir", - help="Location of getdeps.py install dir (With installed mononoke and eden_scm projects)", - ) - parser.add_argument( - "build_dir", help="Location where to put generated manifest.json file" - ) - parser.add_argument( - "tests", - nargs="*", - help="Optional list of tests to run. If provided the --tests default is None", + subs = parser.add_subparsers() + + local_parser = subs.add_parser( + "local", + help=( + "Command to run tests from the current checkout of repo assuming that" + " the required dependencies were already installed by getdeps" + " (getdeps.py build mononoke_integration)." + ), ) - parser.add_argument( - "-t", - "--test-groups", - type=TestGroup, - nargs="*", - choices=list(TestGroup), - help=f"Choose groups of tests to run, default: [{TestGroup.PASSING}]", + local_parser.set_defaults(func=local_cmd) + + getdeps_parser = subs.add_parser( + "getdeps", + help=( + "Command that is invoked by getdeps, you probably don't want to call it" + " directly." + ), ) - parser.add_argument( - "-r", - "--rerun-failed", + getdeps_parser.set_defaults(func=getdeps_cmd) + getdeps_parser.add_argument( + "--generate_manifest", + help="Generate manifest.json file in build directory and don't run tests", action="store_true", - help="Rerun failed tests based on '.testfailed' file", ) + + for p in (local_parser, getdeps_parser): + p.add_argument("getdeps_install_dir", help="Location of getdeps.py install dir") + p.add_argument( + "tests", + nargs="*", + help="Optional list of tests to run. If provided the --test-groups default is None", + ) + p.add_argument( + "-t", + "--test-groups", + type=TestGroup, + nargs="*", + choices=list(TestGroup), + help=f"Choose groups of tests to run, default: [{TestGroup.PASSING}]", + ) + p.add_argument( + "-r", + "--rerun-failed", + action="store_true", + help="Rerun failed tests based on '.testfailed' file", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Just print which tests will be run without running them", + ) + p.add_argument( + "--keep-tmpdir", + action="store_true", + help="Keep temporary directory after running tests", + ) + return parser.parse_args() -def prepare_manifest_deps(install_dir, build_dir, repo_root): +def local_cmd(args): + install_dir = args.getdeps_install_dir + repo_root = dirname(dirname(dirname(dirname(script_dir())))) + + if not args.dry_run: + prepare_manifest_deps(install_dir, repo_root) + run_tests(args, join(install_dir, "../build/mononoke_integration")) + + +def getdeps_cmd(args): + install_dir = args.getdeps_install_dir + mononoke_repo_root = join(install_dir, "mononoke/source") + + if args.generate_manifest: + prepare_manifest_deps(install_dir, mononoke_repo_root) + else: + run_tests(args, join(install_dir, "mononoke_integration")) + + +def prepare_manifest_deps(install_dir, mononoke_repo_root): + build_dir = join(install_dir, "../build/mononoke_integration") + manifest_deps_path = join(script_dir(), "manifest_deps") + exec( "global OSS_DEPS; global MONONOKE_BINS; global EDENSCM_BINS; global EDENSCMLIBEDENAPITOOLS_BINS; " - + open( - join(repo_root, "eden/mononoke/tests/integration/manifest_deps"), "r" - ).read() + + open(manifest_deps_path, "r").read() ) MANIFEST_DEPS = {} for k, v in OSS_DEPS.items(): # noqa: F821 if v.startswith("//"): - MANIFEST_DEPS[k] = join(repo_root, v[2:]) + MANIFEST_DEPS[k] = join(mononoke_repo_root, v[2:]) else: MANIFEST_DEPS[k] = v for k, v in MONONOKE_BINS.items(): # noqa: F821 @@ -87,7 +143,7 @@ def prepare_manifest_deps(install_dir, build_dir, repo_root): f.write(json.dumps(MANIFEST_DEPS, sort_keys=True, indent=4)) -def get_test_groups(repo_root): +def get_test_groups(): test_groups = { TestGroup.TIMING_OUT: { "test-blobimport-lfs.t", @@ -137,8 +193,7 @@ def get_test_groups(repo_root): assert not not_unique, f"The test groups contain not unique tests: {not_unique}" test_groups[TestGroup.ALL] = all_tests = { - basename(p) - for p in iglob(join(repo_root, "eden/mononoke/tests/integration/*.t")) + basename(p) for p in iglob(join(script_dir(), "*.t")) } not_existing = manual_groups - all_tests @@ -156,8 +211,8 @@ def get_test_groups(repo_root): return test_groups -def get_tests_to_run(repo_root, tests, groups_to_run, rerun_failed): - test_groups = get_test_groups(repo_root) +def get_tests_to_run(tests, groups_to_run, rerun_failed, test_root_public): + test_groups = get_test_groups() groups_to_run = set( groups_to_run or ([TestGroup.PASSING] if not (tests or rerun_failed) else []) @@ -173,51 +228,67 @@ def get_tests_to_run(repo_root, tests, groups_to_run, rerun_failed): if rerun_failed: # Based on eden/scm/tests/run-tests.py for title in ("failed", "errored"): - failed = Path(repo_root) / "eden/mononoke/tests/integration/.test{}".format( - title - ) + failed = Path(test_root_public) / ".test{}".format(title) if failed.is_file(): tests_to_run.update(t for t in failed.read_text().splitlines() if t) return tests_to_run -def main(): - args = parse_args() - install_dir = args.install_dir - build_dir = args.build_dir - repo_root = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) +def get_pythonpath(getdeps_install_dir): + paths = [join(getdeps_install_dir, "eden_scm/lib/python2.7/site-packages")] - prepare_manifest_deps(install_dir, build_dir, repo_root) + _, installed, _ = next(os.walk(getdeps_install_dir)) + + packages = ["click", "dulwich"] + + for package in packages: + candidates = [i for i in installed if i.startswith(f"python-{package}-")] + if len(candidates) == 0: + raise Exception( + f"Failed to find 'python-{package}' in installed directory," + " did you run getdeps?" + ) + if len(candidates) > 1: + raise Exception( + f"Found more than one 'python-{package}' package in installed" + "directory, try cleaning the install dir and rerunning getdeps" + ) + paths.append( + join(getdeps_install_dir, candidates[0], f"lib/fb-py-libs/python-{package}") + ) + + pythonpath = os.environ.get("PYTHONPATH") + return ":".join(paths) + (":{}".format(pythonpath) if pythonpath else "") + + +def run_tests(args, manifest_json_dir): + manifest_json_path = join(manifest_json_dir, "manifest.json") + with open(manifest_json_path) as json_file: + manifest = json.load(json_file) + test_root_public = manifest["TEST_ROOT_PUBLIC"] tests_to_run = get_tests_to_run( - repo_root, args.tests, args.test_groups, args.rerun_failed + args.tests, args.test_groups, args.rerun_failed, test_root_public ) env = dict(os.environ.items()) env["NO_LOCAL_PATHS"] = "1" - eden_scm_packages = join(install_dir, "eden_scm/lib/python2.7/site-packages") - pythonpath = env.get("PYTHONPATH") - env["PYTHONPATH"] = eden_scm_packages + ( - ":{}".format(pythonpath) if pythonpath else "" - ) + env["PYTHONPATH"] = get_pythonpath(args.getdeps_install_dir) - if tests_to_run: - sys.exit( - subprocess.run( - [ - sys.executable, - join( - repo_root, - "eden/mononoke/tests/integration/integration_runner_real.py", - ), - join(build_dir, "manifest.json"), - ] - + list(tests_to_run), - env=env, - ).returncode - ) + if args.dry_run: + print("\n".join(tests_to_run)) + elif tests_to_run: + cmd = [ + sys.executable, + join(script_dir(), "integration_runner_real.py"), + manifest_json_path, + ] + if args.keep_tmpdir: + cmd.append("--keep-tmpdir") + sys.exit(subprocess.run(cmd + list(tests_to_run), env=env).returncode) if __name__ == "__main__": - main() + args = parse_args() + args.func(args)