mononoke/integration: create a Makefile to run tests as part of getde…

…ps.py build (#67)

Summary:
Pull Request resolved: #67

With this change it will be possible to build dependencies of and run integration tests using getdeps.py.

Differential Revision: D24253268

fbshipit-source-id: 57d53933731fd421eb901a481a028d14e3ee365f

.github/workflows/mononoke-integration_linux.yml

@@ -33,15 +33,13 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Apt-get dependencies
-      run: |
-        sudo apt-get install nmap tree
     - name: Install system deps
-      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm
+      run: >-
+        sudo python3 build/fbcode_builder/getdeps.py
+        --allow-system-packages
+        install-system-deps
+        --recursive
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         python3 build/fbcode_builder/getdeps.py build
@@ -69,7 +67,6 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Check space before cleanup
@@ -104,27 +101,30 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space before running tests
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      run: >-
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --only-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Build mononoke_integration
       run: >-
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
+      run: >-
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
     - name: Check space after running tests
       run: df -h
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
-      run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed

.github/workflows/mononoke-integration_mac.yml

@@ -25,21 +25,17 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
+    - name: Install curl-openssl
       run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Brew dependencies
-      run: |
-        brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree
+        brew install curl-openssl
     - name: Install system deps
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
         sudo python3 build/fbcode_builder/getdeps.py
         --allow-system-packages
         install-system-deps
         --recursive
-        eden_scm
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
@@ -64,7 +60,6 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Build mononoke dependencies
@@ -89,27 +84,33 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      run: >-
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --only-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Build mononoke_integration
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
       run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
+    - name: Check space after running tests
+      run: df -h

build/fbcode_builder/getdeps/builder.py

@@ -145,12 +145,17 @@ def __init__(
         inst_dir,
         build_args,
         install_args,
+        test_args,
     ):
         super(MakeBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.build_args = build_args or []
         self.install_args = install_args or []
+        self.test_args = test_args
+
+    def _get_prefix(self):
+        return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
 
     def _build(self, install_dirs, reconfigure):
         env = self._compute_env(install_dirs)
@@ -161,17 +166,24 @@ def _build(self, install_dirs, reconfigure):
         cmd = (
             ["make", "-j%s" % self.build_opts.num_jobs]
             + self.build_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
+            + self._get_prefix()
         )
         self._run_cmd(cmd, env=env)
 
-        install_cmd = (
-            ["make"]
-            + self.install_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
-        )
+        install_cmd = ["make"] + self.install_args + self._get_prefix()
         self._run_cmd(install_cmd, env=env)
 
+    def run_tests(
+        self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
+    ):
+        if not self.test_args:
+            return
+
+        env = self._compute_env(install_dirs)
+
+        cmd = ["make"] + self.test_args + self._get_prefix()
+        self._run_cmd(cmd, env=env)
+
 
 class AutoconfBuilder(BuilderBase):
     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):

build/fbcode_builder/getdeps/manifest.py

@@ -88,6 +88,7 @@
     "b2.args": {"optional_section": True},
     "make.build_args": {"optional_section": True},
     "make.install_args": {"optional_section": True},
+    "make.test_args": {"optional_section": True},
     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
     "shipit.pathmap": {"optional_section": True},
     "shipit.strip": {"optional_section": True},
@@ -437,6 +438,7 @@ def create_builder(  # noqa:C901
         if builder == "make":
             build_args = self.get_section_as_args("make.build_args", ctx)
             install_args = self.get_section_as_args("make.install_args", ctx)
+            test_args = self.get_section_as_args("make.test_args", ctx)
             return MakeBuilder(
                 build_options,
                 ctx,
@@ -446,6 +448,7 @@ def create_builder(  # noqa:C901
                 inst_dir,
                 build_args,
                 install_args,
+                test_args,
             )
 
         if builder == "autoconf":

build/fbcode_builder/manifests/mononoke

@@ -34,6 +34,7 @@ tools/rust/ossconfigs = .
 ^fbcode/eden/mononoke/Cargo\.toml$
 ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$
 ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$
+^.*/facebook/.*$
 
 [dependencies]
 fbthrift-source

build/fbcode_builder/manifests/mononoke_integration

@@ -18,14 +18,20 @@ build-getdeps
 [make.install_args]
 install-getdeps
 
+[make.test_args]
+test-getdeps
+
 [shipit.pathmap]
 fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
 
 [shipit.strip]
 ^.*/facebook/.*$
 
 [dependencies]
+eden_scm
+eden_scm_lib_edenapi_tools
 jq
+mononoke
 nmap
 python-click
 python-dulwich

eden/mononoke/tests/integration/Makefile

@@ -13,12 +13,38 @@ all: help
 
 build-getdeps:
 	mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration
-	@echo "building..."
-	touch $(GETDEPS_BUILD_DIR)/mononoke_integration/build
+	# In this step just generate the manifest.json file
+	./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --generate_manifest
 
 install-getdeps:
 	mkdir -p $(GETDEPS_INSTALL_DIR)/mononoke_integration
-	@echo "installing..."
-	touch $(GETDEPS_BUILD_DIR)/mononoke_integration/install
+	# In this step copy the integration/ folder and the manifest.json file
+	# to the installation directory
+	cp -r ../ $(GETDEPS_INSTALL_DIR)/mononoke_integration
+	cp $(GETDEPS_BUILD_DIR)/mononoke_integration/manifest.json $(GETDEPS_INSTALL_DIR)/mononoke_integration
 
-.PHONY: help all build-getdeps install-getdeps
+test-getdeps:
+	# Custom tmp folder inside getdeps scratch path, just to make sure it
+	# has all proper permissions
+	mkdir -p $(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp
+	# Remove the .testfailed and .testerrored files so that after this next
+	# step they are written clean
+	rm -f $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test*
+	# Unsetting http_proxy and https_proxy, because all the traffic from
+	# tests go to localhost (and for some reason the no_proxy=localhost env
+	# variable is not respected).
+	unset http_proxy; \
+	  unset https_proxy; \
+	  export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \
+	  export GETDEPS_BUILD=1; \
+	  ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) || true
+	# Rerunnig the failed test again, because with so many tests run
+	# concurrently there is a certain amount of flakiness involved.
+	cat $(GETDEPS_INSTALL_DIR)/mononoke/source/eden/mononoke/tests/integration/.test* || true
+	unset http_proxy; \
+	  unset https_proxy; \
+	  export TMPDIR=$(GETDEPS_BUILD_DIR)/mononoke_integration/tests-tmp; \
+	  export GETDEPS_BUILD=1; \
+	  ./run_tests_getdeps.py getdeps $(GETDEPS_INSTALL_DIR) --rerun-failed
+
+.PHONY: help all build-getdeps install-getdeps test-getdeps

eden/mononoke/tests/integration/integration_runner_real.py

@@ -152,9 +152,11 @@ def _hg_runner(
     interactive: bool = False,
     quiet: bool = False,
 ):
-    if "SANDCASTLE" in os.environ:
+    if "SANDCASTLE" in os.environ and "GETDEPS_BUILD" not in os.environ:
         # Sandcastle's /tmp might be mounted on a slow device
         # In that case let's move the test tmp dir to /dev/shm
+        # But if this is a getdeps build it might be running in environment
+        # without /dev/shm (like Legocastle's OSX), so leave it be.
         os.environ["TMPDIR"] = "/dev/shm"
 
     with tempfile.TemporaryDirectory() as output_dir: