mononoke/integration: create a Makefile to run tests as part of getde…

…ps.py build (#67)

Summary:
Pull Request resolved: #67

With this change it will be possible to build dependencies of and run integration tests using getdeps.py.

This is the first goal of Q4 as per https://fb.quip.com/v8YzAYNSYgot: "Get Open Source version of integration tests running on Legocastle".

Before this diff:
The OSS integration tests run now on GitHub by:
- Building some test dependencies with getdeps.py
- Building some test dependencies with homebrew/apt-get
- Running tests via python script

The OSS integration tests were not running on Sandcastle.

After this diff:
The OSS integration tests run on Github by:
- Building and executing tests via getdeps.py (execution of tests happens by getdeps.py calling Make calling python script)

The OSS integration tests run on Sandcastle using the same getdeps.py setup as Github.

Reviewed By: krallin

Differential Revision: D24253268

fbshipit-source-id: cae249b72d076222673b8bbe4ec21866dcdbb253

.github/workflows/mononoke-integration_linux.yml

@@ -33,21 +33,20 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Apt-get dependencies
-      run: |
-        sudo apt-get install nmap tree
     - name: Install system deps
-      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm
+      run: >-
+        sudo python3 build/fbcode_builder/getdeps.py
+        --allow-system-packages
+        install-system-deps
+        --recursive
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm
@@ -56,6 +55,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Check space before cleanup
@@ -69,7 +69,7 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
+        --no-tests
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Check space before cleanup
@@ -84,6 +84,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Build mononoke
@@ -92,6 +93,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Check space before cleanup
@@ -104,27 +106,48 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space before running tests
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      # This is a way of getting all the dependencies of mononoke_integration
+      # which were not covered by the dependencies of the previous 3 projects.
+      # We have to do this unfortunately to avoid rebuilding the previous
+      # project as they have been built with "--src-dir=." option.
+      run: >-
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
+        for x in $(
+        sort
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(echo mononoke_integration)
+        | uniq -u
+        )
+        ;do
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-tests
+        "$x"
+        ;done
+    - name: Build mononoke_integration
       run: >-
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
+      run: >-
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
     - name: Check space after running tests
       run: df -h
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
-      run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed

.github/workflows/mononoke-integration_mac.yml

@@ -25,28 +25,25 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
+    - name: Install curl-openssl
       run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Brew dependencies
-      run: |
-        brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree
+        brew install curl-openssl
     - name: Install system deps
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
         sudo python3 build/fbcode_builder/getdeps.py
         --allow-system-packages
         install-system-deps
         --recursive
-        eden_scm
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm
@@ -56,6 +53,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm_lib_edenapi_tools
@@ -64,7 +62,7 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
+        --no-tests
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Build mononoke dependencies
@@ -74,6 +72,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Build mononoke
@@ -83,33 +82,57 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Install Python 3.7
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      # This is a way of getting all the dependencies of mononoke_integration
+      # which were not covered by the dependencies of the previous 3 projects.
+      # We have to do this unfortunately to avoid rebuilding the previous
+      # project as they have been built with "--src-dir=." option.
+      run: >-
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
+        for x in $(
+        sort
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(echo mononoke_integration)
+        | uniq -u
+        )
+        ;do
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-tests
+        "$x"
+        ;done
+    - name: Build mononoke_integration
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
       run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
+    - name: Check space after running tests
+      run: df -h

build/fbcode_builder/getdeps/builder.py

@@ -145,12 +145,17 @@ def __init__(
         inst_dir,
         build_args,
         install_args,
+        test_args,
     ):
         super(MakeBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.build_args = build_args or []
         self.install_args = install_args or []
+        self.test_args = test_args
+
+    def _get_prefix(self):
+        return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
 
     def _build(self, install_dirs, reconfigure):
         env = self._compute_env(install_dirs)
@@ -161,17 +166,24 @@ def _build(self, install_dirs, reconfigure):
         cmd = (
             ["make", "-j%s" % self.build_opts.num_jobs]
             + self.build_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
+            + self._get_prefix()
         )
         self._run_cmd(cmd, env=env)
 
-        install_cmd = (
-            ["make"]
-            + self.install_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
-        )
+        install_cmd = ["make"] + self.install_args + self._get_prefix()
         self._run_cmd(install_cmd, env=env)
 
+    def run_tests(
+        self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
+    ):
+        if not self.test_args:
+            return
+
+        env = self._compute_env(install_dirs)
+
+        cmd = ["make"] + self.test_args + self._get_prefix()
+        self._run_cmd(cmd, env=env)
+
 
 class AutoconfBuilder(BuilderBase):
     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):

build/fbcode_builder/getdeps/manifest.py

@@ -88,6 +88,7 @@
     "b2.args": {"optional_section": True},
     "make.build_args": {"optional_section": True},
     "make.install_args": {"optional_section": True},
+    "make.test_args": {"optional_section": True},
     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
     "shipit.pathmap": {"optional_section": True},
     "shipit.strip": {"optional_section": True},
@@ -437,6 +438,7 @@ def create_builder(  # noqa:C901
         if builder == "make":
             build_args = self.get_section_as_args("make.build_args", ctx)
             install_args = self.get_section_as_args("make.install_args", ctx)
+            test_args = self.get_section_as_args("make.test_args", ctx)
             return MakeBuilder(
                 build_options,
                 ctx,
@@ -446,6 +448,7 @@ def create_builder(  # noqa:C901
                 inst_dir,
                 build_args,
                 install_args,
+                test_args,
             )
 
         if builder == "autoconf":

build/fbcode_builder/manifests/mononoke

@@ -34,6 +34,7 @@ tools/rust/ossconfigs = .
 ^fbcode/eden/mononoke/Cargo\.toml$
 ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$
 ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$
+^.*/facebook/.*$
 
 [dependencies]
 fbthrift-source

build/fbcode_builder/manifests/mononoke_integration

@@ -18,14 +18,20 @@ build-getdeps
 [make.install_args]
 install-getdeps
 
+[make.test_args]
+test-getdeps
+
 [shipit.pathmap]
 fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
 
 [shipit.strip]
 ^.*/facebook/.*$
 
 [dependencies]
+eden_scm
+eden_scm_lib_edenapi_tools
 jq
+mononoke
 nmap
 python-click
 python-dulwich