[sapling] run .t tests for getdeps sapling cli build

.github/workflows/sapling-cli-getdeps_linux.yml

@@ -30,8 +30,14 @@ jobs:
       run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive sapling
     - name: Install packaging system deps
       run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf
+    - name: Install locale-gen
+      run: sudo apt-get install locales
+    - name: Ensure en_US.UTF-8 locale present
+      run: sudo locale-gen en_US.UTF-8
     - name: Install Rust Stable
       uses: dtolnay/rust-toolchain@stable
+    - name: Fetch hexdump
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests hexdump
     - name: Fetch bz2
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests bz2
     - name: Fetch ninja
@@ -104,6 +110,8 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303
     - name: Fetch rust-shed
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed
+    - name: Build hexdump
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests hexdump
     - name: Build bz2
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2
     - name: Build ninja
@@ -185,6 +193,7 @@ jobs:
         name: sapling
         path: _artifacts
     - name: Test sapling
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling  --project-install-prefix sapling:/
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --num-jobs 16 --src-dir=. sapling  --project-install-prefix sapling:/
     - name: Show disk space at end
+      if: always()
       run: df -h

.gitignore

@@ -14,3 +14,4 @@ target/
 
 # Getdeps
 /eden/mononoke/tests/integration/getdeps_build.log
+/build/Testing/Temporary/LastTest.log

build/fbcode_builder/getdeps.py

@@ -963,6 +963,11 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         self.process_project_dir_arguments(args, loader)
         manifest = loader.load_manifest(args.project)
         manifest_ctx = loader.ctx_gen.get_context(manifest.name)
+        run_tests = (args.enable_tests and
+             manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off"
+        )
+        if run_tests:
+            manifest_ctx.set("test", "on")
         run_on = self.get_run_on(args)
 
         # Some projects don't do anything "useful" as a leaf project, only
@@ -1075,10 +1080,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 free_up_disk = ""
 
             allow_sys_arg = ""
-            if (
-                build_opts.allow_system_packages
-                and build_opts.host_type.get_package_manager()
-            ):
+            if run_tests:
                 sudo_arg = "sudo "
                 allow_sys_arg = " --allow-system-packages"
                 if build_opts.host_type.get_package_manager() == "deb":
@@ -1097,6 +1099,14 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                     out.write(
                         f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n"
                     )
+                required_locales = manifest.get("github.actions", "required_locales", ctx=manifest_ctx)
+                if build_opts.host_type.get_package_manager() == "deb" and required_locales:
+                    # ubuntu doesn't include this by default
+                    out.write(f"    - name: Install locale-gen\n")
+                    out.write(f"      run: {sudo_arg}apt-get install locales\n")
+                    for l in required_locales.split():
+                        out.write(f"    - name: Ensure {l} locale present\n")
+                        out.write(f"      run: {sudo_arg}locale-gen {l}\n")
 
             projects = loader.manifests_in_dependency_order()
 
@@ -1188,11 +1198,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            if (
-                args.enable_tests
-                and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
-                != "off"
-            ):
+            if run_tests:
                 num_jobs_arg = ""
                 if args.num_jobs:
                     num_jobs_arg = f"--num-jobs {args.num_jobs} "
@@ -1203,6 +1209,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 )
             if build_opts.free_up_disk and not build_opts.is_windows():
                 out.write("    - name: Show disk space at end\n")
+                out.write("      if: always()\n")
                 out.write("      run: df -h\n")
 
     def setup_project_cmd_parser(self, parser):

build/fbcode_builder/getdeps/manifest.py

@@ -87,6 +87,7 @@
         "optional_section": True,
         "fields": {
             "run_tests": OPTIONAL,
+            "required_locales": OPTIONAL,
         },
     },
     "crate.pathmap": {"optional_section": True},

build/fbcode_builder/manifests/hexdump

@@ -0,0 +1,12 @@
+[manifest]
+name = hexdump
+
+[rpms]
+util-linux
+
+[debs]
+bsdmainutils
+
+# only used from system packages currently
+[build]
+builder = nop

build/fbcode_builder/manifests/sapling

@@ -4,6 +4,9 @@ fbsource_path = fbcode/eden
 shipit_project = eden
 shipit_fbcode_builder = true
 
+[github.actions]
+required_locales = en_US.UTF-8
+
 [git]
 repo_url = https://github.com/facebook/sapling.git
 
@@ -57,6 +60,9 @@ fb303
 fbthrift
 rust-shed
 
+[dependencies.test=on]
+hexdump
+
 [dependencies.not(os=windows)]
 python
 

eden/scm/Makefile

@@ -65,6 +65,9 @@ COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
+MAKE_PID := $(shell echo $$PPID)
+JOBS := $(shell ps T | sed -n 's%.*$(MAKE_PID).*$(MAKE).* \(-j\|--jobs=\) *\([0-9][0-9]*\).*%\2%p')
+
 # Mac Big Sur doesn't find the standard library without this.
 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
 
@@ -172,17 +175,86 @@ install: build
 install-home: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
-install-getdeps: getdepsbuild
+install-getdeps:
 	PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \
 	    GETDEPS_BUILD=1 $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \
-		setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --install-lib="$(PREFIX)/bin" --force
+		setup.py $(PURE) install --skip-build --prefix="$(PREFIX)" --install-scripts="$(PREFIX)/bin"  --install-lib="$(PREFIX)/bin" --force && \
+		cd "$(PREFIX)/bin" && ln -f "$(SL_BIN_NAME)" "$(HG_BIN_NAME)"
+
+# Exclusions for OSS getdeps cli tests.  Newline separated
+# test-cats.t: internal crpyto token test
+# test-check-execute.t: fails on CI, passes locally on ubuntu-22.04
+# test-clone-per-repo-config.t: fails in oss, looks like depends on fb hgrc.dynamic
+# test-clone-resume.t: fails on CI, passes locally on ubuntu-22.04
+# test-commitcloud-sync.t: flaky Last Sync Version on line 812, flips beteen 17 and 16
+# test-config-precedence.t:  output mismatch: DEBUG configloader::hg: spawn ["false"] because * (glob)
+# test-dynamicconfig-unicode.t: output mismatch: cat: .hg/hgrc.dynamic: $ENOENT$
+# test-debugrefreshconfig.t: assumes an internal config location
+# test-fb-ext-fastlog.t: timesout, maybe due to internal endpoint assumptions
+# test-fb-ext-sampling.t: timeout
+# test-fb-ext-smartlog.t: output mismatch
+# test-help.t: help is different vs internal build
+# test-include-fail.t: fails on CI, passes locally on ubuntu-22.04
+# test-matcher-lots-of-globs.t: fails on CI, passes locally on ubuntu-22.04 (needs ~40GiB RAM)
+# test-network-doctor.t: times out
+# test-rust-checkout.t: fails on CI, passes locally on ubuntu-22.04
+# test-smartlog-interactive.t: smartlog format is different causing output mismatch
+# test-smartlog-interactive-highlighting.t: smartlog format is different causing output mismatch
+GETDEPS_TEST_EXCLUSION_LIST := test-cats.t \
+	test-check-execute.t \
+	test-clone-per-repo-config.t \
+	test-clone-resume.t \
+	test-commitcloud-sync.t \
+	test-config-precedence.t \
+	test-dynamicconfig-unicode.t \
+	test-debugrefreshconfig.t \
+	test-fb-ext-fastlog.t \
+	test-fb-ext-sampling.t \
+	test-fb-ext-smartlog.t \
+	test-help.t \
+	test-include-fail.t \
+	test-matcher-lots-of-globs.t \
+	test-network-doctor.t \
+	test-rust-checkout.t \
+	test-smartlog-interactive.t \
+	test-smartlog-interactive-highlighting.t
+
+# convert to a sed expression
+GETDEPS_TEST_EXCLUSIONS := $(subst $() $(),|,$(GETDEPS_TEST_EXCLUSION_LIST))
 
 .PHONY: test-getdeps
 test-getdeps:
-	# Run indicative tests to check the binary is minimally good as will be used later in Mononoke getdeps tests
-	# Running all the tests requires a bit of filtering to run the good set (or deleting flaky ones)
-	cd tests && PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \
-	    $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) run-tests.py -j2 --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" test-status.t test-commit.t
+    # Remove the .testfailed and .testerrored files so that after this next
+	# step they are written clean
+	rm -f ./tests/.test*
+	# ensure that fbpython is present, as some tests depend on it being on PATH
+	if ! which fbpython >/dev/null 2>&1; then \
+		FBPYTHON="$(GETDEPS_INSTALL_DIR)/sapling/bin/fbpython"; \
+		PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \
+		printf "#!/bin/sh\nexec \"$$PYTHON_SYS_EXECUTABLE\" \"\$$@\"\n"  > $$FBPYTHON; \
+		chmod +x "$$FBPYTHON"; \
+	fi;
+	# Run tests and retry any failures
+	export GETDEPS_BUILD=1; \
+	export HGTEST_HG=$(GETDEPS_INSTALL_DIR)/sapling/bin/$(HG_BIN_NAME); \
+	cd tests && export PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \
+	for try in $$(seq 0 $(GETDEPS_TEST_RETRY)); do \
+		RERUN_ARG=""; \
+		GETDEPS_TEST_FILTER=$(GETDEPS_TEST_FILTER); \
+		if [ $$try -gt 0 ]; then \
+			GETDEPS_TEST_FILTER=$$(cat .testfailed .testerrored | sort -u | grep -v '^$$'); \
+			if [ -z "$$GETDEPS_TEST_FILTER" ]; then "Echo no tests found for rerun on try $$try"; exit 2; fi; \
+			echo "Rerunning: $$GETDEPS_TEST_FILTER on try $$try" 1>&2; \
+			rm -f .testfailed .testerrored; \
+		elif [ -z "$$GETDEPS_TEST_FILTER" ]; then \
+		    GETDEPS_TEST_FILTER=$$(echo *.t | sed -Ee 's/($(GETDEPS_TEST_EXCLUSIONS))//g'); \
+		fi; \
+		$$PYTHON_SYS_EXECUTABLE run-tests.py -j $(JOBS) --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" $$GETDEPS_TEST_FILTER; \
+		status=$$?; \
+		# stop if all good \
+		if [ $$status = 0 ]; then echo "passed on try $$try"; exit 0; fi; \
+	done; \
+	exit $$status
 
 check: tests
 

eden/scm/README.md

@@ -4,7 +4,7 @@ Sapling is a fast, easy to use, distributed revision control tool for software
 developers.
 
 
-Basic install:
+# Basic install
 
 ```
 $ make install-oss
@@ -20,6 +20,33 @@ $ make oss        # build for inplace usage
 $ ./sl --version  # should show the latest version
 ```
 
-
 See <https://sapling-scm.com/> for detailed installation instructions,
 platform-specific notes, and Sapling user information.
+
+# Thrift enabled getdeps CLI build for use with Mononoke or EdenFS
+
+Mononoke and EdenFS need the thrift enabled sapling CLI built via getdeps. Check github actions to see current OS version the Sapling CLI Getdeps CI runs with.
+
+This build also provides a way to run the sapling .t tests in github CI and locally.
+
+When building locally you don't need to separately build all the dependencies like the github CI does, command line steps are:
+
+make sure required system packages are installed:
+`./build/fbcode_builder/getdeps.py install-system-deps --recursive sapling`
+
+build sapling (and any dependencies it requires):
+`./build/fbcode_builder/getdeps.py build --allow-system-packages --src-dir=. sapling`
+
+you can find the built binaries via:
+`./build/fbcode_builder/getdeps.py show-inst-dir --allow-system-packages --src-dir=. sapling`
+
+run the tests (you can use --num-jobs=N to adjust concurrency):
+`./build/fbcode_builder/getdeps.py test --allow-system-packages --src-dir=. --num-jobs=64 sapling`
+
+to iterate on one test run with --retry 0 --filter:
+`./build/fbcode_builder/getdeps.py test --allow-system-packages --src-dir=. sapling --retry 0 --filter test-check-execute.t`
+
+to run multiple tests with --filter separate with spaces:
+`./build/fbcode_builder/getdeps.py test --allow-system-packages --src-dir=. sapling --retry 0 --filter "test-include-fail.t test-matcher-lots-of-globs.t"`
+
+The getdeps build doesn't currently build/test ISL or other node integration.

eden/scm/tests/helpers-testrepo.sh

@@ -13,6 +13,7 @@
 # Revert the environment so that running "hg" runs the system hg
 # rather than the test hg installation.
 syshgenv () {
+    # shellcheck disable=SC1090
     . "$HGTEST_RESTOREENV"
     HGPLAIN=1
     export HGPLAIN
@@ -36,11 +37,23 @@ cat >> "$HGRCPATH" << EOF
 evolution = createmarkers
 EOF
 
-# Unconditionally use the system hg to avoid auto migration logic from
-# the in-repo hg.
-testrepohgenv () {
-    syshgenv
-}
+
+SYSTEM_HG_VER=$(syshgenv; hg --version -q 2>/dev/null)
+case "$SYSTEM_HG_VER" in
+    Sapling*)
+        # Use the system hg environment if it has a has a chance 
+        # of reading a sapling repo
+        testrepohgenv () {
+            syshgenv
+        }
+        ;;
+    *)
+        testrepohgenv () {
+            # no suitable system hg, stick current.
+            :
+        }
+        ;;
+esac
 
 testrepohg () {
     (

eden/scm/tests/test-eolfilename.t

@@ -37,27 +37,27 @@ Do error out if the naughty file is explicitly referenced:
   nothing changed
   [1]
   $ echo foo > "$A"
-  $ hg debugwalk
+  $ hg debugwalk 2>&1 | sort
   skipping invalid path 'he\rllo'
   skipping invalid path 'hell\no'
 
   $ echo bla > quickfox
-  $ hg add quickfox
+  $ hg add quickfox 2>&1  | sort
   skipping invalid path 'he\rllo'
   skipping invalid path 'hell\no'
-  $ hg ci -m 2
+  $ hg ci -m 2 2>&1 | sort
   skipping invalid path 'he\rllo'
   skipping invalid path 'hell\no'
   $ A=`printf 'quick\rfox'`
-  $ hg cp quickfox "$A"
+  $ (hg cp quickfox "$A" 2>&1; echo "[$?]" 1>&2) | sort
+  abort: '\n' and '\r' disallowed in filenames: 'quick\rfox'
   skipping invalid path 'he\rllo'
   skipping invalid path 'hell\no'
-  abort: '\n' and '\r' disallowed in filenames: 'quick\rfox'
   [255]
-  $ hg mv quickfox "$A"
+  $ (hg mv quickfox "$A" 2>&1; echo "[$?]" 1>&2) | sort
+  abort: '\n' and '\r' disallowed in filenames: 'quick\rfox'
   skipping invalid path 'he\rllo'
   skipping invalid path 'hell\no'
-  abort: '\n' and '\r' disallowed in filenames: 'quick\rfox'
   [255]
 
 https://bz.mercurial-scm.org/2036
@@ -78,10 +78,8 @@ test issue2039
   $ touch "$A"
   $ touch "$B"
 
-  $ hg status --color=always
-  skipping invalid filename: 'foo
-  bar.baz'
-  skipping invalid filename: 'foo
-  bar'
+  $ hg status --color=always 2>&1 | sed -e 's/foo\n/foo<NEWLINE>/'| sort
+  skipping invalid filename: 'foo<NEWLINE>bar'
+  skipping invalid filename: 'foo<NEWLINE>bar.baz'
 
   $ cd ..

eden/scm/tests/test-identity.t

@@ -97,7 +97,7 @@ Test we prefer ".sl" over ".hg"
 
 Can choose flavor of dot dir using REPO_IDENTITY override:
   $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg hg version -q
-  Sapling 4.4.2_dev
+  Sapling 4.* (glob)
   $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg newrepo
   $ ls .hg/requires
   .hg/requires

eden/scm/tests/test-rust-hooks.t

@@ -136,7 +136,7 @@ Warn about python hooks since we can't fall back to Python:
   [255]
 #else
   $ hg debugtestcommand --echo running
-  * command not found (glob)
+  * not found (glob)
   abort: pre-debugtestcommand hook exited with status 127
   [255]
 #endif