diff --git a/.github/workflows/sapling-cli-getdeps_linux.yml b/.github/workflows/sapling-cli-getdeps_linux.yml index 1e155d8fc3819..880376a63aaee 100644 --- a/.github/workflows/sapling-cli-getdeps_linux.yml +++ b/.github/workflows/sapling-cli-getdeps_linux.yml @@ -185,6 +185,6 @@ jobs: name: sapling path: _artifacts - name: Test sapling - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --project-install-prefix sapling:/ + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --num-jobs 16 --src-dir=. sapling --project-install-prefix sapling:/ - name: Show disk space at end run: df -h diff --git a/.gitignore b/.gitignore index b84d43b241a49..94fcfbee542c7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ target/ # Getdeps /eden/mononoke/tests/integration/getdeps_build.log +/build/Testing/Temporary/ \ No newline at end of file diff --git a/eden/scm/Makefile b/eden/scm/Makefile index 8a1766fcb5900..cbcf27b298793 100644 --- a/eden/scm/Makefile +++ b/eden/scm/Makefile @@ -65,6 +65,9 @@ COMPILERFLAG_tmp_ = COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER) COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}} +MAKE_PID := $(shell echo $$PPID) +JOBS := $(shell ps T | sed -n 's%.*$(MAKE_PID).*$(MAKE).* \(-j\|--jobs=\) *\([0-9][0-9]*\).*%\2%p') + # Mac Big Sur doesn't find the standard library without this. export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk @@ -177,12 +180,64 @@ install-getdeps: getdepsbuild GETDEPS_BUILD=1 $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \ setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --install-lib="$(PREFIX)/bin" --force +# Exclusions for OSS getdeps cli tests. Newline separated +# test-cats.t: internal crpyto token test +# test-check-execute.t: fails on CI, passes locally on ubuntu-22.04 +# test-config-precedence.t: output mismatch: DEBUG configloader::hg: spawn ["false"] because * (glob) +# test-dynamicconfig-unicode.t: output mismatch: cat: .hg/hgrc.dynamic: $ENOENT$ +# test-debugrefreshconfig.t: assumes an internal config location +# test-fb-ext-fastlog.t: timesout, maybe due to internal endpoint assumptions +# test-fb-ext-sampling.t: timeout +# test-fb-ext-smartlog.t: output mismatch +# test-help.t: help is different vs internal build +# test-network-doctor.t: times out +# test-smartlog-interactive.t: smartlog format is different causing output mismatch +# test-smartlog-interactive-highlighting.t: smartlog format is different causing output mismatch +GETDEPS_TEST_EXCLUSION_LIST := test-cats.t \ + test-check-execute.t \ + test-config-precedence.t \ + test-dynamicconfig-unicode.t \ + test-debugrefreshconfig.t \ + test-fb-ext-fastlog.t \ + test-fb-ext-sampling.t \ + test-fb-ext-smartlog.t \ + test-help.t \ + test-network-doctor.t \ + test-smartlog-interactive.t \ + test-smartlog-interactive-highlighting.t + +# convert to a sed expression +GETDEPS_TEST_EXCLUSIONS := $(subst $() $(),|,$(GETDEPS_TEST_EXCLUSION_LIST)) + .PHONY: test-getdeps test-getdeps: - # Run indicative tests to check the binary is minimally good as will be used later in Mononoke getdeps tests - # Running all the tests requires a bit of filtering to run the good set (or deleting flaky ones) - cd tests && PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) \ - $(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)) run-tests.py -j2 --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" test-status.t test-commit.t + # Remove the .testfailed and .testerrored files so that after this next + # step they are written clean + rm -f ./tests/.test* + # ensure that fbpython is present, as some tests depend on it being on PATH + if ! which fbpython >/dev/null 2>&1; then \ + FBPYTHON="$(GETDEPS_INSTALL_DIR)/sapling/bin/fbpython"; \ + PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \ + printf "#!/bin/sh\nexec \"$$PYTHON_SYS_EXECUTABLE\" \"\$$@\"\n" > $$FBPYTHON; \ + chmod +x "$$FBPYTHON"; \ + fi; + export GETDEPS_BUILD=1; \ + export HGTEST_HG=$(GETDEPS_INSTALL_DIR)/sapling/bin/$(HG_BIN_NAME); \ + cd tests && export PYTHON_SYS_EXECUTABLE=$(shell $(PYTHON3) contrib/pick_python.py $(PYTHON3)); \ + for try in $$(seq 0 $(GETDEPS_TEST_RETRY)); do \ + RERUN_ARG=""; \ + GETDEPS_TEST_FILTER=$(GETDEPS_TEST_FILTER); \ + if [ $$try -gt 0 ]; then \ + GETDEPS_TEST_FILTER=$$(sort -u .testfailed | grep -v '^$$'); \ + elif [ -z "$$GETDEPS_TEST_FILTER" ]; then \ + GETDEPS_TEST_FILTER=$$(echo *.t | sed -Ee 's/($(GETDEPS_TEST_EXCLUSIONS))//g'); \ + fi; \ + $$PYTHON_SYS_EXECUTABLE run-tests.py -j $(JOBS) --getdeps-build --with-hg="$(PREFIX)/bin/$(HGNAME)" $$GETDEPS_TEST_FILTER; \ + status=$$?; \ + # stop if all good \ + if [ $$status = 0 ]; then echo "passed on try $$try"; exit 0; fi; \ + done; \ + exit $$status check: tests diff --git a/eden/scm/README.md b/eden/scm/README.md index 0dafea178348e..a9eb95401867d 100644 --- a/eden/scm/README.md +++ b/eden/scm/README.md @@ -4,7 +4,7 @@ Sapling is a fast, easy to use, distributed revision control tool for software developers. -Basic install: +# Basic install ``` $ make install-oss @@ -20,6 +20,23 @@ $ make oss # build for inplace usage $ ./sl --version # should show the latest version ``` - See for detailed installation instructions, platform-specific notes, and Sapling user information. + +# Thrift enabled build for use by Mononoke or EdenFS + +Mononoke and EdenFS need the thrift enabled sapling CLI built via getdeps. Check github actions to see current OS version the Sapling CLI Getdeps CI runs with. + +This build also provides a way to run the sapling .t tests in an open source environment. + +make sure system packages are installed +`./build/fbcode_builder/getdeps.py install-system-deps --recursive sapling` + +build sapling: +`./build/fbcode_builder/getdeps.py build --allow-system-packages --no-facebook-internal --src-dir=. sapling` + +run the tests. 48 jobs was about max concurrency on a 64GB RAM personal linux machine, CI runs specify less. +`./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48` + +to iterate on one test run with --retry 0 --filter: +`./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=48 --retry 0 --filter test-check-execute.t` \ No newline at end of file diff --git a/eden/scm/tests/helpers-testrepo.sh b/eden/scm/tests/helpers-testrepo.sh index b8c0648a79867..48a29e6232f7f 100644 --- a/eden/scm/tests/helpers-testrepo.sh +++ b/eden/scm/tests/helpers-testrepo.sh @@ -13,6 +13,7 @@ # Revert the environment so that running "hg" runs the system hg # rather than the test hg installation. syshgenv () { + # shellcheck disable=SC1090 . "$HGTEST_RESTOREENV" HGPLAIN=1 export HGPLAIN @@ -36,11 +37,23 @@ cat >> "$HGRCPATH" << EOF evolution = createmarkers EOF -# Unconditionally use the system hg to avoid auto migration logic from -# the in-repo hg. -testrepohgenv () { - syshgenv -} + +SYSTEM_HG_VER=$(syshgenv; hg --version -q 2>/dev/null) +case "$SYSTEM_HG_VER" in + Sapling*) + # Use the system hg environment if it has a has a chance + # of reading a sapling repo + testrepohgenv () { + syshgenv + } + ;; + *) + testrepohgenv () { + # no suitable system hg, stick current. + : + } + ;; +esac testrepohg () { ( diff --git a/eden/scm/tests/test-identity.t b/eden/scm/tests/test-identity.t index 23de246699450..023bd79068077 100644 --- a/eden/scm/tests/test-identity.t +++ b/eden/scm/tests/test-identity.t @@ -97,7 +97,7 @@ Test we prefer ".sl" over ".hg" Can choose flavor of dot dir using REPO_IDENTITY override: $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg hg version -q - Sapling 4.4.2_dev + Sapling 4.* (glob) $ SL_IDENTITY=sl SL_REPO_IDENTITY=hg newrepo $ ls .hg/requires .hg/requires diff --git a/eden/scm/tests/test-rust-hooks.t b/eden/scm/tests/test-rust-hooks.t index 7dac07f40ca91..9d6e1ec5bdd49 100644 --- a/eden/scm/tests/test-rust-hooks.t +++ b/eden/scm/tests/test-rust-hooks.t @@ -136,7 +136,7 @@ Warn about python hooks since we can't fall back to Python: [255] #else $ hg debugtestcommand --echo running - * command not found (glob) + * not found (glob) abort: pre-debugtestcommand hook exited with status 127 [255] #endif