mononoke/integration tests: use C locale by default

Summary:
As of D22098359 (facebook@7f15881) the default locale used by integration tests is en_US.UTF-8, but as the comment in code mentiones:
```
The en_US.UTF-8 locale doesn't behave the same on all systems and trying to run
commands like "sed" or "tr" on non-utf8 data will result in "Illegal byte
sequence" error.
That is why we are forcing the "C" locale.
```

Additionally I've changed the test-walker-throttle.t test to use "/bin/date" directly. Previously it was using "/usr/bin/date", but the "/bin/date" is a more standard path as it works on MacOS.

Differential Revision: D22865007

fbshipit-source-id: 8307c82dcb1cf8f23c1832cf54114aaf5cf18b76

eden/mononoke/tests/integration/integration_runner_real.py

@@ -184,6 +184,12 @@ def _hg_runner(
         env.update(
             {
                 "HGPYTHONPATH": manifest_env["RUN_TESTS_LIBRARY"],
+                # Use native byte values when sorting or manipulating data.
+                # Notice that the en_US.UTF-8 locale doesn't behave the same on
+                # all systems and trying to run commands like "sed" or "tr" on
+                # non-utf8 data will result in "Illegal byte sequence" error.
+                # That is why we are forcing the "C" locale.
+                "HGTEST_LOCALE": "C",
                 "PYTHON_SYS_EXECUTABLE": manifest_env["BINARY_HGPYTHON"],
             }
         )

eden/mononoke/tests/integration/test-backsyncer-merges.t

@@ -83,7 +83,7 @@ although the second one became non-merge commit
 
 
 Make sure we have directory from the first move, but not from the second
-  $ LC_ALL=C ls
+  $ ls
   file.txt
   filetoremove
   newrepo

eden/mononoke/tests/integration/test-blobimport-inline.t

@@ -29,7 +29,7 @@
   $ for (( i=0; i < $lines_cnt; i++ ))
   > do
   >  LINE_LENGTH=$(random_int $max_line_length)
-  >  echo $(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
+  >  echo $(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
   > done
 
   $ hg ci -Aqm "commit"$c
@@ -41,7 +41,7 @@
   >   do
   >     LINE_LENGTH=$(random_int $max_line_length)
   >     LINE_NUMBER=$(random_int $lines_cnt)
-  >     CONTENT=$(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
+  >     CONTENT=$(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
   >     sed -i "$LINE_NUMBER""s/.*/$CONTENT/" file
   >   done
   >   hg ci -Aqm "commit"$c

eden/mononoke/tests/integration/test-cross-repo-commit-sync.t

@@ -175,9 +175,9 @@ Disable bookmarks cache because bookmarks are modified by two separate processes
    99c848e3f5ff3ab7746fb71816748e2ba0d7da36 fbsource commit 2
    b0474d400edddcabef0a27ead293a6b99ae59490 ovrsource commit 2
    b06de5da9e40e0da6eda1f7b5c891711106d707b fbsource commit 3
+   e0cb430152c2dcc47b93a516344e3814ece60d4b fbsource commit 10
   default/fbsource/somebook d692e38644b938ccccc4192bd2f507955f3888c5 non-master commit
   default/master_bookmark 8d01dd2e0e909e21d3131b7929787db006de999e fbsource commit resume
-   e0cb430152c2dcc47b93a516344e3814ece60d4b fbsource commit 10
 
 -- Validate the synced entries
   $ REPOIDLARGE=0 validate_commit_sync 17 |& grep "Validated entry"

eden/mononoke/tests/integration/test-edenapi-server-complete-trees.t

@@ -19,7 +19,7 @@ Initialize test repo.
 Create a nested directory structure.
   $ mkdir -p a{1,2}/b{1,2}/c{1,2}
   $ echo "1" | tee a{1,2}/{file,b{1,2}/{file,c{1,2}/file}} > /dev/null
-  $ LC_ALL=C tree
+  $ tree
   .
   |-- a1
   |   |-- b1

eden/mononoke/tests/integration/test-inconsistent-hash.t

@@ -53,7 +53,7 @@
   $ PACK_TO_CORRUPT=".hg/store/packs/dee3d9750ad87ede865d69e20330c34e51ec83d5.datapack"
 # change access to file, as it is readonly
   $ chmod 666 "$PACK_TO_CORRUPT"
-  $ LC_ALL=C sed -i s/hello_world/aaaaaaaaaaa/ "$PACK_TO_CORRUPT"
+  $ sed -i s/hello_world/aaaaaaaaaaa/ "$PACK_TO_CORRUPT"
 
 Do a push, but disable cache verification on the client side, otherwise
 filenode won't be send at all

eden/mononoke/tests/integration/test-walker-throttle.t

@@ -30,15 +30,15 @@ Base case, check can walk fine
   Walked* (glob)
 
 Check reads throttle
-  $ START_SECS=$(date "+%s")
+  $ START_SECS=$(/bin/date "+%s")
   $ mononoke_walker --storage-id=blobstore --readonly-storage --cachelib-only-blobstore --blobstore-read-qps=5 scrub -I deep -q --bookmark master_bookmark 2>&1 | strip_glog
   Walking roots * (glob)
   Walking edge types * (glob)
   Walking node types * (glob)
   Final count: (40, 40)
   Bytes/s,* (glob)
   Walked* (glob)
-  $ END_SECS=$(date "+%s")
+  $ END_SECS=$(/bin/date "+%s")
   $ ELAPSED_SECS=$(( "$END_SECS" - "$START_SECS" ))
   $ if [[ "$ELAPSED_SECS" -ge 4 ]]; then echo Took Long Enough Read; else echo "Too short: $ELAPSED_SECS"; fi
   Took Long Enough Read
@@ -49,7 +49,7 @@ Delete all data from one side of the multiplex
   $ rm blobstore/0/blobs/*
 
 Check writes throttle in Repair mode
-  $ START_SECS=$(date "+%s")
+  $ START_SECS=$(/bin/date "+%s")
   $ mononoke_walker --storage-id=blobstore --readonly-storage --cachelib-only-blobstore --blobstore-write-qps=5 scrub --scrub-blobstore-action=Repair -I deep -q --bookmark master_bookmark 2>&1 | strip_glog | sed -re 's/^(scrub: blobstore_id BlobstoreId.0. repaired for repo0000.).*/\1/' | uniq -c | sed 's/^ *//'
   1 Walking roots * (glob)
   1 Walking edge types * (glob)
@@ -58,7 +58,7 @@ Check writes throttle in Repair mode
   1 Final count: (40, 40)
   1 Bytes/s,* (glob)
   1 Walked* (glob)
-  $ END_SECS=$(date "+%s")
+  $ END_SECS=$(/bin/date "+%s")
   $ ELAPSED_SECS=$(( "$END_SECS" - "$START_SECS" ))
   $ if [[ "$ELAPSED_SECS" -ge 4 ]]; then echo Took Long Enough Repair; else echo "Too short: $ELAPSED_SECS"; fi
   Took Long Enough Repair

eden/scm/tests/run-tests.py

@@ -315,6 +315,7 @@ def t():
     "slowtimeout": ("HGTEST_SLOWTIMEOUT", 1000),
     "port": ("HGTEST_PORT", 20059),
     "shell": ("HGTEST_SHELL", "bash"),
+    "locale": ("HGTEST_LOCALE", "en_US.UTF-8"),
     "maxdifflines": ("HGTEST_MAXDIFFLINES", 200),
 }
 
@@ -472,6 +473,9 @@ def getparser():
     harness.add_argument(
         "--shell", help="shell to use (default: $%s or %s)" % defaults["shell"]
     )
+    harness.add_argument(
+        "--locale", help="locale to use (default: $%s or %s)" % defaults["locale"]
+    )
     harness.add_argument(
         "--showchannels", action="store_true", help="show scheduling channels"
     )
@@ -1516,7 +1520,7 @@ def defineport(i):
 
         # Reset some environment variables to well-known values so that
         # the tests produce repeatable output.
-        env["LANG"] = env["LC_ALL"] = env["LANGUAGE"] = "en_US.UTF-8"
+        env["LANG"] = env["LC_ALL"] = env["LANGUAGE"] = self._options.locale
         env["TZ"] = "GMT"
         env["EMAIL"] = "Foo Bar <[email protected]>"
         env["COLUMNS"] = "80"