Merge remote-tracking branch 'upstream/master' into testing

CHANGELOG.md

@@ -20,16 +20,24 @@ A feature release on top of `1.2.0`, backwards compatible.
 
 - New script `just-deduplicate-repos` to avoid blow up of the
   `repos.json` in the case of chained imports with common dependencies.
+- New subcommand `add-to-cas` to add files and directories to the local
+  CAS and optionally also copy them to the remote-execution endpoint.
 - The built-in `"generic"` rule now supports an argument `"sh -c"`,
   allowing to specify the invocation of the shell (defaulting to
   `["sh", "-c"]`).
 - `just describe` also shows the values of the implicit dependencies.
+- `just-mr` supports a new form of root, called `"foreign file"`.
 - When `just-mr` executes the action to generate the desired tree of a
   `"git tree"` repository, it can be specified that certain variables
   of the environment can be inherited.
+- The just-mr rc file now supports a field `"rc files"` to include
+  other rc files given by location objects; in particular, it is
+  possible to include rc files committed to the workspace.
 - Support for fetching archives from FTP and TFTP was added to `just-mr`
   if it was built with bundled curl. For package builds, libcurl has
   enabled whatever the distro considers suitable.
+- The `gc` subcommand supports an option `--no-rotate` to carry out
+  only local clean up.
 
 ### Fixes
 
@@ -54,6 +62,13 @@ A feature release on top of `1.2.0`, backwards compatible.
   the upgrade.
 - The taintedness of `"configure"` targets is now propagated
   correctly in analysis.
+- It is no longer incorrectly assumed that every `git` URL not
+  starting with `ssh://`, `http://`, nor `https://` is a file on the
+  local disk. Now, only URLs starting with `/`, `./`, or `file://`
+  are considered file URLs. File URLs, as well as URLs starting
+  with `git://`, `http://`, or `https://`, are handled by `just-mr`
+  using `libgit2`; for every other URL, `just-mr` shells out to
+  `git` for fetching and the URL is passed to `git` unchanged.
 - Improved portability and update of the bundled dependencies.
 - Various minor improvements and typo fixes in the documentation.
 - Fixed a race condition in an internal cache of `just execute`

INSTALL.md

@@ -31,9 +31,9 @@ variables. The full list can be obtained via `just-mr describe`.
 | ARCH | x86, x86_64, arm, arm64 | x86_64 |
 | HOST_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* |
 | TARGET_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* |
-| TOOLCHAIN_CONFIG["FAMILY"] | gnu, clang, unknown | unknown |
 | DEBUG | true, false | false |
-| BUILD_STATIC_BINARY | true, false | false |
+| TOOLCHAIN_CONFIG["FAMILY"] | gnu, clang, unknown | unknown |
+| TOOLCHAIN_CONFIG["BUILD_STATIC"] | true, false | false |
 
 Note that you can choose a different stack size for resulting binaries by
 adding `"-Wl,-z,stack-size=<size-in-bytes>"` to variable `"FINAL_LDFLAGS"`

TARGETS

@@ -7,7 +7,6 @@
     , "ADD_CXXFLAGS"
     , "AR"
     , "ARCH"
-    , "BUILD_STATIC_BINARY"
     , "FINAL_LDFLAGS"
     , "CC"
     , "CXX"
@@ -39,14 +38,14 @@
       , "used C and C++ compilers and uses the generic \"cc\" and \"c++\" as"
       , "names for the respective compilers; typically used in conjunction with"
       , "setting \"CC\" and \"CXX\" explicitly and building for host."
+      , ""
+      , "Use field \"BUILD_STATIC\" to specify that binaries should be"
+      , "statically linked. Boolean, default false."
       ]
     , "ENV":
       [ "Map from strings to strings. The build environment ot be used for"
       , "build actions. Typically used to include an unusual value of PATH."
       ]
-    , "BUILD_STATIC_BINARY":
-      [ "Boolean, default false. If set, try to build a (mainly) static binary."
-      ]
     , "FINAL_LDFLAGS": ["Compiler flags for linking the final binary."]
     , "DEBUG": ["Boolean, default false. Whether to build a debug version."]
     , "OS":
@@ -85,7 +84,6 @@
     , "ADD_CFLAGS"
     , "AR"
     , "ENV"
-    , "BUILD_STATIC_BINARY"
     , "FINAL_LDFLAGS"
     , "SOURCE_DATE_EPOCH"
     , "VERSION_EXTRA_SUFFIX"
@@ -136,7 +134,6 @@
     , "ADD_CFLAGS"
     , "AR"
     , "ENV"
-    , "BUILD_STATIC_BINARY"
     , "FINAL_LDFLAGS"
     , "SOURCE_DATE_EPOCH"
     , "VERSION_EXTRA_SUFFIX"

bin/just-import-git.py

@@ -125,8 +125,9 @@ def get_base_config(repository_config: Optional[str]) -> Optional[Json]:
     fail('Could not get base config')
 
 
-def clone(url: str, branch: str,
-          mirrors: List[str]) -> Tuple[str, Dict[str, Any], str]:
+def clone(url: str, branch: str, *,
+          mirrors: List[str], inherit_env: List[str],
+          ) -> Tuple[str, Dict[str, Any], str]:
     # clone the given git repository, checkout the specified
     # branch, and return the checkout location
     workdir: str = tempfile.mkdtemp()
@@ -145,6 +146,8 @@ def clone(url: str, branch: str,
     }
     if mirrors:
         repo = dict(repo, **{"mirrors": mirrors})
+    if inherit_env:
+        repo = dict(repo, **{"inherit env": inherit_env})
     return srcdir, repo, workdir
 
 
@@ -258,7 +261,11 @@ def rewrite_repo(repo_spec: Json, *, remote: Dict[str, Any],
 def handle_import(args: Namespace) -> Json:
     base_config: Json = cast(Json, get_base_config(args.repository_config))
     base_repos: Json = base_config.get("repositories", {})
-    srcdir, remote, to_cleanup = clone(args.URL, args.branch, args.mirrors)
+    srcdir, remote, to_cleanup = clone(
+        args.URL, args.branch,
+        mirrors = args.mirrors,
+        inherit_env = args.inherit_env,
+    )
     if args.foreign_repository_config:
         foreign_config_file = args.foreign_repository_config
     else:
@@ -364,6 +371,13 @@ def main():
                         action="append",
                         default=[],
                         metavar="URL")
+    parser.add_argument(
+        "--inherit-env",
+        dest="inherit_env",
+        help="Environment variables to inherit when calling git to fetch",
+        action="append",
+        default=[],
+        metavar="VAR")
     parser.add_argument('URL')
     parser.add_argument('foreign_repository_name', nargs='?')
     args = parser.parse_args()

doc/concepts/alternative-mirrors.md

@@ -54,7 +54,7 @@ keys in its JSON object.
   mirrors. Those mirrors are always tried first (in the given order)
   before any other URL is contacted.
 - For the optional key `"preferred hostnames"`, if given, a list of
-  host names is specified. When fetching from a non-local URL, URLs
+  host names is specified. When fetching from a non-local mirror, URLs
   with host names in the given list are preferred (in the order
   given) over URLs with other host names.
 

doc/concepts/service-target-cache.md

@@ -236,6 +236,34 @@ provided. They check whether the `just serve` endpoint knows these Git
 objects and, if yes, ensure they are uploaded to the remote CAS, from
 where the client can easily then retrieve them.
 
+#### Auxiliary requests: check and get trees
+
+In `just-mr`, the `to_git` pragma most often is used to make sure a local
+root is available in a content-defined manner as a Git-tree. This allows
+it to be used in the build description of export targets. It would be
+beneficial then for export targets built by a serve endpoint to have access
+to such roots, which often describe exactly the target(s) we want built.
+In order to facilitate this, two more auxiliary requests are added.
+
+The first request supported, given a tree identifier corresponding to a
+root, checks whether the serve endpoint has _direct_ access to it, meaning
+it is available to it locally, and, if found, it makes sure the tree is
+made available in a location where the serve endpoint can build against it.
+The places checked for this tree are, in order: the Git cache, the known
+Git repositories, and the local CAS.
+
+The second request supported, given a tree identifier, retrieves a tree
+from the CAS of the associated remote-execution endpoint and makes it
+available in a location where the serve endpoint can build against it,
+with the understanding that the client has ensured beforehand that the
+tree exists in the remote CAS. This is because, in the typical use case,
+a client will first perform a check via the first request above, and if
+the serve endpoint reports that it doesn't know the tree, the client can
+upload it to the remote CAS and ask the serve endpoint to retrieve it via
+the second request. This ensures that a client can avoid unnecessary uploads
+to the remote CAS, while making sure that the serve endpoint has the roots
+marked absent available to build against.
+
 ### Auxiliary service: configuration
 
 #### Auxiliary request: remote-execution endpoint
@@ -296,6 +324,13 @@ arguments describing a `just serve` endpoint and forwards them as
 early arguments to `just`, in the same way as it does, e.g., with
 `--local-build-root`.
 
+If a `just serve` endpoint is given to `just-mr`, the tool ensures
+however possible that all absent roots it generates are available also to
+the serve endpoint for a subsequent orchestrated remote build. Absent
+roots without providing a serve endpoint can also be generated, however
+this is not a typical use case and the tool provides warnings in this
+regard.
+
 #### `just-mr` to inquire remote execution before fetching
 
 In line with the idea that fetching sources from upstream should

doc/concepts/symlinks.md

@@ -136,7 +136,7 @@ Alternatively, in a *completely resolved tree*, all relative symbolic links
 confined to the tree (whether upwards or not) get resolved, resulting in a
 `git`-tree free of all symbolic links.
 
-For reasons already described, absolute symbolic are never supported.
+For reasons already described, absolute symbolic links are never supported.
 
 As this process acts directly at the repository level, the resulting roots
 remain cacheable and their trees accessible in constant time. Moreover, to

doc/future-designs/cas-objects-import.md

@@ -18,8 +18,8 @@ tree identifier of the root, and secondly, if the tree is not already cached
 then `just-mr` will have to (re)run the (usually expensive) fetch command and
 hash the generated directory in order to store the root in its Git cache.
 
-Proposal
---------
+Implemented Proposal
+--------------------
 
 We propose a new `just add-to-cas` subcommand which takes in a filesystem
 path and provides its Git hash to standard output. A corresponding
@@ -50,15 +50,14 @@ appropriate option.
 
 ### Symbolic links
 
-If the given path points to a symbolic link, by default the link will be
-followed and we will hash the object the symlink resolves to. However, we will
-also support, via an appropriate option, to hash the symlink content as-is
-instead.
+If the given path points to a symbolic link, by default the
+link content will be added to the CAS as blob. If the the option
+`--follow-symlinks` is given, the argument specifying what to add
+will be resolved and the object pointed to will be added to CAS.
 
-If the given path points to a directory, the treatment of contained symbolic
-links will default to allowing only non-upwards symlinks. To mirror options
-available in `just-mr` repository descriptions, we will also support options
-to either ignore, or fully resolve symlinks in the generated Git trees.
+If the given path points to a directory, non-upwards symbolic links
+will be accepted and added to the tree object. Adding directories
+to CAS is only suppored in native mode.
 
 ### Other notes
 
@@ -73,8 +72,14 @@ the command-line `git` tool, a pure computation of the hashes _without_
 generating a CAS entry might still be of interest and available as an option.
 This is, however, not useful in typical situations.
 
-Auxiliary changes
------------------
+Auxiliary changes still to be implemented
+-----------------------------------------
+
+### `just add-to-cas` to support symlink resolving inside directories
+
+To mirror options available in `just-mr` repository descriptions,
+we will also support options to either ignore, or fully resolve
+symlinks in the generated Git trees.
 
 ### `just-mr` to support `archive` subcommand
 

doc/future-designs/git-gc.md

@@ -54,7 +54,7 @@ following way.
    storage overhead will occur. The promoted commit will be tagged
    in the new-generation repository to ensure it stays there
    persistently. As usual, the tag name is encoding the commit id,
-   so that no conflicts occcur.
+   so that no conflicts occur.
 
  - To promote a `git` tree, a commit is created with this tree as
    tree, a commit message that is a function of the tree id, and
@@ -67,8 +67,8 @@ following way.
 ### New command `just-mr gc-repo`
 
 The multi-repository tool `just-mr` will get a new subcommand `gc-repo`,
-a name chosen to not conflict with the laucher functionality; recall
-that `just-mr gc` will simply call `just gc`. This new `just-rm
+a name chosen to not conflict with the launcher functionality; recall
+that `just-mr gc` will simply call `just gc`. This new `just-mr
 gc-repo` command rotates the generations: the old generation will
 be removed and the new one will become (atomically by a rename)
 the old one.

doc/future-designs/large-blobs.md

@@ -111,6 +111,5 @@ a blob via `grpc` without having to use the streaming interface,
 i.e, we chose 2MB. In this way, we already have correct blobs split
 for transfer to an end point that supports blob splicing.
 
-Additionally, the `gc` will get an option `--compactify-only`
-instructing `gc` to only perform the compactification step, without
-rotating generations.
+The compactification step will also be carried out if the `--no-rotate`
+option is given to `gc`.

etc/defaults/CC/TARGETS

@@ -56,5 +56,18 @@
       , "else": ["-O2", "-DNDEBUG"]
       }
     }
+  , "ADD_LDFLAGS":
+    { "type": "if"
+    , "cond":
+      { "type": "lookup"
+      , "map":
+        { "type": "var"
+        , "name": "TOOLCHAIN_CONFIG"
+        , "default": {"type": "empty_map"}
+        }
+      , "key": "BUILD_STATIC"
+      }
+    , "then": ["-static"]
+    }
   }
 }

etc/import/TARGETS.bazel_remote_apis

@@ -31,6 +31,7 @@
     , "The Remote Execution API is an API that, at its most general, allows clients"
     , "to request execution of binaries on a remote system."
     ]
-  , "flexible_config": ["ARCH", "ENV", "HOST_ARCH", "PATCH"]
+  , "flexible_config":
+    ["ARCH", "ENV", "HOST_ARCH", "PATCH", "TOOLCHAIN_CONFIG"]
   }
 }

etc/import/TARGETS.boringssl

@@ -58,7 +58,8 @@
   , "private-hdrs": ["fips_fragments", "crypto_internal_headers"]
   , "srcs": ["crypto_sources", "crypto_sources_asm"]
   , "pure C": ["YES"]
-  , "private-ldflags": ["-pthread"]
+  , "private-ldflags":
+    ["-pthread", "-Wl,--whole-archive,-lpthread,--no-whole-archive"]
   }
 , "ssl-lib":
   { "type": ["@", "rules", "CC", "library"]