From 87d7feea533684d72e75f2c3b09747e279db709d Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 27 Jan 2024 11:59:26 +0100 Subject: [PATCH] replaceDependencies: add support for ca-derivations Unlike regular input-addressed or fixed-output derivations, floating and deferred derivations do not have their store path available at evaluation time, so their outPath is a placeholder. The following changes are needed for replaceDependencies to continue working: * Detect the placeholder and retrieve the store path using another IFD hack when collecting the rewrite plan. * Try to obtain the derivation name needed for replaceDirectDependencies from the derivation arguments if a placeholder is detected. * Move the length mismatch detection to build time, since the placeholder has a fixed length which is unrelated to the store path. --- nixos/tests/replace-dependencies/default.nix | 5 ++- nixos/tests/replace-dependencies/guest.nix | 19 ++++++++ pkgs/build-support/replace-dependencies.nix | 45 +++++++++++-------- .../replace-direct-dependencies.nix | 44 +++++++++++++----- 4 files changed, 82 insertions(+), 31 deletions(-) diff --git a/nixos/tests/replace-dependencies/default.nix b/nixos/tests/replace-dependencies/default.nix index 5e34d34c9b054..1ef20cf9e098d 100644 --- a/nixos/tests/replace-dependencies/default.nix +++ b/nixos/tests/replace-dependencies/default.nix @@ -2,7 +2,10 @@ import ../make-test-python.nix ({ pkgs, ... }: { name = "replace-dependencies"; meta.maintainers = with pkgs.lib.maintainers; [ alois31 ]; - nodes.machine = { ... }: { system.extraDependencies = [ pkgs.stdenvNoCC ]; }; + nodes.machine = { ... }: { + nix.settings.experimental-features = [ "ca-derivations" ]; + system.extraDependencies = [ pkgs.stdenvNoCC ]; + }; testScript = '' start_all() diff --git a/nixos/tests/replace-dependencies/guest.nix b/nixos/tests/replace-dependencies/guest.nix index f8b325c954192..d9dac51c8bdf3 100644 --- a/nixos/tests/replace-dependencies/guest.nix +++ b/nixos/tests/replace-dependencies/guest.nix @@ -20,12 +20,17 @@ let oldDependency = writeShellScriptBin "dependency" '' echo "got old dependency" ''; + oldDependency-ca = oldDependency.overrideAttrs { __contentAddressed = true; }; newDependency = writeShellScriptBin "dependency" '' echo "got new dependency" ''; + newDependency-ca = newDependency.overrideAttrs { __contentAddressed = true; }; basic = writeShellScriptBin "test" '' ${oldDependency}/bin/dependency ''; + basic-ca = writeShellScriptBin "test" '' + ${oldDependency-ca}/bin/dependency + ''; transitive = writeShellScriptBin "test" '' ${basic}/bin/test ''; @@ -57,6 +62,20 @@ in { inherit oldDependency newDependency; }) "got new dependency"; + replacedependency-basic-old-ca = mkCheckOutput "replacedependency-basic" + (replaceDependency { + drv = basic-ca; + oldDependency = oldDependency-ca; + inherit newDependency; + }) "got new dependency"; + + replacedependency-basic-new-ca = mkCheckOutput "replacedependency-basic" + (replaceDependency { + drv = basic; + inherit oldDependency; + newDependency = newDependency-ca; + }) "got new dependency"; + replacedependency-transitive = mkCheckOutput "replacedependency-transitive" (replaceDependency { drv = transitive; diff --git a/pkgs/build-support/replace-dependencies.nix b/pkgs/build-support/replace-dependencies.nix index d4d0a5ec846ab..cd8b43059658d 100644 --- a/pkgs/build-support/replace-dependencies.nix +++ b/pkgs/build-support/replace-dependencies.nix @@ -33,8 +33,8 @@ let inherit (builtins) unsafeDiscardStringContext appendContext; inherit (lib) - trace stringLength listToAttrs attrValues mapAttrs filter hasAttr - mapAttrsToList all; + trace listToAttrs isStorePath readFile attrValues mapAttrs filter hasAttr + mapAttrsToList; inherit (lib.attrsets) mergeAttrsList; toContextlessString = x: unsafeDiscardStringContext (toString x); @@ -64,24 +64,35 @@ let echo }) > $out '').outPath; - knownDerivations = [ drv ] + targetDerivations = [ drv ] ++ map ({ newDependency, ... }: newDependency) replacements; + realisation = drv: + if isStorePath drv then + # Input-addressed and fixed-output derivations have their realisation as outPath. + toContextlessString drv + else + # Floating and deferred derivations have a placeholder outPath. + # The realisation can only be obtained by performing an actual build. + unsafeDiscardStringContext (readFile + (runCommandLocal "realisation" { env = { inherit drv; }; } '' + echo -n "$drv" > $out + '')); referencesMemo = listToAttrs (map (drv: { - name = toContextlessString drv; + name = realisation drv; value = referencesOf drv; - }) knownDerivations); + }) targetDerivations); relevantReferences = mergeAttrsList (attrValues referencesMemo); # Make sure a derivation is returned even when no replacements are actually applied. # Yes, even in the stupid edge case where the root derivation itself is replaced. - storePathOrKnownDerivationMemo = mapAttrs (drv: _references: + storePathOrKnownTargetDerivationMemo = mapAttrs (drv: _references: # builtins.storePath does not work in pure evaluation mode, even though it is not impure. # This reimplementation in Nix works as long as the path is already allowed in the evaluation state. # This is always the case here, because all paths come from the closure of the original derivation. appendContext drv { ${drv}.path = true; }) relevantReferences // listToAttrs (map (drv: { - name = toContextlessString drv; + name = realisation drv; value = drv; - }) knownDerivations); + }) targetDerivations); relevantReplacements = filter ({ oldDependency, newDependency }: if toString oldDependency == toString newDependency then @@ -90,8 +101,8 @@ let # Attempting to replace a dependency by itself is completely useless, and would only lead to infinite recursion. # Hence it must not be attempted to apply this replacement in any case. false - else if !hasAttr (toContextlessString oldDependency) - referencesMemo.${toContextlessString drv} then + else if !hasAttr (realisation oldDependency) + referencesMemo.${realisation drv} then warn "replaceDependencies: ${drv} does not depend on ${oldDependency}" # Handle the corner case where one of the other replacements introduces the dependency. # It would be more correct to not show the warning in this case, but the added complexity is probably not worth it. @@ -113,19 +124,17 @@ let value = rewriteMemo.${reference}; }) rewrittenReferences); in replaceDirectDependencies { - drv = storePathOrKnownDerivationMemo.${drv}; + drv = storePathOrKnownTargetDerivationMemo.${drv}; replacements = mapAttrsToList (name: value: { oldDependency = name; newDependency = value; }) rewrites; }) relevantReferences // listToAttrs (map (drv: { - name = toContextlessString drv; - value = storePathOrKnownDerivationMemo.${toContextlessString drv}; + name = realisation drv; + value = storePathOrKnownTargetDerivationMemo.${realisation drv}; }) cutoffPackages) // listToAttrs (map ({ oldDependency, newDependency }: { - name = toContextlessString oldDependency; - value = rewriteMemo.${toContextlessString newDependency}; + name = realisation oldDependency; + value = rewriteMemo.${realisation newDependency}; }) relevantReplacements); -in assert all ({ oldDependency, newDependency }: - stringLength oldDependency == stringLength newDependency) replacements; -rewriteMemo.${toContextlessString drv} +in rewriteMemo.${realisation drv} diff --git a/pkgs/build-support/replace-direct-dependencies.nix b/pkgs/build-support/replace-direct-dependencies.nix index 8cade88f0dd37..6cf3af89e78aa 100644 --- a/pkgs/build-support/replace-direct-dependencies.nix +++ b/pkgs/build-support/replace-direct-dependencies.nix @@ -3,19 +3,39 @@ # Replace some direct dependencies of drv, not recursing into the dependency tree. # You likely want to use replaceDependencies instead, unless you plan to implement your own recursion mechanism. { drv, replacements ? [ ] }: -let inherit (lib) all stringLength substring concatStringsSep; -in assert all ({ oldDependency, newDependency }: - stringLength oldDependency == stringLength newDependency) replacements; -if replacements == [ ] then +let + inherit (lib) + isStorePath substring stringLength optionalString escapeShellArgs concatMap; +in if replacements == [ ] then drv else - let drvName = substring 33 (stringLength (baseNameOf drv)) (baseNameOf drv); - in runCommandLocal drvName { nixStore = "${nix}/bin/nix-store"; } '' - $nixStore --dump ${drv} | sed 's|${ - baseNameOf drv - }|'$(basename $out)'|g' | sed -e ${ - concatStringsSep " -e " (map ({ oldDependency, newDependency }: - "'s|${baseNameOf oldDependency}|${baseNameOf newDependency}|g'") + let + drvName = if isStorePath drv then + # Reconstruct the name from the actual store path if available. + substring 33 (stringLength (baseNameOf drv)) (baseNameOf drv) + else if drv ? drvAttrs.name then + # Try to get the name from the derivation arguments otherwise (for floating or deferred derivations). + drv.drvAttrs.name + (let outputName = drv.outputName or "out"; + in optionalString (outputName != "out") "-${outputName}") + else + throw "cannot reconstruct the derivation name from ${drv}"; + in runCommandLocal drvName { nativeBuildInputs = [ nix.out ]; } '' + createRewriteScript() { + while [ $# -ne 0 ]; do + oldBasename="$(basename "$1")" + newBasename="$(basename "$2")" + shift 2 + if [ ''${#oldBasename} -ne ''${#newBasename} ]; then + echo "cannot rewrite $oldBasename to $newBasename: length does not match" >&2 + exit 1 + fi + echo "s|$oldBasename|$newBasename|g" >> rewrite.sed + done + } + createRewriteScript ${ + escapeShellArgs ([ drv (placeholder "out") ] ++ concatMap + ({ oldDependency, newDependency }: [ oldDependency newDependency ]) replacements) - } | $nixStore --restore $out + } + nix-store --dump ${drv} | sed -f rewrite.sed | nix-store --restore $out ''