Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build with nixpkgs scope #10963

Merged
merged 11 commits into from
Jun 26, 2024
135 changes: 33 additions & 102 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -153,124 +153,55 @@
'';
};

libgit2-nix = final.libgit2.overrideAttrs (attrs: {
libgit2_nix = final.libgit2.overrideAttrs (attrs: {
src = libgit2;
version = libgit2.lastModifiedDate;
cmakeFlags = attrs.cmakeFlags or []
++ [ "-DUSE_SSH=exec" ];
});

boehmgc-nix = final.boehmgc.override {
boehmgc_nix = final.boehmgc.override {
enableLargeConfig = true;
};

libseccomp-nix = final.libseccomp.overrideAttrs (_: rec {
libseccomp_nix = final.libseccomp.overrideAttrs (_: rec {
version = "2.5.5";
src = final.fetchurl {
url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
};
});

# TODO: define everything here instead of top level?
nix-components = {
inherit (final)
nix-util
nix-util-test-support
nix-util-test
nix-util-c
nix-store
nix-fetchers
nix-perl-bindings
;
};

nix-util = final.callPackage ./src/libutil/package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
};

nix-util-test-support = final.callPackage ./tests/unit/libutil-support/package.nix {
inherit
fileset
stdenv
versionSuffix
;
};

nix-util-test = final.callPackage ./tests/unit/libutil/package.nix {
inherit
fileset
stdenv
versionSuffix
;
};

nix-util-c = final.callPackage ./src/libutil-c/package.nix {
inherit
fileset
stdenv
versionSuffix
;
};
# A new scope, so that we can use `callPackage` to inject our own interdependencies
# without "polluting" the top level "`pkgs`" attrset.
# This also has the benefit of providing us with a distinct set of packages
# we can iterate over.
nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix {
pkgs = final;
inherit stdenv versionSuffix;
});

nix-store = final.callPackage ./src/libstore/package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
libseccomp = final.libseccomp-nix;
# The dependencies are in their own scope, so that they don't have to be
# in Nixpkgs top level `pkgs` or `nixComponents`.
nixDependencies = lib.makeScope final.newScope (scope: {
inherit stdenv versionSuffix;
libseccomp = final.libseccomp_nix;
boehmgc = final.boehmgc_nix;
libgit2 = final.libgit2_nix;
busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
};
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would put out the these deps in packaging/<some-file>.nix. too. Instead of overriding prev/super, they would be overriding the regular scope. (Actually the package set function is the same, up to alpha equivalence, but the caller is different.)

The _nix suffix is just poor mans scoping anyways.

Agreed with using self.callPackage of course

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done:

  • Inline the _nix definitions (ie they're not in the overlay anymore),
  • Move the scope function into a separate file, passing the free variables by attrset
  • Rename final to pkgs.


nix-fetchers = final.callPackage ./src/libfetchers/package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
};

nix =
final.callPackage ./package.nix {
inherit
fileset
stdenv
officialRelease
versionSuffix
;
boehmgc = final.boehmgc-nix;
libgit2 = final.libgit2-nix;
libseccomp = final.libseccomp-nix;
busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
};
});

nix-perl-bindings = final.callPackage ./src/perl/package.nix {
inherit
fileset
stdenv
versionSuffix
;
};
nix = final.nixComponents.nix;

nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix {
inherit
fileset
stdenv
versionSuffix
;
};

nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix {
inherit
fileset
stdenv
versionSuffix
;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would put these in nixComponents too. They only weren't in the list that became them because (as @edolstra reminded us) we didn't want to cross compile them.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, we're not using attrNames nixComponents anyway, so it's actually fine.
Early on in the commit series it would have been suitable for that, but makeScope adds some stuff that'd need to be filtered out, which I think is the right trade-off.

Expand Down Expand Up @@ -340,7 +271,7 @@
"static-" = nixpkgsFor.${system}.static;
})
(nixpkgsPrefix: nixpkgs:
flatMapAttrs nixpkgs.nix-components
flatMapAttrs nixpkgs.nixComponents
(pkgName: pkg:
flatMapAttrs pkg.tests or {}
(testName: test: {
Expand Down Expand Up @@ -373,16 +304,16 @@
}
(pkgName: {}: {
# These attributes go right into `packages.<system>`.
"${pkgName}" = nixpkgsFor.${system}.native.${pkgName};
"${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName};
"${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
"${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
}
// flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
# These attributes go right into `packages.<system>`.
"${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName};
"${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
})
// flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
# These attributes go right into `packages.<system>`.
"${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName};
"${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
})
)
// lib.optionalAttrs (builtins.elem system linux64BitSystems) {
Expand Down Expand Up @@ -444,17 +375,17 @@
};

mesonFlags =
map (transformFlag "libutil") pkgs.nix-util.mesonFlags
++ map (transformFlag "libstore") pkgs.nix-store.mesonFlags
++ map (transformFlag "libfetchers") pkgs.nix-fetchers.mesonFlags
++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nix-perl-bindings.mesonFlags)
map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags
++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags
++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags
++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags)
;

nativeBuildInputs = attrs.nativeBuildInputs or []
++ pkgs.nix-util.nativeBuildInputs
++ pkgs.nix-store.nativeBuildInputs
++ pkgs.nix-fetchers.nativeBuildInputs
++ lib.optionals havePerl pkgs.nix-perl-bindings.nativeBuildInputs
++ pkgs.nixComponents.nix-util.nativeBuildInputs
++ pkgs.nixComponents.nix-store.nativeBuildInputs
++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
++ pkgs.nix-internal-api-docs.nativeBuildInputs
++ pkgs.nix-external-api-docs.nativeBuildInputs
++ [
Expand Down
14 changes: 7 additions & 7 deletions maintainers/hydra.nix
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should move this file to packaging actually. We have a lit of nix files scattered around willy-nilly right now as no directory felt right.

Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
}:
let
inherit (inputs) nixpkgs nixpkgs-regression;
inherit (lib) fileset;

installScriptFor = tarballs:
nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
Expand All @@ -25,14 +24,15 @@ let
lib.versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";

inherit fileset;

test-client = client;
test-daemon = daemon;

doBuild = false;
};

# Technically we could just return `pkgs.nixComponents`, but for Hydra it's
# convention to transpose it, and to transpose it efficiently, we need to
# enumerate them manually, so that we don't evaluate unnecessary package sets.
forAllPackages = lib.genAttrs [
Comment on lines +33 to 36
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Future work: Let's us builtins.attrNames for this list

"nix"
"nix-util"
Expand All @@ -46,16 +46,16 @@ in
{
# Binary package for various platforms.
build = forAllPackages (pkgName:
forAllSystems (system: nixpkgsFor.${system}.native.${pkgName}));
forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}));

shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);

buildStatic = forAllPackages (pkgName:
lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName}));
lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));

buildCross = forAllPackages (pkgName:
forAllCrossSystems (crossSystem:
lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName})));
lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})));

buildNoGc = forAllSystems (system:
self.packages.${system}.nix.override { enableGC = false; }
Expand All @@ -73,7 +73,7 @@ in
);

# Perl bindings for various platforms.
perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix-perl-bindings);
perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);

# Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of
Expand Down
6 changes: 2 additions & 4 deletions package.nix
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
{ lib
, fetchurl
, stdenv
, releaseTools
, autoconf-archive
, autoreconfHook
, aws-sdk-cpp
, boehmgc
, buildPackages
, nlohmann_json
, bison
, boost
Expand All @@ -15,7 +13,6 @@
, curl
, editline
, readline
, fileset
, flex
, git
, gtest
Expand Down Expand Up @@ -50,7 +47,6 @@
, pname ? "nix"

, versionSuffix ? ""
, officialRelease ? false

# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix
, doBuild ? true
Expand Down Expand Up @@ -113,6 +109,8 @@
}:

let
inherit (lib) fileset;

version = lib.fileContents ./.version + versionSuffix;

# selected attributes with defaults, will be used to define some
Expand Down
23 changes: 23 additions & 0 deletions packaging/components.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{pkgs, stdenv, versionSuffix}: scope:
let
inherit (scope) callPackage;
in

# This becomes the pkgs.nixComponents attribute set
{
nix = callPackage ../package.nix { };

nix-util = callPackage ../src/libutil/package.nix { };

nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { };

nix-util-test = callPackage ../tests/unit/libutil/package.nix { };

nix-util-c = callPackage ../src/libutil-c/package.nix { };

nix-store = callPackage ../src/libstore/package.nix { };

nix-fetchers = callPackage ../src/libfetchers/package.nix { };

nix-perl-bindings = callPackage ../src/perl/package.nix { };
}
6 changes: 4 additions & 2 deletions src/external-api-docs/package.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
{ lib
, stdenv
, releaseTools
, fileset

, meson
, ninja
Expand All @@ -12,6 +10,10 @@
, versionSuffix ? ""
}:

let
inherit (lib) fileset;
in

stdenv.mkDerivation (finalAttrs: {
pname = "nix-external-api-docs";
version = lib.fileContents ./.version + versionSuffix;
Expand Down
6 changes: 4 additions & 2 deletions src/internal-api-docs/package.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
{ lib
, stdenv
, releaseTools
, fileset

, meson
, ninja
Expand All @@ -12,6 +10,10 @@
, versionSuffix ? ""
}:

let
inherit (lib) fileset;
in

stdenv.mkDerivation (finalAttrs: {
pname = "nix-internal-api-docs";
version = lib.fileContents ./.version + versionSuffix;
Expand Down
6 changes: 2 additions & 4 deletions src/libfetchers/package.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{ lib
, stdenv
, releaseTools
, fileset

, meson
, ninja
Expand All @@ -16,17 +15,16 @@
# Configuration Options

, versionSuffix ? ""
, officialRelease ? false

# Check test coverage of Nix. Probably want to use with with at least
# one of `doCheck` or `doInstallCheck` enabled.
, withCoverageChecks ? false

# Avoid setting things that would interfere with a functioning devShell
, forDevShell ? false
}:

let
inherit (lib) fileset;

version = lib.fileContents ./.version + versionSuffix;

mkDerivation =
Expand Down
Loading
Loading