Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use fix and extend function for all-packages.nix #14000

Merged
merged 44 commits into from
Mar 24, 2016
Merged

Use fix and extend function for all-packages.nix #14000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
9444d6e
Split miss-indented last lines. to simplify future diffs.
nbp Mar 17, 2016
a39c5ff
Re-indent the top of pkgsFun.
nbp Mar 17, 2016
11a566b
Move 'with helperFunctions' above the set of all packages.
nbp Mar 17, 2016
5f4cb91
Move helperfunctions and stdenvAdapters under pkgsFun.
nbp Mar 17, 2016
f9a2988
Move pkgsOrig outside applyGlobalOverrides and provide it as argument.
nbp Mar 17, 2016
a6e260c
Add an extra argument to the overrider function of applyGlobalOverrides.
nbp Mar 17, 2016
a5188bd
Unify the applyGlobalOverride function with the overridePackages func…
nbp Mar 17, 2016
00963c3
Extract stdenvOverrides function out of mkOverrides function.
nbp Mar 17, 2016
acaa99c
Move applyGlobalOverrides comment to stdenvOverrides, where it fits b…
nbp Mar 18, 2016
71b2fd9
Move mkOverrides under applyGlobalOverrides.
nbp Mar 18, 2016
a8374f2
Under applyGlobalOverrides, inline mkOverrides in overrides.
nbp Mar 18, 2016
a658645
Remove with helperFunction above the list of all packages.
nbp Mar 18, 2016
ffcb668
Under pkgsFun, move the merge of helper functions from self_ to self.
nbp Mar 19, 2016
8d355b8
Under pkgsFun, expand self attribute, and rename self_ to self.
nbp Mar 19, 2016
7f3a4f4
Swap conditions of stdenvOverrides to prevent infinite loops while ev…
nbp Mar 20, 2016
020bb40
Split stdenv attribute set.
nbp Mar 19, 2016
be3531a
Replace with statements of stdenvDefault to rely on pkgs instead of s…
nbp Mar 20, 2016
f7c86ee
Move merge of stdenvDefault from the self attribute to the returned v…
nbp Mar 20, 2016
0b67f7c
Move tweakAlias function into the aliases.nix file, and rename it.
nbp Mar 20, 2016
a09683e
Remove unused 'ncat' alias, as a package with the same name already e…
nbp Mar 20, 2016
f043ffc
Move the set of aliases after 'self', such that we can later use the …
nbp Mar 20, 2016
0543277
Move stdenvDefault into its own file.
nbp Mar 20, 2016
007500f
Move the merge of the overrides from pkgsFun to applyGlobalOverrides.
nbp Mar 20, 2016
5e3ce63
Remove applyGlobalOverrides's pkgsOrig argument.
nbp Mar 20, 2016
32e96bb
Within applyGlobalOverrides, alias pkgsFun calls.
nbp Mar 20, 2016
07e549e
Rename applyGlobalOverrides to pkgsWithOverrides
nbp Mar 20, 2016
a543a57
Reformat pkgsWithOverrides to later facilitate the identitication wit…
nbp Mar 20, 2016
ad31783
Extract the top-level logic out of all-packages.nix into pkgs/top-lev…
nbp Mar 20, 2016
335a969
Inline pkgsFun in pkgsWithOverrides.
nbp Mar 20, 2016
ae0471b
Split consecutive merge operators to simplify the identification of e…
nbp Mar 20, 2016
5cdaa7b
Remove all-packages.nix helperFunctions dependency.
nbp Mar 20, 2016
ff72cf1
Split helperFunctions to simplify the identification with the extend …
nbp Mar 20, 2016
01693ac
Move stdenvAdapaters attribute from all-packages.nix into the stdenvA…
nbp Mar 20, 2016
f8dedbb
Build aliases based on the previous set including all-packages, inste…
nbp Mar 20, 2016
5783f66
Internalize the recursion over the set of all packages.
nbp Mar 20, 2016
a190baa
In top-level/default.nix, rename self to allPackages.
nbp Mar 20, 2016
598ed87
Add extra argument to better identity with the extend function later.
nbp Mar 20, 2016
9e8c520
Add bootStdenv condition around the overrider.
nbp Mar 20, 2016
21b8007
Move the customOverrides after the stdenvOverrides.
nbp Mar 20, 2016
800766f
Change the customOverrides to use the stdenvOverrides instead of the …
nbp Mar 20, 2016
1bcefcd
Under pkgsWithOverrides, replace the last uses of pkgs by the corresp…
nbp Mar 20, 2016
5ae7356
Replace the merge operators by the fix' and extends functions.
nbp Mar 20, 2016
aa7f0fc
Move stdenvOverrides under pkgsWithOverrides.
nbp Mar 20, 2016
87ad35e
Fix comments typos.
nbp Mar 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ if ! builtins ? nixVersion || builtins.compareVersions requiredVersion builtins.

else

import ./pkgs/top-level/all-packages.nix
import ./pkgs/top-level
4 changes: 2 additions & 2 deletions pkgs/build-support/dotnetbuildhelpers/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{ helperFunctions, mono, pkgconfig }:
helperFunctions.runCommand
{ runCommand, mono, pkgconfig }:
runCommand
"dotnetbuildhelpers"
{ preferLocalBuild = true; }
''
Expand Down
15 changes: 13 additions & 2 deletions pkgs/top-level/aliases.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,21 @@ self:

with self;

let
# Removing recurseForDerivation prevents derivations of aliased attribute
# set to appear while listing all the packages available.
removeRecurseForDerivations = _n: alias: with lib;
if alias.recurseForDerivations or false then
removeAttrs alias ["recurseForDerivations"]
else alias;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function can be simplified to:

_n: alias: removeAttrs alias ["recurseForDerivations"]

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought of that, but I have not done it because this adds extra computations, which is not desirable. For example if you have a large attribute set, you would decompose it to recompose it without one attribute.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't removeAttrs return the same set if nothing has changed ? In any case this is not new code so it can stay the same.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/NixOS/nix/blob/master/src/libexpr/primops.cc#L1060-L1080

Yes, the attribute will contain the same content, but this would be a new one.
I guess this would be easy to fix in Nix.


doNotDisplayTwice = aliases:
lib.mapAttrs removeRecurseForDerivations aliases;
in

### Deprecated aliases - for backward compatibility

rec {
doNotDisplayTwice rec {
accounts-qt = qt5.accounts-qt; # added 2015-12-19
adobeReader = adobe-reader;
aircrackng = aircrack-ng; # added 2016-01-14
Expand Down Expand Up @@ -49,7 +61,6 @@ rec {
mssys = ms-sys; # added 2015-12-13
multipath_tools = multipath-tools; # added 2016-01-21
mupen64plus1_5 = mupen64plus; # added 2016-02-12
ncat = nmap; # added 2016-01-26
nfsUtils = nfs-utils; # added 2014-12-06
phonon_qt5 = qt5.phonon; # added 2015-12-19
phonon_qt5_backend_gstreamer = qt5.phonon-backend-gstreamer; # added 2015-12-19
Expand Down
215 changes: 26 additions & 189 deletions pkgs/top-level/all-packages.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,133 +1,16 @@
/* This file composes the Nix Packages collection. That is, it
imports the functions that build the various packages, and calls
them with appropriate arguments. The result is a set of all the
packages in the Nix Packages collection for some particular
platform. */
{ system, bootStdenv, noSysDirs, gccWithCC, gccWithProfiling
, config, crossSystem, platform, lib
, pkgsWithOverrides
, ... }:
self: pkgs:


{ # The system (e.g., `i686-linux') for which to build the packages.
system ? builtins.currentSystem

, # The standard environment to use. Only used for bootstrapping. If
# null, the default standard environment is used.
bootStdenv ? null

, # Non-GNU/Linux OSes are currently "impure" platforms, with their libc
# outside of the store. Thus, GCC, GFortran, & co. must always look for
# files in standard system directories (/usr/include, etc.)
noSysDirs ? (system != "x86_64-freebsd" && system != "i686-freebsd"
&& system != "x86_64-solaris"
&& system != "x86_64-kfreebsd-gnu")

# More flags for the bootstrapping of stdenv.
, gccWithCC ? true
, gccWithProfiling ? true

, # Allow a configuration attribute set to be passed in as an
# argument. Otherwise, it's read from $NIXPKGS_CONFIG or
# ~/.nixpkgs/config.nix.
config ? null

, crossSystem ? null
, platform ? null
}:


let config_ = config; platform_ = platform; in # rename the function arguments
with pkgs;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be self? pkgs = super, right? I am reading 295b7a6 and getting even more confused :).

While you've argued well that aliases should refer to the unoverriden packages, dependencies should definitely be overridden.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the final result, self is equivalent to rec, and pkgs is equivalent to the self of the fix-point. super would be all the stdenvAdapaters and the trivialBuilders.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I missed https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/default.nix#L119

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

grep --color -E 'self( |$|\.)' pkgs/top-level/all-packages.nix Convinces me we could remove the self arg altogether from all-packages and put everything in aliases.


let
defaultScope = pkgs // pkgs.xorg;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as #14000 (comment), shouldn't defaultScope be made from self?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As self corresponds to rec at the moment, we should not do that as this will break the packageOverride mechanism, and the security update branch.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok based on other thread it is the fixed point being used here as I wanted all along :).

in

lib = import ../../lib;

# The contents of the configuration file found at $NIXPKGS_CONFIG or
# $HOME/.nixpkgs/config.nix.
# for NIXOS (nixos-rebuild): use nixpkgs.config option
config =
let
toPath = builtins.toPath;
getEnv = x: if builtins ? getEnv then builtins.getEnv x else "";
pathExists = name:
builtins ? pathExists && builtins.pathExists (toPath name);

configFile = getEnv "NIXPKGS_CONFIG";
homeDir = getEnv "HOME";
configFile2 = homeDir + "/.nixpkgs/config.nix";

configExpr =
if config_ != null then config_
else if configFile != "" && pathExists configFile then import (toPath configFile)
else if homeDir != "" && pathExists configFile2 then import (toPath configFile2)
else {};

in
# allow both:
# { /* the config */ } and
# { pkgs, ... } : { /* the config */ }
if builtins.isFunction configExpr
then configExpr { inherit pkgs; }
else configExpr;

# Allow setting the platform in the config file. Otherwise, let's use a reasonable default (pc)

platformAuto = let
platforms = (import ./platforms.nix);
in
if system == "armv6l-linux" then platforms.raspberrypi
else if system == "armv7l-linux" then platforms.armv7l-hf-multiplatform
else if system == "armv5tel-linux" then platforms.sheevaplug
else if system == "mips64el-linux" then platforms.fuloong2f_n32
else if system == "x86_64-linux" then platforms.pc64
else if system == "i686-linux" then platforms.pc32
else platforms.pcBase;

platform = if platform_ != null then platform_
else config.platform or platformAuto;

# Helper functions that are exported through `pkgs'.
helperFunctions =
stdenvAdapters //
(import ../build-support/trivial-builders.nix { inherit lib; inherit (pkgs) stdenv; inherit (pkgs.xorg) lndir; });

stdenvAdapters =
import ../stdenv/adapters.nix pkgs;


# Allow packages to be overriden globally via the `packageOverrides'
# configuration option, which must be a function that takes `pkgs'
# as an argument and returns a set of new or overriden packages.
# The `packageOverrides' function is called with the *original*
# (un-overriden) set of packages, allowing packageOverrides
# attributes to refer to the original attributes (e.g. "foo =
# ... pkgs.foo ...").
pkgs = applyGlobalOverrides (config.packageOverrides or (pkgs: {}));

mkOverrides = pkgsOrig: overrides: overrides //
(lib.optionalAttrs (pkgsOrig.stdenv ? overrides && crossSystem == null) (pkgsOrig.stdenv.overrides pkgsOrig));

# Return the complete set of packages, after applying the overrides
# returned by the `overrider' function (see above). Warning: this
# function is very expensive!
applyGlobalOverrides = overrider:
let
# Call the overrider function. We don't want stdenv overrides
# in the case of cross-building, or otherwise the basic
# overrided packages will not be built with the crossStdenv
# adapter.
overrides = mkOverrides pkgsOrig (overrider pkgsOrig);

# The un-overriden packages, passed to `overrider'.
pkgsOrig = pkgsFun pkgs {};

# The overriden, final packages.
pkgs = pkgsFun pkgs overrides;
in pkgs;


# The package compositions. Yes, this isn't properly indented.
pkgsFun = pkgs: overrides:
with helperFunctions;
let defaultScope = pkgs // pkgs.xorg; self = self_ // overrides;
self_ = with self; helperFunctions // {
{

# Make some arguments passed to all-packages.nix available
inherit system platform;
Expand Down Expand Up @@ -157,11 +40,7 @@ let
#
# The result is `pkgs' where all the derivations depending on `foo'
# will use the new version.
overridePackages = f:
let
newpkgs = pkgsFun newpkgs overrides;
overrides = mkOverrides pkgs (f newpkgs pkgs);
in newpkgs;
overridePackages = f: pkgsWithOverrides f;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This replaces packageOverrides in config with something else. but sometimes one wants to chain overrides. I'd like to see a

let # for syntax coloring
  reoverridePackages = f: fix' (extend f self.__unfix__);

(bikeshed name of course)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure what should be done with this overridePackages function. I personally have no uses of it, and I am sure we can discuss that, or just remove it if not used.

I did not went for using the unfix attribute for one simple reason. The current version of overridePackages does not contain any of the modifications made in ~/.nixpkgs/config.nix. Thus, adding such a thing would probably be bad, as one might rely on such feature to provide its own set of overridden packages inside a repository.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough, this can be dealt with separately. The fact that nixpkgs even has such ~/.nixpkgs.config.nix etc hardcodings kinda scares me, but at least it's cordoned from all-packages now.


# Override system. This is useful to build i686 packages on x86_64-linux.
forceSystem = system: kernel: (import ./../..) {
Expand All @@ -171,19 +50,31 @@ let
crossSystem;
};


# Used by wine, firefox with debugging version of Flash, ...
pkgsi686Linux = forceSystem "i686-linux" "i386";

callPackage_i686 = lib.callPackageWith (pkgsi686Linux // pkgsi686Linux.xorg);

forceNativeDrv = drv : if crossSystem == null then drv else
(drv // { crossDrv = drv.nativeDrv; });

stdenvCross = lowPrio (makeStdenvCross defaultStdenv crossSystem binutilsCross gccCrossStageFinal);

# A stdenv capable of building 32-bit binaries. On x86_64-linux,
# it uses GCC compiled with multilib support; on i686-linux, it's
# just the plain stdenv.
stdenv_32bit = lowPrio (
if system == "x86_64-linux" then
overrideCC stdenv gcc_multi
else
stdenv);

# For convenience, allow callers to get the path to Nixpkgs.
path = ../..;


### Helper functions.
inherit lib config stdenvAdapters;
inherit lib config;

inherit (lib) lowPrio hiPrio appendToName makeOverridable;
inherit (misc) versionedDerivation;
Expand All @@ -206,49 +97,6 @@ let
nixpkgs-lint = callPackage ../../maintainers/scripts/nixpkgs-lint.nix { };


### STANDARD ENVIRONMENT


allStdenvs = import ../stdenv {
inherit system platform config lib;
allPackages = args: import ./../.. ({ inherit config system; } // args);
};

defaultStdenv = allStdenvs.stdenv // { inherit platform; };

stdenvCross = lowPrio (makeStdenvCross defaultStdenv crossSystem binutilsCross gccCrossStageFinal);

stdenv =
if bootStdenv != null then (bootStdenv // {inherit platform;}) else
if crossSystem != null then
stdenvCross
else
let
changer = config.replaceStdenv or null;
in if changer != null then
changer {
# We import again all-packages to avoid recursivities.
pkgs = import ./../.. {
# We remove packageOverrides to avoid recursivities
config = removeAttrs config [ "replaceStdenv" ];
};
}
else
defaultStdenv;

forceNativeDrv = drv : if crossSystem == null then drv else
(drv // { crossDrv = drv.nativeDrv; });

# A stdenv capable of building 32-bit binaries. On x86_64-linux,
# it uses GCC compiled with multilib support; on i686-linux, it's
# just the plain stdenv.
stdenv_32bit = lowPrio (
if system == "x86_64-linux" then
overrideCC stdenv gcc_multi
else
stdenv);


### BUILD SUPPORT

attrSetToDir = arg: callPackage ../build-support/upstream-updater/attrset-to-dir.nix {
Expand Down Expand Up @@ -303,9 +151,7 @@ let
dotnetfx = dotnetfx40;
};

dotnetbuildhelpers = callPackage ../build-support/dotnetbuildhelpers {
inherit helperFunctions;
};
dotnetbuildhelpers = callPackage ../build-support/dotnetbuildhelpers { };

dispad = callPackage ../tools/X11/dispad { };

Expand Down Expand Up @@ -16468,14 +16314,5 @@ let

mg = callPackage ../applications/editors/mg { };

}; # self_ =


aliases = import ./aliases.nix self;

tweakAlias = _n: alias: with lib;
if alias.recurseForDerivations or false then
removeAttrs alias ["recurseForDerivations"]
else alias;
}

in lib.mapAttrs tweakAlias aliases // self; in pkgs
Loading