[nix profile] Changes to support format changes from nix 2.20 (#1770)

## Summary

The latest nix version (2.20) changed how the nix profile output is
represented:

From the [release
notes](https://nixos.org/manual/nix/stable/release-notes/rl-2.20):

> nix profile now allows referring to elements by human-readable names
NixOS/nix#8678

> [nix
profile](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile)
now uses names to refer to installed packages when running
[list](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-list),
[remove](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-remove)
or
[upgrade](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-upgrade)
as opposed to indices. Profile element names are generated when a
package is installed and remain the same until the package is removed.

> Warning: The manifest.nix file used to record the contents of profiles
has changed. Nix will automatically upgrade profiles to the new version
when you modify the profile. After that, the profile can no longer be
used by older versions of Nix.

and for `nix search`:
> Disallow empty search regex in nix search
[#9481](NixOS/nix#9481)

> [nix
search](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-search)
now requires a search regex to be passed. To show all packages, use ^.


TODOs:
- [x] update `nix.readManifest` to handle the new format
- [x] `nix search` requires a regex to be passed
- [x] manually test on nix < 2.20 on devbox.sh to verify the older nix
still works

Fixes #1767 

## How was it tested?

CICD should pass

Installed nix 2.20.1 locally and am using Devbox with it to add, remove
packages and run scripts and shell.

verified flake updating works:
1. `examples/flakes/remote`. Did `devbox shell`, dropped the `v0.43.1`
from process-compose flake, did `devbox update`, and verified that
`process-compose` now had the latest version (IIRC `0.80+`)
2. `examples/flakes/php`. Did `devbox shell`, edited the flake to drop
`ds` and did `devbox update`. Verified no `ds` in `php -m | grep ds`

.github/workflows/cli-tests.yaml

@@ -129,8 +129,9 @@ jobs:
         run-project-tests: ["project-tests", "project-tests-off"]
         # Run tests on:
         # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0)
-        # 2. latest nix version
-        nix-version: ["2.12.0", "2.19.2"]
+        # 2. nix 2.19.2: version before nix profile changes
+        # 2. latest nix version (note, 2.20.1 introduced a new profile change)
+        nix-version: ["2.12.0", "2.19.2", "2.20.1"]
         exclude:
           - is-main: "not-main"
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"

internal/nix/nixprofile/item.go

@@ -16,6 +16,11 @@ type NixProfileListItem struct {
 	// invocations of nix profile remove and nix profile upgrade.
 	index int
 
+	// name of the package
+	// nix 2.20 introduced a new format for the output of nix profile list, which includes the package name.
+	// This field is used instead of index for `list`, `remove` and `upgrade` subcommands of `nix profile`.
+	name string
+
 	// The original ("unlocked") flake reference and output attribute path used at installation time.
 	// NOTE that this will be empty if the package was added to the nix profile via store path.
 	unlockedReference string
@@ -74,10 +79,10 @@ func (i *NixProfileListItem) addedByStorePath() bool {
 	return i.unlockedReference == ""
 }
 
-// String serializes the NixProfileListItem back into the format printed by `nix profile list`
+// String serializes the NixProfileListItem for debuggability
 func (i *NixProfileListItem) String() string {
-	return fmt.Sprintf("{%d %s %s %s}",
-		i.index,
+	return fmt.Sprintf("{nameOrIndex:%s unlockedRef:%s lockedRef:%s, nixStorePaths:%s}",
+		i.NameOrIndex(),
 		i.unlockedReference,
 		i.lockedReference,
 		i.nixStorePaths,
@@ -87,3 +92,13 @@ func (i *NixProfileListItem) String() string {
 func (i *NixProfileListItem) StorePaths() []string {
 	return i.nixStorePaths
 }
+
+// NameOrIndex is a helper method to get the name of the package if it exists, or the index if it doesn't.
+// `nix profile` subcommands `list`, `remove`, and `upgrade` use either name (nix >= 2.20) or index (nix < 2.20)
+// to identify the package.
+func (i *NixProfileListItem) NameOrIndex() string {
+	if i.name != "" {
+		return i.name
+	}
+	return fmt.Sprintf("%d", i.index)
+}

internal/nix/nixprofile/profile.go

@@ -42,24 +42,46 @@ func ProfileListItems(
 		URL         string   `json:"url"`
 	}
 	type ProfileListOutput struct {
-		Elements []ProfileListElement `json:"elements"`
-		Version  int                  `json:"version"`
+		Elements map[string]ProfileListElement `json:"elements"`
+		Version  int                           `json:"version"`
 	}
 
+	// Modern nix profiles: nix >= 2.20
 	var structOutput ProfileListOutput
-	if err := json.Unmarshal([]byte(output), &structOutput); err != nil {
-		return nil, err
+	if err := json.Unmarshal([]byte(output), &structOutput); err == nil {
+		items := []*NixProfileListItem{}
+		for name, element := range structOutput.Elements {
+			items = append(items, &NixProfileListItem{
+				name:              name,
+				unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""),
+				lockedReference:   lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""),
+				nixStorePaths:     element.StorePaths,
+			})
+		}
+		return items, nil
 	}
+	// Fall back to trying format for nix < version 2.20
 
+	// ProfileListOutputJSONLegacy is for parsing `nix profile list --json` in nix < version 2.20
+	// that relied on index instead of name for each package installed.
+	type ProfileListOutputJSONLegacy struct {
+		Elements []ProfileListElement `json:"elements"`
+		Version  int                  `json:"version"`
+	}
+	var structOutput2 ProfileListOutputJSONLegacy
+	if err := json.Unmarshal([]byte(output), &structOutput2); err != nil {
+		return nil, err
+	}
 	items := []*NixProfileListItem{}
-	for index, element := range structOutput.Elements {
+	for index, element := range structOutput2.Elements {
 		items = append(items, &NixProfileListItem{
 			index:             index,
 			unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""),
 			lockedReference:   lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""),
 			nixStorePaths:     element.StorePaths,
 		})
 	}
+
 	return items, nil
 }
 
@@ -88,7 +110,7 @@ func profileListLegacy(
 		if line == "" {
 			continue
 		}
-		item, err := parseNixProfileListItem(line)
+		item, err := parseNixProfileListItemLegacy(line)
 		if err != nil {
 			return nil, err
 		}
@@ -98,7 +120,7 @@ func profileListLegacy(
 	return items, nil
 }
 
-type ProfileListIndexArgs struct {
+type ProfileListNameOrIndexArgs struct {
 	// For performance, you can reuse the same list in multiple operations if you
 	// are confident index has not changed.
 	Items      []*NixProfileListItem
@@ -108,21 +130,21 @@ type ProfileListIndexArgs struct {
 	ProfileDir string
 }
 
-// ProfileListIndex returns the index of args.Package in the nix profile specified by args.ProfileDir,
-// or -1 if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again.
-func ProfileListIndex(args *ProfileListIndexArgs) (int, error) {
+// ProfileListNameOrIndex returns the name or index of args.Package in the nix profile specified by args.ProfileDir,
+// or nix.ErrPackageNotFound if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again.
+func ProfileListNameOrIndex(args *ProfileListNameOrIndexArgs) (string, error) {
 	var err error
 	items := args.Items
 	if items == nil {
 		items, err = ProfileListItems(args.Writer, args.ProfileDir)
 		if err != nil {
-			return -1, err
+			return "", err
 		}
 	}
 
 	inCache, err := args.Package.IsInBinaryCache()
 	if err != nil {
-		return -1, err
+		return "", err
 	}
 
 	if !inCache && args.Package.IsDevboxPackage {
@@ -131,28 +153,29 @@ func ProfileListIndex(args *ProfileListIndexArgs) (int, error) {
 		// of an existing profile item.
 		ref, err := args.Package.NormalizedDevboxPackageReference()
 		if err != nil {
-			return -1, errors.Wrapf(err, "failed to get installable for %s", args.Package.String())
+			return "", errors.Wrapf(err, "failed to get installable for %s", args.Package.String())
 		}
 
 		for _, item := range items {
 			if ref == item.unlockedReference {
-				return item.index, nil
+				return item.NameOrIndex(), nil
 			}
 		}
-		return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
+		return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
 	}
 
 	for _, item := range items {
 		if item.Matches(args.Package, args.Lockfile) {
-			return item.index, nil
+			return item.NameOrIndex(), nil
 		}
 	}
-	return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
+	return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String())
 }
 
-// parseNixProfileListItem reads each line of output (from `nix profile list`) and converts
+// parseNixProfileListItemLegacy reads each line of output (from `nix profile list`) and converts
 // into a golang struct. Refer to NixProfileListItem struct definition for explanation of each field.
-func parseNixProfileListItem(line string) (*NixProfileListItem, error) {
+// NOTE: this API is for legacy nix. Newer nix versions use --json output.
+func parseNixProfileListItemLegacy(line string) (*NixProfileListItem, error) {
 	scanner := bufio.NewScanner(strings.NewReader(line))
 	scanner.Split(bufio.ScanWords)
 

internal/nix/nixprofile/profile_test.go

@@ -15,7 +15,10 @@ type expectedTestData struct {
 	packageName string
 }
 
-func TestNixProfileListItem(t *testing.T) {
+// TestNixProfileListItemLegacy tests the parsing of legacy nix profile list items.
+// It only applies to much older nix versions. Newer nix versions rely on the --json output
+// instead parsing the legacy output.
+func TestNixProfileListItemLegacy(t *testing.T) {
 	testCases := map[string]struct {
 		line     string
 		expected expectedTestData
@@ -49,10 +52,10 @@ func TestNixProfileListItem(t *testing.T) {
 			),
 			expected: expectedTestData{
 				item: &NixProfileListItem{
-					2,
-					"github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
-					"github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
-					[]string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"},
+					index:             2,
+					unlockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
+					lockedReference:   "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy",
+					nixStorePaths:     []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"},
 				},
 				attrPath:    "legacyPackages.x86_64-darwin.python39Packages.numpy",
 				packageName: "python39Packages.numpy",
@@ -68,7 +71,7 @@ func TestNixProfileListItem(t *testing.T) {
 }
 
 func testItem(t *testing.T, line string, expected expectedTestData) {
-	item, err := parseNixProfileListItem(line)
+	item, err := parseNixProfileListItemLegacy(line)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}

internal/nix/nixprofile/upgrade.go

@@ -12,8 +12,8 @@ import (
 )
 
 func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) error {
-	idx, err := ProfileListIndex(
-		&ProfileListIndexArgs{
+	nameOrIndex, err := ProfileListNameOrIndex(
+		&ProfileListNameOrIndexArgs{
 			Lockfile:   lock,
 			Writer:     os.Stderr,
 			Package:    pkg,
@@ -24,5 +24,5 @@ func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) err
 		return err
 	}
 
-	return nix.ProfileUpgrade(ProfileDir, idx)
+	return nix.ProfileUpgrade(ProfileDir, nameOrIndex)
 }

internal/nix/profiles.go

@@ -94,8 +94,8 @@ func ProfileRemove(profilePath string, indexes ...string) error {
 
 type manifest struct {
 	Elements []struct {
-		Priority int `json:"priority"`
-	} `json:"elements"`
+		Priority int
+	}
 }
 
 func readManifest(profilePath string) (manifest, error) {
@@ -107,8 +107,37 @@ func readManifest(profilePath string) (manifest, error) {
 		return manifest{}, err
 	}
 
-	var m manifest
-	return m, json.Unmarshal(data, &m)
+	type manifestModern struct {
+		Elements map[string]struct {
+			Priority int `json:"priority"`
+		} `json:"elements"`
+	}
+	var modernMani manifestModern
+	if err := json.Unmarshal(data, &modernMani); err == nil {
+		// Convert to the result format
+		result := manifest{}
+		for _, e := range modernMani.Elements {
+			result.Elements = append(result.Elements, struct{ Priority int }{e.Priority})
+		}
+		return result, nil
+	}
+
+	type manifestLegacy struct {
+		Elements []struct {
+			Priority int `json:"priority"`
+		} `json:"elements"`
+	}
+	var legacyMani manifestLegacy
+	if err := json.Unmarshal(data, &legacyMani); err != nil {
+		return manifest{}, err
+	}
+
+	// Convert to the result format
+	result := manifest{}
+	for _, e := range legacyMani.Elements {
+		result.Elements = append(result.Elements, struct{ Priority int }{e.Priority})
+	}
+	return result, nil
 }
 
 const DefaultPriority = 5

internal/nix/search.go

@@ -98,15 +98,22 @@ func searchSystem(url, system string) (map[string]*Info, error) {
 		_ = EnsureNixpkgsPrefetched(writer, hash)
 	}
 
-	cmd := exec.Command("nix", "search", "--json", url)
+	// The `^` is added to indicate we want to show all packages
+	cmd := exec.Command("nix", "search", url, "^" /*regex*/, "--json")
 	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
 	if system != "" {
 		cmd.Args = append(cmd.Args, "--system", system)
 	}
 	debug.Log("running command: %s\n", cmd)
 	out, err := cmd.Output()
 	if err != nil {
+		if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) {
+			err = fmt.Errorf("nix search exit code: %d, stderr: %s, original error: %w", exitErr.ExitCode(), exitErr.Stderr, err)
+		}
+
 		// for now, assume all errors are invalid packages.
+		// TODO: check the error string for "did not find attribute" and
+		// return ErrPackageNotFound only for that case.
 		return nil, fmt.Errorf("error searching for pkg %s: %w", url, err)
 	}
 	return parseSearchResults(out), nil

internal/nix/upgrade.go

@@ -4,7 +4,6 @@
 package nix
 
 import (
-	"fmt"
 	"os"
 	"os/exec"
 
@@ -13,11 +12,11 @@ import (
 	"go.jetpack.io/devbox/internal/vercheck"
 )
 
-func ProfileUpgrade(ProfileDir string, idx int) error {
+func ProfileUpgrade(ProfileDir, indexOrName string) error {
 	cmd := command(
 		"profile", "upgrade",
 		"--profile", ProfileDir,
-		fmt.Sprintf("%d", idx),
+		indexOrName,
 	)
 	out, err := cmd.CombinedOutput()
 	if err != nil {