PackageManager: Store packages hierarchically, by version

As explained in the changelog, this hierarchy makes much more sense
than the current one, and will make it possible to read the version
from the directory name.

changelog/hierarchy.dd

@@ -0,0 +1,14 @@
+The way packages are stored internally has changed
+
+Previous versions of dub stored packages in the following format:
+`$CACHE_PATH/$PACKAGE_NAME-$PACKAGE_VERSION/$PACKAGE_NAME/`
+Starting from this version, the format will be:
+`$CACHE_PATH/$PACKAGE_NAME/$PACKAGE_VERSION/$PACKAGE_NAME`.
+
+Introducing a new level will help users quickly list what packages
+they actually have installed, and reduce visibility of packages that
+might update frequently. It will render various commands (e.g. `du`)
+more useful, and pave the way for a package GC function.
+More importantly, it will allow future version of dub to infer the
+version from the path to the package, removing the need to read (or edit)
+the recipe file on every dub invocation.

source/dub/packagemanager.d

@@ -1298,29 +1298,8 @@ private struct Location {
 		if (!path.existsDirectory())
 			return;
 
-		logDebug("iterating dir %s", path.toNativeString());
-		try foreach (pdir; iterateDirectory(path)) {
-			logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
-			if (!pdir.isDirectory) continue;
-
-			// Old / flat directory structure, used in non-standard path
-			// Packages are stored in $ROOT/$SOMETHING/`
-			auto pack_path = path ~ (pdir.name ~ "/");
-			auto packageFile = Package.findPackageFile(pack_path);
-
-			// New (since 2015) managed structure:
-			// $ROOT/$NAME-$VERSION/$NAME
-			// This is the most common code path
-			if (mgr.isManagedPath(path) && packageFile.empty) {
-				foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/")))
-					if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {
-						pack_path ~= subdir.name ~ "/";
-						packageFile = Package.findPackageFile(pack_path);
-						break;
-					}
-			}
-
-			if (packageFile.empty) continue;
+		void loadInternal (NativePath pack_path, NativePath packageFile)
+		{
 			Package p;
 			try {
 				foreach (pp; existing_packages)
@@ -1338,6 +1317,38 @@ private struct Location {
 				logDiagnostic("Full error: %s", e.toString().sanitize());
 			}
 		}
+
+		logDebug("iterating dir %s", path.toNativeString());
+		try foreach (pdir; iterateDirectory(path)) {
+			logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
+			if (!pdir.isDirectory) continue;
+
+			// Old / flat directory structure, used in non-standard path
+			// Packages are stored in $ROOT/$SOMETHING/`
+			const pack_path = path ~ (pdir.name ~ "/");
+			auto packageFile = Package.findPackageFile(pack_path);
+			if (!packageFile.empty) {
+				// Deprecated unmanaged directory structure
+				logWarn("Package at path '%s' should be under '%s'",
+						pack_path.toNativeString().color(Mode.bold),
+						(pack_path ~ "$VERSION" ~ pdir.name).toNativeString().color(Mode.bold));
+				logWarn("The package will no longer be detected starting from v1.42.0");
+				loadInternal(pack_path, packageFile);
+			}
+
+			// Managed structure: $ROOT/$NAME/$VERSION/$NAME
+			// This is the most common code path
+			else if (mgr.isManagedPath(path)) {
+				// Iterate over versions of a package
+				foreach (versdir; iterateDirectory(pack_path)) {
+					if (!versdir.isDirectory) continue;
+					auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/");
+					if (!vers_path.existsDirectory()) continue;
+					packageFile = Package.findPackageFile(vers_path);
+					loadInternal(vers_path, packageFile);
+				}
+			}
+		}
 		catch (Exception e)
 			logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
 	}
@@ -1417,9 +1428,7 @@ private struct Location {
 	 */
 	private NativePath getPackagePath (string name, string vers)
 	{
-		// + has special meaning for Optlink
-		string clean_vers = vers.chompPrefix("~").replace("+", "_");
-		NativePath result = this.packagePath ~ (name ~ "-" ~ clean_vers);
+		NativePath result = this.packagePath ~ name ~ vers;
 		result.endsWithSlash = true;
 		return result;
 	}

test/dpath-variable.sh

@@ -6,7 +6,7 @@ rm -rf "$DPATH"
 cd "${CURR_DIR}/dpath-variable"
 "${DUB}" upgrade
 
-if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
+if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
 	die $LINENO 'Did not get dependencies installed into $DPATH.'
 fi
 
@@ -24,6 +24,6 @@ trap cleanup EXIT
 
 "${DUB}" upgrade
 
-if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
+if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
 	die $LINENO 'Did not get dependencies installed into dubHome (set from config).'
 fi

test/interactive-remove.sh

@@ -9,32 +9,32 @@
 # we need to nuke every `dub` version in the user cache...
 $DUB remove dub -n || true
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 
 echo 1 | $DUB remove dub | tr -d '\n' | grep --ignore-case 'select.*1\.9\.0.*1\.10\.0.*'
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ]; then
     die $LINENO 'Failed to remove dub-1.9.0'
 fi
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
 # EOF aborts remove
 echo -xn '' | $DUB remove dub
-if [ ! -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ ! -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Aborted dub still removed a package'
 fi
 
 # validates input
 echo -e 'abc\n4\n-1\n3' | $DUB remove dub
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to remove all version of dub'
 fi
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 # is non-interactive with a <version-spec>
 $DUB remove [email protected]
 $DUB remove [email protected]
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to non-interactively remove specified versions'
 fi

test/version-spec.sh

@@ -43,10 +43,10 @@ $DUB add-local "$CURR_DIR/version-spec/oldfoo"
 $DUB remove-local "$CURR_DIR/version-spec/newfoo"
 $DUB remove-local "$CURR_DIR/version-spec/oldfoo"
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 $DUB remove [email protected]
 $DUB remove dub=1.10.0
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to remove specified versions'
 fi