Move to a unified file system, and many related optimisations (#338)

.gitignore

@@ -28,3 +28,4 @@ bin/
 # Testing
 run/
 logs/
+quiltloader.log

proguard.conf

@@ -5,6 +5,6 @@
 -keepparameternames
 -keepattributes *
 
--keep class !org.quiltmc.loader.impl.lib {
+-keep class !org.quiltmc.loader.impl.lib.** {
     *;
 }

src/main/java/org/quiltmc/loader/api/ExtendedFileSystem.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2023 QuiltMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.quiltmc.loader.api;
+
+import java.io.IOException;
+import java.nio.file.CopyOption;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.NotLinkException;
+import java.nio.file.Path;
+
+/** A {@link FileSystem} which may support additional features, beyond those which normal file systems support. Similar
+ * to regular file systems, you should generally use {@link ExtendedFiles} to perform these operations. */
+public interface ExtendedFileSystem extends FasterFileSystem {
+
+	/** Copies the source file to the target file. If the source file system is read-only then this will
+	 * {@link #mount(Path, Path, MountOption...)} the given file with {@link MountOption#COPY_ON_WRITE}.
+	 * 
+	 * @param source A {@link Path}, which might not be in this {@link FileSystem}.
+	 * @param target A {@link Path} which must be from this {@link ExtendedFileSystem}
+	 * @return target */
+	default Path copyOnWrite(Path source, Path target, CopyOption... options) throws IOException {
+		return Files.copy(source, target, options);
+	}
+
+	/** Mounts the given source file on the target file, such that all reads and writes will actually read and write the
+	 * source file. (The exact behaviour depends on the options given).
+	 * <p>
+	 * This is similar to {@link Files#createSymbolicLink(Path, Path, java.nio.file.attribute.FileAttribute...)} except
+	 * the source and target files don't need to be on the same filesystem.
+	 * <p>
+	 * Note that this does not support mounting folders.
+	 * 
+	 * @param source A path from any {@link FileSystem}.
+	 * @param target A path from this {@link ExtendedFileSystem}.
+	 * @param options Options which control how the file is mounted.
+	 * @throws UnsupportedOperationException if this filesystem doesn't support file mounts. */
+	default Path mount(Path source, Path target, MountOption... options) throws IOException {
+		throw new UnsupportedOperationException(getClass() + " doesn't support ExtendedFileSystem.mount");
+	}
+
+	/** @return True if the file has been mounted with {@link #mount(Path, Path, MountOption...)}. */
+	default boolean isMountedFile(Path file) {
+		return false;
+	}
+
+	/** @return True if the given file was created by {@link #mount(Path, Path, MountOption...)} with
+	 *         {@link MountOption#COPY_ON_WRITE}, and the file has not been modified since it was copied. */
+	default boolean isCopyOnWrite(Path file) {
+		return false;
+	}
+
+	/** Reads the target of a mounted file, if it was created by {@link #mount(Path, Path, MountOption...)}.
+	 * 
+	 * @throws NotLinkException if the given file is not a {@link #isMountedFile(Path)}.
+	 * @throws UnsupportedOperationException if this filesystem doesn't support file mounts. */
+	default Path readMountTarget(Path file) throws IOException {
+		throw new UnsupportedOperationException(getClass() + " doesn't support ExtendedFileSystem.mount");
+	}
+}

src/main/java/org/quiltmc/loader/api/ExtendedFiles.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 QuiltMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.quiltmc.loader.api;
+
+import java.io.IOException;
+import java.nio.file.CopyOption;
+import java.nio.file.Files;
+import java.nio.file.NotLinkException;
+import java.nio.file.Path;
+
+/** Similar to {@link Files}, but for {@link ExtendedFileSystem}. Unlike {@link Files}, most operations can take
+ * {@link Path}s from any file system. */
+public class ExtendedFiles {
+
+	/** Copies the source file to the target file. If the source file system is read-only then the target file may
+	 * become a link to the source file, which is fully copied when it is modified.
+	 * <p>
+	 * This method is a safe alternative to {@link #mount(Path, Path, MountOption...)}, when passing them
+	 * {@link MountOption#COPY_ON_WRITE}, in the sense that it will copy the file if the filesystem doesn't support
+	 * mounts. */
+	public static Path copyOnWrite(Path source, Path target, CopyOption... options) throws IOException {
+		if (target.getFileSystem() instanceof ExtendedFileSystem) {
+			return ((ExtendedFileSystem) target.getFileSystem()).copyOnWrite(source, target, options);
+		} else {
+			return Files.copy(source, target, options);
+		}
+	}
+
+	/** Attempts to mount the source file onto the target file, such that all reads and writes to the target file
+	 * actually read and write the source file. (The exact behaviour depends on the options given).
+	 * <p>
+	 * This is similar to {@link Files#createSymbolicLink(Path, Path, java.nio.file.attribute.FileAttribute...)}, but
+	 * the source file and target file don't need to be on the same filesystem.
+	 * <p>
+	 * This does not support mounting folders.
+	 * 
+	 * @throws UnsupportedOperationException if the filesystem doesn't support this operation.
+	 * @throws IOException if anything goes wrong while mounting the file. */
+	public static Path mount(Path source, Path target, MountOption... options) throws IOException {
+		if (target.getFileSystem() instanceof ExtendedFileSystem) {
+			return ((ExtendedFileSystem) target.getFileSystem()).mount(source, target, options);
+		} else {
+			throw new UnsupportedOperationException(target.getFileSystem() + " does not support file mounts!");
+		}
+	}
+
+	/** @return True if the file has been mounted with {@link #mount(Path, Path, MountOption...)}. */
+	public static boolean isMountedFile(Path file) {
+		if (file.getFileSystem() instanceof ExtendedFileSystem) {
+			return ((ExtendedFileSystem) file.getFileSystem()).isMountedFile(file);
+		} else {
+			return false;
+		}
+	}
+
+	/** @return True if the given file was created by {@link #mount(Path, Path, MountOption...)} with
+	 *         {@link MountOption#COPY_ON_WRITE}, and the file has not been modified since it was copied. */
+	public static boolean isCopyOnWrite(Path file) {
+		if (file.getFileSystem() instanceof ExtendedFileSystem) {
+			return ((ExtendedFileSystem) file.getFileSystem()).isCopyOnWrite(file);
+		} else {
+			return false;
+		}
+	}
+
+	/** Reads the target of a mounted file, if it was created by {@link #mount(Path, Path, MountOption...)}.
+	 * 
+	 * @throws NotLinkException if the given file is not a {@link #isMountedFile(Path)}.
+	 * @throws UnsupportedOperationException if this filesystem doesn't support file mounts. */
+	public static Path readMountTarget(Path file) throws IOException {
+		if (file.getFileSystem() instanceof ExtendedFileSystem) {
+			return ((ExtendedFileSystem) file.getFileSystem()).readMountTarget(file);
+		} else {
+			throw new UnsupportedOperationException(file + " is not a mounted file!");
+		}
+	}
+}

src/main/java/org/quiltmc/loader/api/FasterFileSystem.java

@@ -17,6 +17,7 @@
 package org.quiltmc.loader.api;
 
 import java.io.IOException;
+import java.nio.file.CopyOption;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -41,6 +42,13 @@ default Path createDirectories(Path dir, FileAttribute<?>... attrs) throws IOExc
 		return Files.createDirectories(dir, attrs);
 	}
 
+	/** @param source A {@link Path}, which might not be in this {@link FileSystem}.
+	 * @param target A {@link Path} which must be from this {@link FileSystem}
+	 * @return target */
+	default Path copy(Path source, Path target, CopyOption... options) throws IOException {
+		return Files.copy(source, target, options);
+	}
+
 	default boolean isSymbolicLink(Path path) {
 		return Files.isSymbolicLink(path);
 	}

src/main/java/org/quiltmc/loader/api/FasterFiles.java

@@ -17,6 +17,7 @@
 package org.quiltmc.loader.api;
 
 import java.io.IOException;
+import java.nio.file.CopyOption;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -57,6 +58,14 @@ public static Path createDirectories(Path dir, FileAttribute<?>... attrs) throws
 		}
 	}
 
+	public static Path copy(Path source, Path target, CopyOption... options) throws IOException {
+		if (target.getFileSystem() instanceof FasterFileSystem) {
+			return ((FasterFileSystem) target.getFileSystem()).copy(source, target, options);
+		} else {
+			return Files.copy(source, target, options);
+		}
+	}
+
 	public static boolean isSymbolicLink(Path path) {
 		if (path.getFileSystem() instanceof FasterFileSystem) {
 			return ((FasterFileSystem) path.getFileSystem()).isSymbolicLink(path);

src/main/java/org/quiltmc/loader/api/MountOption.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 QuiltMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.quiltmc.loader.api;
+
+/** Options for {@link ExtendedFiles#mount(java.nio.file.Path, java.nio.file.Path, MountOption...)} */
+public enum MountOption {
+
+	/** Replace an existing file if it exists when mounting. This cannot replace a non-empty directory. */
+	REPLACE_EXISTING,
+
+	/** Indicates that the mounted file will not permit writes.
+	 * <p>
+	 * This option is incompatible with {@link #COPY_ON_WRITE} */
+	READ_ONLY,
+
+	/** Indicates that the mounted file will copy to a new, separate file when written to.
+	 * <p>
+	 * This option is incompatible with {@link #READ_ONLY} */
+	COPY_ON_WRITE,
+}

src/main/java/org/quiltmc/loader/impl/QuiltLoaderImpl.java

@@ -424,6 +424,13 @@ private void setup() throws ModResolutionException {
 			addMod(modOption.convertToMod(resourceRoot));
 		}
 
+		try {
+			transformedModBundle.getFileSystem().close();
+		} catch (IOException e) {
+			// TODO!
+			throw new Error(e);
+		}
+
 		temporaryPluginSolveResult = null;
 		temporaryOrderedModList = null;
 		temporarySourcePaths = null;

src/main/java/org/quiltmc/loader/impl/discovery/RuntimeModRemapper.java

@@ -20,36 +20,28 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystem;
-import java.nio.file.FileVisitResult;
-import java.nio.file.FileVisitor;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 import java.util.stream.Collectors;
 
 import org.objectweb.asm.commons.Remapper;
+import org.quiltmc.loader.api.ExtendedFiles;
 import org.quiltmc.loader.api.FasterFiles;
+import org.quiltmc.loader.api.MountOption;
 import org.quiltmc.loader.api.plugin.solver.ModLoadOption;
 import org.quiltmc.loader.impl.QuiltLoaderImpl;
-import org.quiltmc.loader.impl.filesystem.QuiltMemoryFileSystem;
 import org.quiltmc.loader.impl.launch.common.QuiltLauncher;
 import org.quiltmc.loader.impl.launch.common.QuiltLauncherBase;
-import org.quiltmc.loader.impl.util.FileSystemUtil;
 import org.quiltmc.loader.impl.util.QuiltLoaderInternal;
 import org.quiltmc.loader.impl.util.QuiltLoaderInternalType;
 import org.quiltmc.loader.impl.util.SystemProperties;
 import org.quiltmc.loader.impl.util.mappings.TinyRemapperMappingsHelper;
 
-import net.fabricmc.accesswidener.AccessWidenerFormatException;
 import net.fabricmc.accesswidener.AccessWidenerReader;
 import net.fabricmc.accesswidener.AccessWidenerRemapper;
 import net.fabricmc.accesswidener.AccessWidenerWriter;
@@ -61,6 +53,8 @@
 @QuiltLoaderInternal(QuiltLoaderInternalType.LEGACY_EXPOSED)
 public final class RuntimeModRemapper {
 
+	static final boolean COPY_ON_WRITE = true;
+
 	public static void remap(Path cache, List<ModLoadOption> modList) {
 		List<ModLoadOption> modsToRemap = modList.stream()
 				.filter(modLoadOption -> modLoadOption.namespaceMappingFrom() != null)
@@ -92,7 +86,11 @@ public static void remap(Path cache, List<ModLoadOption> modList) {
 						Path dst = modDst.resolve(sub.toString().replace(modSrc.getFileSystem().getSeparator(), modDst.getFileSystem().getSeparator()));
 						try {
 							FasterFiles.createDirectories(dst.getParent());
-							Files.copy(path, dst);
+							if (COPY_ON_WRITE) {
+								ExtendedFiles.mount(path, dst, MountOption.COPY_ON_WRITE);
+							} else {
+								FasterFiles.copy(path, dst);
+							}
 						} catch (IOException e) {
 							throw new Error(e);
 						}
@@ -165,7 +163,6 @@ public static void remap(Path cache, List<ModLoadOption> modList) {
 
 				if (info.accessWideners != null) {
 					for (Map.Entry<String, byte[]> entry : info.accessWideners.entrySet()) {
-						Files.delete(info.outputPath.resolve(entry.getKey()));
 						Files.write(info.outputPath.resolve(entry.getKey()), entry.getValue());
 					}
 				}