From bd0aadf9d99ebff8be3181dee3dc08f8752c69a1 Mon Sep 17 00:00:00 2001 From: Jozufozu Date: Sat, 28 Sep 2024 22:05:38 -0700 Subject: [PATCH] Streamlined pipelines - Make UberShaderComponent#build NotNull - Move index update and key creation logic to PipelineCompiler - Always update index when a resource location is requested to fix MaterialEncoder misses - Indices trigger pipeline compiler deletion when updated --- .../backend/MaterialShaderIndices.java | 25 ++++-- .../backend/compile/IndirectPrograms.java | 20 +---- .../backend/compile/InstancingPrograms.java | 22 +---- .../backend/compile/PipelineCompiler.java | 80 ++++++++++++++++--- .../backend/compile/PipelineProgramKey.java | 17 ---- .../component/UberShaderComponent.java | 16 +--- 6 files changed, 94 insertions(+), 86 deletions(-) delete mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineProgramKey.java diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/MaterialShaderIndices.java b/common/src/backend/java/dev/engine_room/flywheel/backend/MaterialShaderIndices.java index c962359f0..71fcbd112 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/MaterialShaderIndices.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/MaterialShaderIndices.java @@ -6,6 +6,7 @@ import dev.engine_room.flywheel.api.material.CutoutShader; import dev.engine_room.flywheel.api.material.FogShader; +import dev.engine_room.flywheel.backend.compile.PipelineCompiler; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectArrayList; @@ -45,23 +46,31 @@ private Index() { this.sources = new ObjectArrayList<>(); } - public void add(ResourceLocation source) { - if (sources2Index.putIfAbsent(source, sources.size()) == -1) { - sources.add(source); - } + public ResourceLocation get(int index) { + return sources.get(index); } public int index(ResourceLocation source) { - return sources2Index.getInt(source); - } + var out = sources2Index.getInt(source); - public ResourceLocation get(int index) { - return sources.get(index); + if (out == -1) { + add(source); + PipelineCompiler.deleteAll(); + return sources2Index.getInt(source); + } + + return out; } @Unmodifiable public List all() { return sources; } + + private void add(ResourceLocation source) { + if (sources2Index.putIfAbsent(source, sources.size()) == -1) { + sources.add(source); + } + } } } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/IndirectPrograms.java b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/IndirectPrograms.java index e12e2137c..39d211ef9 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/IndirectPrograms.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/IndirectPrograms.java @@ -9,12 +9,10 @@ import dev.engine_room.flywheel.api.Flywheel; import dev.engine_room.flywheel.api.instance.InstanceType; import dev.engine_room.flywheel.api.material.Material; -import dev.engine_room.flywheel.backend.MaterialShaderIndices; import dev.engine_room.flywheel.backend.compile.component.InstanceStructComponent; import dev.engine_room.flywheel.backend.compile.component.SsboInstanceComponent; import dev.engine_room.flywheel.backend.compile.core.CompilationHarness; import dev.engine_room.flywheel.backend.compile.core.Compile; -import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms; import dev.engine_room.flywheel.backend.engine.uniform.Uniforms; import dev.engine_room.flywheel.backend.gl.GlCompat; import dev.engine_room.flywheel.backend.gl.shader.GlProgram; @@ -44,11 +42,11 @@ public class IndirectPrograms extends AtomicReferenceCounted { @Nullable private static IndirectPrograms instance; - private final CompilationHarness pipeline; + private final PipelineCompiler pipeline; private final CompilationHarness> culling; private final CompilationHarness utils; - private IndirectPrograms(CompilationHarness pipeline, CompilationHarness> culling, CompilationHarness utils) { + private IndirectPrograms(PipelineCompiler pipeline, CompilationHarness> culling, CompilationHarness utils) { this.pipeline = pipeline; this.culling = culling; this.utils = utils; @@ -151,19 +149,7 @@ public static void kill() { } public GlProgram getIndirectProgram(InstanceType instanceType, ContextShader contextShader, Material material) { - var light = material.light(); - var cutout = material.cutout(); - var shaders = material.shaders(); - var fog = material.fog(); - - var fogIndex = MaterialShaderIndices.fogSources(); - if (fogIndex.index(fog.source()) == -1) { - fogIndex.add(fog.source()); - pipeline.delete(); - PipelineCompiler.createFogComponent(); - } - - return pipeline.get(new PipelineProgramKey(instanceType, contextShader, light, cutout, shaders, FrameUniforms.debugOn())); + return pipeline.get(instanceType, contextShader, material); } public GlProgram getCullingProgram(InstanceType instanceType) { diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/InstancingPrograms.java b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/InstancingPrograms.java index fa51637f0..c1e736b1e 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/InstancingPrograms.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/InstancingPrograms.java @@ -8,9 +8,6 @@ import dev.engine_room.flywheel.api.instance.InstanceType; import dev.engine_room.flywheel.api.material.Material; -import dev.engine_room.flywheel.backend.MaterialShaderIndices; -import dev.engine_room.flywheel.backend.compile.core.CompilationHarness; -import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms; import dev.engine_room.flywheel.backend.gl.GlCompat; import dev.engine_room.flywheel.backend.gl.shader.GlProgram; import dev.engine_room.flywheel.backend.glsl.GlslVersion; @@ -24,9 +21,9 @@ public class InstancingPrograms extends AtomicReferenceCounted { @Nullable private static InstancingPrograms instance; - private final CompilationHarness pipeline; + private final PipelineCompiler pipeline; - private InstancingPrograms(CompilationHarness pipeline) { + private InstancingPrograms(PipelineCompiler pipeline) { this.pipeline = pipeline; } @@ -73,20 +70,7 @@ public static void kill() { } public GlProgram get(InstanceType instanceType, ContextShader contextShader, Material material) { - var light = material.light(); - var cutout = material.cutout(); - var materialShaders = material.shaders(); - - var fog = material.fog(); - - var fogIndex = MaterialShaderIndices.fogSources(); - if (fogIndex.index(fog.source()) == -1) { - fogIndex.add(fog.source()); - pipeline.delete(); - PipelineCompiler.createFogComponent(); - } - - return pipeline.get(new PipelineProgramKey(instanceType, contextShader, light, cutout, materialShaders, FrameUniforms.debugOn())); + return pipeline.get(instanceType, contextShader, material); } @Override diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineCompiler.java b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineCompiler.java index a38dbee23..9804bd48d 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineCompiler.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineCompiler.java @@ -4,6 +4,10 @@ import java.util.List; import dev.engine_room.flywheel.api.Flywheel; +import dev.engine_room.flywheel.api.instance.InstanceType; +import dev.engine_room.flywheel.api.material.LightShader; +import dev.engine_room.flywheel.api.material.Material; +import dev.engine_room.flywheel.api.material.MaterialShaders; import dev.engine_room.flywheel.backend.BackendConfig; import dev.engine_room.flywheel.backend.InternalVertex; import dev.engine_room.flywheel.backend.MaterialShaderIndices; @@ -12,6 +16,7 @@ import dev.engine_room.flywheel.backend.compile.component.UberShaderComponent; import dev.engine_room.flywheel.backend.compile.core.CompilationHarness; import dev.engine_room.flywheel.backend.compile.core.Compile; +import dev.engine_room.flywheel.backend.engine.uniform.FrameUniforms; import dev.engine_room.flywheel.backend.engine.uniform.Uniforms; import dev.engine_room.flywheel.backend.gl.GlCompat; import dev.engine_room.flywheel.backend.gl.shader.GlProgram; @@ -25,6 +30,8 @@ import net.minecraft.resources.ResourceLocation; public final class PipelineCompiler { + private static final List ALL = List.of(); + private static final Compile PIPELINE = new Compile<>(); private static UberShaderComponent FOG; @@ -33,11 +40,48 @@ public final class PipelineCompiler { private static final ResourceLocation API_IMPL_VERT = Flywheel.rl("internal/api_impl.vert"); private static final ResourceLocation API_IMPL_FRAG = Flywheel.rl("internal/api_impl.frag"); - static CompilationHarness create(ShaderSources sources, Pipeline pipeline, List vertexComponents, List fragmentComponents, Collection extensions) { + private final CompilationHarness harness; + + public PipelineCompiler(CompilationHarness harness) { + this.harness = harness; + } + + public GlProgram get(InstanceType instanceType, ContextShader contextShader, Material material) { + var light = material.light(); + var cutout = material.cutout(); + var shaders = material.shaders(); + var fog = material.fog(); + + // Tell fogSources to index the fog shader if we haven't seen it before. + // If it is new, this will trigger a deletion of all programs. + MaterialShaderIndices.fogSources() + .index(fog.source()); + + boolean useCutout = cutout != CutoutShaders.OFF; + + if (useCutout) { + // Same thing for cutout. + MaterialShaderIndices.cutoutSources() + .index(cutout.source()); + } + + return harness.get(new PipelineProgramKey(instanceType, contextShader, light, shaders, useCutout, FrameUniforms.debugOn())); + } + + public void delete() { + harness.delete(); + } + + public static void deleteAll() { + createFogComponent(); + createCutoutComponent(); + ALL.forEach(PipelineCompiler::delete); + } + + static PipelineCompiler create(ShaderSources sources, Pipeline pipeline, List vertexComponents, List fragmentComponents, Collection extensions) { // We could technically compile every version of light smoothness ahead of time, // but that seems unnecessary as I doubt most folks will be changing this option often. - var lightSmoothness = BackendConfig.INSTANCE.lightSmoothness(); - return PIPELINE.program() + var harness = PIPELINE.program() .link(PIPELINE.shader(GlCompat.MAX_GLSL_VERSION, ShaderType.VERTEX) .nameMapper(key -> { var instance = ResourceUtil.toDebugFileNameNoExtension(key.instanceType() @@ -53,7 +97,8 @@ static CompilationHarness create(ShaderSources sources, Pipe .requireExtensions(extensions) .onCompile((key, comp) -> key.contextShader() .onCompile(comp)) - .onCompile((key, comp) -> lightSmoothness.onCompile(comp)) + .onCompile((key, comp) -> BackendConfig.INSTANCE.lightSmoothness() + .onCompile(comp)) .onCompile((key, comp) -> { if (key.debugEnabled()) { comp.define("_FLW_DEBUG"); @@ -78,26 +123,25 @@ static CompilationHarness create(ShaderSources sources, Pipe var material = ResourceUtil.toDebugFileNameNoExtension(key.materialShaders() .fragmentSource()); - var cutout = ResourceUtil.toDebugFileNameNoExtension(key.cutout() - .source()); - var light = ResourceUtil.toDebugFileNameNoExtension(key.light() .source()); var debug = key.debugEnabled() ? "_debug" : ""; - return "pipeline/" + pipeline.compilerMarker() + "/frag/" + material + "/" + light + "_" + cutout + "_" + context + debug; + var cutout = key.useCutout() ? "_cutout" : ""; + return "pipeline/" + pipeline.compilerMarker() + "/frag/" + material + "/" + light + "_" + context + cutout + debug; }) .requireExtensions(extensions) .enableExtension("GL_ARB_conservative_depth") .onCompile((key, comp) -> key.contextShader() .onCompile(comp)) - .onCompile((key, comp) -> lightSmoothness.onCompile(comp)) + .onCompile((key, comp) -> BackendConfig.INSTANCE.lightSmoothness() + .onCompile(comp)) .onCompile((key, comp) -> { if (key.debugEnabled()) { comp.define("_FLW_DEBUG"); } }) .onCompile((key, comp) -> { - if (key.cutout() != CutoutShaders.OFF) { + if (key.useCutout()) { comp.define("_FLW_USE_DISCARD"); } }) @@ -108,8 +152,7 @@ static CompilationHarness create(ShaderSources sources, Pipe .withComponent(key -> FOG) .withResource(key -> key.light() .source()) - .withResource(key -> key.cutout() - .source()) + .with((key, fetcher) -> (key.useCutout() ? CUTOUT : fetcher.get(CutoutShaders.OFF.source()))) .withResource(pipeline.fragmentMain())) .preLink((key, program) -> { program.bindAttribLocation("_flw_aPos", 0); @@ -135,6 +178,8 @@ static CompilationHarness create(ShaderSources sources, Pipe GlProgram.unbind(); }) .harness(pipeline.compilerMarker(), sources); + + return new PipelineCompiler(harness); } public static void createFogComponent() { @@ -162,4 +207,15 @@ private static void createCutoutComponent() { .switchOn(GlslExpr.variable("_flw_uberCutoutIndex")) .build(FlwPrograms.SOURCES); } + + /** + * Represents the entire context of a program's usage. + * + * @param instanceType The instance shader to use. + * @param contextShader The context shader to use. + * @param light The light shader to use. + */ + public record PipelineProgramKey(InstanceType instanceType, ContextShader contextShader, LightShader light, + MaterialShaders materialShaders, boolean useCutout, boolean debugEnabled) { + } } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineProgramKey.java b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineProgramKey.java deleted file mode 100644 index ac004c1c4..000000000 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/PipelineProgramKey.java +++ /dev/null @@ -1,17 +0,0 @@ -package dev.engine_room.flywheel.backend.compile; - -import dev.engine_room.flywheel.api.instance.InstanceType; -import dev.engine_room.flywheel.api.material.CutoutShader; -import dev.engine_room.flywheel.api.material.LightShader; -import dev.engine_room.flywheel.api.material.MaterialShaders; - -/** - * Represents the entire context of a program's usage. - * - * @param instanceType The instance shader to use. - * @param contextShader The context shader to use. - * @param light The light shader to use. - */ -public record PipelineProgramKey(InstanceType instanceType, ContextShader contextShader, LightShader light, - CutoutShader cutout, MaterialShaders materialShaders, boolean debugEnabled) { -} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/component/UberShaderComponent.java b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/component/UberShaderComponent.java index e708943d4..ac3d41119 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/compile/component/UberShaderComponent.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/compile/component/UberShaderComponent.java @@ -136,7 +136,6 @@ public Builder switchOn(GlslExpr expr) { return this; } - @Nullable public UberShaderComponent build(ShaderSources sources) { if (switchArg == null) { throw new NullPointerException("Switch argument must be set"); @@ -144,24 +143,15 @@ public UberShaderComponent build(ShaderSources sources) { var transformed = ImmutableList.builder(); - boolean errored = false; int index = 0; for (var rl : materialSources) { SourceFile sourceFile = sources.get(rl); - if (sourceFile != null) { - final int finalIndex = index; - var adapterMap = createAdapterMap(adaptedFunctions, fnName -> "_" + fnName + "_" + finalIndex); - transformed.add(new StringSubstitutionComponent(sourceFile, adapterMap)); - } else { - errored = true; - } + final int finalIndex = index; + var adapterMap = createAdapterMap(adaptedFunctions, fnName -> "_" + fnName + "_" + finalIndex); + transformed.add(new StringSubstitutionComponent(sourceFile, adapterMap)); index++; } - if (errored) { - return null; - } - return new UberShaderComponent(name, switchArg, adaptedFunctions, transformed.build()); }