From 35c18007e85b608e74f9f543c861b8f6a6058cd1 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 15 Apr 2024 16:59:24 +0200 Subject: [PATCH 01/16] Partial points-to analysis starting from base analysis results. --- .../com/oracle/graal/pointsto/api/HostVM.java | 4 ++++ .../pointsto/flow/MethodTypeFlowBuilder.java | 18 ++++++++++++++++++ .../com/oracle/svm/core/SubstrateOptions.java | 3 +++ .../src/com/oracle/svm/hosted/SVMHost.java | 7 +++++++ 4 files changed, 32 insertions(+) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java index d108b62a6c5c..e9b94ba822f8 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java @@ -163,6 +163,10 @@ public boolean useBaseLayer() { return false; } + public boolean partialPointsToAnalysis() { + return false; + } + /** * Check if an {@link AnalysisType} is initialized. */ diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java index 53ec2b07a9b1..a46bf9196289 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java @@ -620,6 +620,24 @@ protected void apply(boolean forceReparse, Object reason) { assert !processed : "can only call apply once per MethodTypeFlowBuilder"; processed = true; + if (bb.getHostVM().useBaseLayer() && bb.getHostVM().partialPointsToAnalysis()) { + if (method.isInBaseLayer()) { + /* + * We don't need to analyze this method. We already know it's return type state from + * the open world analysis. We just install a return flow to link it with its uses. + */ + AnalysisType returnType = TypeFlow.filterUncheckedInterface(method.getSignature().getReturnType()); + if (returnType.getJavaKind().isObject()) { + // TODO the return type state should not be all-instantiated, it should be the + // persisted result of the open-world analysis [GR-52421] + insertAllInstantiatedTypesReturn(); + } + // TODO verify that tracked parameter state is subset of persisted state [GR-52421] + insertPlaceholderParamAndReturnFlows(); + return; + } + } + // assert method.getAnnotation(Fold.class) == null : method; if (handleNodeIntrinsic()) { assert !method.getReturnsAllInstantiatedTypes() : method; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index e8fa77820495..b0c1059ad48b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -1084,6 +1084,9 @@ public static boolean includeAll() { @Option(help = "Run layered image base layer open-world analysis. Includes all public types and methods that can be reached using normal Java access rules.")// public static final HostedOptionKey LayeredBaseImageAnalysis = new HostedOptionKey<>(false); + @Option(help = "Enable partial points-to analysis starting from results of base layer analysis.")// + public static final HostedOptionKey PartialPointsToAnalysis = new HostedOptionKey<>(false); + @Option(help = "Support for calls via the Java Foreign Function and Memory API", type = Expert) // public static final HostedOptionKey ForeignAPISupport = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java index 9d841716122e..181f4d8d3a04 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java @@ -178,6 +178,7 @@ public class SVMHost extends HostVM { private final FieldValueInterceptionSupport fieldValueInterceptionSupport; private final boolean useBaseLayer; + private final boolean partialPointsToAnalysis; private Set excludedFields; @SuppressWarnings("this-escape") @@ -207,6 +208,7 @@ public SVMHost(OptionValues options, ImageClassLoader loader, ClassInitializatio fieldValueInterceptionSupport = new FieldValueInterceptionSupport(annotationSubstitutions, classInitializationSupport); ImageSingletons.add(FieldValueInterceptionSupport.class, fieldValueInterceptionSupport); useBaseLayer = SubstrateOptions.LoadImageLayer.hasBeenSet(); + partialPointsToAnalysis = SubstrateOptions.PartialPointsToAnalysis.hasBeenSet(); if (SubstrateOptions.includeAll()) { initializeExcludedFields(); } @@ -217,6 +219,11 @@ public boolean useBaseLayer() { return useBaseLayer; } + @Override + public boolean partialPointsToAnalysis() { + return partialPointsToAnalysis; + } + protected InlineBeforeAnalysisPolicyUtils getInlineBeforeAnalysisPolicyUtils() { return new InlineBeforeAnalysisPolicyUtils(); } From 27a0a665a7b4d64408389eb2e2742720ebfb15f9 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Wed, 17 Apr 2024 21:34:11 +0200 Subject: [PATCH 02/16] Print parsing reason in call tree. --- .../src/com/oracle/graal/pointsto/meta/AnalysisMethod.java | 2 +- .../com/oracle/graal/pointsto/reports/CallTreePrinter.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java index a6e5af1f400b..f42f49977714 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java @@ -517,7 +517,7 @@ public boolean isImplementationInvoked() { return !Modifier.isAbstract(getModifiers()) && (isIntrinsicMethod() || AtomicUtils.isSet(this, isImplementationInvokedUpdater)); } - protected Object getImplementationInvokedReason() { + public Object getImplementationInvokedReason() { return isImplementationInvoked; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java index ed6d1bc3afce..c3efb9274258 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/reports/CallTreePrinter.java @@ -61,6 +61,7 @@ import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.meta.InvokeInfo; +import com.oracle.graal.pointsto.meta.PointsToAnalysisMethod; import com.oracle.graal.pointsto.util.AnalysisError; import jdk.graal.compiler.java.LambdaUtils; @@ -275,7 +276,8 @@ private void printMethods(PrintWriter out) { while (iterator.hasNext()) { MethodNode node = iterator.next(); boolean lastEntryPoint = !iterator.hasNext(); - out.format("%s%s %s %n", lastEntryPoint ? LAST_CHILD : CHILD, "entry", node.format()); + out.format("%s%s %s, parsing reason: %s %n", lastEntryPoint ? LAST_CHILD : CHILD, "entry", node.format(), + PointsToAnalysisMethod.unwrapInvokeReason(node.method.getImplementationInvokedReason())); printCallTreeNode(out, lastEntryPoint ? EMPTY_INDENT : CONNECTING_INDENT, node); } out.println(); From 9cb3aa4c006a7dc3357d4c6c6022235faf7958dc Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Wed, 17 Apr 2024 13:23:29 +0200 Subject: [PATCH 03/16] Prototype ImageLayer command. --- .../com/oracle/svm/core/BuildArtifacts.java | 2 + .../com/oracle/svm/core/SubstrateOptions.java | 20 +++++- .../com/oracle/svm/driver/NativeImage.java | 3 +- .../hosted/NativeImageGeneratorRunner.java | 9 ++- .../svm/hosted/image/AbstractImage.java | 19 +++++- .../svm/hosted/image/CCLinkerInvocation.java | 2 + .../svm/hosted/image/ImageLayerViaCC.java | 61 +++++++++++++++++++ .../svm/hosted/image/NativeImageViaCC.java | 15 ++--- 8 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageLayerViaCC.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java index 06d4f2fe0a16..f5e976452ebd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java @@ -40,6 +40,8 @@ enum ArtifactType { /* For all executables needed at run-time. */ EXECUTABLE("executables"), + /* Native image layer. */ + IMAGE_LAYER("image_layer"), /* For all shared libraries that are not JDK-related and needed at run-time. */ SHARED_LIBRARY("shared_libraries"), diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index b0c1059ad48b..54c69485bc0d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -103,6 +103,16 @@ public class SubstrateOptions { @Option(help = "Build shared library")// public static final HostedOptionKey SharedLibrary = new HostedOptionKey<>(false); + @Option(help = "Build a Native Image layer.")// + public static final HostedOptionKey ImageLayer = new HostedOptionKey<>(false) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + LayeredBaseImageAnalysis.update(values, newValue); + ClosedTypeWorld.update(values, !newValue); + PersistImageLayer.update(values, newValue); + } + }; + @APIOption(name = "static")// @Option(help = "Build statically linked executable (requires static libc and zlib)")// public static final HostedOptionKey StaticExecutable = new HostedOptionKey<>(false, key -> { @@ -1105,7 +1115,15 @@ public static boolean closedTypeWorld() { public static final HostedOptionKey AbortOnNameConflict = new HostedOptionKey<>(false); @Option(help = "Names of layer snapshots produced by PersistImageLayer", type = OptionType.Debug) // - public static final HostedOptionKey LoadImageLayer = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.build()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey LoadImageLayer = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.build()) { + @Override + public void update(EconomicMap, Object> values, Object boxedValue) { + super.update(values, boxedValue); + PartialPointsToAnalysis.update(values, true); + ClosedTypeWorld.update(values, false); + } + }; public static class TruffleStableOptions { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 9109d68635e7..dadbc8c7005c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -257,6 +257,7 @@ private static String oR(OptionKey option) { final String oHUseLibC = oH(SubstrateOptions.UseLibC); final String oHEnableStaticExecutable = oHEnabled(SubstrateOptions.StaticExecutable); final String oHEnableSharedLibraryFlagPrefix = oHEnabled + SubstrateOptions.SharedLibrary.getName(); + final String oHEnableImageLayerFlagPrefix = oHEnabled + SubstrateOptions.ImageLayer.getName(); final String oHColor = oH(SubstrateOptions.Color); final String oHEnableBuildOutputProgress = oHEnabledByDriver(SubstrateOptions.BuildOutputProgress); final String oHEnableBuildOutputLinks = oHEnabledByDriver(SubstrateOptions.BuildOutputLinks); @@ -1168,7 +1169,7 @@ private int completeImageBuild() { imageBuilderJavaArgs.addAll(getAgentArguments()); mainClass = getHostedOptionArgumentValue(imageBuilderArgs, oHClass); - buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.startsWith(oHEnableSharedLibraryFlagPrefix)); + buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.startsWith(oHEnableSharedLibraryFlagPrefix) || arg.startsWith(oHEnableImageLayerFlagPrefix)); staticExecutable = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oHEnableStaticExecutable)); boolean listModules = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oH + "+" + "ListModules")); printFlags |= imageBuilderArgs.stream().anyMatch(arg -> arg.matches("-H:MicroArchitecture(@[^=]*)?=list")); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java index 6ba7a931bff9..b2ce35238117 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java @@ -397,11 +397,14 @@ private int buildImage(ImageClassLoader classLoader) { NativeImageKind imageKind; boolean isStaticExecutable = SubstrateOptions.StaticExecutable.getValue(parsedHostedOptions); boolean isSharedLibrary = SubstrateOptions.SharedLibrary.getValue(parsedHostedOptions); - if (isStaticExecutable && isSharedLibrary) { - throw UserError.abort("Cannot pass both option: %s and %s", SubstrateOptionsParser.commandArgument(SubstrateOptions.SharedLibrary, "+"), - SubstrateOptionsParser.commandArgument(SubstrateOptions.StaticExecutable, "+")); + boolean isImageLayer = SubstrateOptions.ImageLayer.getValue(parsedHostedOptions); + if ((isStaticExecutable && isSharedLibrary) || (isStaticExecutable && isImageLayer) || (isSharedLibrary && isImageLayer)) { + throw UserError.abort("Cannot pass multiple options: %s, %s, %s", SubstrateOptionsParser.commandArgument(SubstrateOptions.SharedLibrary, "+"), + SubstrateOptionsParser.commandArgument(SubstrateOptions.StaticExecutable, "+"), SubstrateOptionsParser.commandArgument(SubstrateOptions.ImageLayer, "+")); } else if (isSharedLibrary) { imageKind = NativeImageKind.SHARED_LIBRARY; + } else if (isImageLayer) { + imageKind = NativeImageKind.IMAGE_LAYER; } else if (isStaticExecutable) { imageKind = NativeImageKind.STATIC_EXECUTABLE; } else { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java index 1937ff5c7081..b543ce3b7fcf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/AbstractImage.java @@ -27,8 +27,6 @@ import java.nio.file.Path; import java.util.List; -import jdk.graal.compiler.debug.DebugContext; - import com.oracle.objectfile.ObjectFile; import com.oracle.svm.core.LinkerInvocation; import com.oracle.svm.hosted.FeatureImpl.BeforeImageWriteAccessImpl; @@ -37,6 +35,8 @@ import com.oracle.svm.hosted.meta.HostedMethod; import com.oracle.svm.hosted.meta.HostedUniverse; +import jdk.graal.compiler.debug.DebugContext; + public abstract class AbstractImage { protected final HostedMetaAccess metaAccess; @@ -50,6 +50,13 @@ public abstract class AbstractImage { protected int debugInfoSize = -1; // for build output reporting public enum NativeImageKind { + /* IMAGE_LAYER mimics a SHARED_LIBRARY. */ + IMAGE_LAYER(false, true) { + @Override + public String getFilenameSuffix() { + return ".gso"; // Graal shared object + } + }, SHARED_LIBRARY(false) { @Override public String getFilenameSuffix() { @@ -65,10 +72,16 @@ public String getFilenameSuffix() { STATIC_EXECUTABLE(true); public final boolean isExecutable; + public final boolean isImageLayer; public final String mainEntryPointName; NativeImageKind(boolean executable) { + this(executable, false); + } + + NativeImageKind(boolean executable, boolean imageLayer) { isExecutable = executable; + isImageLayer = imageLayer; mainEntryPointName = executable ? "main" : "run_main"; } @@ -126,6 +139,8 @@ public static AbstractImage create(NativeImageKind k, HostedUniverse universe, H return switch (k) { case SHARED_LIBRARY -> new SharedLibraryImageViaCC(universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); + case IMAGE_LAYER -> + new ImageLayerViaCC(universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); case EXECUTABLE, STATIC_EXECUTABLE -> new ExecutableImageViaCC(k, universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, classLoader); }; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java index 09b96331a626..d37ba21e4f68 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java @@ -316,6 +316,7 @@ protected void setOutputKind(List cmd) { cmd.add("-static"); } break; + case IMAGE_LAYER: case SHARED_LIBRARY: cmd.add("-shared"); break; @@ -475,6 +476,7 @@ protected void setOutputKind(List cmd) { // Must use /MD in order to link with JDK native libraries built that way cmd.add("/MD"); break; + case IMAGE_LAYER: case SHARED_LIBRARY: cmd.add("/MD"); cmd.add("/LD"); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageLayerViaCC.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageLayerViaCC.java new file mode 100644 index 000000000000..eac643ead4eb --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageLayerViaCC.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.hosted.image; + +import java.lang.reflect.Method; +import java.nio.file.Path; +import java.util.List; + +import com.oracle.svm.core.LinkerInvocation; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.FeatureImpl.BeforeImageWriteAccessImpl; +import com.oracle.svm.hosted.c.NativeLibraries; +import com.oracle.svm.hosted.meta.HostedMetaAccess; +import com.oracle.svm.hosted.meta.HostedMethod; +import com.oracle.svm.hosted.meta.HostedUniverse; + +import jdk.graal.compiler.debug.DebugContext; + +public class ImageLayerViaCC extends NativeImageViaCC { + + public ImageLayerViaCC(HostedUniverse universe, HostedMetaAccess metaAccess, NativeLibraries nativeLibs, NativeImageHeap heap, NativeImageCodeCache codeCache, + List entryPoints, ClassLoader imageLoader) { + super(NativeImageKind.IMAGE_LAYER, universe, metaAccess, nativeLibs, heap, codeCache, entryPoints, imageLoader); + } + + @Override + public String[] makeLaunchCommand(NativeImageKind k, String imageName, Path binPath, Path workPath, Method method) { + throw VMError.intentionallyUnimplemented(); // ExcludeFromJacocoGeneratedReport + } + + @Override + public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tempDirectory, String imageName, BeforeImageWriteAccessImpl config) { + LinkerInvocation inv = super.write(debug, outputDirectory, tempDirectory, imageName, config); + writeHeaderFiles(outputDirectory, imageName, false); + writeHeaderFiles(outputDirectory, imageName, true); + return inv; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java index b9bd9b8c66bc..1ad8e3e63929 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageViaCC.java @@ -38,8 +38,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import jdk.graal.compiler.debug.DebugContext; -import jdk.graal.compiler.debug.Indent; import org.graalvm.nativeimage.Platform; import com.oracle.objectfile.ObjectFile; @@ -57,6 +55,9 @@ import com.oracle.svm.hosted.meta.HostedMethod; import com.oracle.svm.hosted.meta.HostedUniverse; +import jdk.graal.compiler.debug.DebugContext; +import jdk.graal.compiler.debug.Indent; + public abstract class NativeImageViaCC extends NativeImage { public NativeImageViaCC(NativeImageKind k, HostedUniverse universe, HostedMetaAccess metaAccess, NativeLibraries nativeLibs, NativeImageHeap heap, NativeImageCodeCache codeCache, @@ -110,11 +111,11 @@ public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tem try { List cmd = inv.getCommand(); - runLinkerCommand(imageName, inv, cmd, imageKind.isExecutable); + runLinkerCommand(imageName, inv, cmd, imageKind); } catch (RuntimeException e) { if (inv.shouldRunFallback(e.getMessage())) { List cmd = inv.getFallbackCommand(); - runLinkerCommand(imageName, inv, cmd, imageKind.isExecutable); + runLinkerCommand(imageName, inv, cmd, imageKind); } else { throw e; } @@ -124,7 +125,7 @@ public LinkerInvocation write(DebugContext debug, Path outputDirectory, Path tem } } - private void runLinkerCommand(String imageName, LinkerInvocation inv, List cmd, boolean imageKindIsExecutable) { + private void runLinkerCommand(String imageName, LinkerInvocation inv, List cmd, NativeImageKind kind) { Process linkerProcess = null; String commandLine = SubstrateUtil.getShellCommandString(cmd, false); try { @@ -149,9 +150,9 @@ private void runLinkerCommand(String imageName, LinkerInvocation inv, List Date: Thu, 11 Apr 2024 18:21:46 +0200 Subject: [PATCH 04/16] Update progress reporter. --- .../oracle/svm/hosted/ProgressReporter.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 7f182b95a495..b5d7dd646fa8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -96,6 +96,7 @@ import com.oracle.svm.hosted.util.JDKArgsUtils; import com.oracle.svm.hosted.util.VMErrorReporter; import com.oracle.svm.util.ImageBuildStatistics; +import com.sun.management.OperatingSystemMXBean; import jdk.graal.compiler.options.OptionDescriptor; import jdk.graal.compiler.options.OptionKey; @@ -474,7 +475,12 @@ public void closeAction() { private void printAnalysisStatistics(AnalysisUniverse universe, Collection libraries) { String actualFormat = "%,9d "; String totalFormat = " (%4.1f%% of %,8d total)"; - long reachableTypes = universe.getTypes().stream().filter(AnalysisType::isReachable).count(); + long reachableTypes; + if (universe.hostVM().useBaseLayer()) { + reachableTypes = universe.getTypes().stream().filter(AnalysisType::isReachable).filter(t -> !t.isInBaseLayer()).count(); + } else { + reachableTypes = universe.getTypes().stream().filter(AnalysisType::isReachable).count(); + } long totalTypes = universe.getTypes().size(); recordJsonMetric(AnalysisResults.TYPES_TOTAL, totalTypes); recordJsonMetric(AnalysisResults.DEPRECATED_CLASS_TOTAL, totalTypes); @@ -483,14 +489,24 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection fields = universe.getFields(); - long reachableFields = fields.stream().filter(AnalysisField::isAccessed).count(); + long reachableFields; + if (universe.hostVM().useBaseLayer()) { + reachableFields = fields.stream().filter(AnalysisField::isAccessed).filter(f -> !f.isInBaseLayer()).count(); + } else { + reachableFields = fields.stream().filter(AnalysisField::isAccessed).count(); + } int totalFields = fields.size(); recordJsonMetric(AnalysisResults.FIELD_TOTAL, totalFields); recordJsonMetric(AnalysisResults.FIELD_REACHABLE, reachableFields); l().a(actualFormat, reachableFields).doclink("reachable fields", "#glossary-reachability").a(" ") .a(totalFormat, Utils.toPercentage(reachableFields, totalFields), totalFields).println(); Collection methods = universe.getMethods(); - long reachableMethods = methods.stream().filter(AnalysisMethod::isReachable).count(); + long reachableMethods; + if (universe.hostVM().useBaseLayer()) { + reachableMethods = methods.stream().filter(AnalysisMethod::isReachable).filter(m -> !m.isInBaseLayer()).count(); + } else { + reachableMethods = methods.stream().filter(AnalysisMethod::isReachable).count(); + } int totalMethods = methods.size(); recordJsonMetric(AnalysisResults.METHOD_TOTAL, totalMethods); recordJsonMetric(AnalysisResults.METHOD_REACHABLE, reachableMethods); @@ -859,8 +875,8 @@ private static Timer getTimer(TimerCollection.Registry type) { return TimerCollection.singleton().get(type); } - private static com.sun.management.OperatingSystemMXBean getOperatingSystemMXBean() { - return (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + private static OperatingSystemMXBean getOperatingSystemMXBean() { + return (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); } private static class Utils { From de28e93aa8e4dac7052dc041b6edba09e30f5ab2 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Fri, 22 Mar 2024 16:49:36 +0100 Subject: [PATCH 05/16] Skip compilation for base layer methods. --- .../graal/pointsto/meta/AnalysisMethod.java | 3 ++ .../oracle/svm/hosted/code/CompileQueue.java | 43 +++++++++++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java index f42f49977714..5a705096cdee 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java @@ -775,6 +775,9 @@ public Type[] getGenericParameterTypes() { @Override public boolean canBeInlined() { + if (isInBaseLayer) { + return false; + } return !hasNeverInlineDirective(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index a07bc1f7dcef..1d6224cb2931 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -629,12 +629,15 @@ private void parseAheadOfTimeCompiledMethods() { */ continue; } - if (hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || - hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) { + if ((hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || + hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) && !hMethod.wrapped.isInBaseLayer()) { ensureParsed(hMethod, null, new EntryPointReason()); } if (hMethod.wrapped.isVirtualRootMethod()) { for (HostedMethod impl : hMethod.getImplementations()) { + if (impl.wrapped.isInBaseLayer()) { + continue; + } VMError.guarantee(impl.wrapped.isImplementationInvoked()); ensureParsed(impl, null, new EntryPointReason()); } @@ -645,11 +648,14 @@ private void parseAheadOfTimeCompiledMethods() { SubstrateForeignCallsProvider foreignCallsProvider = (SubstrateForeignCallsProvider) runtimeConfig.getProviders().getForeignCalls(); for (SubstrateForeignCallLinkage linkage : foreignCallsProvider.getForeignCalls().values()) { HostedMethod method = (HostedMethod) linkage.getDescriptor().findMethod(runtimeConfig.getProviders().getMetaAccess()); - if (method.wrapped.isDirectRootMethod() && method.wrapped.isSimplyImplementationInvoked()) { + if (method.wrapped.isDirectRootMethod() && method.wrapped.isSimplyImplementationInvoked() && !method.wrapped.isInBaseLayer()) { ensureParsed(method, null, new EntryPointReason()); } if (method.wrapped.isVirtualRootMethod()) { for (HostedMethod impl : method.getImplementations()) { + if (impl.wrapped.isInBaseLayer()) { + continue; + } VMError.guarantee(impl.wrapped.isImplementationInvoked()); ensureParsed(impl, null, new EntryPointReason()); } @@ -697,7 +703,7 @@ protected void inlineTrivialMethods(DebugContext debug) throws InterruptedExcept assert method.isOriginalMethod(); for (MultiMethod multiMethod : method.getAllMultiMethods()) { HostedMethod hMethod = (HostedMethod) multiMethod; - if (hMethod.compilationInfo.getCompilationGraph() != null) { + if (hMethod.compilationInfo.getCompilationGraph() != null && !hMethod.wrapped.isInBaseLayer()) { executor.execute(new TrivialInlineTask(hMethod)); } } @@ -743,6 +749,15 @@ class InliningGraphDecoder extends PEGraphDecoder { protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, BytecodeProvider intrinsicBytecodeProvider) { return ((HostedMethod) method).compilationInfo.getCompilationGraph().getEncodedGraph(); } + + @Override + protected LoopScope trySimplifyInvoke(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { + if (((HostedMethod) callTarget.targetMethod()).wrapped.isInBaseLayer()) { + /* Cannot inline base layer method. */ + return null; + } + return super.trySimplifyInvoke(methodScope, loopScope, invokeData, callTarget); + } } // Wrapper to clearly identify phase @@ -909,14 +924,17 @@ public void scheduleEntryPoints() { continue; } - if (hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || - hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) { + if ((hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || + hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) && !hMethod.wrapped.isInBaseLayer()) { ensureCompiled(hMethod, new EntryPointReason()); } if (hMethod.wrapped.isVirtualRootMethod()) { MultiMethod.MultiMethodKey key = hMethod.getMultiMethodKey(); assert key != DEOPT_TARGET_METHOD && key != SubstrateCompilationDirectives.RUNTIME_COMPILED_METHOD : "unexpected method as virtual root " + hMethod; for (HostedMethod impl : hMethod.getImplementations()) { + if (impl.wrapped.isInBaseLayer()) { + continue; + } VMError.guarantee(impl.wrapped.isImplementationInvoked()); ensureCompiled(impl, new EntryPointReason()); } @@ -1028,6 +1046,9 @@ private void defaultParseFunction(DebugContext debug, HostedMethod method, Compi private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetNode targetNode, HostedMethod invokeTarget, boolean isIndirect) { if (isIndirect) { for (HostedMethod invokeImplementation : invokeTarget.getImplementations()) { + if (invokeImplementation.wrapped.isInBaseLayer()) { + continue; + } handleSpecialization(method, targetNode, invokeTarget, invokeImplementation); ensureParsed(invokeImplementation, method, new VirtualCallReason(method, invokeImplementation, reason)); } @@ -1042,7 +1063,7 @@ private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetN * already applied during parsing before we reach this point, so we look at the "simple" * implementation invoked status. */ - if (invokeTarget.wrapped.isSimplyImplementationInvoked()) { + if (invokeTarget.wrapped.isSimplyImplementationInvoked() && !invokeTarget.wrapped.isInBaseLayer()) { handleSpecialization(method, targetNode, invokeTarget, invokeTarget); ensureParsed(invokeTarget, method, new DirectCallReason(method, reason)); } @@ -1276,10 +1297,13 @@ protected void ensureCalleesCompiled(HostedMethod method, CompileReason reason, for (Infopoint infopoint : result.getInfopoints()) { if (infopoint instanceof Call call) { HostedMethod callTarget = (HostedMethod) call.target; - if (call.direct || isDynamicallyResolvedCall(result, call)) { + if ((call.direct || isDynamicallyResolvedCall(result, call)) && !callTarget.wrapped.isInBaseLayer()) { ensureCompiled(callTarget, new DirectCallReason(method, reason)); } else if (callTarget != null && callTarget.getImplementations() != null) { for (HostedMethod impl : callTarget.getImplementations()) { + if (impl.wrapped.isInBaseLayer()) { + continue; + } ensureCompiled(impl, new VirtualCallReason(method, callTarget, reason)); } } @@ -1304,6 +1328,9 @@ protected final void ensureCompiledForMethodPointerConstants(HostedMethod method if (constant instanceof SubstrateMethodPointerConstant) { MethodPointer pointer = ((SubstrateMethodPointerConstant) constant).pointer(); HostedMethod referencedMethod = (HostedMethod) pointer.getMethod(); + if (referencedMethod.wrapped.isInBaseLayer()) { + continue; + } ensureCompiled(referencedMethod, new MethodPointerConstantReason(method, referencedMethod, reason)); } } From 2080bed54786a9df1bc71ead97a4f60db1281c63 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 25 Mar 2024 22:49:18 +0100 Subject: [PATCH 06/16] Refactor image heap error reporting. --- .../svm/hosted/image/NativeImageHeap.java | 19 ++++++++++++++----- .../hosted/image/NativeImageHeapWriter.java | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java index 37ee2791f2bb..2177d9eb3853 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java @@ -356,7 +356,7 @@ public void addConstant(final JavaConstant constant, boolean immutableFromParent * image that the static analysis has not seen - so this check actually protects * against much more than just missing class initialization information. */ - throw reportIllegalType(hUniverse.getSnippetReflection().asObject(Object.class, constant), reason); + throw reportIllegalType(hub, reason, "Missing class initialization info for " + hub.getName() + " type."); } } @@ -599,15 +599,24 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable } private static HostedType requireType(Optional optionalType, Object object, Object reason) { - if (!optionalType.isPresent() || !optionalType.get().isInstantiated()) { - throw reportIllegalType(object, reason); + if (optionalType.isEmpty()) { + throw reportIllegalType(object, reason, "Analysis type is missing for hosted object of " + object.getClass().getTypeName() + " class."); } - return optionalType.get(); + HostedType hostedType = optionalType.get(); + if (!hostedType.isInstantiated()) { + throw reportIllegalType(object, reason, "Type " + hostedType.toJavaName() + " was not marked instantiated."); + } + return hostedType; } static RuntimeException reportIllegalType(Object object, Object reason) { + throw reportIllegalType(object, reason, ""); + } + + static RuntimeException reportIllegalType(Object object, Object reason, String problem) { StringBuilder msg = new StringBuilder(); - msg.append("Image heap writing found a class not seen during static analysis. "); + msg.append("Problem during heap layout: ").append(problem).append(" "); + msg.append("The static analysis may have missed a type. "); msg.append("Did a static field or an object referenced from a static field change during native image generation? "); msg.append("For example, a lazily initialized cache could have been initialized during image generation, in which case "); msg.append("you need to force eager initialization of the cache before static analysis or reset the cache using a field "); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java index 6fbb938a271e..acb80b0cd8d4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java @@ -136,7 +136,7 @@ private void mustBeReferenceAligned(int index) { private static void verifyTargetDidNotChange(Object target, Object reason, Object targetInfo) { if (targetInfo == null) { - throw NativeImageHeap.reportIllegalType(target, reason); + throw NativeImageHeap.reportIllegalType(target, reason, "Inconsistent image heap."); } } From 022a9778e556477d08aa0382fb4d9128e34e9915 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 15 Apr 2024 22:40:41 +0200 Subject: [PATCH 07/16] Skip base layer constants when writing the image. --- .../svm/hosted/image/LIRNativeImageCodeCache.java | 5 +++++ .../svm/hosted/image/NativeImageCodeCache.java | 5 +++++ .../oracle/svm/hosted/image/NativeImageHeap.java | 14 +++++++++++++- .../svm/hosted/image/NativeImageHeapWriter.java | 11 +++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java index 694707325b24..989d0e1b2dad 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java @@ -36,6 +36,7 @@ import org.graalvm.collections.Pair; import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.objectfile.ObjectFile; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.config.ConfigurationValues; @@ -355,6 +356,10 @@ public void patchMethods(DebugContext debug, RelocatableBuffer relocs, ObjectFil } else if (codeAnnotation instanceof HostedImageHeapConstantPatch) { HostedImageHeapConstantPatch patch = (HostedImageHeapConstantPatch) codeAnnotation; + if (patch.constant instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { + // TODO use object offset in base layer heap [GR-52911] + continue; + } ObjectInfo objectInfo = imageHeap.getConstantInfo(patch.constant); long objectAddress = objectInfo.getOffset(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java index b6b4db14bc9e..c114b39921dc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java @@ -57,6 +57,7 @@ import com.oracle.graal.pointsto.AbstractAnalysisEngine; import com.oracle.graal.pointsto.BigBang; +import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.graal.pointsto.infrastructure.WrappedElement; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; @@ -666,6 +667,10 @@ protected boolean verifyMethods(DebugContext debug, HostedUniverse hUniverse, Co public void writeConstants(NativeImageHeapWriter writer, RelocatableBuffer buffer) { ByteBuffer bb = buffer.getByteBuffer(); dataSection.buildDataSection(bb, (position, constant) -> { + if (constant instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { + // TODO use object offset in base layer heap [GR-52911] + return; + } writer.writeReference(buffer, position, (JavaConstant) constant, "VMConstant: " + constant); }); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java index 2177d9eb3853..30ae580f92db 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java @@ -215,7 +215,8 @@ public void addTrailingObjects() { // Process any remaining objects on the worklist, especially that might intern strings. processAddObjectWorklist(); - boolean usesInternedStrings = hMetaAccess.lookupJavaField(StringInternSupport.getInternedStringsField()).isAccessed(); + HostedField hostedField = hMetaAccess.optionalLookupJavaField(StringInternSupport.getInternedStringsField()); + boolean usesInternedStrings = hostedField != null && hostedField.isAccessed(); if (usesInternedStrings) { /* * Ensure that the hub of the String[] array (used for the interned objects) is written. @@ -274,6 +275,9 @@ private void addStaticFields() { * fields manually. */ for (HostedField field : hUniverse.getFields()) { + if (field.wrapped.isInBaseLayer()) { + continue; + } if (Modifier.isStatic(field.getModifiers()) && field.hasLocation() && field.getType().getStorageKind() == JavaKind.Object && field.isRead()) { assert field.isWritten() || !field.isValueAvailable() || MaterializedConstantFields.singleton().contains(field.wrapped); addConstant(readConstantField(field, null), false, field); @@ -340,6 +344,14 @@ public void addObject(final Object original, boolean immutableFromParent, final public void addConstant(final JavaConstant constant, boolean immutableFromParent, final Object reason) { assert addObjectsPhase.isAllowed() : "Objects cannot be added at phase: " + addObjectsPhase.toString() + " with reason: " + reason; + if (constant instanceof ImageHeapConstant hc && hc.isInBaseLayer() && !hMetaAccess.isInstanceOf(constant, Class.class)) { + /* + * Skip base layer constants, but not the hubs. We need the object info in + * NativeImageHeapWriter.writeObjectHeader() + */ + return; + } + if (constant.getJavaKind().isPrimitive() || constant.isNull() || hMetaAccess.isInstanceOf(constant, WordBase.class)) { return; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java index acb80b0cd8d4..9a7d2d17c4a0 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java @@ -118,6 +118,9 @@ private void writeStaticFields(RelocatableBuffer buffer) { ObjectInfo primitiveFields = heap.getObjectInfo(StaticFieldsSupport.getStaticPrimitiveFields()); ObjectInfo objectFields = heap.getObjectInfo(StaticFieldsSupport.getStaticObjectFields()); for (HostedField field : heap.hUniverse.getFields()) { + if (field.wrapped.isInBaseLayer()) { + continue; + } if (Modifier.isStatic(field.getModifiers()) && field.hasLocation() && field.isRead()) { assert field.isWritten() || !field.isValueAvailable() || MaterializedConstantFields.singleton().contains(field.wrapped); ObjectInfo fields = (field.getStorageKind() == JavaKind.Object) ? objectFields : primitiveFields; @@ -152,6 +155,10 @@ private void writeField(RelocatableBuffer buffer, ObjectInfo fields, HostedField if (value instanceof RelocatableConstant) { addNonDataRelocation(buffer, index, prepareRelocatable(info, value)); } else { + if (value instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { + // TODO use object offset in base layer heap [GR-52911] + return; + } write(buffer, index, value, info != null ? info : field); } } @@ -197,6 +204,10 @@ private void writeConstant(RelocatableBuffer buffer, int index, JavaKind kind, J } else { con = constant; } + if (con instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { + // TODO use object offset in base layer heap [GR-52911] + return; + } write(buffer, index, con, info); } From 0981685e2e50cff73d3b46cd98cdfe9c3224ac71 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Thu, 18 Apr 2024 17:11:48 +0200 Subject: [PATCH 08/16] Force indirect calls. --- .../graal/pointsto/meta/AnalysisMethod.java | 2 +- .../graal/amd64/SubstrateAMD64Backend.java | 23 ++++++-- .../svm/core/c/BoxedRelocatedPointer.java | 4 ++ .../graal/snippets/NonSnippetLowerings.java | 26 ++++++++- .../oracle/svm/core/meta/SharedMethod.java | 6 ++ .../svm/graal/meta/SubstrateMethod.java | 11 ++++ .../hosted/ExtensionLayerImageFeature.java | 57 +++++++++++++++++++ .../code/CEntryPointLiteralFeature.java | 4 +- .../oracle/svm/hosted/code/CompileQueue.java | 2 +- .../hosted/image/LIRNativeImageCodeCache.java | 1 + .../oracle/svm/hosted/image/NativeImage.java | 2 +- .../oracle/svm/hosted/meta/HostedMethod.java | 25 +++++++- .../svm/hosted/meta/RelocatableConstant.java | 9 +++ 13 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java index 5a705096cdee..2755e01e8ea9 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisMethod.java @@ -301,7 +301,7 @@ private static JavaType getCatchType(AnalysisUniverse universe, ResolvedJavaMeth } @Override - protected AnalysisUniverse getUniverse() { + public AnalysisUniverse getUniverse() { /* Access the universe via the declaring class to avoid storing it here. */ return declaringClass.getUniverse(); } diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java index ec6c762d2bcd..6985427a521d 100644 --- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java +++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java @@ -79,6 +79,7 @@ import com.oracle.svm.core.graal.nodes.ComputedIndirectCallTargetNode.Computation; import com.oracle.svm.core.graal.nodes.ComputedIndirectCallTargetNode.FieldLoad; import com.oracle.svm.core.graal.nodes.ComputedIndirectCallTargetNode.FieldLoadIfZero; +import com.oracle.svm.core.graal.snippets.NonSnippetLowerings; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.SubstrateReferenceMapBuilder; import com.oracle.svm.core.meta.CompressedNullConstant; @@ -199,6 +200,7 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.JavaType; +import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.Value; @@ -664,12 +666,23 @@ protected boolean getDestroysCallerSavedRegisters(ResolvedJavaMethod targetMetho @Override protected Value emitIndirectForeignCallAddress(ForeignCallLinkage linkage) { + SubstrateForeignCallLinkage callTarget = (SubstrateForeignCallLinkage) linkage; + SharedMethod targetMethod = (SharedMethod) callTarget.getMethod(); + if (SubstrateUtil.HOSTED && targetMethod.forceIndirectCall()) { + // Emit a load for the BoxedRelocatedPointer.pointer field holding the + // MethodPointer to the target method + ResolvedJavaField boxedPointerField = getMetaAccess().lookupJavaField(NonSnippetLowerings.boxedRelocatedPointerField); + int displacement = boxedPointerField.getOffset(); + JavaConstant boxedPointerBase = targetMethod.getMethodPointer(); + RegisterValue heapBaseRegister = ReservedRegisters.singleton().getHeapBaseRegister().asValue(); + AMD64AddressValue boxedRelocatedPointerBaseAddress = new AMD64AddressValue(getLIRKindTool().getWordKind(), heapBaseRegister, Value.ILLEGAL, + Stride.S1, displacement + SubstrateAMD64Backend.addressDisplacement(boxedPointerBase, getConstantReflection()), + SubstrateAMD64Backend.addressDisplacementAnnotation(boxedPointerBase)); + return getArithmetic().emitLoad(getLIRKindTool().getWordKind(), boxedRelocatedPointerBaseAddress, null, MemoryOrderMode.PLAIN, MemoryExtendKind.DEFAULT); + } if (!shouldEmitOnlyIndirectCalls()) { return null; } - SubstrateForeignCallLinkage callTarget = (SubstrateForeignCallLinkage) linkage; - SharedMethod targetMethod = (SharedMethod) callTarget.getMethod(); - Value codeOffsetInImage = emitConstant(getLIRKindTool().getWordKind(), JavaConstant.forLong(targetMethod.getImageCodeOffset())); Value codeInfo = emitJavaConstant(SubstrateObjectConstant.forObject(targetMethod.getImageCodeInfo())); Value codeStartField = new AMD64AddressValue(getLIRKindTool().getWordKind(), asAllocatable(codeInfo), KnownOffsets.singleton().getImageCodeInfoCodeStartOffset()); @@ -684,9 +697,9 @@ protected void emitForeignCallOp(ForeignCallLinkage linkage, Value targetAddress Value exceptionTemp = getExceptionTemp(info != null && info.exceptionEdge != null); vzeroupperBeforeCall(this, arguments, info, targetMethod); - if (shouldEmitOnlyIndirectCalls()) { + if (shouldEmitOnlyIndirectCalls() || targetMethod.forceIndirectCall()) { AllocatableValue targetRegister = AMD64.rax.asValue(FrameAccess.getWordStamp().getLIRKind(getLIRKindTool())); - emitMove(targetRegister, targetAddress); + emitMove(targetRegister, targetAddress); // targetAddress is a CFunctionPointer append(new SubstrateAMD64IndirectCallOp(targetMethod, result, arguments, temps, targetRegister, info, Value.ILLEGAL, Value.ILLEGAL, StatusSupport.STATUS_ILLEGAL, getDestroysCallerSavedRegisters(targetMethod), exceptionTemp, null)); } else { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/BoxedRelocatedPointer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/BoxedRelocatedPointer.java index 7bd7a987b537..821e8a94798e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/BoxedRelocatedPointer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/BoxedRelocatedPointer.java @@ -26,12 +26,16 @@ import org.graalvm.nativeimage.c.function.RelocatedPointer; +import com.oracle.svm.core.BuildPhaseProvider.AfterCompilation; +import com.oracle.svm.core.heap.UnknownPrimitiveField; + /** * A variant of {@link BoxedPointer} that is immutable, but has a non-final field, intended to work * around limitations on folding {@link RelocatedPointer} into a constant in call stub code. */ public final class BoxedRelocatedPointer { + @UnknownPrimitiveField(availability = AfterCompilation.class)// private RelocatedPointer pointer; public BoxedRelocatedPointer(RelocatedPointer pointer) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/NonSnippetLowerings.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/NonSnippetLowerings.java index 58ecce3c1285..18e44b4f7a3c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/NonSnippetLowerings.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/NonSnippetLowerings.java @@ -26,6 +26,7 @@ import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.HAS_SIDE_EFFECT; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.EnumMap; import java.util.List; @@ -37,6 +38,7 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.c.BoxedRelocatedPointer; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.code.SubstrateBackend; import com.oracle.svm.core.graal.meta.KnownOffsets; @@ -50,6 +52,7 @@ import com.oracle.svm.core.snippets.SnippetRuntime; import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.util.ReflectionUtil; import jdk.graal.compiler.core.common.memory.BarrierType; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; @@ -106,12 +109,14 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.JavaType; +import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; public abstract class NonSnippetLowerings { public static final SnippetRuntime.SubstrateForeignCallDescriptor REPORT_VERIFY_TYPES_ERROR = SnippetRuntime.findForeignCall(NonSnippetLowerings.class, "reportVerifyTypesError", HAS_SIDE_EFFECT, LocationIdentity.any()); + public static final Field boxedRelocatedPointerField = ReflectionUtil.lookupField(BoxedRelocatedPointer.class, "pointer"); private final Predicate mustNotAllocatePredicate; @@ -364,7 +369,26 @@ public void lower(FixedNode node, LoweringTool tool) { targetMethod = implementations[0]; } - if (!SubstrateBackend.shouldEmitOnlyIndirectCalls()) { + if (targetMethod.forceIndirectCall()) { + /* + * Lower cross layer boundary direct calls to indirect calls. First emit a + * load for the BoxedRelocatedPointer.pointer field holding the + * MethodPointer to the target method, then emit an indirect call to that + * pointer. + */ + ResolvedJavaField boxedPointerField = tool.getMetaAccess().lookupJavaField(NonSnippetLowerings.boxedRelocatedPointerField); + ConstantNode boxedPointerFieldOffset = ConstantNode.forIntegerKind(ConfigurationValues.getWordKind(), boxedPointerField.getOffset(), graph); + ConstantNode boxedPointerBase = ConstantNode.forConstant(targetMethod.getMethodPointer(), tool.getMetaAccess(), graph); + + AddressNode methodPointerAddress = graph.unique(new OffsetAddressNode(boxedPointerBase, boxedPointerFieldOffset)); + /* + * Use the ANY location identity to prevent ReadNode.canonicalizeRead() to + * try to constant fold the method address. + */ + ReadNode entry = graph.add(new ReadNode(methodPointerAddress, LocationIdentity.any(), FrameAccess.getWordStamp(), BarrierType.NONE, MemoryOrderMode.PLAIN)); + loweredCallTarget = createIndirectCall(graph, callTarget, parameters, method, signature, callType, invokeKind, entry); + graph.addBeforeFixed(node, entry); + } else if (!SubstrateBackend.shouldEmitOnlyIndirectCalls()) { loweredCallTarget = createDirectCall(graph, callTarget, parameters, signature, callType, invokeKind, targetMethod, node); } else if (!targetMethod.hasImageCodeOffset()) { /* diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java index 83e1f4b99367..d5b6a5cb30a9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/meta/SharedMethod.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.graal.code.SubstrateCallingConventionKind; import com.oracle.svm.core.graal.code.SubstrateCallingConventionType; +import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.ResolvedJavaMethod; /** @@ -81,4 +82,9 @@ public interface SharedMethod extends ResolvedJavaMethod { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getImageCodeDeoptOffset(); + /** Always call this method indirectly, even if it is normally called directly. */ + boolean forceIndirectCall(); + + /** Return a boxed pointer to this method. */ + JavaConstant getMethodPointer(); } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMethod.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMethod.java index 00f9d436e37e..ac65d02f38ff 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMethod.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/SubstrateMethod.java @@ -60,6 +60,7 @@ import jdk.vm.ci.meta.ConstantPool; import jdk.vm.ci.meta.DefaultProfilingInfo; import jdk.vm.ci.meta.ExceptionHandler; +import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.LineNumberTable; import jdk.vm.ci.meta.LocalVariableTable; import jdk.vm.ci.meta.ProfilingInfo; @@ -224,6 +225,16 @@ public int getImageCodeDeoptOffset() { return imageCodeDeoptOffset; } + @Override + public boolean forceIndirectCall() { + return false; + } + + @Override + public JavaConstant getMethodPointer() { + throw VMError.intentionallyUnimplemented(); // ExcludeFromJacocoGeneratedReport + } + @Override public int getEncodedGraphStartOffset() { return encodedGraphStartOffset; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java new file mode 100644 index 000000000000..69288a8c7707 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.c.BoxedRelocatedPointer; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; +import com.oracle.svm.util.ReflectionUtil; + +/** + * This feature contains some configs currently necessary to build an extension layer. We'll need + * better mechanisms to avoid these workarounds. + */ +@AutomaticallyRegisteredFeature +final class ExtensionLayerImageFeature implements InternalFeature { + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return SubstrateOptions.LoadImageLayer.hasBeenSet(); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess a) { + BeforeAnalysisAccessImpl access = (BeforeAnalysisAccessImpl) a; + /* + * BoxedRelocatedPointer is used for implementing the indirect calls between layers. Since + * the box object itself is only reachable late, after compilation, we need to mark it as + * allocated and the pointer field as accessed. + */ + access.registerAsInHeap(BoxedRelocatedPointer.class); + access.registerAsAccessed(ReflectionUtil.lookupField(BoxedRelocatedPointer.class, "pointer")); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointLiteralFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointLiteralFeature.java index 82157ab018d7..835c780c1afb 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointLiteralFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CEntryPointLiteralFeature.java @@ -36,9 +36,9 @@ import com.oracle.graal.pointsto.BigBang; import com.oracle.graal.pointsto.infrastructure.UniverseMetaAccess; import com.oracle.graal.pointsto.meta.AnalysisMethod; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.meta.MethodPointer; -import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.util.UserError; import com.oracle.svm.hosted.FeatureImpl.CompilationAccessImpl; import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl; @@ -76,7 +76,7 @@ public Object apply(Object source) { AnalysisMethod aStub = CEntryPointCallStubSupport.singleton().getStubForMethod(aMethod); HostedMethod hStub = (HostedMethod) metaAccess.getUniverse().lookup(aStub); assert hStub.wrapped.isEntryPoint(); - assert hStub.isCompiled(); + assert hStub.isCompiled() || hStub.wrapped.isInBaseLayer(); /* * Only during compilation and native image writing, we do the actual * replacement. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index 1d6224cb2931..d5c46fe3d7f4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -703,7 +703,7 @@ protected void inlineTrivialMethods(DebugContext debug) throws InterruptedExcept assert method.isOriginalMethod(); for (MultiMethod multiMethod : method.getAllMultiMethods()) { HostedMethod hMethod = (HostedMethod) multiMethod; - if (hMethod.compilationInfo.getCompilationGraph() != null && !hMethod.wrapped.isInBaseLayer()) { + if (hMethod.compilationInfo.getCompilationGraph() != null) { executor.execute(new TrivialInlineTask(hMethod)); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java index 989d0e1b2dad..94dad1420ae1 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java @@ -386,6 +386,7 @@ public void patchMethods(DebugContext debug, RelocatableBuffer relocs, ObjectFil // which is also in the code cache (a.k.a. a section-local call). // This will change, and we will have to case-split here... but not yet. HostedMethod callTarget = (HostedMethod) call.target; + VMError.guarantee(!callTarget.wrapped.isInBaseLayer(), "Unexpected direct call to base layer method %s. These calls are currently lowered to indirect calls.", callTarget); int callTargetStart = callTarget.getCodeAddressOffset(); if (trampolineOffsetMap != null && trampolineOffsetMap.containsKey(callTarget)) { callTargetStart = trampolineOffsetMap.get(callTarget); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index 457f76e03d85..9882479874a7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -594,7 +594,7 @@ private void markFunctionRelocationSite(final ProgbitsSectionImpl sectionImpl, f MethodPointer methodPointer = (MethodPointer) info.getTargetObject(); ResolvedJavaMethod method = methodPointer.getMethod(); HostedMethod target = (method instanceof HostedMethod) ? (HostedMethod) method : heap.hUniverse.lookup(method); - if (!target.isCompiled()) { + if (!target.isCompiled() && !target.wrapped.isInBaseLayer()) { target = metaAccess.lookupJavaMethod(InvalidMethodPointerHandler.METHOD_POINTER_NOT_COMPILED_HANDLER_METHOD); } // A reference to a method. Mark the relocation site using the symbol name. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java index 4b8b8cd4557b..12ef7c99672e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMethod.java @@ -51,6 +51,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.BoxedRelocatedPointer; import com.oracle.svm.core.code.ImageCodeInfo; import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; @@ -77,6 +78,7 @@ import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.ConstantPool; import jdk.vm.ci.meta.ExceptionHandler; +import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaMethod; import jdk.vm.ci.meta.JavaType; import jdk.vm.ci.meta.LineNumberTable; @@ -92,6 +94,8 @@ public final class HostedMethod extends HostedElement implements SharedMethod, W public static final String METHOD_NAME_COLLISION_SEPARATOR = "%"; public final AnalysisMethod wrapped; + /** A boxed relocated pointer to this method. */ + public JavaConstant methodPointer; private final HostedType holder; private final ResolvedSignature signature; @@ -208,6 +212,14 @@ private HostedMethod(AnalysisMethod wrapped, HostedType holder, ResolvedSignatur this.uniqueShortName = uniqueShortName; this.multiMethodKey = multiMethodKey; this.multiMethodMap = multiMethodMap; + /* + * Cache a method pointer for base layer methods. Cross layer direct calls are currently + * lowered to indirect calls. + */ + if (wrapped.isInBaseLayer()) { + BoxedRelocatedPointer pointer = new BoxedRelocatedPointer(new MethodPointer(wrapped)); + this.methodPointer = wrapped.getUniverse().getSnippetReflection().forObject(pointer); + } } @Override @@ -266,6 +278,17 @@ public ImageCodeInfo getImageCodeInfo() { throw intentionallyUnimplemented(); // ExcludeFromJacocoGeneratedReport } + @Override + public boolean forceIndirectCall() { + return wrapped.isInBaseLayer(); + } + + @Override + public JavaConstant getMethodPointer() { + assert forceIndirectCall(); + return methodPointer; + } + @Override public boolean hasImageCodeOffset() { throw intentionallyUnimplemented(); // ExcludeFromJacocoGeneratedReport @@ -479,7 +502,7 @@ public Type[] getGenericParameterTypes() { @Override public boolean canBeInlined() { - return !hasNeverInlineDirective(); + return wrapped.canBeInlined(); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/RelocatableConstant.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/RelocatableConstant.java index 35289610e8c4..6e7295ff8a94 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/RelocatableConstant.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/RelocatableConstant.java @@ -28,6 +28,7 @@ import com.oracle.graal.pointsto.heap.TypedConstant; import com.oracle.graal.pointsto.meta.AnalysisType; +import com.oracle.svm.core.meta.MethodPointer; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; @@ -109,4 +110,12 @@ public boolean equals(Object obj) { } return false; } + + @Override + public String toValueString() { + if (pointer instanceof MethodPointer mp) { + return "relocatable method pointer: " + mp.getMethod().format("%H.%n(%p)") + ", isAbsolute: " + mp.isAbsolute(); + } + return "relocatable constant"; + } } From 10b7372c14944699713f05c5335998b3395e7885 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Thu, 18 Apr 2024 17:49:39 +0200 Subject: [PATCH 09/16] Register base layer methods as external symbols in the extension layer. --- .../svm/hosted/HostedConfiguration.java | 2 +- .../oracle/svm/hosted/code/CompileQueue.java | 64 +++++++++++++++---- .../hosted/image/LIRNativeImageCodeCache.java | 13 ++-- .../oracle/svm/hosted/image/NativeImage.java | 53 +++++++++------ .../hosted/image/NativeImageCodeCache.java | 17 ++++- 5 files changed, 113 insertions(+), 36 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index c58ad7d62299..f314a9679ea8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -331,7 +331,7 @@ public NativeImageCodeCacheFactory newCodeCacheFactory() { return new NativeImageCodeCacheFactory() { @Override public NativeImageCodeCache newCodeCache(CompileQueue compileQueue, NativeImageHeap heap, Platform targetPlatform, Path tempDir) { - return new LIRNativeImageCodeCache(compileQueue.getCompilationResults(), heap); + return new LIRNativeImageCodeCache(compileQueue.getCompilationResults(), compileQueue.getBaseLayerMethods(), heap); } }; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java index d5c46fe3d7f4..70cd62bedaf3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/CompileQueue.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -164,6 +165,8 @@ protected PhaseSuite getAfterParseSuite() { protected final List policies; protected CompletionExecutor executor; protected final ConcurrentMap compilations; + /** Collect referenced base layer methods. They will be registered as external symbols. */ + protected final Set baseLayerMethods; protected final RuntimeConfiguration runtimeConfig; protected final MetaAccessProvider metaAccess; private Suites regularSuites = null; @@ -371,6 +374,12 @@ public CompileQueue(DebugContext debug, FeatureHandler featureHandler, HostedUni this.graphTransplanter = createGraphTransplanter(); this.defaultParseHooks = new ParseHooks(this); + if (universe.hostVM().useBaseLayer()) { + this.baseLayerMethods = ConcurrentHashMap.newKeySet(); + } else { + this.baseLayerMethods = null; + } + callForReplacements(debug, runtimeConfig); } @@ -629,13 +638,18 @@ private void parseAheadOfTimeCompiledMethods() { */ continue; } - if ((hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || - hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) && !hMethod.wrapped.isInBaseLayer()) { - ensureParsed(hMethod, null, new EntryPointReason()); + if (hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || + hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) { + if (hMethod.wrapped.isInBaseLayer()) { + baseLayerMethods.add(hMethod); + } else { + ensureParsed(hMethod, null, new EntryPointReason()); + } } if (hMethod.wrapped.isVirtualRootMethod()) { for (HostedMethod impl : hMethod.getImplementations()) { if (impl.wrapped.isInBaseLayer()) { + baseLayerMethods.add(impl); continue; } VMError.guarantee(impl.wrapped.isImplementationInvoked()); @@ -648,12 +662,17 @@ private void parseAheadOfTimeCompiledMethods() { SubstrateForeignCallsProvider foreignCallsProvider = (SubstrateForeignCallsProvider) runtimeConfig.getProviders().getForeignCalls(); for (SubstrateForeignCallLinkage linkage : foreignCallsProvider.getForeignCalls().values()) { HostedMethod method = (HostedMethod) linkage.getDescriptor().findMethod(runtimeConfig.getProviders().getMetaAccess()); - if (method.wrapped.isDirectRootMethod() && method.wrapped.isSimplyImplementationInvoked() && !method.wrapped.isInBaseLayer()) { - ensureParsed(method, null, new EntryPointReason()); + if (method.wrapped.isDirectRootMethod() && method.wrapped.isSimplyImplementationInvoked()) { + if (method.wrapped.isInBaseLayer()) { + baseLayerMethods.add(method); + } else { + ensureParsed(method, null, new EntryPointReason()); + } } if (method.wrapped.isVirtualRootMethod()) { for (HostedMethod impl : method.getImplementations()) { if (impl.wrapped.isInBaseLayer()) { + baseLayerMethods.add(impl); continue; } VMError.guarantee(impl.wrapped.isImplementationInvoked()); @@ -924,21 +943,29 @@ public void scheduleEntryPoints() { continue; } - if ((hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || - hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) && !hMethod.wrapped.isInBaseLayer()) { - ensureCompiled(hMethod, new EntryPointReason()); + if (hMethod.isEntryPoint() || SubstrateCompilationDirectives.singleton().isForcedCompilation(hMethod) || + hMethod.wrapped.isDirectRootMethod() && hMethod.wrapped.isSimplyImplementationInvoked()) { + if (hMethod.wrapped.isInBaseLayer()) { + baseLayerMethods.add(hMethod); + } else { + ensureCompiled(hMethod, new EntryPointReason()); + } } if (hMethod.wrapped.isVirtualRootMethod()) { MultiMethod.MultiMethodKey key = hMethod.getMultiMethodKey(); assert key != DEOPT_TARGET_METHOD && key != SubstrateCompilationDirectives.RUNTIME_COMPILED_METHOD : "unexpected method as virtual root " + hMethod; for (HostedMethod impl : hMethod.getImplementations()) { if (impl.wrapped.isInBaseLayer()) { + baseLayerMethods.add(impl); continue; } VMError.guarantee(impl.wrapped.isImplementationInvoked()); ensureCompiled(impl, new EntryPointReason()); } } + if (hMethod.wrapped.isIntrinsicMethod() && hMethod.wrapped.isInBaseLayer()) { + baseLayerMethods.add(hMethod); + } } } } @@ -1047,6 +1074,7 @@ private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetN if (isIndirect) { for (HostedMethod invokeImplementation : invokeTarget.getImplementations()) { if (invokeImplementation.wrapped.isInBaseLayer()) { + baseLayerMethods.add(invokeImplementation); continue; } handleSpecialization(method, targetNode, invokeTarget, invokeImplementation); @@ -1063,7 +1091,11 @@ private void ensureParsed(HostedMethod method, CompileReason reason, CallTargetN * already applied during parsing before we reach this point, so we look at the "simple" * implementation invoked status. */ - if (invokeTarget.wrapped.isSimplyImplementationInvoked() && !invokeTarget.wrapped.isInBaseLayer()) { + if (invokeTarget.wrapped.isSimplyImplementationInvoked()) { + if (invokeTarget.wrapped.isInBaseLayer()) { + baseLayerMethods.add(invokeTarget); + return; + } handleSpecialization(method, targetNode, invokeTarget, invokeTarget); ensureParsed(invokeTarget, method, new DirectCallReason(method, reason)); } @@ -1297,11 +1329,16 @@ protected void ensureCalleesCompiled(HostedMethod method, CompileReason reason, for (Infopoint infopoint : result.getInfopoints()) { if (infopoint instanceof Call call) { HostedMethod callTarget = (HostedMethod) call.target; - if ((call.direct || isDynamicallyResolvedCall(result, call)) && !callTarget.wrapped.isInBaseLayer()) { - ensureCompiled(callTarget, new DirectCallReason(method, reason)); + if (call.direct || isDynamicallyResolvedCall(result, call)) { + if (callTarget.wrapped.isInBaseLayer()) { + baseLayerMethods.add(callTarget); + } else { + ensureCompiled(callTarget, new DirectCallReason(method, reason)); + } } else if (callTarget != null && callTarget.getImplementations() != null) { for (HostedMethod impl : callTarget.getImplementations()) { if (impl.wrapped.isInBaseLayer()) { + baseLayerMethods.add(impl); continue; } ensureCompiled(impl, new VirtualCallReason(method, callTarget, reason)); @@ -1329,6 +1366,7 @@ protected final void ensureCompiledForMethodPointerConstants(HostedMethod method MethodPointer pointer = ((SubstrateMethodPointerConstant) constant).pointer(); HostedMethod referencedMethod = (HostedMethod) pointer.getMethod(); if (referencedMethod.wrapped.isInBaseLayer()) { + baseLayerMethods.add(referencedMethod); continue; } ensureCompiled(referencedMethod, new MethodPointerConstantReason(method, referencedMethod, reason)); @@ -1337,6 +1375,10 @@ protected final void ensureCompiledForMethodPointerConstants(HostedMethod method } } + public Set getBaseLayerMethods() { + return baseLayerMethods; + } + public Map getCompilationResults() { Map result = new TreeMap<>(HostedUniverse.METHOD_COMPARATOR); for (Entry entry : compilations.entrySet()) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java index 94dad1420ae1..eb4c479a309e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java @@ -30,6 +30,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -71,8 +72,8 @@ public class LIRNativeImageCodeCache extends NativeImageCodeCache { private final TargetDescription target; @SuppressWarnings("this-escape") - public LIRNativeImageCodeCache(Map compilations, NativeImageHeap imageHeap) { - super(compilations, imageHeap); + public LIRNativeImageCodeCache(Map compilations, Set baseLayerMethods, NativeImageHeap imageHeap) { + super(compilations, imageHeap, baseLayerMethods); target = ConfigurationValues.getTarget(); trampolineMap = new HashMap<>(); orderedTrampolineMap = new HashMap<>(); @@ -491,8 +492,12 @@ private NativeTextSectionImpl(RelocatableBuffer buffer, ObjectFile objectFile, N @Override protected void defineMethodSymbol(String name, boolean global, ObjectFile.Element section, HostedMethod method, CompilationResult result) { - final int size = result == null ? 0 : result.getTargetCodeSize(); - objectFile.createDefinedSymbol(name, section, method.getCodeAddressOffset(), size, true, global); + if (method.wrapped.isInBaseLayer()) { + objectFile.createUndefinedSymbol(name, 0, true); + } else { + final int size = result == null ? 0 : result.getTargetCodeSize(); + objectFile.createDefinedSymbol(name, section, method.getCodeAddressOffset(), size, true, global); + } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index 9882479874a7..ee15de0772aa 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -915,27 +915,21 @@ protected void writeTextSection(DebugContext debug, final Section textSection, f } final Map methodsBySignature = new HashMap<>(); // 1. fq with return type + + if (codeCache.getBaseLayerMethods() != null) { + // define base layer methods symbols + for (HostedMethod current : codeCache.getBaseLayerMethods()) { + final String symName = localSymbolNameForMethod(current); + final String signatureString = current.getUniqueShortName(); + defineMethodSymbol(textSection, current, methodsBySignature, signatureString, symName, null); + } + } + for (Pair pair : codeCache.getOrderedCompilations()) { - final String symName = localSymbolNameForMethod(pair.getLeft()); - final String signatureString = pair.getLeft().getUniqueShortName(); - final HostedMethod existing = methodsBySignature.get(signatureString); HostedMethod current = pair.getLeft(); - if (existing != null) { - /* - * We've hit a signature with multiple methods. Choose the "more specific" - * of the two methods, i.e. the overriding covariant signature. - */ - HostedType existingReturnType = existing.getSignature().getReturnType(); - HostedType currentReturnType = current.getSignature().getReturnType(); - if (existingReturnType.isAssignableFrom(currentReturnType)) { - /* current is more specific than existing */ - final HostedMethod replaced = methodsBySignature.put(signatureString, current); - assert replaced.equals(existing); - } - } else { - methodsBySignature.put(signatureString, current); - } - defineMethodSymbol(symName, false, textSection, current, pair.getRight()); + final String symName = localSymbolNameForMethod(current); + final String signatureString = current.getUniqueShortName(); + defineMethodSymbol(textSection, current, methodsBySignature, signatureString, symName, pair.getRight()); } // 2. fq without return type -- only for entry points! for (Map.Entry ent : methodsBySignature.entrySet()) { @@ -978,6 +972,27 @@ protected void writeTextSection(DebugContext debug, final Section textSection, f } } + private void defineMethodSymbol(Section textSection, HostedMethod current, Map methodsBySignature, + String signatureString, String symName, CompilationResult compilationResult) { + final HostedMethod existing = methodsBySignature.get(signatureString); + if (existing != null) { + /* + * We've hit a signature with multiple methods. Choose the "more specific" of the + * two methods, i.e. the overriding covariant signature. + */ + HostedType existingReturnType = existing.getSignature().getReturnType(); + HostedType currentReturnType = current.getSignature().getReturnType(); + if (existingReturnType.isAssignableFrom(currentReturnType)) { + /* current is more specific than existing */ + final HostedMethod replaced = methodsBySignature.put(signatureString, current); + assert replaced.equals(existing); + } + } else { + methodsBySignature.put(signatureString, current); + } + defineMethodSymbol(symName, false, textSection, current, compilationResult); + } + protected NativeTextSectionImpl(RelocatableBuffer relocatableBuffer, ObjectFile objectFile, NativeImageCodeCache codeCache) { // TODO: Do not separate the byte[] from the RelocatableBuffer. super(relocatableBuffer.getBackingArray()); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java index c114b39921dc..9aa3e12e2063 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java @@ -126,6 +126,7 @@ public abstract class NativeImageCodeCache { private final Map embeddedConstants = new HashMap<>(); + private final Set baseLayerMethods; public static class Options { @Option(help = "Verify that all possible deoptimization entry points have been properly compiled and registered in the metadata")// @@ -147,7 +148,11 @@ public static class Options { private final Map constantReasons = new HashMap<>(); public NativeImageCodeCache(Map compilationResultMap, NativeImageHeap imageHeap) { - this(compilationResultMap, imageHeap, ImageSingletons.lookup(Platform.class)); + this(compilationResultMap, imageHeap, ImageSingletons.lookup(Platform.class), null); + } + + public NativeImageCodeCache(Map compilationResultMap, NativeImageHeap imageHeap, Set baseLayerMethods) { + this(compilationResultMap, imageHeap, ImageSingletons.lookup(Platform.class), baseLayerMethods); } public void purge() { @@ -157,8 +162,14 @@ public void purge() { @SuppressWarnings("this-escape")// public NativeImageCodeCache(Map compilations, NativeImageHeap imageHeap, Platform targetPlatform) { + this(compilations, imageHeap, targetPlatform, null); + } + + @SuppressWarnings("this-escape")// + public NativeImageCodeCache(Map compilations, NativeImageHeap imageHeap, Platform targetPlatform, Set baseLayerMethods) { this.compilations = compilations; this.imageHeap = imageHeap; + this.baseLayerMethods = baseLayerMethods; this.dataSection = new DataSection(); this.targetPlatform = targetPlatform; this.orderedCompilations = computeCompilationOrder(compilations); @@ -194,6 +205,10 @@ public List> getOrderedCompilations() { return orderedCompilations; } + public Set getBaseLayerMethods() { + return baseLayerMethods; + } + public abstract int codeSizeFor(HostedMethod method); protected CompilationResult compilationResultFor(HostedMethod method) { From 2b50f89b2879a0cb7d1cabc8a25ff616d5447c65 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 15 Apr 2024 17:32:10 +0200 Subject: [PATCH 10/16] Preserve base layer symbols. --- .../src/com/oracle/svm/core/SubstrateOptions.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 54c69485bc0d..c6d382ef342e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -110,6 +110,8 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o LayeredBaseImageAnalysis.update(values, newValue); ClosedTypeWorld.update(values, !newValue); PersistImageLayer.update(values, newValue); + DeleteLocalSymbols.update(values, !newValue); + StripDebugInfo.update(values, !newValue); } }; From dd62f22b88f51b1ebbc6509ef4ef52163e9380bb Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 15 Apr 2024 17:34:57 +0200 Subject: [PATCH 11/16] Add image layers options handler. --- .../src/com/oracle/svm/core/SubstrateOptions.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index c6d382ef342e..41e0946cb064 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -188,6 +188,7 @@ protected void onValueUpdate(EconomicMap, Object> values, String ol public static final String IMAGE_MODULEPATH_PREFIX = "-imagemp"; public static final String KEEP_ALIVE_PREFIX = "-keepalive"; private static ValueUpdateHandler optimizeValueUpdateHandler; + private static OptionEnabledHandler imageLayerEnabledHandler; @Fold public static boolean getSourceLevelDebug() { @@ -336,10 +337,18 @@ public interface ValueUpdateHandler { void onValueUpdate(EconomicMap, Object> values, T newValue); } + public interface OptionEnabledHandler { + void onOptionEnabled(EconomicMap, Object> values); + } + public static void setOptimizeValueUpdateHandler(ValueUpdateHandler updateHandler) { SubstrateOptions.optimizeValueUpdateHandler = updateHandler; } + public static void setImageLayerEnabledHandler(OptionEnabledHandler updateHandler) { + SubstrateOptions.imageLayerEnabledHandler = updateHandler; + } + @Option(help = "Track NodeSourcePositions during runtime-compilation")// public static final HostedOptionKey IncludeNodeSourcePositions = new HostedOptionKey<>(false); @@ -1124,6 +1133,9 @@ public void update(EconomicMap, Object> values, Object boxedValue) super.update(values, boxedValue); PartialPointsToAnalysis.update(values, true); ClosedTypeWorld.update(values, false); + if (imageLayerEnabledHandler != null) { + imageLayerEnabledHandler.onOptionEnabled(values); + } } }; From 3b19e1b63ac7b78f9290378af898fbfe90aafa71 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 25 Mar 2024 22:49:59 +0100 Subject: [PATCH 12/16] Disable threads feature assertion when using base layer. --- .../oracle/svm/hosted/thread/HostedJavaThreadsFeature.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java index f224e89a96c8..d0c1abf71b41 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java @@ -38,6 +38,7 @@ import com.oracle.svm.core.util.ConcurrentIdentityHashMap; import com.oracle.svm.core.util.UserError; import com.oracle.svm.hosted.FeatureImpl; +import com.oracle.svm.hosted.FeatureImpl.AfterAnalysisAccessImpl; import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.ReflectionUtil; @@ -162,7 +163,8 @@ public void duringAnalysis(DuringAnalysisAccess access) { } @Override - public void afterAnalysis(AfterAnalysisAccess access) { + public void afterAnalysis(AfterAnalysisAccess a) { + AfterAnalysisAccessImpl access = (AfterAnalysisAccessImpl) a; /* * No more changes to the reachable threads and thread groups are allowed after the * analysis. @@ -182,7 +184,8 @@ public void afterAnalysis(AfterAnalysisAccess access) { maxThreadId = Math.max(maxThreadId, threadId(thread)); maxAutonumber = Math.max(maxAutonumber, autonumberOf(thread)); } - assert maxThreadId >= 1 : "main thread with id 1 must always be found"; + // TODO load maxThreadId from base layer [GR-52413] + assert access.getBigBang().getHostVM().useBaseLayer() || maxThreadId >= 1 : "main thread with id 1 must always be found"; setThreadSeqNumber(maxThreadId); setThreadInitNumber(maxAutonumber); } From 2abfd224b52a18aa40a03c259d3709a05f729a14 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Mon, 15 Apr 2024 22:02:50 +0200 Subject: [PATCH 13/16] Add extension layer workarounds. --- .../svm/hosted/ExtensionLayerImageFeature.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java index 69288a8c7707..98657dc4c0fb 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java @@ -26,6 +26,7 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.c.BoxedRelocatedPointer; +import com.oracle.svm.core.code.ImageCodeInfo; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; @@ -53,5 +54,22 @@ public void beforeAnalysis(BeforeAnalysisAccess a) { */ access.registerAsInHeap(BoxedRelocatedPointer.class); access.registerAsAccessed(ReflectionUtil.lookupField(BoxedRelocatedPointer.class, "pointer")); + + /* + * ImageCodeInfo.codeStart, used by KnownOffsetsFeature, is not normally reachable for a + * minimal extension layer. + */ + access.registerAsAccessed(ReflectionUtil.lookupField(ImageCodeInfo.class, "codeStart")); + + /* + * In an extension layer build ConcurrentHashMap$CounterCell is not marked as allocated by + * the analysis since ConcurrentHashMap.fullAddCount() is not analyzed. However, an instance + * of this type may still be reachable when scanning ClassLoader.packages, but its + * allocation is non-deterministic, and it depends on the contention on the map. This can + * lead to + * "image heap writing found an object whose type was not marked as instantiated by the static analysis" + * transient errors when writing the heap of the extension image. + */ + access.registerAsInHeap(ReflectionUtil.lookupClass(false, "java.util.concurrent.ConcurrentHashMap$CounterCell")); } } From 6d4543fd2abe95e3cde17a92a360036978d3668c Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Wed, 17 Apr 2024 21:33:33 +0200 Subject: [PATCH 14/16] Remove PartialPointsToAnalysisOption. --- .../com/oracle/graal/pointsto/api/HostVM.java | 4 --- .../pointsto/flow/MethodTypeFlowBuilder.java | 28 +++++++++---------- .../com/oracle/svm/core/SubstrateOptions.java | 4 --- .../src/com/oracle/svm/hosted/SVMHost.java | 7 ----- 4 files changed, 13 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java index e9b94ba822f8..d108b62a6c5c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java @@ -163,10 +163,6 @@ public boolean useBaseLayer() { return false; } - public boolean partialPointsToAnalysis() { - return false; - } - /** * Check if an {@link AnalysisType} is initialized. */ diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java index a46bf9196289..3c84fc60d0b7 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java @@ -620,22 +620,20 @@ protected void apply(boolean forceReparse, Object reason) { assert !processed : "can only call apply once per MethodTypeFlowBuilder"; processed = true; - if (bb.getHostVM().useBaseLayer() && bb.getHostVM().partialPointsToAnalysis()) { - if (method.isInBaseLayer()) { - /* - * We don't need to analyze this method. We already know it's return type state from - * the open world analysis. We just install a return flow to link it with its uses. - */ - AnalysisType returnType = TypeFlow.filterUncheckedInterface(method.getSignature().getReturnType()); - if (returnType.getJavaKind().isObject()) { - // TODO the return type state should not be all-instantiated, it should be the - // persisted result of the open-world analysis [GR-52421] - insertAllInstantiatedTypesReturn(); - } - // TODO verify that tracked parameter state is subset of persisted state [GR-52421] - insertPlaceholderParamAndReturnFlows(); - return; + if (bb.getHostVM().useBaseLayer() && method.isInBaseLayer()) { + /* + * We don't need to analyze this method. We already know it's return type state from the + * open world analysis. We just install a return flow to link it with its uses. + */ + AnalysisType returnType = TypeFlow.filterUncheckedInterface(method.getSignature().getReturnType()); + if (returnType.getJavaKind().isObject()) { + // TODO the return type state should not be all-instantiated, it should be the + // persisted result of the open-world analysis [GR-52421] + insertAllInstantiatedTypesReturn(); } + // TODO verify that tracked parameter state is subset of persisted state [GR-52421] + insertPlaceholderParamAndReturnFlows(); + return; } // assert method.getAnnotation(Fold.class) == null : method; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 41e0946cb064..88f9f046df68 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -1105,9 +1105,6 @@ public static boolean includeAll() { @Option(help = "Run layered image base layer open-world analysis. Includes all public types and methods that can be reached using normal Java access rules.")// public static final HostedOptionKey LayeredBaseImageAnalysis = new HostedOptionKey<>(false); - @Option(help = "Enable partial points-to analysis starting from results of base layer analysis.")// - public static final HostedOptionKey PartialPointsToAnalysis = new HostedOptionKey<>(false); - @Option(help = "Support for calls via the Java Foreign Function and Memory API", type = Expert) // public static final HostedOptionKey ForeignAPISupport = new HostedOptionKey<>(false); @@ -1131,7 +1128,6 @@ public static boolean closedTypeWorld() { @Override public void update(EconomicMap, Object> values, Object boxedValue) { super.update(values, boxedValue); - PartialPointsToAnalysis.update(values, true); ClosedTypeWorld.update(values, false); if (imageLayerEnabledHandler != null) { imageLayerEnabledHandler.onOptionEnabled(values); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java index 181f4d8d3a04..9d841716122e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java @@ -178,7 +178,6 @@ public class SVMHost extends HostVM { private final FieldValueInterceptionSupport fieldValueInterceptionSupport; private final boolean useBaseLayer; - private final boolean partialPointsToAnalysis; private Set excludedFields; @SuppressWarnings("this-escape") @@ -208,7 +207,6 @@ public SVMHost(OptionValues options, ImageClassLoader loader, ClassInitializatio fieldValueInterceptionSupport = new FieldValueInterceptionSupport(annotationSubstitutions, classInitializationSupport); ImageSingletons.add(FieldValueInterceptionSupport.class, fieldValueInterceptionSupport); useBaseLayer = SubstrateOptions.LoadImageLayer.hasBeenSet(); - partialPointsToAnalysis = SubstrateOptions.PartialPointsToAnalysis.hasBeenSet(); if (SubstrateOptions.includeAll()) { initializeExcludedFields(); } @@ -219,11 +217,6 @@ public boolean useBaseLayer() { return useBaseLayer; } - @Override - public boolean partialPointsToAnalysis() { - return partialPointsToAnalysis; - } - protected InlineBeforeAnalysisPolicyUtils getInlineBeforeAnalysisPolicyUtils() { return new InlineBeforeAnalysisPolicyUtils(); } From c77a4b65cf814a4511ec6dfb8972f71add7f1af6 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Wed, 17 Apr 2024 21:37:26 +0200 Subject: [PATCH 15/16] Refactor comments. --- .../oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java | 6 +++--- .../oracle/svm/hosted/image/LIRNativeImageCodeCache.java | 2 +- .../com/oracle/svm/hosted/image/NativeImageCodeCache.java | 2 +- .../com/oracle/svm/hosted/image/NativeImageHeapWriter.java | 4 ++-- .../oracle/svm/hosted/thread/HostedJavaThreadsFeature.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java index 3c84fc60d0b7..d090e4eb329f 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java @@ -627,11 +627,11 @@ protected void apply(boolean forceReparse, Object reason) { */ AnalysisType returnType = TypeFlow.filterUncheckedInterface(method.getSignature().getReturnType()); if (returnType.getJavaKind().isObject()) { - // TODO the return type state should not be all-instantiated, it should be the - // persisted result of the open-world analysis [GR-52421] + // GR-52421: the return type state should not be all-instantiated, it should be the + // persisted result of the open-world analysis insertAllInstantiatedTypesReturn(); } - // TODO verify that tracked parameter state is subset of persisted state [GR-52421] + // GR-52421: verify that tracked parameter state is subset of persisted state insertPlaceholderParamAndReturnFlows(); return; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java index eb4c479a309e..c81dde7266ce 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/LIRNativeImageCodeCache.java @@ -358,7 +358,7 @@ public void patchMethods(DebugContext debug, RelocatableBuffer relocs, ObjectFil HostedImageHeapConstantPatch patch = (HostedImageHeapConstantPatch) codeAnnotation; if (patch.constant instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { - // TODO use object offset in base layer heap [GR-52911] + // GR-52911: use object offset in base layer heap continue; } ObjectInfo objectInfo = imageHeap.getConstantInfo(patch.constant); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java index 9aa3e12e2063..aefc17284ddc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java @@ -683,7 +683,7 @@ public void writeConstants(NativeImageHeapWriter writer, RelocatableBuffer buffe ByteBuffer bb = buffer.getByteBuffer(); dataSection.buildDataSection(bb, (position, constant) -> { if (constant instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { - // TODO use object offset in base layer heap [GR-52911] + // GR-52911: use object offset in base layer heap return; } writer.writeReference(buffer, position, (JavaConstant) constant, "VMConstant: " + constant); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java index 9a7d2d17c4a0..6ab683c3bcc5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java @@ -156,7 +156,7 @@ private void writeField(RelocatableBuffer buffer, ObjectInfo fields, HostedField addNonDataRelocation(buffer, index, prepareRelocatable(info, value)); } else { if (value instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { - // TODO use object offset in base layer heap [GR-52911] + // GR-52911: use object offset in base layer heap return; } write(buffer, index, value, info != null ? info : field); @@ -205,7 +205,7 @@ private void writeConstant(RelocatableBuffer buffer, int index, JavaKind kind, J con = constant; } if (con instanceof ImageHeapConstant hc && hc.isInBaseLayer()) { - // TODO use object offset in base layer heap [GR-52911] + // GR-52911: use object offset in base layer heap return; } write(buffer, index, con, info); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java index d0c1abf71b41..17da926c95d4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/thread/HostedJavaThreadsFeature.java @@ -184,7 +184,7 @@ public void afterAnalysis(AfterAnalysisAccess a) { maxThreadId = Math.max(maxThreadId, threadId(thread)); maxAutonumber = Math.max(maxAutonumber, autonumberOf(thread)); } - // TODO load maxThreadId from base layer [GR-52413] + // GR-52413: load maxThreadId from base layer assert access.getBigBang().getHostVM().useBaseLayer() || maxThreadId >= 1 : "main thread with id 1 must always be found"; setThreadSeqNumber(maxThreadId); setThreadInitNumber(maxAutonumber); From 59ab0fd1152670b74cfd1a79aa62b781b233f66f Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Thu, 18 Apr 2024 18:07:13 +0200 Subject: [PATCH 16/16] Remove unnecessary filter. --- .../com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java index d090e4eb329f..f83d937e3072 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/MethodTypeFlowBuilder.java @@ -622,10 +622,10 @@ protected void apply(boolean forceReparse, Object reason) { if (bb.getHostVM().useBaseLayer() && method.isInBaseLayer()) { /* - * We don't need to analyze this method. We already know it's return type state from the + * We don't need to analyze this method. We already know its return type state from the * open world analysis. We just install a return flow to link it with its uses. */ - AnalysisType returnType = TypeFlow.filterUncheckedInterface(method.getSignature().getReturnType()); + AnalysisType returnType = method.getSignature().getReturnType(); if (returnType.getJavaKind().isObject()) { // GR-52421: the return type state should not be all-instantiated, it should be the // persisted result of the open-world analysis