From a6ae59cbf641e95b80430ed548917d5624ab18e0 Mon Sep 17 00:00:00 2001 From: Tomasz Pasternak Date: Fri, 11 Oct 2024 12:01:24 +0400 Subject: [PATCH] feat: Query Sync for Python 1/n - non-analysis mode --- .../blaze/base/qsync/LanguageClasses.java | 4 +- examples/python/with_numpy/app/main.py | 1 + examples/python/with_numpy/lib/BUILD.bazel | 5 ++ examples/python/with_numpy/lib/symbols.py | 1 + python/BUILD | 3 + .../python/resolve/BlazePyResolverUtils.java | 5 ++ .../AbstractPyImportResolverStrategy.java | 86 ++++++++++++++++++- querysync/BUILD | 1 + .../idea/blaze/qsync/BlazeQueryParser.java | 18 ++++ .../qsync/project/QuerySyncLanguage.java | 3 +- .../blaze/qsync/project/language_class.proto | 1 + .../idea/blaze/qsync/query/QuerySummary.java | 4 +- .../idea/blaze/qsync/query/querysummary.proto | 1 + .../google/idea/blaze/common/RuleKinds.java | 7 ++ 14 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 examples/python/with_numpy/lib/BUILD.bazel create mode 100644 examples/python/with_numpy/lib/symbols.py diff --git a/base/src/com/google/idea/blaze/base/qsync/LanguageClasses.java b/base/src/com/google/idea/blaze/base/qsync/LanguageClasses.java index f0c3664dfae..95a461a2ec4 100644 --- a/base/src/com/google/idea/blaze/base/qsync/LanguageClasses.java +++ b/base/src/com/google/idea/blaze/base/qsync/LanguageClasses.java @@ -32,7 +32,9 @@ public class LanguageClasses { ImmutableBiMap.of( QuerySyncLanguage.JAVA, LanguageClass.JAVA, QuerySyncLanguage.KOTLIN, LanguageClass.KOTLIN, - QuerySyncLanguage.CC, LanguageClass.C); + QuerySyncLanguage.CC, LanguageClass.C, + QuerySyncLanguage.PYTHON, LanguageClass.PYTHON + ); private LanguageClasses() {} diff --git a/examples/python/with_numpy/app/main.py b/examples/python/with_numpy/app/main.py index 8055ff16093..4ed94ef776e 100644 --- a/examples/python/with_numpy/app/main.py +++ b/examples/python/with_numpy/app/main.py @@ -1,3 +1,4 @@ from numpy import abs +from symbols import ALPHA print(abs(-2)) diff --git a/examples/python/with_numpy/lib/BUILD.bazel b/examples/python/with_numpy/lib/BUILD.bazel new file mode 100644 index 00000000000..df94a699323 --- /dev/null +++ b/examples/python/with_numpy/lib/BUILD.bazel @@ -0,0 +1,5 @@ +py_library( + name = "symbols", + srcs = glob(["*.py"]), + imports = ["."], +) diff --git a/examples/python/with_numpy/lib/symbols.py b/examples/python/with_numpy/lib/symbols.py new file mode 100644 index 00000000000..d763504d80c --- /dev/null +++ b/examples/python/with_numpy/lib/symbols.py @@ -0,0 +1 @@ +ALPHA="alpha" \ No newline at end of file diff --git a/python/BUILD b/python/BUILD index 104ed53826a..19ddccea9e5 100644 --- a/python/BUILD +++ b/python/BUILD @@ -29,7 +29,9 @@ java_library( "//intellij_platform_sdk:jsr305", "//intellij_platform_sdk:plugin_api", "//proto:proto_deps", + "//querysync", "//sdkcompat", + "//shared", "//third_party/python", ], ) @@ -85,6 +87,7 @@ intellij_integration_test_suite( "//intellij_platform_sdk:jsr305", "//intellij_platform_sdk:plugin_api_for_tests", "//intellij_platform_sdk:test_libs", + "//querysync", "//third_party/python:python_for_tests", "//third_party/toml:toml_for_tests", "@com_google_guava_guava//jar", diff --git a/python/src/com/google/idea/blaze/python/resolve/BlazePyResolverUtils.java b/python/src/com/google/idea/blaze/python/resolve/BlazePyResolverUtils.java index 3c7ab833310..7b1c135fb0f 100644 --- a/python/src/com/google/idea/blaze/python/resolve/BlazePyResolverUtils.java +++ b/python/src/com/google/idea/blaze/python/resolve/BlazePyResolverUtils.java @@ -20,6 +20,8 @@ import com.google.idea.blaze.base.io.VirtualFileSystemProvider; import com.google.idea.blaze.base.model.BlazeProjectData; import com.google.idea.blaze.base.model.RemoteOutputArtifacts; +import com.google.idea.blaze.base.settings.Blaze; +import com.google.idea.blaze.base.settings.BlazeImportSettings; import com.google.idea.blaze.base.sync.data.BlazeProjectDataManager; import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolver; import com.intellij.openapi.project.Project; @@ -51,6 +53,9 @@ public static PsiElement resolveGenfilesPath( private static Optional resolveGenfilesPath(Project project, String relativePath) { BlazeProjectData projectData = BlazeProjectDataManager.getInstance(project).getBlazeProjectData(); + if(Blaze.getProjectType(project) == BlazeImportSettings.ProjectType.QUERY_SYNC) { + return Optional.empty(); + } if (projectData == null) { return Optional.empty(); } diff --git a/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java b/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java index 73178643bf4..9320cc3bb45 100644 --- a/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java +++ b/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java @@ -26,12 +26,17 @@ import com.google.idea.blaze.base.ideinfo.TargetIdeInfo; import com.google.idea.blaze.base.model.BlazeProjectData; import com.google.idea.blaze.base.model.primitives.LanguageClass; +import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; +import com.google.idea.blaze.base.qsync.QuerySyncManager; import com.google.idea.blaze.base.settings.Blaze; import com.google.idea.blaze.base.settings.BlazeImportSettings.ProjectType; import com.google.idea.blaze.base.sync.SyncCache; import com.google.idea.blaze.base.sync.workspace.ArtifactLocationDecoder; import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolver; +import com.google.idea.blaze.common.Label; import com.google.idea.blaze.python.resolve.BlazePyResolverUtils; +import com.google.idea.blaze.qsync.project.ProjectTarget; +import com.google.idea.blaze.qsync.query.Query; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.PsiElement; @@ -50,6 +55,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; @@ -129,7 +135,7 @@ public final void addImportCandidates( @Nullable private PySourcesIndex getSourcesIndex(Project project) { if (Blaze.getProjectType(project) == ProjectType.QUERY_SYNC) { - return null; + return SyncCache.getInstance(project).get(getClass(), this::buildSourcesIndexQuerySync); } return SyncCache.getInstance(project).get(getClass(), this::buildSourcesIndex); } @@ -161,6 +167,45 @@ PySourcesIndex buildSourcesIndex(Project project, BlazeProjectData projectData) return new PySourcesIndex(shortNames.build(), ImmutableMap.copyOf(map)); } + // exposed package-private for testing + @SuppressWarnings("unused") + PySourcesIndex buildSourcesIndexQuerySync(Project project, BlazeProjectData projectData) { + ImmutableSetMultimap.Builder shortNames = ImmutableSetMultimap.builder(); + Map map = new HashMap<>(); + + var currentSnapshot =QuerySyncManager.getInstance(project) + .getLoadedProject() + .flatMap(it -> it.getSnapshotHolder().getCurrent()) + .orElse(null); + if(currentSnapshot == null) { + return null; + } + var workspaceRoot = WorkspaceRoot.fromProjectSafe(project); + if (workspaceRoot == null) { + return null; + } + + for(var target: currentSnapshot.getTargetMap().entrySet()){ + List importRoots = assembleImportRootsQuerySync(target.getKey(), project); + for (var source : target.getValue().sourceLabels().get(ProjectTarget.SourceType.REGULAR)) { + List sourceImports = assembleSourceImportsFromImportRoots(importRoots, + fromRelativePath(source.toFilePath().toString())); + var file = workspaceRoot.path().resolve(source.toFilePath()); + for (QualifiedName sourceImport : sourceImports) { + if (null != sourceImport.getLastComponent()) { + shortNames.put(sourceImport.getLastComponent(), sourceImport); + PsiElementProvider psiProvider = mapToPsiProviderQuerySync(file.toFile()); + map.put(sourceImport, psiProvider); + if (includeParentDirectory(file.toFile())) { + map.put(sourceImport.removeTail(1), PsiElementProvider.getParent(psiProvider)); + } + } + } + } + } + return new PySourcesIndex(shortNames.build(), ImmutableMap.copyOf(map)); + } + /** * This method will extract sources from the supplied target. If any of the sources * are a directory rather than a file then it will descend through the directory @@ -280,6 +325,10 @@ private static boolean includeParentDirectory(ArtifactLocation source) { return source.getRelativePath().endsWith(".py"); } + private static boolean includeParentDirectory(File source) { + return source.getName().endsWith(".py"); + } + static QualifiedName fromRelativePath(String relativePath) { relativePath = StringUtil.trimEnd(relativePath, File.separator + PyNames.INIT_DOT_PY); relativePath = StringUtil.trimExtensions(relativePath); @@ -361,6 +410,41 @@ private static List assembleImportRoots(TargetIdeInfo target) { return resultBuilder.build(); } + private static List assembleImportRootsQuerySync(Label label, Project project) { + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + + // In query sync, when analysis is disabled, we rely on the `imports` attribute + // of the rules. Once it's enabled we should switch to what we got from providers + var imports = QuerySyncManager.getInstance(project).getLoadedProject() + .flatMap(it -> it.getSnapshotHolder().getCurrent()) + .map(it -> it.queryData().querySummary().getRulesMap()) + .flatMap(it -> Optional.ofNullable(it.get(label))) + .map(Query.Rule::getImportsList) + .map(it -> it.stream().toList()) + .orElse(ImmutableList.of()); + + var buildParentPath = label.getPackage(); + for (String imp : imports) { + Path impPath = buildParentPath.resolve(imp).normalize(); + String[] impPathParts = new String[impPath.getNameCount()]; + for (int i = impPath.getNameCount() - 1; i >= 0; i--) { + impPathParts[i] = impPath.getName(i).toString(); + } + resultBuilder.add(QualifiedName.fromComponents(impPathParts)); + } + return resultBuilder.build(); + } + + private PsiElementProvider mapToPsiProviderQuerySync(File file) { + return (manager) -> { + if (PyNames.INIT_DOT_PY.equals(file.getName())) { + return BlazePyResolverUtils.resolveFile(manager, file.getParentFile()); + } else { + return BlazePyResolverUtils.resolveFile(manager, file); + } + }; + } + /** * For each of the importRoots, see if it matches as a prefix on the * sourceImport and then trim off the prefix; yielding the true module name. diff --git a/querysync/BUILD b/querysync/BUILD index 09427dc9142..e1b27d51e37 100644 --- a/querysync/BUILD +++ b/querysync/BUILD @@ -9,6 +9,7 @@ package(default_visibility = [ "//java/com/google/devtools/intellij/blaze/plugin/base:__subpackages__", "//java/com/google/devtools/intellij/blaze/plugin/cpp:__subpackages__", "//java/com/google/devtools/intellij/blaze/plugin/querysync:__subpackages__", + "//python:__subpackages__", "//querysync/javatests:__subpackages__", ]) diff --git a/querysync/java/com/google/idea/blaze/qsync/BlazeQueryParser.java b/querysync/java/com/google/idea/blaze/qsync/BlazeQueryParser.java index 66b6f68c342..18db6faa36a 100644 --- a/querysync/java/com/google/idea/blaze/qsync/BlazeQueryParser.java +++ b/querysync/java/com/google/idea/blaze/qsync/BlazeQueryParser.java @@ -137,6 +137,9 @@ public BuildGraphData parse() { if (RuleKinds.isProtoSource(ruleClass)) { visitProtoRule(ruleEntry.getValue(), targetBuilder); } + if (RuleKinds.isPythonSource(ruleClass)) { + visitPythonRule(ruleEntry.getKey(), ruleEntry.getValue(), targetBuilder); + } if (alwaysBuildRuleKinds.contains(ruleClass)) { projectTargetsToBuild.add(ruleEntry.getKey()); } @@ -174,6 +177,21 @@ public BuildGraphData parse() { return graph; } + private void visitPythonRule(Label label, Rule rule, ProjectTarget.Builder targetBuilder) { + graphBuilder.allTargetsBuilder().add(label); + targetBuilder.languagesBuilder().add(QuerySyncLanguage.PYTHON); + targetBuilder + .sourceLabelsBuilder() + .putAll(SourceType.REGULAR, expandFileGroupValues(rule.getSourcesList())); + + + Set