Implement SPICE-0009 External Readers

[SPICE-0009](apple/pkl-evolution#10)

New close flow

bench/src/jmh/java/org/pkl/core/ListSort.java

@@ -28,7 +28,7 @@
 import org.pkl.core.repl.ReplRequest;
 import org.pkl.core.repl.ReplResponse;
 import org.pkl.core.repl.ReplServer;
-import org.pkl.core.resource.ResourceReaders;
+import org.pkl.core.resource.ResourceReaderFactories;
 import org.pkl.core.util.IoUtils;
 
 @Warmup(iterations = 5, time = 2)
@@ -43,7 +43,7 @@ public class ListSort {
           HttpClient.dummyClient(),
           Loggers.stdErr(),
           List.of(ModuleKeyFactories.standardLibrary),
-          List.of(ResourceReaders.file()),
+          List.of(ResourceReaderFactories.file()),
           Map.of(),
           Map.of(),
           null,

docs/src/test/kotlin/DocSnippetTests.kt

@@ -22,10 +22,10 @@ import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
-import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
 import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
 import kotlin.io.path.isRegularFile

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -134,6 +135,12 @@ data class CliBaseOptions(
 
   /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
   val httpNoProxy: List<String>? = null,
+
+  /** External module reader process specs */
+  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+
+  /** External resource reader process specs */
+  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
 ) {
 
   companion object {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -21,6 +21,7 @@ import java.util.regex.Pattern
 import kotlin.io.path.isRegularFile
 import org.pkl.core.*
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
+import org.pkl.core.externalProcess.ExternalProcessImpl
 import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
@@ -108,12 +109,16 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected val allowedModules: List<Pattern> by lazy {
     cliOptions.allowedModules
-      ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
+      ?: evaluatorSettings?.allowedModules
+        ?: (SecurityManagers.defaultAllowedModules +
+        externalModuleReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val allowedResources: List<Pattern> by lazy {
     cliOptions.allowedResources
-      ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
+      ?: evaluatorSettings?.allowedResources
+        ?: (SecurityManagers.defaultAllowedResources +
+        externalResourceReaders.keys.map { Pattern.compile("$it:") }.toList())
   }
 
   protected val rootDir: Path? by lazy {
@@ -169,6 +174,26 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy
   }
 
+  private val externalModuleReaders by lazy {
+    (project?.evaluatorSettings?.externalModuleReaders
+      ?: emptyMap()) + cliOptions.externalModuleReaders
+  }
+
+  private val externalResourceReaders by lazy {
+    (project?.evaluatorSettings?.externalResourceReaders
+      ?: emptyMap()) + cliOptions.externalResourceReaders
+  }
+
+  private val externalProcesses by lazy {
+    // share ExternalProcessImpl instances between configured external resource/module readers with
+    // the same spec
+    // this avoids spawning multiple subprocesses if the same reader implements both reader types
+    // and/or multiple schemes
+    (externalModuleReaders + externalResourceReaders).values.associateWith {
+      ExternalProcessImpl(it)
+    }
+  }
+
   private fun HttpClient.Builder.addDefaultCliCertificates() {
     val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts")
     var certsAdded = false
@@ -213,6 +238,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
     return buildList {
+      externalModuleReaders.forEach { (key, value) ->
+        add(ModuleKeyFactories.external(key, externalProcesses[value]!!))
+      }
       add(ModuleKeyFactories.standardLibrary)
       add(ModuleKeyFactories.modulePath(modulePathResolver))
       add(ModuleKeyFactories.pkg)
@@ -226,6 +254,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
   private fun resourceReaders(modulePathResolver: ModulePathResolver): List<ResourceReader> {
     return buildList {
+      externalResourceReaders.forEach { (key, value) ->
+        add(ResourceReaders.external(key, externalProcesses[value]!!))
+      }
       add(ResourceReaders.environmentVariable())
       add(ResourceReaders.externalProperty())
       add(ResourceReaders.modulePath(modulePathResolver))

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -28,6 +28,8 @@ import java.time.Duration
 import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
+import org.pkl.commons.shlex
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
 
@@ -207,6 +209,34 @@ class BaseOptions : OptionGroup() {
       .single()
       .split(",")
 
+  val externalModuleReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-module"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for module URI schemes"
+      )
+      .splitPair("=")
+      .convert { // in newer clikt versions this can be done in one call using associateWith
+        val cmd = shlex(it.second)
+        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+      }
+      .multiple()
+      .toMap()
+
+  val externalResourceReaders: Map<String, ExternalReader> by
+    option(
+        names = arrayOf("--external-resource"),
+        metavar = "<scheme>='<executable>[ <arguments>]'",
+        help = "External reader registrations for resource URI schemes"
+      )
+      .splitPair("=")
+      .convert { // in newer clikt versions this can be done in one call using associateWith
+        val cmd = shlex(it.second)
+        Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+      }
+      .multiple()
+      .toMap()
+
   // hidden option used by native tests
   private val testPort: Int by
     option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
@@ -239,7 +269,9 @@ class BaseOptions : OptionGroup() {
       noProject = projectOptions?.noProject ?: false,
       caCertificates = caCertificates,
       httpProxy = proxy,
-      httpNoProxy = noProxy ?: emptyList()
+      httpNoProxy = noProxy ?: emptyList(),
+      externalModuleReaders = externalModuleReaders,
+      externalResourceReaders = externalResourceReaders,
     )
   }
 }

pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt

@@ -18,7 +18,9 @@ package org.pkl.commons
 import java.io.File
 import java.net.URI
 import java.nio.file.Path
+import java.util.*
 import java.util.regex.Pattern
+import kotlin.collections.ArrayList
 
 fun String.toPath(): Path = Path.of(this)
 
@@ -36,3 +38,53 @@ fun String.toUri(): URI {
   }
   return URI(null, null, this, null)
 }
+
+/** Lex a string into tokens similar to how a shell would */
+fun shlex(input: String): List<String> {
+  val result = ArrayList<String>()
+  var inEscape = false
+  var quote: Char? = null
+  var lastCloseQuoteIndex = Int.MIN_VALUE
+  var current = StringBuilder()
+
+  for ((idx, char) in input.toCharArray().withIndex()) {
+    when {
+      // if in an escape always append the next character
+      inEscape -> {
+        inEscape = false
+        current.append(char)
+      }
+      // enter an escape on \ if not in a quote or in a non-single quote
+      char == '\\' && quote != '\'' -> inEscape = true
+      // if in a quote and encounter the delimiter, tentatively exit the quote
+      // this handles cases with adjoining quotes e.g. `abc'123''xyz'`
+      quote == char -> {
+        quote = null
+        lastCloseQuoteIndex = idx
+      }
+      // if not in a quote and encounter a quote charater, enter a quote
+      quote == null && (char == '\'' || char == '"') -> {
+        quote = char
+      }
+      // if not in a quote and whitespace is encountered
+      quote == null && char.isWhitespace() -> {
+        // if the current token isn't empty or if a quote has just ended, finalize the current token
+        // otherwise do nothing, which handles multiple whitespace cases e.g. `abc     123`
+        if (current.isNotEmpty() || lastCloseQuoteIndex == (idx - 1)) {
+          result.add(current.toString())
+          current = StringBuilder()
+        }
+      }
+      // in other cases, append to the current token
+      else -> current.append(char)
+    }
+  }
+  // clean up last token
+  // if the current token isn't empty or if a quote has just ended, finalize the token
+  // if this condition is false, the input likely ended in whitespace
+  if (current.isNotEmpty() || lastCloseQuoteIndex == (input.length - 1)) {
+    result.add(current.toString())
+  }
+
+  return result
+}

pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt

@@ -0,0 +1,78 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+
+class ShlexTest {
+
+  @Test
+  fun `empty input produces empty output`() {
+    assertThat(shlex("")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `whitespace input produces empty output`() {
+    assertThat(shlex("  \n \t  ")).isEqualTo(emptyList<String>())
+  }
+
+  @Test
+  fun `regular token parsing`() {
+    assertThat(shlex("\nabc  def\tghi ")).isEqualTo(listOf("abc", "def", "ghi"))
+  }
+
+  @Test
+  fun `single quoted token parsing`() {
+    assertThat(shlex("'this is a single token'")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `double quoted token parsing`() {
+    assertThat(shlex("\"this is a single token\"")).isEqualTo(listOf("this is a single token"))
+  }
+
+  @Test
+  fun `escaping handles double quotes`() {
+    assertThat(shlex(""""\"this is a single double quoted token\"""""))
+      .isEqualTo(listOf("\"this is a single double quoted token\""))
+  }
+
+  @Test
+  fun `escaping does not apply within single quotes`() {
+    assertThat(shlex("""'this is a single \" token'"""))
+      .isEqualTo(listOf("""this is a single \" token"""))
+  }
+
+  @Test
+  fun `adjacent quoted strings are one token`() {
+    assertThat(shlex(""""single"' joined 'token""")).isEqualTo(listOf("single joined token"))
+    assertThat(shlex(""""single"' 'token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `space escapes do not split tokens`() {
+    assertThat(shlex("""single\ token""")).isEqualTo(listOf("single token"))
+  }
+
+  @Test
+  fun `empty quotes produce a single empty token`() {
+    assertThat(shlex("\"\"")).isEqualTo(listOf(""))
+    assertThat(shlex("''")).isEqualTo(listOf(""))
+    assertThat(shlex("'' ''")).isEqualTo(listOf("", ""))
+    assertThat(shlex("''''")).isEqualTo(listOf(""))
+  }
+}

pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java

@@ -20,6 +20,7 @@
 import java.util.*;
 import java.util.regex.Pattern;
 import org.pkl.core.SecurityManagers.StandardBuilder;
+import org.pkl.core.externalProcess.ExternalProcessImpl;
 import org.pkl.core.http.HttpClient;
 import org.pkl.core.module.ModuleKeyFactories;
 import org.pkl.core.module.ModuleKeyFactory;
@@ -478,6 +479,18 @@ public EvaluatorBuilder applyFromProject(Project project) {
     } else if (settings.moduleCacheDir() != null) {
       setModuleCacheDir(settings.moduleCacheDir());
     }
+    if (settings.externalModuleReaders() != null) {
+      for (var entry : settings.externalModuleReaders().entrySet()) {
+        var process = new ExternalProcessImpl(entry.getValue());
+        addModuleKeyFactory(ModuleKeyFactories.external(entry.getKey(), process));
+      }
+    }
+    if (settings.externalResourceReaders() != null) {
+      for (var entry : settings.externalResourceReaders().entrySet()) {
+        var process = new ExternalProcessImpl(entry.getValue());
+        addResourceReader(ResourceReaders.external(entry.getKey(), process));
+      }
+    }
     return this;
   }
 

pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java

@@ -55,6 +55,7 @@
 import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
 import org.pkl.core.ast.member.*;
 import org.pkl.core.ast.type.*;
+import org.pkl.core.externalProcess.ExternalProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.ModuleKeys;
 import org.pkl.core.module.ResolvedModuleKey;
@@ -1847,6 +1848,12 @@ private URI resolveImport(String importUri, StringConstantContext importUriCtx)
           .withHint(e.getHint())
           .withSourceSection(createSourceSection(importUriCtx))
           .build();
+    } catch (ExternalProcessException e) {
+      throw exceptionBuilder()
+          .evalError("externalReaderFailure")
+          .withCause(e.getCause())
+          .withSourceSection(createSourceSection(importUriCtx))
+          .build();
     }
 
     if (!resolvedUri.isAbsolute()) {

pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java

@@ -23,6 +23,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalProcess.ExternalProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.VmContext;
@@ -75,6 +76,8 @@ private URI resolveResource(ModuleKey moduleKey, String resourceUri) {
           .build();
     } catch (PackageLoadError | SecurityManagerException e) {
       throw exceptionBuilder().withCause(e).build();
+    } catch (ExternalProcessException e) {
+      throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
     }
 
     if (!resolvedUri.isAbsolute()) {