Implement SPICE-0009 External Readers

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .build
+.index-build
 /Packages
 /*.xcodeproj
 xcuserdata/

Sources/PklSwift/EvaluatorManager.swift

@@ -108,7 +108,7 @@ public actor EvaluatorManager {
 
     // note; when our C bindings are released, change `init()` based on compiler flags.
     public init() {
-        self.init(transport: ChildProcessMessageTransport())
+        self.init(transport: ServerMessageTransport())
     }
 
     // Used for testing only.
@@ -371,8 +371,10 @@ enum PklBugError: Error {
 
 let pklVersion0_25 = SemanticVersion("0.25.0")!
 let pklVersion0_26 = SemanticVersion("0.26.0")!
+let pklVersion0_27 = SemanticVersion("0.27.0")!
 
 let supportedPklVersions = [
     pklVersion0_25,
     pklVersion0_26,
+    pklVersion0_27,
 ]

Sources/PklSwift/EvaluatorOptions.swift

@@ -32,7 +32,9 @@ public struct EvaluatorOptions {
         logger: Logger = Loggers.noop,
         projectBaseURI: URL? = nil,
         http: Http? = nil,
-        declaredProjectDependencies: [String: ProjectDependency]? = nil
+        declaredProjectDependencies: [String: ProjectDependency]? = nil,
+        externalModuleReaders: [String: ExternalReader]? = nil,
+        externalResourceReaders: [String: ExternalReader]? = nil
     ) {
         self.allowedModules = allowedModules
         self.allowedResources = allowedResources
@@ -49,6 +51,8 @@ public struct EvaluatorOptions {
         self.projectBaseURI = projectBaseURI
         self.http = http
         self.declaredProjectDependencies = declaredProjectDependencies
+        self.externalModuleReaders = externalModuleReaders
+        self.externalResourceReaders = externalResourceReaders
     }
 
     /// Regular expression patterns that control what modules are allowed to be imported in a Pkl program.
@@ -123,6 +127,18 @@ public struct EvaluatorOptions {
     /// When importing dependencies, a `PklProject.deps.json` file must exist within ``projectBaseURI``
     /// that contains the project's resolved dependencies.
     public var declaredProjectDependencies: [String: ProjectDependency]?
+
+    /// Registered external commands that implement module reader schemes.
+    ///
+    /// Added in Pkl 0.27.
+    /// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used.
+    public var externalModuleReaders: [String: ExternalReader]?
+
+    /// Registered external commands that implement resource reader schemes.
+    ///
+    /// Added in Pkl 0.27.
+    /// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used.
+    public var externalResourceReaders: [String: ExternalReader]?
 }
 
 extension EvaluatorOptions {
@@ -162,7 +178,9 @@ extension EvaluatorOptions {
             cacheDir: self.cacheDir,
             outputFormat: self.outputFormat,
             project: self.project(),
-            http: self.http
+            http: self.http,
+            externalModuleReaders: self.externalModuleReaders,
+            externalResourceReaders: self.externalResourceReaders
         )
     }
 
@@ -245,6 +263,8 @@ extension EvaluatorOptions {
         options.cacheDir = evaluatorSettings.noCache != nil ? nil : (evaluatorSettings.moduleCacheDir ?? self.cacheDir)
         options.rootDir = evaluatorSettings.rootDir ?? self.rootDir
         options.http = evaluatorSettings.http ?? self.http
+        options.externalModuleReaders = evaluatorSettings.externalModuleReaders ?? options.externalModuleReaders
+        options.externalResourceReaders = evaluatorSettings.externalResourceReaders ?? options.externalResourceReaders
         return options
     }
 

Sources/PklSwift/ExternalReaderRuntime.swift

@@ -0,0 +1,209 @@
+// ===----------------------------------------------------------------------===//
+// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ===----------------------------------------------------------------------===//
+
+import Foundation
+import MessagePack
+
+// Example usage:
+// import PklSwift
+// @main
+// struct Main {
+//     static func main() async throws {
+//         let runtime = ExternalReaderRuntime(
+//             options: ExternalReaderRuntimeOptions(
+//                 resourceReaders: [MyResourceReader()]
+//             ))
+//         try await runtime.run()
+//     }
+// }
+
+public struct ExternalReaderRuntimeOptions {
+    /// Reader to receive requests.
+    public var requestReader: Reader = FileHandle.standardInput
+
+    /// Writer to publish responses.
+    public var responseWriter: Writer = FileHandle.standardOutput
+
+    /// Readers that allow reading custom resources in Pkl.
+    public var moduleReaders: [ModuleReader] = []
+
+    /// Readers that allow importing custom modules in Pkl.
+    public var resourceReaders: [ResourceReader] = []
+
+    public init(
+        requestReader: Reader = FileHandle.standardInput,
+        responseWriter: Writer = FileHandle.standardOutput,
+        moduleReaders: [ModuleReader] = [],
+        resourceReaders: [ResourceReader] = []
+    ) {
+        self.requestReader = requestReader
+        self.responseWriter = responseWriter
+        self.moduleReaders = moduleReaders
+        self.resourceReaders = resourceReaders
+    }
+}
+
+public class ExternalReaderRuntime {
+    private let moduleReaders: [ModuleReader]
+    private let resourceReaders: [ResourceReader]
+    private let transport: MessageTransport
+
+    public init(options: ExternalReaderRuntimeOptions) {
+        self.moduleReaders = options.moduleReaders
+        self.resourceReaders = options.resourceReaders
+        self.transport = ExternalReaderMessageTransport(
+            reader: options.requestReader, writer: options.responseWriter
+        )
+    }
+
+    public func run() async throws {
+        for try await message in try self.transport.getMessages() {
+            switch message {
+            case let message as InitializeModuleReaderRequest:
+                try await self.handleInitializeModuleReaderRequest(request: message)
+            case let message as InitializeResourceReaderRequest:
+                try await self.handleInitializeResourceReaderRequest(request: message)
+            case let message as ReadModuleRequest:
+                try await self.handleReadModuleRequest(request: message)
+            case let message as ReadResourceRequest:
+                try await self.handleReadResourceRequest(request: message)
+            case let message as ListModulesRequest:
+                try await self.handleListModulesRequest(request: message)
+            case let message as ListResourcesRequest:
+                try await self.handleListResourcesRequest(request: message)
+            case _ as CloseExternalProcess:
+                self.close()
+            default:
+                throw PklBugError.unknownMessage("Got request for unknown message: \(message)")
+            }
+        }
+    }
+
+    public func close() {
+        self.transport.close()
+    }
+
+    func handleInitializeModuleReaderRequest(request: InitializeModuleReaderRequest) async throws {
+        var response = InitializeModuleReaderResponse(requestId: request.requestId, spec: nil)
+        guard let reader = moduleReaders.first(where: { $0.scheme == request.scheme }) else {
+            try self.transport.send(response)
+            return
+        }
+        response.spec = reader.toMessage()
+        try self.transport.send(response)
+    }
+
+    func handleInitializeResourceReaderRequest(request: InitializeResourceReaderRequest)
+        async throws {
+        var response = InitializeResourceReaderResponse(requestId: request.requestId, spec: nil)
+        guard let reader = resourceReaders.first(where: { $0.scheme == request.scheme }) else {
+            try self.transport.send(response)
+            return
+        }
+        response.spec = reader.toMessage()
+        try self.transport.send(response)
+    }
+
+    func handleReadModuleRequest(request: ReadModuleRequest) async throws {
+        var response = ReadModuleResponse(
+            requestId: request.requestId,
+            evaluatorId: request.evaluatorId,
+            contents: nil,
+            error: nil
+        )
+        guard let reader = moduleReaders.first(where: { $0.scheme == request.uri.scheme }) else {
+            response.error = "No module reader found for scheme \(request.uri.scheme!)"
+            try self.transport.send(response)
+            return
+        }
+        do {
+            let result = try await reader.read(url: request.uri)
+            response.contents = result
+            try self.transport.send(response)
+        } catch {
+            response.error = "\(error)"
+            try self.transport.send(response)
+        }
+    }
+
+    func handleReadResourceRequest(request: ReadResourceRequest) async throws {
+        var response = ReadResourceResponse(
+            requestId: request.requestId,
+            evaluatorId: request.evaluatorId,
+            contents: nil,
+            error: nil
+        )
+        guard
+            let reader = resourceReaders.first(where: { $0.scheme == request.uri.scheme }) else {
+            response.error = "No resource reader found for scheme \(request.uri.scheme!)"
+            try self.transport.send(response)
+            return
+        }
+        do {
+            let result = try await reader.read(url: request.uri)
+            response.contents = result
+            try self.transport.send(response)
+        } catch {
+            response.error = "\(error)"
+            try self.transport.send(response)
+        }
+    }
+
+    func handleListModulesRequest(request: ListModulesRequest) async throws {
+        var response = ListModulesResponse(
+            requestId: request.requestId,
+            evaluatorId: request.evaluatorId,
+            pathElements: nil,
+            error: nil
+        )
+        guard let reader = moduleReaders.first(where: { $0.scheme == request.uri.scheme }) else {
+            response.error = "No module reader found for scheme \(request.uri.scheme!)"
+            try self.transport.send(response)
+            return
+        }
+        do {
+            let elems = try await reader.listElements(uri: request.uri)
+            response.pathElements = elems.map { $0.toMessage() }
+            try self.transport.send(response)
+        } catch {
+            response.error = "\(error)"
+            try self.transport.send(response)
+        }
+    }
+
+    func handleListResourcesRequest(request: ListResourcesRequest) async throws {
+        var response = ListResourcesResponse(
+            requestId: request.requestId,
+            evaluatorId: request.evaluatorId,
+            pathElements: nil,
+            error: nil
+        )
+        guard
+            let reader = resourceReaders.first(where: { $0.scheme == request.uri.scheme }) else {
+            response.error = "No resource reader found for scheme \(request.uri.scheme!)"
+            try self.transport.send(response)
+            return
+        }
+        do {
+            let elems = try await reader.listElements(uri: request.uri)
+            response.pathElements = elems.map { $0.toMessage() }
+            try self.transport.send(response)
+        } catch {
+            response.error = "\(error)"
+            try self.transport.send(response)
+        }
+    }
+}

Sources/PklSwift/Message.swift

@@ -62,8 +62,8 @@ enum MessageType: Int, Codable {
     case CREATE_EVALUATOR_REQUEST = 0x20
     case CREATE_EVALUATOR_RESPONSE = 0x21
     case CLOSE_EVALUATOR = 0x22
-    case EVALUATOR_REQUEST = 0x23
-    case EVALUATOR_RESPONSE = 0x24
+    case EVALUATE_REQUEST = 0x23
+    case EVALUATE_RESPONSE = 0x24
     case LOG_MESSAGE = 0x25
     case READ_RESOURCE_REQUEST = 0x26
     case READ_RESOURCE_RESPONSE = 0x27
@@ -73,6 +73,11 @@ enum MessageType: Int, Codable {
     case LIST_RESOURCES_RESPONSE = 0x2B
     case LIST_MODULES_REQUEST = 0x2C
     case LIST_MODULES_RESPONSE = 0x2D
+    case INITIALIZE_MODULE_READER_REQUEST = 0x2E
+    case INITIALIZE_MODULE_READER_RESPONSE = 0x2F
+    case INITIALIZE_RESOURCE_READER_REQUEST = 0x30
+    case INITIALIZE_RESOURCE_READER_RESPONSE = 0x31
+    case CLOSE_EXTERNAL_PROCESS = 0x32
 }
 
 extension MessageType {
@@ -85,9 +90,9 @@ extension MessageType {
         case is CloseEvaluatorRequest:
             return MessageType.CLOSE_EVALUATOR
         case is EvaluateRequest:
-            return MessageType.EVALUATOR_REQUEST
+            return MessageType.EVALUATE_REQUEST
         case is EvaluateResponse:
-            return MessageType.EVALUATOR_RESPONSE
+            return MessageType.EVALUATE_RESPONSE
         case is LogMessage:
             return MessageType.LOG_MESSAGE
         case is ListResourcesRequest:
@@ -102,6 +107,16 @@ extension MessageType {
             return MessageType.READ_MODULE_RESPONSE
         case is ReadResourceResponse:
             return MessageType.READ_RESOURCE_RESPONSE
+        case is InitializeModuleReaderRequest:
+            return MessageType.INITIALIZE_MODULE_READER_REQUEST
+        case is InitializeModuleReaderResponse:
+            return MessageType.INITIALIZE_MODULE_READER_RESPONSE
+        case is InitializeResourceReaderRequest:
+            return MessageType.INITIALIZE_RESOURCE_READER_REQUEST
+        case is InitializeResourceReaderResponse:
+            return MessageType.INITIALIZE_RESOURCE_READER_RESPONSE
+        case is CloseExternalProcess:
+            return MessageType.CLOSE_EXTERNAL_PROCESS
         default:
             preconditionFailure("Unreachable code")
         }
@@ -123,6 +138,8 @@ struct CreateEvaluatorRequest: ClientRequestMessage {
     var outputFormat: String?
     var project: ProjectOrDependency?
     var http: Http?
+    var externalModuleReaders: [String: ExternalReader]?
+    var externalResourceReaders: [String: ExternalReader]?
 }
 
 struct ProjectOrDependency: Codable {
@@ -231,3 +248,25 @@ struct LogMessage: ServerOneWayMessage {
     // NOTE: not guaranteed to conform to URL. This might have been transformed by a stack frame transformer.
     let frameUri: String
 }
+
+struct InitializeModuleReaderRequest: ServerRequestMessage {
+    var requestId: Int64
+    let scheme: String
+}
+
+struct InitializeModuleReaderResponse: ClientResponseMessage {
+    var requestId: Int64
+    var spec: ModuleReaderSpec?
+}
+
+struct InitializeResourceReaderRequest: ServerRequestMessage {
+    var requestId: Int64
+    let scheme: String
+}
+
+struct InitializeResourceReaderResponse: ClientResponseMessage {
+    var requestId: Int64
+    var spec: ResourceReaderSpec?
+}
+
+struct CloseExternalProcess: ServerOneWayMessage {}