Skip to content

Commit

Permalink
Implement SPICE-0009 External Readers
Browse files Browse the repository at this point in the history 
[SPICE-0009](apple/pkl-evolution#10)

* Add `EvaluatorOptions.ExternalModuleReaders` and `EvaluatorOptions.ExternalResourceReaders`.
* Add `ExternalReaderRuntime` to host the child process side of the external reader workflow.
  • Loading branch information
@HT154
HT154 committed Sep 24, 2024
1 parent fa8c241 commit 7bd1266
Show file tree
Hide file tree
Showing 10 changed files with 555 additions and 72 deletions.
128 changes: 97 additions & 31 deletions pkl/evaluator_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,18 @@ type EvaluatorOptions struct {
// Added in Pkl 0.26.
// If the underlying Pkl does not support HTTP options, NewEvaluator will return with an error.
Http *Http

// ExternalModuleReaders registers external commands that implement module reader schemes.
//
// Added in Pkl 0.27
// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used.
ExternalModuleReaders map[string]ExternalReader

// ExternalResourceReaders registers external commands that implement resource reader schemes.
//
// Added in Pkl 0.27
// If the underlying Pkl does not support external readers, evaluation will fail when a registered scheme is used.
ExternalResourceReaders map[string]ExternalReader
}

type ProjectRemoteDependency struct {
Expand Down Expand Up @@ -251,37 +263,37 @@ func (p *Proxy) toMessage() *msgapi.Proxy {
}
}

func (e *EvaluatorOptions) toMessage() *msgapi.CreateEvaluator {
var resourceReaders []*msgapi.ResourceReader
for _, reader := range e.ResourceReaders {
resourceReaders = append(resourceReaders, &msgapi.ResourceReader{
Scheme: reader.Scheme(),
IsGlobbable: reader.IsGlobbable(),
HasHierarchicalUris: reader.HasHierarchicalUris(),
})
}
var moduleReaders []*msgapi.ModuleReader
for _, reader := range e.ModuleReaders {
moduleReaders = append(moduleReaders, &msgapi.ModuleReader{
Scheme: reader.Scheme(),
IsGlobbable: reader.IsGlobbable(),
HasHierarchicalUris: reader.HasHierarchicalUris(),
IsLocal: reader.IsLocal(),
})
type ExternalReader struct {
Executable string
Arguments []string
}

func (r *ExternalReader) toMessage() *msgapi.ExternalReader {
if r == nil {
return nil
}
return &msgapi.ExternalReader{
Executable: r.Executable,
Arguments: r.Arguments,
}
}

func (e *EvaluatorOptions) toMessage() *msgapi.CreateEvaluator {
return &msgapi.CreateEvaluator{
ResourceReaders: resourceReaders,
ModuleReaders: moduleReaders,
Env: e.Env,
Properties: e.Properties,
ModulePaths: e.ModulePaths,
AllowedModules: e.AllowedModules,
AllowedResources: e.AllowedResources,
CacheDir: e.CacheDir,
OutputFormat: e.OutputFormat,
RootDir: e.RootDir,
Project: e.project(),
Http: e.Http.toMessage(),
ResourceReaders: resourceReadersToMessage(e.ResourceReaders),
ModuleReaders: moduleReadersToMessage(e.ModuleReaders),
Env: e.Env,
Properties: e.Properties,
ModulePaths: e.ModulePaths,
AllowedModules: e.AllowedModules,
AllowedResources: e.AllowedResources,
CacheDir: e.CacheDir,
OutputFormat: e.OutputFormat,
RootDir: e.RootDir,
Project: e.project(),
Http: e.Http.toMessage(),
ExternalModuleReaders: externalReadersToMessage(e.ExternalModuleReaders),
ExternalResourceReaders: externalReadersToMessage(e.ExternalResourceReaders),
}
}

Expand Down Expand Up @@ -317,6 +329,9 @@ func buildEvaluatorOptions(version *semver, fns ...func(*EvaluatorOptions)) (*Ev
if o.Http != nil && pklVersion0_26.isGreaterThan(version) {
return nil, fmt.Errorf("http options are not supported on Pkl versions lower than 0.26")
}
if (len(o.ExternalModuleReaders) > 0 || len(o.ExternalResourceReaders) > 0) && pklVersion0_27.isGreaterThan(version) {
return nil, fmt.Errorf("external reader options are not supported on Pkl versions lower than 0.27")
}
return o, nil
}

Expand Down Expand Up @@ -394,14 +409,45 @@ var WithProjectEvaluatorSettings = func(project *Project) func(opts *EvaluatorOp
}
opts.Properties = evaluatorSettings.ExternalProperties
opts.Env = evaluatorSettings.Env
opts.AllowedModules = evaluatorSettings.AllowedModules
opts.AllowedResources = evaluatorSettings.AllowedResources
if evaluatorSettings.AllowedModules != nil {
opts.AllowedModules = *evaluatorSettings.AllowedModules
}
if evaluatorSettings.AllowedResources != nil {
opts.AllowedResources = *evaluatorSettings.AllowedResources
}
if evaluatorSettings.NoCache != nil && *evaluatorSettings.NoCache {
opts.CacheDir = ""
} else {
opts.CacheDir = evaluatorSettings.ModuleCacheDir
}
opts.RootDir = evaluatorSettings.RootDir
if evaluatorSettings.Http != nil {
opts.Http = &Http{}
if evaluatorSettings.Http.Proxy != nil {
opts.Http.Proxy = &Proxy{NoProxy: opts.Http.Proxy.NoProxy}
if evaluatorSettings.Http.Proxy.Address != nil {
opts.Http.Proxy.Address = *evaluatorSettings.Http.Proxy.Address
}
}
}
if evaluatorSettings.ExternalModuleReaders != nil {
opts.ExternalModuleReaders = make(map[string]ExternalReader, len(evaluatorSettings.ExternalModuleReaders))
for scheme, reader := range evaluatorSettings.ExternalModuleReaders {
opts.ExternalModuleReaders[scheme] = ExternalReader(reader)
if evaluatorSettings.AllowedModules == nil { // if no explicit allowed modules are set in the project, allow declared external module readers
opts.AllowedModules = append(opts.AllowedModules, scheme+":")
}
}
}
if evaluatorSettings.ExternalResourceReaders != nil {
opts.ExternalResourceReaders = make(map[string]ExternalReader, len(evaluatorSettings.ExternalResourceReaders))
for scheme, reader := range evaluatorSettings.ExternalResourceReaders {
opts.ExternalResourceReaders[scheme] = ExternalReader(reader)
if evaluatorSettings.AllowedResources == nil { // if no explicit allowed resources are set in the project, allow declared external resource readers
opts.AllowedResources = append(opts.AllowedResources, scheme+":")
}
}
}
}
}

Expand All @@ -420,6 +466,26 @@ var WithProject = func(project *Project) func(opts *EvaluatorOptions) {
}
}

var WithExternalModuleReader = func(scheme string, spec ExternalReader) func(opts *EvaluatorOptions) {
return func(opts *EvaluatorOptions) {
if opts.ExternalModuleReaders == nil {
opts.ExternalModuleReaders = map[string]ExternalReader{}
}
opts.ExternalModuleReaders[scheme] = spec
opts.AllowedModules = append(opts.AllowedModules, scheme+":")
}
}

var WithExternalResourceReader = func(scheme string, spec ExternalReader) func(opts *EvaluatorOptions) {
return func(opts *EvaluatorOptions) {
if opts.ExternalResourceReaders == nil {
opts.ExternalResourceReaders = map[string]ExternalReader{}
}
opts.ExternalResourceReaders[scheme] = spec
opts.AllowedResources = append(opts.AllowedResources, scheme+":")
}
}

// PreconfiguredOptions configures an evaluator with:
// - allowance for "file", "http", "https", "env", "prop", "package resource schemes
// - allowance for "repl", "file", "http", "https", "pkl", "package" module schemes
Expand Down
Loading

0 comments on commit 7bd1266

Please sign in to comment.