feat: "dependencies" support in Application spec

Signed-off-by: aavarghese <[email protected]>

cmd/subcommands/dependson.go

@@ -0,0 +1,24 @@
+//go:build full || deploy
+
+package subcommands
+
+import (
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/subcommands/dependson"
+)
+
+func newDependsOnCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "dependson",
+		Short: "Commands for installing dependencies",
+		Long:  "Commands for installing dependencies",
+	}
+}
+
+func init() {
+	cmd := newDependsOnCmd()
+	rootCmd.AddCommand(cmd)
+	cmd.AddCommand(dependson.Minio())
+	cmd.AddCommand(dependson.Python())
+}

cmd/subcommands/dependson/minio.go

@@ -0,0 +1,30 @@
+package dependson
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/dependson"
+)
+
+func Minio() *cobra.Command {
+	var opts dependson.DependsOnOptions
+	cmd := &cobra.Command{
+		Use:   "minio",
+		Short: "Install minio",
+		Long:  "Install minio",
+		Args:  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		opts.Verbose = logOpts.Verbose
+		opts.Debug = logOpts.Debug
+		return dependson.InstallMinio(context.Background(), args[0], opts)
+	}
+
+	return cmd
+}

cmd/subcommands/dependson/python.go

@@ -0,0 +1,30 @@
+package dependson
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/dependson"
+)
+
+func Python() *cobra.Command {
+	var opts dependson.DependsOnOptions
+	cmd := &cobra.Command{
+		Use:   "python",
+		Short: "Install conda/python environment",
+		Long:  "Install conda/python environment",
+		Args:  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		opts.Verbose = logOpts.Verbose
+		opts.Debug = logOpts.Debug
+		return dependson.InstallPython(context.Background(), args[0], opts)
+	}
+
+	return cmd
+}

pkg/be/local/shell/stage.go

@@ -1,15 +1,12 @@
 package shell
 
 import (
-	"errors"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 
 	"lunchpail.io/pkg/ir/hlir"
 	"lunchpail.io/pkg/ir/llir"
-	"lunchpail.io/pkg/lunchpail"
 )
 
 func stage(c llir.ShellComponent) (string, string, error) {
@@ -30,33 +27,9 @@ func stage(c llir.ShellComponent) (string, string, error) {
 	//command = strings.Replace(command, "lunchpail", os.Args[0] + " ", 1)
 	//}
 
-	// hmm, hacky attempts to get intrinsic prereqs
-	switch c.C() {
-	case lunchpail.MinioComponent:
-		if err := ensureMinio(); err != nil {
-			return "", "", err
-		}
-	}
-
 	return workdir, command, nil
 }
 
 func saveToWorkdir(workdir string, code hlir.Code) error {
 	return os.WriteFile(filepath.Join(workdir, code.Name), []byte(code.Source), 0700)
 }
-
-func ensureMinio() error {
-	if err := setenvForMinio(); err != nil {
-		return err
-	}
-
-	if _, err := exec.LookPath("minio"); err != nil {
-		if errors.Is(err, exec.ErrNotFound) {
-			return installMinio()
-		} else if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/fe/transformer/api/minio/lower.go

@@ -1,6 +1,8 @@
 package minio
 
 import (
+	"fmt"
+
 	"lunchpail.io/pkg/compilation"
 	"lunchpail.io/pkg/fe/transformer/api/shell"
 	"lunchpail.io/pkg/ir/hlir"
@@ -18,6 +20,8 @@ func Lower(compilationName, runname string, model hlir.HLIR, ir llir.LLIR, opts
 		return nil, err
 	}
 
+	app.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE dependson minio %s --verbose=%v; %s`, app.Spec.DependsOn.Minio, opts.Log.Verbose, app.Spec.Command)
+
 	component, err := shell.LowerAsComponent(
 		compilationName,
 		runname,

pkg/fe/transformer/api/minio/transpile.go

@@ -20,6 +20,7 @@ func transpile(runname string, ir llir.LLIR) (hlir.Application, error) {
 	app.Spec.Expose = []string{fmt.Sprintf("%d:%d", ir.Queue.Port, ir.Queue.Port)}
 	app.Spec.Command = fmt.Sprintf("$LUNCHPAIL_EXE minio server --port %d", ir.Queue.Port)
 
+	app.Spec.DependsOn.Minio = "latest"
 	prefixIncludingBucket := api.QueuePrefixPath(ir.Queue, runname)
 	A := strings.Split(prefixIncludingBucket, "/")
 	prefixExcludingBucket := filepath.Join(A[1:]...)

pkg/fe/transformer/api/shell/lower.go

@@ -26,6 +26,8 @@ func Lower(compilationName, runname string, app hlir.Application, ir llir.LLIR,
 
 func LowerAsComponent(compilationName, runname string, app hlir.Application, ir llir.LLIR, component llir.ShellComponent, opts compilation.Options) (llir.Component, error) {
 	component.Application = app
+
+	//lunchpail dependson minio
 	if component.Sizing.Workers == 0 {
 		component.Sizing = api.ApplicationSizing(app, opts)
 	}

pkg/fe/transformer/api/workerpool/lower.go

@@ -30,8 +30,9 @@ func Lower(compilationName, runname string, app hlir.Application, pool hlir.Work
 	}
 	app.Spec.Env["LUNCHPAIL_STARTUP_DELAY"] = strconv.Itoa(startupDelay)
 
-	app.Spec.Command = fmt.Sprintf(`trap "$LUNCHPAIL_EXE worker prestop" EXIT
-$LUNCHPAIL_EXE worker run --debug=%v -- %s`, opts.Log.Debug, app.Spec.Command)
+	app.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE dependson python %s --verbose=%v;
+trap "$LUNCHPAIL_EXE worker prestop" EXIT
+$LUNCHPAIL_EXE worker run --debug=%v -- %s`, app.Spec.DependsOn.Python, opts.Log.Verbose, opts.Log.Debug, app.Spec.Command)
 
 	return shell.LowerAsComponent(
 		compilationName,

pkg/ir/hlir/application.go

@@ -7,6 +7,12 @@ type Code struct {
 	Source string
 }
 
+type DependsOn struct {
+	Minio        string
+	Python       string
+	Requirements string
+}
+
 type Application struct {
 	ApiVersion string `yaml:"apiVersion"`
 	Kind       string
@@ -25,6 +31,7 @@ type Application struct {
 		Datasets                 []Dataset                `yaml:"datasets,omitempty"`
 		SecurityContext          SecurityContext          `yaml:"securityContext,omitempty"`
 		ContainerSecurityContext ContainerSecurityContext `yaml:"containerSecurityContext,omitempty"`
+		DependsOn                DependsOn                `yaml:"dependson,omitempty"`
 	}
 }
 

pkg/runtime/dependson/dependson.go

@@ -0,0 +1,9 @@
+package dependson
+
+type DependsOnOptions struct {
+	// Verbose output
+	Verbose bool
+
+	// Debug output
+	Debug bool
+}

pkg/runtime/dependson/install_darwin.go

@@ -0,0 +1,36 @@
+package dependson
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+func installMinio(version string, verbose bool) error {
+	if version == "latest" {
+		if verbose {
+			fmt.Fprintf(os.Stderr, "Installing the latest minio release\n")
+		}
+		return brewInstall("minio/stable/minio", verbose)
+	} else {
+		return nil //Todo
+	}
+}
+
+func installPython(version string, verbose bool) error {
+	return brewInstall("python", verbose)
+}
+
+func brewInstall(pkg string, verbose bool) error {
+	cmd := exec.Command("brew", "install", pkg)
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func setenv() error {
+	// nothing to do
+	return nil
+}

pkg/be/local/shell/install_linux.go → pkg/runtime/dependson/install_linux.go

@@ -1,4 +1,4 @@
-package shell
+package dependson
 
 import (
 	"os"
@@ -14,7 +14,7 @@ func bindir() (string, error) {
 	return filepath.Join(cachedir, "lunchpail", "bin"), nil
 }
 
-func installMinio() error {
+func installMinio(version string, verbose bool) error {
 	dir, err := bindir()
 	if err != nil {
 		return err
@@ -26,7 +26,32 @@ func installMinio() error {
 
 	cmd := exec.Command("wget", "https://dl.min.io/server/minio/release/linux-amd64/minio")
 	cmd.Dir = dir
-	cmd.Stdout = os.Stdout
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return os.Chmod(filepath.Join(dir, "minio"), 0755)
+}
+
+func installPython(version string, verbose bool) error {
+	dir, err := bindir()
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+
+	cmd := exec.Command("dnf", "install", "python3")
+	cmd.Dir = dir
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
 		return err
@@ -35,7 +60,7 @@ func installMinio() error {
 	return os.Chmod(filepath.Join(dir, "minio"), 0755)
 }
 
-func setenvForMinio() error {
+func setenv() error {
 	dir, err := bindir()
 	if err != nil {
 		return err

pkg/runtime/dependson/minio.go

@@ -0,0 +1,24 @@
+package dependson
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+)
+
+func InstallMinio(ctx context.Context, version string, opts DependsOnOptions) error {
+	if err := setenv(); err != nil {
+		return err
+	}
+
+	if _, err := exec.LookPath("minio"); err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			//Todo: use context?
+			return installMinio(version, opts.Verbose)
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/runtime/dependson/python.go

@@ -0,0 +1,24 @@
+package dependson
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+)
+
+func InstallPython(ctx context.Context, version string, opts DependsOnOptions) error {
+	if err := setenv(); err != nil {
+		return err
+	}
+
+	if _, err := exec.LookPath("python"); err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			//Todo: use context?
+			return installPython(version, opts.Verbose)
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tests/tests/test7-python/README.md

@@ -0,0 +1,3 @@
+# test7-python
+
+This is a variant of python-basic, but using `dependson` Application support