feat: "dependencies" support in Application spec

Signed-off-by: aavarghese <[email protected]>

cmd/subcommands/dependencies.go

@@ -0,0 +1,40 @@
+//go:build full || deploy
+
+package subcommands
+
+import (
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/cmd/subcommands/dependencies"
+)
+
+func newDependenciesCmd() *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "dependencies",
+		Short: "Commands for installing pre-requisites",
+		Long:  "Commands for installing pre-requisites",
+	}
+
+	opts, err := options.RestoreCompilationOptions()
+	if err != nil {
+		return nil
+	}
+
+	options.AddTargetOptionsTo(cmd, &opts)
+	options.AddLogOptionsTo(cmd, &opts)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		//Todo
+		return nil
+	}
+
+	return cmd
+}
+
+func init() {
+	cmd := newDependenciesCmd()
+	rootCmd.AddCommand(cmd)
+	cmd.AddCommand(dependencies.Minio())
+	cmd.AddCommand(dependencies.Python())
+}

cmd/subcommands/dependencies/minio.go

@@ -0,0 +1,24 @@
+package dependencies
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/pkg/runtime/dependencies"
+)
+
+func Minio() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "minio",
+		Short: "Install minio",
+		Long:  "Install minio",
+		Args:  cobra.MatchAll(cobra.ExactArgs(0), cobra.OnlyValidArgs),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return dependencies.InstallMinio(context.Background())
+		},
+	}
+
+	return cmd
+}

cmd/subcommands/dependencies/python.go

@@ -0,0 +1,24 @@
+package dependencies
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/pkg/runtime/dependencies"
+)
+
+func Python() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "python",
+		Short: "Install conda/python environment",
+		Long:  "Install conda/python environment",
+		Args:  cobra.MatchAll(cobra.ExactArgs(0), cobra.OnlyValidArgs),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return dependencies.InstallPython(context.Background())
+		},
+	}
+
+	return cmd
+}

pkg/be/local/shell/install_darwin.go → pkg/be/dependencies/install_darwin.go

@@ -1,4 +1,4 @@
-package shell
+package dependencies
 
 import (
 	"os"
@@ -9,6 +9,10 @@ func installMinio() error {
 	return brewInstall("minio/stable/minio")
 }
 
+func installPython() error {
+	return brewInstall("python")
+}
+
 func brewInstall(pkg string) error {
 	cmd := exec.Command("brew", "install", pkg)
 	cmd.Stdout = os.Stdout

pkg/be/local/shell/install_linux.go → pkg/be/dependencies/install_linux.go

@@ -1,4 +1,4 @@
-package shell
+package dependencies
 
 import (
 	"os"
@@ -35,6 +35,27 @@ func installMinio() error {
 	return os.Chmod(filepath.Join(dir, "minio"), 0755)
 }
 
+func installPython() error {
+	dir, err := bindir()
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+
+	cmd := exec.Command("dnf", "install", "python3")
+	cmd.Dir = dir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return os.Chmod(filepath.Join(dir, "minio"), 0755)
+}
+
 func setenvForMinio() error {
 	dir, err := bindir()
 	if err != nil {

pkg/be/dependencies/minio.go

@@ -0,0 +1,22 @@
+package dependencies
+
+import (
+	"errors"
+	"os/exec"
+)
+
+func DepsMinio() error {
+	if err := setenvForMinio(); err != nil {
+		return err
+	}
+
+	if _, err := exec.LookPath("minio"); err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			return installMinio()
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/be/dependencies/python.go

@@ -0,0 +1,18 @@
+package dependencies
+
+import (
+	"errors"
+	"os/exec"
+)
+
+func DepsPython() error {
+	if _, err := exec.LookPath("python"); err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			return installPython()
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/be/local/shell/stage.go

@@ -1,15 +1,12 @@
 package shell
 
 import (
-	"errors"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 
 	"lunchpail.io/pkg/ir/hlir"
 	"lunchpail.io/pkg/ir/llir"
-	"lunchpail.io/pkg/lunchpail"
 )
 
 func stage(c llir.ShellComponent) (string, string, error) {
@@ -30,33 +27,9 @@ func stage(c llir.ShellComponent) (string, string, error) {
 	//command = strings.Replace(command, "lunchpail", os.Args[0] + " ", 1)
 	//}
 
-	// hmm, hacky attempts to get intrinsic prereqs
-	switch c.C() {
-	case lunchpail.MinioComponent:
-		if err := ensureMinio(); err != nil {
-			return "", "", err
-		}
-	}
-
 	return workdir, command, nil
 }
 
 func saveToWorkdir(workdir string, code hlir.Code) error {
 	return os.WriteFile(filepath.Join(workdir, code.Name), []byte(code.Source), 0700)
 }
-
-func ensureMinio() error {
-	if err := setenvForMinio(); err != nil {
-		return err
-	}
-
-	if _, err := exec.LookPath("minio"); err != nil {
-		if errors.Is(err, exec.ErrNotFound) {
-			return installMinio()
-		} else if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/fe/transformer/api/minio/lower.go

@@ -1,6 +1,8 @@
 package minio
 
 import (
+	"fmt"
+
 	"lunchpail.io/pkg/compilation"
 	"lunchpail.io/pkg/fe/transformer/api/shell"
 	"lunchpail.io/pkg/ir/hlir"
@@ -18,6 +20,8 @@ func Lower(compilationName, runname string, model hlir.AppModel, ir llir.LLIR, o
 		return nil, err
 	}
 
+	app.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE dependencies minio %s --verbose=%v; %s`, app.Spec.Dependencies.Minio, opts.Log.Verbose, app.Spec.Command)
+
 	component, err := shell.LowerAsComponent(
 		compilationName,
 		runname,

pkg/fe/transformer/api/minio/transpile.go

@@ -20,6 +20,7 @@ func transpile(runname string, ir llir.LLIR) (hlir.Application, error) {
 	app.Spec.Expose = []string{fmt.Sprintf("%d:%d", ir.Queue.Port, ir.Queue.Port)}
 	app.Spec.Command = fmt.Sprintf("$LUNCHPAIL_EXE minio server --port %d", ir.Queue.Port)
 
+	app.Spec.Dependencies.Minio = "latest"
 	prefixIncludingBucket := api.QueuePrefixPath(ir.Queue, runname)
 	A := strings.Split(prefixIncludingBucket, "/")
 	prefixExcludingBucket := filepath.Join(A[1:]...)

pkg/fe/transformer/api/shell/lower.go

@@ -26,6 +26,8 @@ func Lower(compilationName, runname string, app hlir.Application, ir llir.LLIR,
 
 func LowerAsComponent(compilationName, runname string, app hlir.Application, ir llir.LLIR, component llir.ShellComponent, opts compilation.Options) (llir.Component, error) {
 	component.Application = app
+
+	//lunchpail dependencies minio
 	if component.Sizing.Workers == 0 {
 		component.Sizing = api.ApplicationSizing(app, opts)
 	}

pkg/fe/transformer/api/workerpool/lower.go

@@ -30,8 +30,9 @@ func Lower(compilationName, runname string, app hlir.Application, pool hlir.Work
 	}
 	app.Spec.Env["LUNCHPAIL_STARTUP_DELAY"] = strconv.Itoa(startupDelay)
 
-	app.Spec.Command = fmt.Sprintf(`trap "$LUNCHPAIL_EXE worker prestop" EXIT
-$LUNCHPAIL_EXE worker run --debug=%v -- %s`, opts.Log.Debug, app.Spec.Command)
+	app.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE dependencies python %s --verbose=%v;
+trap "$LUNCHPAIL_EXE worker prestop" EXIT
+$LUNCHPAIL_EXE worker run --debug=%v -- %s`, app.Spec.Dependencies.Python, opts.Log.Verbose, opts.Log.Debug, app.Spec.Command)
 
 	return shell.LowerAsComponent(
 		compilationName,

pkg/ir/hlir/application.go

@@ -7,6 +7,12 @@ type Code struct {
 	Source string
 }
 
+type Dependencies struct {
+	Minio        string
+	Python       string
+	Requirements string
+}
+
 type Application struct {
 	ApiVersion string `yaml:"apiVersion"`
 	Kind       string
@@ -25,6 +31,7 @@ type Application struct {
 		Datasets                 []Dataset                `yaml:"datasets,omitempty"`
 		SecurityContext          SecurityContext          `yaml:"securityContext,omitempty"`
 		ContainerSecurityContext ContainerSecurityContext `yaml:"containerSecurityContext,omitempty"`
+		Dependencies             Dependencies             `yaml:"dependencies,omitempty"`
 	}
 }
 

pkg/runtime/dependencies/minio.go

@@ -0,0 +1,11 @@
+package dependencies
+
+import (
+	"context"
+
+	"lunchpail.io/pkg/be/dependencies"
+)
+
+func InstallMinio(ctx context.Context) error {
+	return dependencies.DepsMinio()
+}

pkg/runtime/dependencies/python.go

@@ -0,0 +1,11 @@
+package dependencies
+
+import (
+	"context"
+
+	"lunchpail.io/pkg/be/dependencies"
+)
+
+func InstallPython(ctx context.Context) error {
+	return dependencies.DepsPython()
+}

tests/tests/test7-python/README.md

@@ -0,0 +1,3 @@
+# test7-python
+
+This is a variant of python-basic, but using `dependencies` Application support

tests/tests/test7-python/add-data-to-queue.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+
+export NAMESPACE=$1
+
+# number of task
+N=${2-10}
+
+# name of s3 bucket in which to store the tasks
+BUCKET=${3-test7}
+RUN_NAME=$BUCKET
+
+B=$(mktemp -d)/$BUCKET # bucket path
+D=$B/$BUCKET # data path; in this case the bucket name and the folder name are both the run name
+mkdir -p $D
+echo "Staging to $D" 1>&2
+
+for idx in $(seq 1 $N) # for each iteration
+do
+    # if we are doing a test, then make sure to use a
+    # repeatable name for the task files, so that we know what
+    # to look for when confirming that the tasks were
+    # processed by the workers
+    if [[ -n "$CI" ]] || [[ -n "$RUNNING_LUNCHPAIL_TESTS" ]]; then
+        id=$idx
+    else
+        # otherwise, use a more random name, so that we can
+        # inject multiple batches of tasks across executions
+        # of this script
+        id=$(uuidgen)
+    fi
+
+    echo "this is task idx=$idx" > $D/task.$id.txt
+done
+
+"$SCRIPTDIR"/../../../tests/bin/add-data.sh $B

tests/tests/test7-python/init.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+
+# make sure these values are compatible with the values in ./settings.sh
+NUM_TASKS=6
+
+# $1: namespace
+
+"$SCRIPTDIR"/add-data-to-queue.sh \
+            $1 \
+            $NUM_TASKS \
+            ${TEST_NAME-test7}