feat: "dependencies" support in Application spec

Signed-off-by: aavarghese <[email protected]>

cmd/subcommands/needs.go

@@ -0,0 +1,20 @@
+package subcommands
+
+import (
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/subcommands/needs"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:     "needs",
+		GroupID: internalGroup.ID,
+		Short:   "Commands for installing dependencies to run the application",
+		Long:    "Commands for installing dependencies to run the application",
+	}
+
+	rootCmd.AddCommand(cmd)
+	cmd.AddCommand(needs.Minio())
+	cmd.AddCommand(needs.Python())
+}

cmd/subcommands/needs/minio.go

@@ -0,0 +1,34 @@
+package needs
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/needs"
+)
+
+func Minio() *cobra.Command {
+	var opts needs.NeedsOptions
+	cmd := &cobra.Command{
+		Use:   "minio",
+		Short: "Install minio",
+		Long:  "Install minio",
+		Args:  cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		opts.Verbose = logOpts.Verbose
+		opts.Debug = logOpts.Debug
+		version := "latest"
+		if len(args) > 0 {
+			version = args[0]
+		}
+		return needs.InstallMinio(context.Background(), version, opts)
+	}
+
+	return cmd
+}

cmd/subcommands/needs/python.go

@@ -0,0 +1,37 @@
+package needs
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/needs"
+)
+
+func Python() *cobra.Command {
+	var opts needs.NeedsOptions
+	var requirementsFlag string
+	cmd := &cobra.Command{
+		Use:   "python",
+		Short: "Install conda/python environment",
+		Long:  "Install conda/python environment",
+		Args:  cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+
+	cmd.Flags().StringVarP(&requirementsFlag, "requirements", "r", requirementsFlag, "Install from the given requirements file")
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		opts.Verbose = logOpts.Verbose
+		opts.Debug = logOpts.Debug
+		version := "latest"
+		if len(args) > 0 {
+			version = args[0]
+		}
+		return needs.InstallPython(context.Background(), version, requirementsFlag, opts)
+	}
+
+	return cmd
+}

pkg/be/local/shell/workdir.go

@@ -1,15 +1,12 @@
 package shell
 
 import (
-	"errors"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 
 	"lunchpail.io/pkg/ir/hlir"
 	"lunchpail.io/pkg/ir/llir"
-	"lunchpail.io/pkg/lunchpail"
 )
 
 func PrepareWorkdirForComponent(c llir.ShellComponent) (string, string, error) {
@@ -26,33 +23,9 @@ func PrepareWorkdirForComponent(c llir.ShellComponent) (string, string, error) {
 
 	command := c.Application.Spec.Command
 
-	// hmm, hacky attempts to get intrinsic prereqs
-	switch c.C() {
-	case lunchpail.MinioComponent:
-		if err := ensureMinio(); err != nil {
-			return "", "", err
-		}
-	}
-
 	return workdir, command, nil
 }
 
 func saveCodeToWorkdir(workdir string, code hlir.Code) error {
 	return os.WriteFile(filepath.Join(workdir, code.Name), []byte(code.Source), 0700)
 }
-
-func ensureMinio() error {
-	if err := setenvForMinio(); err != nil {
-		return err
-	}
-
-	if _, err := exec.LookPath("minio"); err != nil {
-		if errors.Is(err, exec.ErrNotFound) {
-			return installMinio()
-		} else if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/fe/transformer/api/minio/transpile.go

@@ -20,6 +20,8 @@ func transpile(runname string, ir llir.LLIR) (hlir.Application, error) {
 	app.Spec.Expose = []string{fmt.Sprintf("%d:%d", ir.Queue.Port, ir.Queue.Port)}
 	app.Spec.Command = fmt.Sprintf("$LUNCHPAIL_EXE component minio server --port %d", ir.Queue.Port)
 
+	app.Spec.Needs = []hlir.Needs{
+		{Name: "minio", Version: "latest"}}
 	prefixIncludingBucket := api.QueuePrefixPath(ir.Queue, runname)
 	A := strings.Split(prefixIncludingBucket, "/")
 	prefixExcludingBucket := filepath.Join(A[1:]...)

pkg/fe/transformer/api/shell/lower.go

@@ -3,6 +3,7 @@ package shell
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"lunchpail.io/pkg/build"
 	"lunchpail.io/pkg/fe/linker/queue"
@@ -36,6 +37,18 @@ func LowerAsComponent(buildName, runname string, app hlir.Application, ir llir.L
 		component.InstanceName = runname
 	}
 
+	for _, needs := range app.Spec.Needs {
+		var reqs string
+		if needs.Requirements != "" {
+			//Todo
+			fmt.Println(needs.Requirements)
+		}
+		component.Spec.Command = strings.Join([]string{
+			fmt.Sprintf(`$LUNCHPAIL_EXE needs %s %s %s --verbose=%v`, needs.Name, needs.Version, reqs, opts.Log.Verbose),
+			component.Spec.Command,
+		}, "\n")
+	}
+
 	for _, dataset := range app.Spec.Datasets {
 		if dataset.S3.Rclone.RemoteName != "" && dataset.S3.CopyIn.Path != "" {
 			// We were asked to copy data in from s3, so

pkg/ir/hlir/application.go

@@ -7,6 +7,12 @@ type Code struct {
 	Source string
 }
 
+type Needs struct {
+	Name         string
+	Version      string
+	Requirements string
+}
+
 type Application struct {
 	ApiVersion string `yaml:"apiVersion"`
 	Kind       string
@@ -25,6 +31,7 @@ type Application struct {
 		Datasets                 []Dataset                `yaml:"datasets,omitempty"`
 		SecurityContext          SecurityContext          `yaml:"securityContext,omitempty"`
 		ContainerSecurityContext ContainerSecurityContext `yaml:"containerSecurityContext,omitempty"`
+		Needs                    []Needs                  `yaml:"needs,omitempty"`
 	}
 }
 

pkg/runtime/needs/install_darwin.go

@@ -0,0 +1,88 @@
+package needs
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+)
+
+func homedir() (string, error) {
+	currentUser, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	homeDir := currentUser.HomeDir
+	return homeDir, nil
+}
+
+func installMinio(version string, verbose bool) error {
+	if version == "latest" {
+		if verbose {
+			fmt.Fprintf(os.Stdout, "Installing the latest minio release\n")
+		}
+		return brewInstall("minio/stable/minio", verbose)
+	} else {
+		return nil //Todo
+	}
+}
+
+func installPython(version string, requirements string, verbose bool) error {
+	if version == "latest" {
+		if verbose {
+			fmt.Fprintf(os.Stdout, "Installing the latest minio release\n")
+		}
+	} else {
+		//TODO
+	}
+	err := brewInstall("python3", verbose)
+	if err != nil {
+		return err
+	}
+	return requirementsInstall(requirements, verbose)
+}
+
+func brewInstall(pkg string, verbose bool) error {
+	var cmd *exec.Cmd
+	if verbose {
+		cmd = exec.Command("brew", "install", "--verbose", "--debug", pkg)
+		cmd.Stdout = os.Stdout
+	} else {
+		cmd = exec.Command("brew", "install", pkg)
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func requirementsInstall(requirements string, verbose bool) error {
+	var cmd *exec.Cmd
+	if verbose {
+		cmd = exec.Command("python3", "-m", "pip", "install", "--upgrade", "pip", "--verbose")
+		cmd.Stdout = os.Stdout
+	} else {
+		cmd = exec.Command("python3", "-m", "pip", "install", "--upgrade", "pip")
+	}
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	if verbose {
+		cmd = exec.Command("pip", "install", "--requirement", requirements) //TODO: convert string to a file
+		cmd.Stdout = os.Stdout
+	} else {
+		cmd = exec.Command("pip", "install", "--requirement", requirements) //TODO: convert string to a file
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func setenv() error {
+	dir, err := homedir()
+	if err != nil {
+		return err
+	}
+	return os.Setenv("HOME", dir)
+}