Add file streams for teeing Job stdout/err to disc

share/wake/lib/system/job.wake

@@ -114,6 +114,10 @@ export tuple Plan =
     export Stdout: LogLevel
     # How should standard error be displayed during a build
     export Stderr: LogLevel
+    # The workspace-relative paths to which the stdout will be appended
+    export TeeStdout: List String
+    # The workspace-relative paths to which the stderr will be appended
+    export TeeStderr: List String
     # Echo the command to this stream
     export Echo: LogLevel
     # See Persistence table above
@@ -227,7 +231,7 @@ export def editPlanShare (f: Boolean => Boolean): Plan => Plan =
 
 # Get a unique hash-code for the job
 export def getPlanHash (plan: Plan): Integer =
-    def Plan _ cmd _ env dir stdin _ _ _ _ _ _ _ _ isAtty = plan
+    def Plan _ cmd _ env dir stdin _ _ _ _ _ _ _ _ _ _ isAtty = plan
     def isAttyStr = if isAtty then "true" else "false"
 
     def sig =
@@ -275,7 +279,7 @@ def bToInt b =
 
 # Set reasonable defaults for all Plan arguments
 export def makeExecPlan (cmd: List String) (visible: List Path): Plan =
-    Plan "" cmd visible environment "." "" logInfo logWarning logEcho Share Nil defaultUsage id id False
+    Plan "" cmd visible environment "." "" logInfo logWarning Nil Nil logEcho Share Nil defaultUsage id id False
 
 export def makeShellPlan (script: String) (visible: List Path): Plan =
     makeExecPlan (which "dash", "-c", script, Nil) visible
@@ -571,8 +575,8 @@ export def virtualRunner: Runner =
 def implode l =
     cat (foldr (_, "\0", _) Nil l)
 
-def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo stdout stderr label isatty: Job =
-    def create label dir stdin env cmd signature visible keep echo stdout stderr isatty =
+def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echoLevel stdoutLevel stderrLevel stdoutFiles stderrFiles label isatty: Job =
+    def create label dir stdin env cmd signature visible keep echoLevel stdoutLevel stderrLevel stdoutFilesFlat stderrFilesFlat isatty =
         prim "job_create"
 
     def finish job inputs outputs all_outputs status runtime cputime membytes ibytes obytes =
@@ -583,6 +587,13 @@ def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo st
     def signature cmd res fni fno keep = prim "hash"
     def hash = signature cmd res finputs foutputs keep
 
+    def catTeeFiles paths =
+        # Null byte added here rather than via `catWith "\0"` in order to have a null byte after the
+        # *final* path as well, as required by `split_null`.
+        map ("{simplify _}\0") paths
+        | distinctBy scmp
+        | cat
+
     def build Unit =
         def visStrings = map getPathName vis
 
@@ -596,9 +607,11 @@ def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo st
             hash
             visStrings.implode
             (bToInt keep)
-            echo
-            stdout
-            stderr
+            echoLevel
+            stdoutLevel
+            stderrLevel
+            (catTeeFiles stdoutFiles)
+            (catTeeFiles stderrFiles)
             (bToInt isatty)
 
         def prefix = str (getJobId job)
@@ -620,6 +633,7 @@ def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo st
 
                 def output =
                     foutputs outputs
+                    | filter (\f !exists (f ==* _) (stdoutFiles ++ stderrFiles))
                     | computeHashes prefix
                     | implode
 
@@ -655,10 +669,10 @@ def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo st
                 Pair Nil last -> confirm False last (build Unit)
 
 # Only run if the first four arguments differ
-target runOnce cmd env dir stdin vis isatty run \ res usage finputs foutputs keep echo stdout stderr label =
-    runAlways cmd env dir stdin res usage finputs foutputs vis keep run echo stdout stderr label isatty
+target runOnce cmd env dir stdin vis isatty run \ res usage finputs foutputs keep echoLevel stdoutLevel stderrLevel stdoutFiles stderrFiles label =
+    runAlways cmd env dir stdin res usage finputs foutputs vis keep run echoLevel stdoutLevel stderrLevel stdoutFiles stderrFiles label isatty
 
-export def runJobImp label cmd env dir stdin res usage finputs foutputs vis pers run (LogLevel echo) (LogLevel stdout) (LogLevel stderr) isatty =
+export def runJobImp label cmd env dir stdin res usage finputs foutputs vis pers run (LogLevel echo) (LogLevel stdout) (LogLevel stderr) stdoutFiles stderrFiles isatty =
     if isOnce pers then
         runOnce
         cmd
@@ -676,6 +690,8 @@ export def runJobImp label cmd env dir stdin res usage finputs foutputs vis pers
         echo
         stdout
         stderr
+        stdoutFiles
+        stderrFiles
         label
     else
         runAlways
@@ -693,11 +709,13 @@ export def runJobImp label cmd env dir stdin res usage finputs foutputs vis pers
         echo
         stdout
         stderr
+        stdoutFiles
+        stderrFiles
         label
         isatty
 
-export def runJobWith (Runner _ _ run) (Plan label cmd vis env dir stdin stdout stderr echo pers res usage finputs foutputs isatty) =
-    runJobImp label cmd env dir stdin res usage finputs foutputs vis pers run echo stdout stderr isatty
+export def runJobWith (Runner _ _ run) (Plan label cmd vis env dir stdin stdoutLevel stderrLevel stdoutFiles stderrFiles echoLevel pers res usage finputs foutputs isatty) =
+    runJobImp label cmd env dir stdin res usage finputs foutputs vis pers run echoLevel stdoutLevel stderrLevel stdoutFiles stderrFiles isatty
 
 # Set the value of a tag on a Job
 # This is useful for post-build reflection into the database

share/wake/lib/system/path.wake

@@ -179,7 +179,7 @@ def computeHashes (prefix: String) (files: List String): List String =
     def add f h = prim "add_hash"
 
     def hashPlan cmd vis =
-        Plan "<hash>" cmd vis Nil "." "" logNever logError logDebug ReRun Nil hashUsage id id False
+        Plan "<hash>" cmd vis Nil "." "" logNever logError Nil Nil logDebug ReRun Nil hashUsage id id False
 
     def stdin_file_path = "to_hash.{prefix}.stdin"
 
@@ -240,7 +240,7 @@ target hashcode (f: String): String =
         reuse
     else
         def hashPlan cmd =
-            Plan "" cmd Nil Nil "." "" logNever logError logDebug ReRun Nil hashUsage id id False
+            Plan "" cmd Nil Nil "." "" logNever logError Nil Nil logDebug ReRun Nil hashUsage id id False
 
         def job =
             hashPlan ("<hash>", f, Nil)
@@ -259,7 +259,7 @@ target hashcode (f: String): String =
 # Allow an untracked file to be removed via `wake --clean`
 export target markFileCleanable (filepath: String): Result Unit Error =
     def hashPlan cmd =
-        Plan "" cmd Nil Nil "." "" logNever logError logDebug ReRun Nil hashUsage id id False
+        Plan "" cmd Nil Nil "." "" logNever logError Nil Nil logDebug ReRun Nil hashUsage id id False
 
     def job =
         hashPlan ("<hash>", filepath, Nil)

share/wake/lib/system/plan_scorer.wake

@@ -25,7 +25,7 @@ publish runner =
 
 # Run a job, via a Runner chosen based on 'score' functions.
 export def runJob (p: Plan): Job = match p
-    Plan label cmd vis env dir stdin stdout stderr echo pers res usage finputs foutputs isatty ->
+    Plan label cmd vis env dir stdin stdoutLevel stderrLevel stdoutFiles stderrFiles echoLevel pers res usage finputs foutputs isatty ->
         def implode l = cat (foldr (_, "\0", _) Nil l)
         def bToInt b = if b then 1 else 0
 
@@ -50,16 +50,18 @@ export def runJob (p: Plan): Job = match p
 
         match (opts | foldl best (Pair 0.0 None) | getPairSecond)
             Some r ->
-                runJobImp label cmd env dir stdin res usage finputs foutputs vis pers r echo stdout stderr isatty
+                runJobImp label cmd env dir stdin res usage finputs foutputs vis pers r echoLevel stdoutLevel stderrLevel stdoutFiles stderrFiles isatty
             None ->
-                def create label dir stdin env cmd signature visible keep echo stdout stderr isatty =
+                def create label dir stdin env cmd signature visible keep echoLevel stdoutLevel stderrLevel stdoutFilesFlat stderrFilesFlat isatty =
                     prim "job_create"
 
                 def badfinish job e = prim "job_fail_finish"
                 def badlaunch job e = prim "job_fail_launch"
+                def stdoutFilesFlat = catWith "\0" stdoutFiles
+                def stderrFilesFlat = catWith "\0" stderrFiles
 
                 def job =
-                    create label dir stdin env.implode cmd.implode 0 "" 0 "echo" "info" "error" (bToInt isatty)
+                    create label dir stdin env.implode cmd.implode 0 "" 0 "echo" "info" stdoutFilesFlat "error" stderrFilesFlat (bToInt isatty)
 
                 def error =
                     def pretty = match _

src/compat/spawn.c → src/compat/spawn.cpp

@@ -21,13 +21,24 @@
 
 #include "spawn.h"
 
+#include <iostream>
 #include <sys/types.h>
 #include <unistd.h>
 
-pid_t wake_spawn(const char *cmd, char **cmdline, char **environ) {
+pid_t wake_spawn(const std::string cmd, std::vector<std::string> cmdline, std::vector<std::string> environ) {
   pid_t pid = vfork();
   if (pid == 0) {
-    execve(cmdline[0], cmdline, environ);
+    std::vector<char*> cmdline_c = std::vector<char*>();
+    for (auto str : cmdline) {
+      cmdline_c.push_back(&str.front());
+    }
+    cmdline_c.push_back(nullptr);
+    std::vector<char*> environ_c = std::vector<char*>();
+    for (auto str : environ) {
+      environ_c.push_back(&str.front());
+    }
+    environ_c.push_back(nullptr);
+    execve(cmd.c_str(), cmdline_c.data(), environ_c.data());
     _exit(127);
   }
   return pid;

src/compat/spawn.h

@@ -18,16 +18,10 @@
 #ifndef SPAWN_H
 #define SPAWN_H
 
+#include <string>
 #include <sys/types.h>
+#include <vector>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-pid_t wake_spawn(const char *cmd, char **cmdline, char **environ);
-
-#ifdef __cplusplus
-};
-#endif
+pid_t wake_spawn(const std::string cmd, std::vector<std::string> cmdline, std::vector<std::string> environ);
 
 #endif