Introduce pkg/etk, a new TUI framework.

The framework has the following interesting features:

- It has a first-class Elvish binding with the same expressive power as the Go
  version of the framework.

- It has an immediate-mode API, meaning that a component is represented by a
  function that gets invoked every time the state changes.

For more on the motivation and tradeoffs of the design, see the slidedeck in
website/slides/draft-etk.md.

This package will eventually replace pkg/cli/tk as Elvish's internal TUI
framework, and Elvish's TUI will be rewritten to use it.

This commit also incorporates the following changes that overlapped with the code change necessary for implementing etk:

- pkg/ui: De-export TextFromSegment. The styled builtin used to require it, but no longer does now that Text.Scan is implemented.

apps.elv

@@ -0,0 +1,70 @@
+use etk
+
+# mkdir prompt - what can be done today:
+var w
+set w = (edit:new-codearea [&prompt=(styled 'mkdir:' inverse)' ' &on-submit={ mkdir (edit:get-state $w)[buffer][content] }])
+edit:push-addon $w
+
+# mkdir prompt - slightly cleaned up version
+edit:push-addon (etk:new-codearea [&prompt='mkdir: ' &on-submit={|s| mkdir $s[buffer][content] }])
+
+# mkdir prompt - state management version
+var dirname
+edit:push-addon {
+  etk:textbox [&prompt='mkdir: ' &on-submit={ mkdir $dirname }] ^
+              [&buffer=[&content=(bind dirname)]]
+}
+
+# Temperature conversion
+var c = ''
+etk:run-app {
+  var f = (/ (- $c 32) 1.8)
+  etk:vbox [&children=[
+    (etk:textbox [&prompt='input: '] [&buffer=[&content=(bind c)]])
+    (etk:label $c' ℉ = '$f' ℃')
+  ]]
+}
+
+# Elvish configuration helper
+var tasks = [
+  [&name='Use readline binding'
+   &detail='Readline binding enables keys like Ctrl-N, Ctrl-F'
+   &eval-code=''
+   &rc-code='use readline-binding']
+
+  [&name='Install Carapace'
+   &detail='Carapace provides completions.'
+   &eval-code='brew install carapace'
+   &rc-code='eval (carapace init elvish)']
+]
+
+fn execute-task {|task|
+  eval $task[eval-code]
+  eval $task[rc-code]
+  echo $task[rc-code] >> $runtime:rc-file
+}
+
+var i = (num 0)
+etk:run-app {
+  etk:hbox [&children=[
+    (etk:list [&items=$tasks &display={|t| put $t[name]} &on-submit=$execute-task~] ^
+              [&selected=(bind i)])
+    (etk:label $tasks[i][detail])
+  ]]
+}
+
+# Markdown-driven presentation
+var filename = 'a.md'
+var @slides = (slurp < $filename |
+               re:split '\n {0,3}((?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})\n' (one))
+
+var i = (num 0)
+etk:run-app {
+  etk:vbox [
+    &binding=[&Left={|_| set i = (- $i 1) } &Right={|_| set i = (+ $i 1) }]
+    &children=[
+      (etk:label $slides[i])
+      (etk:label (+ 1 $i)/(count $slides))
+    ]
+  ]
+}

pkg/edit/custom_widget.go

@@ -0,0 +1,13 @@
+package edit
+
+import (
+	"src.elv.sh/pkg/cli"
+	"src.elv.sh/pkg/eval"
+)
+
+func initCustomWidgetAPI(app cli.App, nb eval.NsBuilder) {
+	nb.AddGoFns(map[string]any{
+		"push-addon": app.PushAddon,
+		"pop-addon":  app.PopAddon,
+	})
+}

pkg/edit/editor.go

@@ -88,6 +88,7 @@ func NewEditor(tty cli.TTY, ev *eval.Evaler, st storedefs.Store) *Editor {
 	initMiscBuiltins(ed, nb)
 	initStateAPI(ed.app, nb)
 	initStoreAPI(ed.app, nb, hs)
+	initCustomWidgetAPI(ed.app, nb)
 
 	ed.ns = nb.Ns()
 	initElvishState(ev, ed.ns)

pkg/edit/highlight.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"src.elv.sh/pkg/cli"
 	"src.elv.sh/pkg/edit/highlight"
@@ -21,7 +22,8 @@ func initHighlighter(appSpec *cli.AppSpec, ed *Editor, ev *eval.Evaler, nb eval.
 			ed.autofix.Store(autofix)
 			return autofix, eval.UnpackCompilationErrors(err)
 		},
-		HasCommand: func(cmd string) bool { return hasCommand(ev, cmd) },
+		HasCommand:         func(cmd string) bool { return hasCommand(ev, cmd) },
+		HasCommandMaxBlock: func() time.Duration { return 10 * time.Millisecond },
 		AutofixTip: func(autofix string) ui.Text {
 			return bindingTips(ed.ns, "insert:binding",
 				bindingTip("autofix: "+autofix, "apply-autofix"),

pkg/edit/highlight/highlight.go

@@ -12,9 +12,10 @@ import (
 
 // Config keeps configuration for highlighting code.
 type Config struct {
-	Check      func(n parse.Tree) (string, []*eval.CompilationError)
-	HasCommand func(name string) bool
-	AutofixTip func(autofix string) ui.Text
+	Check              func(n parse.Tree) (string, []*eval.CompilationError)
+	HasCommand         func(name string) bool
+	HasCommandMaxBlock func() time.Duration
+	AutofixTip         func(autofix string) ui.Text
 }
 
 // Information collected about a command region, used for asynchronous
@@ -24,9 +25,6 @@ type cmdRegion struct {
 	cmd string
 }
 
-// Maximum wait time to block for late results. Can be changed for test cases.
-var maxBlockForLate = 10 * time.Millisecond
-
 // Highlights a piece of Elvish code.
 func highlight(code string, cfg Config, lateCb func(ui.Text)) (ui.Text, []ui.Text) {
 	var tips []ui.Text
@@ -113,17 +111,22 @@ func highlight(code string, cfg Config, lateCb func(ui.Text)) (ui.Text, []ui.Tex
 			}
 			lateCh <- newText
 		}()
-		// Block a short while for the late text to arrive, in order to reduce
-		// flickering. Otherwise, return the text already computed, and pass the
-		// late result to lateCb in another goroutine.
-		select {
-		case late := <-lateCh:
-			return late, tips
-		case <-time.After(maxBlockForLate):
-			go func() {
-				lateCb(<-lateCh)
-			}()
-			return text, tips
+		if cfg.HasCommandMaxBlock != nil {
+			// Block a short while for the late text to arrive, in order to
+			// reduce flickering. Otherwise, return the text already computed,
+			// and pass the late result to lateCb in another goroutine.
+			select {
+			case late := <-lateCh:
+				return late, tips
+			case <-time.After(cfg.HasCommandMaxBlock()):
+				go func() {
+					lateCb(<-lateCh)
+				}()
+				return text, tips
+			}
+		} else {
+			// If cfg.HasCommandMaxBlock is not populated, block indefinitely.
+			return <-lateCh, tips
 		}
 	}
 	return text, tips

pkg/edit/highlight/highlighter_test.elvts

@@ -1,6 +1,4 @@
-//each:highlight-in-global
-//each:with-known-commands echo var set tmp del if for try
-//each:with-max-block-for-late 100ms
+//each:highlight-in-global 100ms echo var set tmp del if for try
 
 ///////////////////////////////
 # Simple lexical highlighting #

pkg/edit/highlight/highlighter_test.go

@@ -13,7 +13,7 @@ import (
 	"src.elv.sh/pkg/ui"
 )
 
-var any = anyMatcher{}
+var anything = anyMatcher{}
 var noTips []ui.Text
 
 var styles = ui.RuneStylesheet{
@@ -26,8 +26,6 @@ var styles = ui.RuneStylesheet{
 var Args = tt.Args
 
 func TestHighlighter_HighlightRegions(t *testing.T) {
-	// Force commands to be delivered synchronously.
-	testutil.Set(t, &maxBlockForLate, testutil.Scaled(100*time.Millisecond))
 	hl := NewHighlighter(Config{
 		HasCommand: func(name string) bool { return name == "ls" },
 	})
@@ -76,8 +74,8 @@ func TestHighlighter_ParseErrors(t *testing.T) {
 				"vv $? ?"),
 			matchTexts("1:5", "1:7")),
 		// Errors at the end are ignored
-		Args("ls $").Rets(any, noTips),
-		Args("ls [").Rets(any, noTips),
+		Args("ls $").Rets(anything, noTips),
+		Args("ls [").Rets(anything, noTips),
 	)
 }
 
@@ -106,7 +104,7 @@ func TestHighlighter_AutofixesAndCheckErrors(t *testing.T) {
 				"vv ?? ?? "),
 			matchTexts("1:4", "1:7")),
 		// Check errors at the end are ignored
-		Args("set _").Rets(any, noTips),
+		Args("set _").Rets(anything, noTips),
 
 		// Autofix
 		Args("nop $mod1:").Rets(
@@ -149,15 +147,16 @@ func testThat(t *testing.T, hl *Highlighter, c c) {
 }
 
 func TestHighlighter_HasCommand_LateResult_Async(t *testing.T) {
-	// When the HasCommand callback takes longer than maxBlockForLate, late
+	// When the HasCommand callback takes longer than HasCommandMaxBlock, late
 	// results are delivered asynchronously.
-	testutil.Set(t, &maxBlockForLate, testutil.Scaled(time.Millisecond))
 	hl := NewHighlighter(Config{
 		// HasCommand is slow and only recognizes "ls".
 		HasCommand: func(cmd string) bool {
 			time.Sleep(testutil.Scaled(10 * time.Millisecond))
 			return cmd == "ls"
-		}})
+		},
+		HasCommandMaxBlock: constantly(time.Duration(0)),
+	})
 
 	testThat(t, hl, c{
 		given:       "ls",
@@ -172,16 +171,26 @@ func TestHighlighter_HasCommand_LateResult_Async(t *testing.T) {
 }
 
 func TestHighlighter_HasCommand_LateResult_Sync(t *testing.T) {
-	// When the HasCommand callback takes shorter than maxBlockForLate, late
-	// results are delivered asynchronously.
-	testutil.Set(t, &maxBlockForLate, testutil.Scaled(100*time.Millisecond))
+	// When the HasCommand callback takes shorter than HasCommandMaxBlock, late
+	// results are delivered synchronously.
 	hl := NewHighlighter(Config{
-		// HasCommand is fast and only recognizes "ls".
-		HasCommand: func(cmd string) bool {
-			time.Sleep(testutil.Scaled(time.Millisecond))
-			return cmd == "ls"
-		}})
+		HasCommand:         func(cmd string) bool { return cmd == "ls" },
+		HasCommandMaxBlock: constantly(testutil.Scaled(100 * time.Millisecond)),
+	})
+
+	testThat(t, hl, c{
+		given:       "ls",
+		wantInitial: ui.T("ls", ui.FgGreen),
+	})
+	testThat(t, hl, c{
+		given:       "echo",
+		wantInitial: ui.T("echo", ui.FgRed),
+	})
 
+	// Missing HasCommandMaxBlock means block definitely.
+	hl = NewHighlighter(Config{
+		HasCommand: func(cmd string) bool { return cmd == "ls" },
+	})
 	testThat(t, hl, c{
 		given:       "ls",
 		wantInitial: ui.T("ls", ui.FgGreen),
@@ -198,9 +207,6 @@ func TestHighlighter_HasCommand_LateResultOutOfOrder(t *testing.T) {
 	// first and then "ls". The late result for "l" is delivered after that of
 	// "ls" and is dropped.
 
-	// Make sure that the HasCommand callback takes longer than maxBlockForLate.
-	testutil.Set(t, &maxBlockForLate, testutil.Scaled(time.Millisecond))
-
 	hlSecond := make(chan struct{})
 	hl := NewHighlighter(Config{
 		HasCommand: func(cmd string) bool {
@@ -214,7 +220,9 @@ func TestHighlighter_HasCommand_LateResultOutOfOrder(t *testing.T) {
 			time.Sleep(testutil.Scaled(10 * time.Millisecond))
 			close(hlSecond)
 			return cmd == "ls"
-		}})
+		},
+		HasCommandMaxBlock: constantly(time.Duration(0)),
+	})
 
 	hl.Get("l")
 
@@ -256,3 +264,5 @@ func (m textsMatcher) Match(v tt.RetValue) bool {
 	}
 	return true
 }
+
+func constantly[T any](v T) func() T { return func() T { return v } }

pkg/edit/highlight/transcripts_test.go

@@ -12,7 +12,6 @@ import (
 	"src.elv.sh/pkg/eval"
 	"src.elv.sh/pkg/eval/evaltest"
 	"src.elv.sh/pkg/must"
-	"src.elv.sh/pkg/testutil"
 	"src.elv.sh/pkg/ui"
 	"src.elv.sh/pkg/ui/styledown"
 )
@@ -21,25 +20,22 @@ import (
 var transcripts embed.FS
 
 func TestTranscripts(t *testing.T) {
-	var validCommands []string
 	evaltest.TestTranscriptsInFS(t, transcripts,
-		"with-max-block-for-late", func(t *testing.T, s string) {
-			testutil.Set(t, highlight.MaxBlockForLate,
-				testutil.Scaled(must.OK1(time.ParseDuration(s))))
+		"highlight-in-global", func(ev *eval.Evaler, arg string) {
+			fields := strings.Fields(arg)
+			maxBlock, validCommands := fields[0], fields[1:]
+			evaltest.GoFnInGlobal("highlight", func(fm *eval.Frame, s string) {
+				hl := highlight.NewHighlighter(highlight.Config{
+					HasCommand:         func(name string) bool { return slices.Contains(validCommands, name) },
+					HasCommandMaxBlock: func() time.Duration { return must.OK1(time.ParseDuration(maxBlock)) },
+				})
+				text, tips := hl.Get(s)
+				fmt.Fprint(fm.ByteOutput(), toStyledown(text))
+				for i, tip := range tips {
+					fmt.Fprintf(fm.ByteOutput(), "= tip %d:\n%s", i, toStyledown(tip))
+				}
+			})(ev)
 		},
-		"with-known-commands", func(arg string) {
-			validCommands = strings.Fields(arg)
-		},
-		"highlight-in-global", evaltest.GoFnInGlobal("highlight", func(fm *eval.Frame, s string) {
-			hl := highlight.NewHighlighter(highlight.Config{
-				HasCommand: func(name string) bool { return slices.Contains(validCommands, name) },
-			})
-			text, tips := hl.Get(s)
-			fmt.Fprint(fm.ByteOutput(), toStyledown(text))
-			for i, tip := range tips {
-				fmt.Fprintf(fm.ByteOutput(), "= tip %d:\n%s", i, toStyledown(tip))
-			}
-		}),
 	)
 }
 

pkg/elvdoc/highlight.go

@@ -8,9 +8,10 @@ import (
 	"src.elv.sh/pkg/ui"
 )
 
-// With an empty highlight.Config, this highlighter does not check for
-// compilation errors or non-existent commands.
-var highlighter = highlight.NewHighlighter(highlight.Config{})
+// Assume all commands are valid.
+var highlighter = highlight.NewHighlighter(highlight.Config{
+	HasCommand: func(string) bool { return true },
+})
 
 // HighlightCodeBlock highlights a code block from Markdown. It handles thea
 // elvish and elvish-transcript languages. It also removes comment and directive