Skip to content

Commit

Permalink
Add build time math rendering
Browse files Browse the repository at this point in the history
While very useful on its own (and combined with the passthrough render hooks), this also serves as a proof of concept of using WASI (WebAssembly System Interface) modules in Hugo.

This will be marked _experimental_ in the documentation. Not because it will be removed or changed in a dramatic way, but we need to think a little more how to best set up/configure similar services, define where these WASM files gets stored, maybe we can allow user provided WASM files plugins via Hugo Modules mounts etc.

See these issues for more context:

* #12736
* #12737

See #11927
  • Loading branch information
bep committed Aug 9, 2024
1 parent 0c3a1c7 commit 33c0938
Show file tree
Hide file tree
Showing 26 changed files with 1,598 additions and 13 deletions.
12 changes: 11 additions & 1 deletion cache/filecache/filecache_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ const (
CacheKeyAssets = "assets"
CacheKeyModules = "modules"
CacheKeyGetResource = "getresource"
CacheKeyMisc = "misc"
)

type Configs map[string]FileCacheConfig
Expand All @@ -70,10 +71,14 @@ var defaultCacheConfigs = Configs{
MaxAge: -1,
Dir: resourcesGenDir,
},
CacheKeyGetResource: FileCacheConfig{
CacheKeyGetResource: {
MaxAge: -1, // Never expire
Dir: cacheDirProject,
},
CacheKeyMisc: {
MaxAge: -1,
Dir: cacheDirProject,
},
}

type FileCacheConfig struct {
Expand Down Expand Up @@ -120,6 +125,11 @@ func (f Caches) AssetsCache() *Cache {
return f[CacheKeyAssets]
}

// MiscCache gets the file cache for miscellaneous stuff.
func (f Caches) MiscCache() *Cache {
return f[CacheKeyMisc]
}

// GetResourceCache gets the file cache for remote resources.
func (f Caches) GetResourceCache() *Cache {
return f[CacheKeyGetResource]
Expand Down
6 changes: 3 additions & 3 deletions cache/filecache/filecache_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ dir = "/path/to/c4"
c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6)
c.Assert(len(decoded), qt.Equals, 7)

c2 := decoded["getcsv"]
c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s")
Expand Down Expand Up @@ -106,7 +106,7 @@ dir = "/path/to/c4"
c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6)
c.Assert(len(decoded), qt.Equals, 7)

for _, v := range decoded {
c.Assert(v.MaxAge, qt.Equals, time.Duration(0))
Expand All @@ -129,7 +129,7 @@ func TestDecodeConfigDefault(t *testing.T) {

fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6)
c.Assert(len(decoded), qt.Equals, 7)

imgConfig := decoded[filecache.CacheKeyImages]
jsonConfig := decoded[filecache.CacheKeyGetJSON]
Expand Down
30 changes: 30 additions & 0 deletions common/hugio/writers.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,33 @@ func ToReadCloser(r io.Reader) io.ReadCloser {
io.NopCloser(nil),
}
}

type ReadWriteCloser interface {
io.Reader
io.Writer
io.Closer
}

// PipeReadWriteCloser is a convenience type to create a pipe with a ReadCloser and a WriteCloser.
type PipeReadWriteCloser struct {
*io.PipeReader
*io.PipeWriter
}

// NewPipeReadWriteCloser creates a new PipeReadWriteCloser.
func NewPipeReadWriteCloser() PipeReadWriteCloser {
pr, pw := io.Pipe()
return PipeReadWriteCloser{pr, pw}
}

func (c PipeReadWriteCloser) Close() (err error) {
if err = c.PipeReader.Close(); err != nil {
return
}
err = c.PipeWriter.Close()
return
}

func (c PipeReadWriteCloser) WriteString(s string) (int, error) {
return c.PipeWriter.Write([]byte(s))
}
8 changes: 8 additions & 0 deletions deps/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/internal/warpc"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/postpub"
Expand Down Expand Up @@ -93,6 +94,10 @@ type Deps struct {
// This is common/global for all sites.
BuildState *BuildState

// Holds RPC dispatchers for Katex etc.
// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
WasmDispatchers *warpc.Dispatchers

*globalErrHandler
}

Expand Down Expand Up @@ -343,6 +348,9 @@ func (d *Deps) Close() error {
if d.MemCache != nil {
d.MemCache.Stop()
}
if d.WasmDispatchers != nil {
d.WasmDispatchers.Close()
}
return d.BuildClosers.Close()
}

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/tdewolff/minify/v2 v2.20.37
github.com/tdewolff/parse/v2 v2.7.15
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec
github.com/yuin/goldmark v1.7.4
github.com/yuin/goldmark-emoji v1.0.3
go.uber.org/automaxprocs v1.5.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,6 @@ github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ
github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0 h1:MNdY6hYCTQEekY0oAfsxWZU1CDt6iH+tMLgyMJQh/sg=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0/go.mod h1:oBdBVuiZ0fv9xd8xflUgt53QxW5jOCb1S+xntcN4SKo=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0/go.mod h1:g9CCh+Ci2IMbPUrVJuXbBTrA+rIIx5+hDQ4EXYaQDoM=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0 h1:7PY5PIJ2mck7v6R52yCFvvYHvsPMEbulgRviw3I9lP4=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0/go.mod h1:r8g5S7bHfdj0+9ShBog864ufCsVODKQZNjYYY8OnJpM=
github.com/gohugoio/locales v0.14.0 h1:Q0gpsZwfv7ATHMbcTNepFd59H7GoykzWJIxi113XGDc=
Expand Down Expand Up @@ -461,6 +459,8 @@ github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec h1:KeQseLFSWb9qjW4PSWxciTBk1hbG7KsVx3rs1hIQnbQ=
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
11 changes: 11 additions & 0 deletions hugolib/site_new.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"fmt"
"html/template"
"os"
"path/filepath"
"sort"
"time"

Expand All @@ -34,6 +35,7 @@ import (
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/internal/warpc"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/langs/i18n"
"github.com/gohugoio/hugo/lazy"
Expand Down Expand Up @@ -157,6 +159,15 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
MemCache: memCache,
TemplateProvider: tplimpl.DefaultTemplateProvider,
TranslationProvider: i18n.NewTranslationProvider(),
WasmDispatchers: warpc.AllDispatchers(
warpc.Options{
CompilationCacheDir: filepath.Join(conf.Dirs().CacheDir, "_warpc"),

// Katex is relatively slow.
PoolSize: 8,
Infof: logger.InfoCommand("wasm").Logf,
},
),
}

if err := firstSiteDeps.Init(); err != nil {
Expand Down
5 changes: 5 additions & 0 deletions internal/warpc/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# TODO1 clean up when done.
go generate ./gen
javy compile js/greet.bundle.js -d -o wasm/greet.wasm
javy compile js/renderkatex.bundle.js -d -o wasm/renderkatex.wasm
touch warpc_test.go
55 changes: 55 additions & 0 deletions internal/warpc/gen/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
//go:generate go run main.go
package main

import (
"fmt"
"log"
"os"
"path/filepath"
"strings"

"github.com/evanw/esbuild/pkg/api"
)

var scripts = []string{
"greet.js",
"renderkatex.js",
}

func main() {
for _, script := range scripts {
filename := filepath.Join("../js", script)
err := buildJSBundle(filename)
if err != nil {
log.Fatal(err)
}
}
}

func buildJSBundle(filename string) error {
minify := true
result := api.Build(
api.BuildOptions{
EntryPoints: []string{filename},
Bundle: true,
MinifyWhitespace: minify,
MinifyIdentifiers: minify,
MinifySyntax: minify,
Target: api.ES2020,
Outfile: strings.Replace(filename, ".js", ".bundle.js", 1),
SourceRoot: "../js",
})

if len(result.Errors) > 0 {
return fmt.Errorf("build failed: %v", result.Errors)
}
if len(result.OutputFiles) != 1 {
return fmt.Errorf("expected 1 output file, got %d", len(result.OutputFiles))
}

of := result.OutputFiles[0]
if err := os.WriteFile(filepath.FromSlash(of.Path), of.Contents, 0o644); err != nil {
return fmt.Errorf("write file failed: %v", err)
}
return nil
}
2 changes: 2 additions & 0 deletions internal/warpc/js/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
node_modules/
package-lock.json
56 changes: 56 additions & 0 deletions internal/warpc/js/common.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Read JSONL from stdin.
export function readInput(handle) {
const buffSize = 1024;
let currentLine = [];
const buffer = new Uint8Array(buffSize);

// Read all the available bytes
while (true) {
// Stdin file descriptor
const fd = 0;
let bytesRead = 0;
try {
bytesRead = Javy.IO.readSync(fd, buffer);
} catch (e) {
// IO.readSync fails with os error 29 when stdin closes.
if (e.message.includes('os error 29')) {
break;
}
throw new Error('Error reading from stdin');
}

if (bytesRead < 0) {
throw new Error('Error reading from stdin');
break;
}

if (bytesRead === 0) {
break;
}

currentLine = [...currentLine, ...buffer.subarray(0, bytesRead)];

// Split array into chunks by newline.
let i = 0;
for (let j = 0; i < currentLine.length; i++) {
if (currentLine[i] === 10) {
const chunk = currentLine.splice(j, i + 1);
const arr = new Uint8Array(chunk);
const json = JSON.parse(new TextDecoder().decode(arr));
handle(json);
j = i + 1;
}
}
// Remove processed data.
currentLine = currentLine.slice(i);
}
}

// Write JSONL to stdout
export function writeOutput(output) {
const encodedOutput = new TextEncoder().encode(JSON.stringify(output) + '\n');
const buffer = new Uint8Array(encodedOutput);
// Stdout file descriptor
const fd = 1;
Javy.IO.writeSync(fd, buffer);
}
2 changes: 2 additions & 0 deletions internal/warpc/js/greet.bundle.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions internal/warpc/js/greet.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { readInput, writeOutput } from './common';

const greet = function (input) {
writeOutput({ header: input.header, data: { greeting: 'Hello ' + input.data.name + '!' } });
};

console.log('Greet module loaded');

readInput(greet);
14 changes: 14 additions & 0 deletions internal/warpc/js/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"name": "js",
"version": "1.0.0",
"main": "greet.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"description": "",
"devDependencies": {
"katex": "^0.16.11"
}
}
Loading

0 comments on commit 33c0938

Please sign in to comment.