diff --git a/Dockerfile b/Dockerfile index a466b38..021d3a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,22 @@ FROM node:18.17.1-bookworm-slim RUN apt-get update && \ - apt-get install --yes --no-install-recommends p7zip-full && \ + apt-get install --yes --no-install-recommends \ + ca-certificates curl p7zip-full python3 python3-pip pipx && \ rm -rf /var/lib/apt/lists/* +ENV PIPX_HOME /opt/pipx +ENV PIPX_BIN_DIR /usr/bin +RUN pipx install bs-highlighter + COPY --from=ghcr.io/whatwg/wattsi:latest /whatwg/wattsi/bin/wattsi /bin/wattsi WORKDIR /app +# TODO: update this to use gcr.io version +COPY --from=whatwg-html:latest /whatwg/html-build/build.sh /whatwg/html-build/lint.sh ./ +COPY --from=whatwg-html:latest /bin/html-build /bin/ +COPY --from=whatwg-html:latest /whatwg/html-build/entities ./entities/ + COPY . . RUN npm install --omit=dev diff --git a/README.md b/README.md index b7381bb..1c1c493 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # build.whatwg.org -This app is a build server to allow you to run [Wattsi](https://github.com/whatwg/wattsi) without having to actually install it locally. Which is really useful, since not everyone has a Free Pascal compiler lying around. +This app is a build server to allow you to run [html-build](https://github.com/whatwg/html-build) and [Wattsi](https://github.com/whatwg/wattsi) without having to actually install many dependencies locally. -Currently it is hosted on build.whatwg.org.Currently it is hosted on build.whatwg.org. +Currently it is hosted on build.whatwg.org. ## Endpoints @@ -24,6 +24,32 @@ If the resulting status code is 200, the result will be a ZIP file containing th The response will have a header, `Exit-Code`, which gives the exit code of Wattsi. This will always be `0` for a 200 OK response, but a 400 Bad Request could give a variety of different values, depending on how Wattsi failed. +### `/html-build` + +The `/html-build` endpoint accepts POSTs with the following request body fields: + +- `html`, a ZIP file, containing your local checkout of [whatwg/html](https://github.com/whatwg/html). We recommend excluding the unneeded `.git/` and `review-drafts/` directories; there are other unneeded files, but those are the large ones. + + ```sh + zip -r html.zip . --exclude .\* review-drafts/\* + ``` + +- `sha`, a string, the Git commit hash of the whatwg/html repository + +You can send the following query string parameters, which correspond to the same-named html-build options: + +- `no-update` +- `no-lint` +- `no-highlight` +- `single-page` +- `fast` +- `quiet` +- `verbose` + +If the resulting status code is 200, the result will be a ZIP file containing the output, as well as an `output.txt` containing the stdout/stderr output. If the resulting status code is 400, the body text will be the error message. + +The response will have a header, `Exit-Code`, which gives the exit code of html-build. This will always be `0` for a 200 OK response, but a 400 Bad Request could give a variety of different values, depending on how html-build failed. + ### `/version` This endpoint responds to GET requests so you can check to see if the server is working. It returns a `text/plain` response of the latest-deployed Git commit SHA. @@ -34,7 +60,12 @@ This server requires the following to run: - [Node.js](https://nodejs.org/) 18.17.1 or later - [7zip](http://www.7-zip.org/) in your path as `7za` -- And, of course, [Wattsi](https://github.com/whatwg/wattsi), in your `$PATH` as `wattsi` +- [Wattsi](https://github.com/whatwg/wattsi), in your `$PATH` as `wattsi` +- Several files from [html-build](https://github.com/whatwg/html-build) in your `$PATH`: + - `build.sh` + - `lint.sh` + - `entities/` + - `html-build`, the executable built from the Rust preprocessor portions It will expose itself on the port given by the `$PORT` environment variable. diff --git a/lib/app.js b/lib/app.js index 0c7ed1c..ed70709 100644 --- a/lib/app.js +++ b/lib/app.js @@ -77,6 +77,68 @@ router.post("/wattsi", bodyParser, async ctx => { } }); +router.post("/html-build", bodyParser, async ctx => { + const args = booleanArgsFromQuery( + ctx.request.query, + ["no-update", "no-lint", "no-highlight", "single-page", "fast", "quiet", "verbose"] + ); + + const sha = ctx.request.body.sha || "(sha not provided)"; + + const htmlZipPath = ctx.request.files.html?.filepath ?? ctx.throw(400, "Expected a html file"); + + const htmlDirectory = newTempDirectoryName(); + const outDirectory = newTempDirectoryName(); + const cacheDirectory = newTempDirectoryName(); + await mkdir(outDirectory, { recursive: true }); + + const env = { + HTML_SOURCE: htmlDirectory, + HTML_OUTPUT: outDirectory, + HTML_CACHE: cacheDirectory, + SHA_OVERRIDE: sha, + SKIP_BUILD_UPDATE_CHECK: true, + PROCESS_WITH_RUST: true + }; + + try { + await execFile("7za", ["x", htmlZipPath, `-o${htmlDirectory}`]); + + try { + console.log(`Running build.sh ${args.join(" ")}`); + const result = await promisedSpawnWhileCapturingOutput("build.sh", args, { env, shell: "bash" }); + + const outputFile = path.join(outDirectory, "output.txt"); + await writeFile(outputFile, result, { encoding: "utf-8" }); + console.log(` build.sh succeeded`); + } catch (e) { + const errorBody = e.output ?? e.stack; + console.log(` build.sh or file-writing failed:`); + console.log(errorBody); + const headers = typeof e.code === "number" ? { "Exit-Code": e.code } : {}; + ctx.throw(400, errorBody, { headers }); + } + + ctx.response.set("Exit-Code", "0"); + const zipFilePath = `${outDirectory}.zip`; + console.log(` zipping result`); + await execFile("7za", ["a", "-tzip", "-r", zipFilePath, `${outDirectory}/*`]); + console.log(` zipping succeeded`); + + ctx.response.type = "application/zip"; + ctx.response.body = createReadStream(zipFilePath); + + finished(ctx, () => rm(zipFilePath)); + } finally { + await cleanupLoggingRejections([ + ...requestFinalRemovalPromises(ctx.request), + rm(outDirectory, { recursive: true }), + rm(cacheDirectory, { recursive: true }), + rm(htmlDirectory, { recursive: true }) + ]); + } +}); + app .use(router.routes()) .use(router.allowedMethods())