diff --git a/README.md b/README.md index e5519544..e18a58e1 100644 --- a/README.md +++ b/README.md @@ -26,44 +26,47 @@ For full usage, run `docker run arxivvanity/engrafo engrafo --help`. ## Development environment -### CSS development environment - -To convert LaTeX documents, you need to build a huge development environment, with several gigabytes of LaTeX junk. If you're just developing CSS, we've got a few sample conversions that you can develop with. - First, install [Node](https://nodejs.org/en/) and [Yarn](https://yarnpkg.com/en/docs/install#mac-stable). Then, install the Node dependencies: $ yarn +### CSS development environment + +To convert LaTeX documents, you need to build a huge development environment, with several gigabytes of LaTeX junk. If you're just developing CSS, we've got a few sample conversions that you can develop with. + The sample documents are `samples/*/index.html` You can use Parcel to serve these documents and compile the frontend code. For example: $ yarn run parcel serve samples/1707.08952/index.html ### Setting up a full development environment -In development, you can build an image locally and use a script to run the image: +In development, the LaTeXML and LaTeX toolchain is run inside Docker. So, install Docker on your system. + +Next, build the Docker image and install the Node dependencies: $ script/build - $ script/engrafo -o output/ tests/documents/sample2e.tex + $ yarn -You can also run a server for developing CSS. It renders a file then runs a server that will automatically reload the CSS when you change it. Start it by running: +You can convert documents with `yarn run convert`: +$ yarn run convert -o output/ tests/documents/sample2e.tex - $ script/server tests/documents/sample2e.tex +You can also run a server for developing CSS. It renders a file then runs a server that will automatically reload the CSS when you change it. Start it by running: -And it will be available at [http://localhost:8010/](http://localhost:8010/). + $ yarn run server tests/documents/sample2e.tex ## Tests Run the main test suite: - $ script/test + $ yarn test -You can run entire suites: +You can run particular suites: - $ script/test integration-tests/images.test.js + $ yarn test tests/integration.test.js -Or individual tests by matching a string: +Or particular tests by matching a string: - $ script/test -t "titles and headings" + $ yarn test -t "titles and headings" ### Writing integration tests @@ -91,24 +94,12 @@ Then, write `tests/documents/bold.tex`: Now, run the test passing the `-u` option to write out a snapshot of what is rendered: - $ script/test -t "bold text renders correctly" -u + $ yarn test -t "bold text renders correctly" -u Check the output looks correct in `tests/__snapshots__/integration.test.js.snap`. You can re-run that command without the `-u` option to ensure the test passes. The test will fail if the output changes in the future. If the change is expected, then you can simply re-run the test with `-u` to overwrite the snapshot and fix the test. -## Installing new yarn packages - -All the Node dependencies are inside the Docker container, which makes managing dependencies a bit unusual. To add a new dependency, use `script/yarn` and rebuild the image: - - $ script/yarn add leftpad - $ script/build - -Similarly, - - $ script/yarn remove leftpad - $ script/build - ## Sponsors Thanks to our generous sponsors for supporting the development of Arxiv Vanity! [Sponsor us to get your logo here.](https://www.patreon.com/arxivvanity) diff --git a/codeship-steps.yml b/codeship-steps.yml index 7d2c5d9b..31fc4c9d 100644 --- a/codeship-steps.yml +++ b/codeship-steps.yml @@ -1,8 +1,8 @@ - type: parallel steps: - - name: integration + - name: tests service: engrafo - command: yarn test + command: jest - name: "Push Git commit tag to Docker Hub" service: engrafo type: push diff --git a/package.json b/package.json index 0aa70deb..f0775c8e 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,10 @@ "license": "Apache-2.0", "scripts": { "build": "parcel build src/assets/css/index.scss src/assets/javascript/index.js", + "convert": "LATEXML_DOCKER=true bin/engrafo", "prettier": "prettier --write {src,tests}/**/*.js", - "test": "jest" + "server": "LATEXML_DOCKER=true bin/engrafo-server", + "test": "LATEXML_DOCKER=true jest" }, "dependencies": { "aws-sdk": "^2.127.0", diff --git a/script/build b/script/docker-build similarity index 100% rename from script/build rename to script/docker-build diff --git a/script/run b/script/docker-run similarity index 100% rename from script/run rename to script/docker-run diff --git a/script/engrafo b/script/engrafo deleted file mode 100755 index 1eb2f0be..00000000 --- a/script/engrafo +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -ENGRAFO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" - -script/run yarn build -exec docker run \ - --init \ - -v "$(pwd)":/workdir -w /workdir \ - -v "$ENGRAFO_DIR:/app" \ - --rm \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_S3_REGION_NAME \ - engrafo \ - engrafo "$@" diff --git a/script/server b/script/server deleted file mode 100755 index 97bc6c95..00000000 --- a/script/server +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -ENGRAFO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" - -exec docker run \ - --init \ - -v "$ENGRAFO_DIR:/app" \ - -w /app \ - -p 8000:8000 \ - -p 8001:8001 \ - --rm \ - -it \ - --entrypoint node \ - engrafo \ - bin/engrafo-server "$@" diff --git a/script/test b/script/test deleted file mode 100755 index b08914a7..00000000 --- a/script/test +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -ENGRAFO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" - -script/run yarn build -exec docker run \ - --init \ - -v "$ENGRAFO_DIR:/app" \ - --rm \ - -it \ - --cap-add=SYS_ADMIN \ - engrafo \ - jest "$@" diff --git a/script/yarn b/script/yarn deleted file mode 100755 index 583fa2f8..00000000 --- a/script/yarn +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -exec script/run yarn --modules-folder=/node_modules "$@" - -# TODO: commit container that just run so we don't have to build image again diff --git a/src/converter/index.js b/src/converter/index.js index 12a1d2c1..e8efae40 100644 --- a/src/converter/index.js +++ b/src/converter/index.js @@ -49,14 +49,14 @@ async function render({ const texPath = await io.pickLatexFile(inputDir); const outputDir = await io.prepareOutputDirectory(output); - // If there is external CSS, don't let LaTeXML copy it to the output + // If there are external assets, don't let LaTeXML copy it to the output // directory - we will handle it ourselves - const cssPath = externalCSS - ? null - : path.join(__dirname, "../../dist/css/index.css"); + // Otherwise, link directly to the built asset. Absolute path, assuming + // latexml is always run in Docker. + const cssPath = externalCSS ? null : "/app/dist/css/index.css"; const javaScriptPath = externalJavaScript ? null - : path.join(__dirname, "../../dist/javascript/index.js"); + : "/app/dist/javascript/index.js"; console.log(`Rendering tex file ${texPath} to ${outputDir}`); const htmlPath = await latexml.render({ diff --git a/src/converter/io.js b/src/converter/io.js index 79612007..172b18b2 100644 --- a/src/converter/io.js +++ b/src/converter/io.js @@ -70,16 +70,17 @@ async function extractGzipToTmpdir(gzipPath) { cwd: tmpDir.path }); } catch (err) { - if ( - err.stderr && - err.stderr - .toString() - .indexOf("tar: This does not look like a tar archive") !== -1 - ) { - console.log( - "Input file is gzipped but not a tarball, assuming it is a .tex file" - ); - await fs.rename(gunzippedPath, path.join(tmpDir.path, "main.tex")); + if (err.stderr) { + const errorMessage = err.stderr.toString(); + if ( + errorMessage.includes("tar: This does not look like a tar archive") || + errorMessage.includes("tar: Unrecognized archive format") + ) { + console.log( + "Input file is gzipped but not a tarball, assuming it is a .tex file" + ); + await fs.rename(gunzippedPath, path.join(tmpDir.path, "main.tex")); + } } else { throw err; } diff --git a/src/converter/latexml.js b/src/converter/latexml.js index 9c9cef88..089bd960 100644 --- a/src/converter/latexml.js +++ b/src/converter/latexml.js @@ -13,13 +13,15 @@ function unlinkIfExists(path) { } } -// render a document with latexml -function render({ texPath, outputDir, cssPath, javaScriptPath }) { - const htmlPath = path.join(outputDir, "index.html"); - +function createChildProcess({ + cssPath, + javaScriptPath, + htmlPath, + texPath, + outputDir +}) { // prettier-ignore - const args = [ - "--dest", htmlPath, + const latexmlArgs = [ "--format", "html5", "--nodefaultresources", "--mathtex", @@ -32,18 +34,51 @@ function render({ texPath, outputDir, cssPath, javaScriptPath }) { ]; if (cssPath) { - args.push("--css", cssPath); + latexmlArgs.push("--css", cssPath); } if (javaScriptPath) { - args.push("--javascript", javaScriptPath); + latexmlArgs.push("--javascript", javaScriptPath); + } + + latexmlArgs.push(path.basename(texPath)); + + if (process.env.LATEXML_DOCKER) { + // prettier-ignore + const dockerArgs = [ + "run", + "--init", + "-v", `${path.dirname(texPath)}:/input`, + "-v", `${outputDir}:/output`, + "-w", "/input", + "--rm", + "engrafo", + ]; + + latexmlArgs.push("--dest", "/output/index.html"); + + const args = dockerArgs.concat(["latexmlc"], latexmlArgs); + return childProcess.spawn("docker", args); } - args.push(texPath); + latexmlArgs.push("--dest", htmlPath); - const latexmlc = childProcess.spawn("latexmlc", args, { + return childProcess.spawn("latexmlc", latexmlArgs, { cwd: path.dirname(texPath) }); +} + +// render a document with latexml +function render({ texPath, outputDir, cssPath, javaScriptPath }) { + const htmlPath = path.join(outputDir, "index.html"); + + const latexmlc = createChildProcess({ + texPath, + outputDir, + htmlPath, + cssPath, + javaScriptPath + }); const stdoutReadline = readline.createInterface({ input: latexmlc.stdout }); stdoutReadline.on("line", console.log); diff --git a/src/converter/server.js b/src/converter/server.js index b578a134..8b65853c 100644 --- a/src/converter/server.js +++ b/src/converter/server.js @@ -5,7 +5,7 @@ const path = require("path"); const tmp = require("tmp-promise"); module.exports.start = async input => { - const tmpDir = await tmp.dir(); + const tmpDir = await tmp.dir({ dir: "/tmp" }); const htmlPath = await render({ input: input, output: tmpDir.path, diff --git a/tests/utils.js b/tests/utils.js index 99170212..0fc3b401 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -17,7 +17,7 @@ expect.extend({ toMatchImageSnapshot }); exports.renderToDom = async input => { input = path.join(__dirname, input); - const tmpDir = await tmp.dir({ unsafeCleanup: true }); + const tmpDir = await tmp.dir({ unsafeCleanup: true, dir: "/tmp" }); const htmlPath = await converter.render({ input: input,