From f835f0b5f145b11efc446fa03760fe4806aed6fa Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 17 Apr 2023 14:07:37 +0200 Subject: [PATCH 01/12] Use a fully static seshat build --- .github/workflows/build_linux.yaml | 16 +- hak/matrix-seshat/build.ts | 284 ++--------------------------- hak/matrix-seshat/check.ts | 17 -- hak/matrix-seshat/fetchDeps.ts | 129 ------------- hak/matrix-seshat/hak.json | 9 +- package.json | 2 +- scripts/hak/README.md | 1 - scripts/hak/index.ts | 8 +- 8 files changed, 25 insertions(+), 441 deletions(-) delete mode 100644 hak/matrix-seshat/fetchDeps.ts diff --git a/.github/workflows/build_linux.yaml b/.github/workflows/build_linux.yaml index 866e2cfb5..4e8db7268 100644 --- a/.github/workflows/build_linux.yaml +++ b/.github/workflows/build_linux.yaml @@ -150,17 +150,11 @@ jobs: LIBS=$(readelf -d dist/**/*.node | grep NEEDED) echo "$LIBS" - if [ "$SQLCIPHER_STATIC" == "1" ]; then - if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then - exit 2 - fi - else - if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then - exit 3 - fi - if ! grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then - exit 4 - fi + if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then + exit 2 + fi + if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then + exit 3 fi env: ARCH: ${{ steps.config.outputs.arch }} diff --git a/hak/matrix-seshat/build.ts b/hak/matrix-seshat/build.ts index aba67b25d..19204ef38 100644 --- a/hak/matrix-seshat/build.ts +++ b/hak/matrix-seshat/build.ts @@ -22,101 +22,22 @@ import fsExtra from "fs-extra"; import HakEnv from "../../scripts/hak/hakEnv"; import { DependencyInfo } from "../../scripts/hak/dep"; -type WinConfiguration = - | "VC-WIN32" - | "VC-WIN64A" - | "VC-WIN64-ARM" - | "VC-WIN64-CLANGASM-ARM" - | "VC-CLANG-WIN64-CLANGASM-ARM" - | "VC-WIN32-HYBRIDCRT" - | "VC-WIN64A-HYBRIDCRT"; - export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - if (hakEnv.isWin()) { - await buildOpenSslWin(hakEnv, moduleInfo); - await buildSqlCipherWin(hakEnv, moduleInfo); - } else if (hakEnv.wantsStaticSqlCipherUnix()) { - await buildSqlCipherUnix(hakEnv, moduleInfo); - } - await buildMatrixSeshat(hakEnv, moduleInfo); -} - -async function buildOpenSslWin(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - const version = moduleInfo.cfg.dependencies.openssl; - const openSslDir = path.join(moduleInfo.moduleTargetDotHakDir, `openssl-${version}`); + const env = hakEnv.makeGypEnv(); - let openSslArch: WinConfiguration; - switch (hakEnv.getTargetArch()) { - case "x64": - openSslArch = "VC-WIN64A"; - break; - case "ia32": - openSslArch = "VC-WIN32"; - break; - case "arm64": - openSslArch = "VC-WIN64-ARM"; - break; + if (!hakEnv.isHost()) { + env.CARGO_BUILD_TARGET = hakEnv.getTargetId(); } - console.log("Building openssl in " + openSslDir); + console.log("Running yarn install"); await new Promise((resolve, reject) => { const proc = childProcess.spawn( - "perl", - [ - "Configure", - "--prefix=" + moduleInfo.depPrefix, - // sqlcipher only uses about a tiny part of openssl. We link statically - // so will only pull in the symbols we use, but we may as well turn off - // as much as possible to save on build time. - "no-afalgeng", - "no-capieng", - "no-cms", - "no-ct", - "no-deprecated", - "no-dgram", - "no-dso", - "no-ec", - "no-ec2m", - "no-gost", - "no-nextprotoneg", - "no-ocsp", - "no-sock", - "no-srp", - "no-srtp", - "no-tests", - "no-ssl", - "no-tls", - "no-dtls", - "no-shared", - "no-aria", - "no-camellia", - "no-cast", - "no-chacha", - "no-cmac", - "no-des", - "no-dh", - "no-dsa", - "no-ecdh", - "no-ecdsa", - "no-idea", - "no-md4", - "no-mdc2", - "no-ocb", - "no-poly1305", - "no-rc2", - "no-rc4", - "no-rmd160", - "no-scrypt", - "no-seed", - "no-siphash", - "no-sm2", - "no-sm3", - "no-sm4", - "no-whirlpool", - openSslArch, - ], + "yarn" + (hakEnv.isWin() ? ".cmd" : ""), + ["install"], { - cwd: openSslDir, + cwd: moduleInfo.moduleBuildDir, + env, + shell: true, stdio: "inherit", }, ); @@ -125,194 +46,15 @@ async function buildOpenSslWin(hakEnv: HakEnv, moduleInfo: DependencyInfo): Prom }); }); - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("nmake", ["build_libs"], { - cwd: openSslDir, - stdio: "inherit", - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); - - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("nmake", ["install_dev"], { - cwd: openSslDir, - stdio: "inherit", - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); -} - -async function buildSqlCipherWin(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - const version = moduleInfo.cfg.dependencies.sqlcipher; - const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`); - const buildDir = path.join(sqlCipherDir, "bld"); - - await mkdirp(buildDir); - - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("nmake", ["/f", path.join("..", "Makefile.msc"), "libsqlite3.lib", "TOP=.."], { - cwd: buildDir, - stdio: "inherit", - env: Object.assign({}, process.env, { - CCOPTS: "-DSQLITE_HAS_CODEC -I" + path.join(moduleInfo.depPrefix, "include"), - LTLIBPATHS: "/LIBPATH:" + path.join(moduleInfo.depPrefix, "lib"), - LTLIBS: "libcrypto.lib", - }), - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); - - await fsExtra.copy(path.join(buildDir, "libsqlite3.lib"), path.join(moduleInfo.depPrefix, "lib", "sqlcipher.lib")); - - await fsExtra.copy(path.join(buildDir, "sqlite3.h"), path.join(moduleInfo.depPrefix, "include", "sqlcipher.h")); -} - -async function buildSqlCipherUnix(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - const version = moduleInfo.cfg.dependencies.sqlcipher; - const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`); - - const args = [ - "--prefix=" + moduleInfo.depPrefix + "", - "--enable-tempstore=yes", - "--enable-shared=no", - "--enable-tcl=no", - ]; - - if (hakEnv.isMac()) { - args.push("--with-crypto-lib=commoncrypto"); - } - - if (hakEnv.wantsStaticSqlCipherUnix()) { - args.push("--enable-tcl=no"); - - if (hakEnv.isLinux()) { - args.push("--with-pic=yes"); - } - } - - if (!hakEnv.isHost()) { - // In the nonsense world of `configure`, it is assumed you are building - // a compiler like `gcc`, so the `host` option actually means the target - // the build output runs on. - args.push(`--host=${hakEnv.getTargetId()}`); - } - - const cflags = ["-DSQLITE_HAS_CODEC"]; - - // If the caller has specified CFLAGS then we shouldn't specify target - // as their compiler may be incompatible (gcc) - if (!hakEnv.isHost() && !process.env.CFLAGS) { - // `clang` uses more logical option naming. - cflags.push(`--target=${hakEnv.getTargetId()}`); - } - - if (process.env.CFLAGS) cflags.unshift(process.env.CFLAGS); - args.push(`CFLAGS=${cflags.join(" ")}`); - - const ldflags: string[] = []; - - if (hakEnv.isMac()) { - ldflags.push("-framework Security"); - ldflags.push("-framework Foundation"); - } - - if (ldflags.length) { - if (process.env.LDFLAGS) ldflags.unshift(process.env.LDFLAGS); - args.push(`LDFLAGS=${ldflags.join(" ")}`); - } - - await new Promise((resolve, reject) => { - const proc = childProcess.spawn(path.join(sqlCipherDir, "configure"), args, { - cwd: sqlCipherDir, - stdio: "inherit", - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); - - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("make", [], { - cwd: sqlCipherDir, - stdio: "inherit", - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); - - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("make", ["install"], { - cwd: sqlCipherDir, - stdio: "inherit", - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - }); -} - -async function buildMatrixSeshat(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - // seshat now uses n-api so we shouldn't need to specify a node version to - // build against, but it does seems to still need something in here, so leaving - // it for now: we should confirm how much of this it still actually needs. - const env = hakEnv.makeGypEnv(); - - if (!hakEnv.isLinux() || hakEnv.wantsStaticSqlCipherUnix()) { - Object.assign(env, { - SQLCIPHER_STATIC: 1, - SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, "lib"), - SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, "include"), - }); - } - - if (hakEnv.isLinux() && hakEnv.wantsStaticSqlCipherUnix()) { - // Ensure Element uses the statically-linked seshat build, and prevent other applications - // from attempting to use this one. Detailed explanation: - // - // RUSTFLAGS - // An environment variable containing a list of arguments to pass to rustc. - // -Clink-arg=VALUE - // A rustc argument to pass a single argument to the linker. - // -Wl, - // gcc syntax to pass an argument (from gcc) to the linker (ld). - // -Bsymbolic: - // Prefer local/statically linked symbols over those in the environment. - // Prevent overriding native libraries by LD_PRELOAD etc. - // --exclude-libs ALL - // Prevent symbols from being exported by any archive libraries. - // Reduces output filesize and prevents being dynamically linked against. - env.RUSTFLAGS = "-Clink-arg=-Wl,-Bsymbolic -Clink-arg=-Wl,--exclude-libs,ALL"; - } - - if (hakEnv.isWin()) { - env.RUSTFLAGS = "-Ctarget-feature=+crt-static -Clink-args=libcrypto.lib"; - // Note that in general, you can specify targets in Rust without having to have - // the matching toolchain, however for this, cargo gets confused when building - // the build scripts since they run on the host, but vcvarsall.bat sets the c - // compiler in the path to be the one for the target, so we just use the matching - // toolchain for the target architecture which makes everything happy. - env.RUSTUP_TOOLCHAIN = `stable-${hakEnv.getTargetId()}`; - } - - if (!hakEnv.isHost()) { - env.CARGO_BUILD_TARGET = hakEnv.getTargetId(); - } - - console.log("Running neon with env", env); + console.log("Running yarn build"); await new Promise((resolve, reject) => { const proc = childProcess.spawn( - path.join(moduleInfo.nodeModuleBinDir, "neon" + (hakEnv.isWin() ? ".cmd" : "")), - ["build", "--release"], + "yarn" + (hakEnv.isWin() ? ".cmd" : ""), + ["run", "build-bundled"], { cwd: moduleInfo.moduleBuildDir, env, + shell: true, stdio: "inherit", }, ); diff --git a/hak/matrix-seshat/check.ts b/hak/matrix-seshat/check.ts index a4405e6f3..90863d7a0 100644 --- a/hak/matrix-seshat/check.ts +++ b/hak/matrix-seshat/check.ts @@ -21,23 +21,6 @@ import HakEnv from "../../scripts/hak/hakEnv"; import { DependencyInfo } from "../../scripts/hak/dep"; export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - if (hakEnv.wantsStaticSqlCipher()) { - // of course tcl doesn't have a --version - await new Promise((resolve, reject) => { - const proc = childProcess.spawn("tclsh", [], { - stdio: ["pipe", "ignore", "ignore"], - }); - proc.on("exit", (code) => { - if (code !== 0) { - reject("Can't find tclsh - have you installed TCL?"); - } else { - resolve(); - } - }); - proc.stdin.end(); - }); - } - const tools = [ ["rustc", "--version"], ["python", "--version"], // node-gyp uses python for reasons beyond comprehension diff --git a/hak/matrix-seshat/fetchDeps.ts b/hak/matrix-seshat/fetchDeps.ts deleted file mode 100644 index fb0863945..000000000 --- a/hak/matrix-seshat/fetchDeps.ts +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import path from "path"; -import childProcess from "child_process"; -import fs from "fs"; -import fsProm from "fs/promises"; -import tar from "tar"; -import fetch from "node-fetch"; -import { promises as stream } from "stream"; - -import HakEnv from "../../scripts/hak/hakEnv"; -import { DependencyInfo } from "../../scripts/hak/dep"; - -async function download(url: string, filename: string): Promise { - const resp = await fetch(url); - if (!resp.ok) throw new Error(`unexpected response ${resp.statusText}`); - if (!resp.body) throw new Error(`unexpected response has no body ${resp.statusText}`); - await stream.pipeline(resp.body, fs.createWriteStream(filename)); -} - -export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - if (hakEnv.wantsStaticSqlCipher()) { - await getSqlCipher(hakEnv, moduleInfo); - } - - if (hakEnv.isWin()) { - await getOpenSsl(hakEnv, moduleInfo); - } -} - -async function getSqlCipher(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - const version = moduleInfo.cfg.dependencies.sqlcipher; - const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`); - - let haveSqlcipher: boolean; - try { - await fsProm.stat(sqlCipherDir); - haveSqlcipher = true; - } catch (e) { - haveSqlcipher = false; - } - - if (haveSqlcipher) return; - - const sqlCipherTarball = path.join(moduleInfo.moduleDotHakDir, `sqlcipher-${version}.tar.gz`); - let haveSqlcipherTar: boolean; - try { - await fsProm.stat(sqlCipherTarball); - haveSqlcipherTar = true; - } catch (e) { - haveSqlcipherTar = false; - } - if (!haveSqlcipherTar) { - await download(`https://github.com/sqlcipher/sqlcipher/archive/v${version}.tar.gz`, sqlCipherTarball); - } - - // Extract the tarball to per-target directories, then we avoid cross-contaiminating archs - await tar.x({ - file: sqlCipherTarball, - cwd: moduleInfo.moduleTargetDotHakDir, - }); - - if (hakEnv.isWin()) { - // On Windows, we need to patch the makefile because it forces TEMP_STORE to - // default to files (1) but the README specifically says you '*must*' set it - // set it to 2 (default to memory). - const patchFile = path.join(moduleInfo.moduleHakDir, `sqlcipher-${version}-win.patch`); - - await new Promise((resolve, reject) => { - const readStream = fs.createReadStream(patchFile); - - const proc = childProcess.spawn("patch", ["-p1"], { - cwd: sqlCipherDir, - stdio: ["pipe", "inherit", "inherit"], - }); - proc.on("exit", (code) => { - code ? reject(code) : resolve(); - }); - readStream.pipe(proc.stdin); - }); - } -} - -async function getOpenSsl(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise { - const version = moduleInfo.cfg.dependencies.openssl; - const openSslDir = path.join(moduleInfo.moduleTargetDotHakDir, `openssl-${version}`); - - let haveOpenSsl: boolean; - try { - await fsProm.stat(openSslDir); - haveOpenSsl = true; - } catch (e) { - haveOpenSsl = false; - } - - if (haveOpenSsl) return; - - const openSslTarball = path.join(moduleInfo.moduleDotHakDir, `openssl-${version}.tar.gz`); - let haveOpenSslTar: boolean; - try { - await fsProm.stat(openSslTarball); - haveOpenSslTar = true; - } catch (e) { - haveOpenSslTar = false; - } - if (!haveOpenSslTar) { - await download(`https://www.openssl.org/source/openssl-${version}.tar.gz`, openSslTarball); - } - - console.log("extracting " + openSslTarball + " in " + moduleInfo.moduleTargetDotHakDir); - await tar.x({ - file: openSslTarball, - cwd: moduleInfo.moduleTargetDotHakDir, - }); -} diff --git a/hak/matrix-seshat/hak.json b/hak/matrix-seshat/hak.json index d6f5adf47..9902eaa21 100644 --- a/hak/matrix-seshat/hak.json +++ b/hak/matrix-seshat/hak.json @@ -1,13 +1,8 @@ { "scripts": { "check": "check.ts", - "fetchDeps": "fetchDeps.ts", "build": "build.ts" }, - "prune": "native", - "copy": "native/index.node", - "dependencies": { - "openssl": "1.1.1f", - "sqlcipher": "4.3.0" - } + "prune": ".", + "copy": "index.node" } diff --git a/package.json b/package.json index 0e0d793be..b01133259 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "typescript": "5.0.4" }, "hakDependencies": { - "matrix-seshat": "^2.3.3", + "matrix-seshat": "^3.0.0", "keytar": "^7.9.0" }, "resolutions": { diff --git a/scripts/hak/README.md b/scripts/hak/README.md index 04f3676c5..472cd1c2a 100644 --- a/scripts/hak/README.md +++ b/scripts/hak/README.md @@ -62,7 +62,6 @@ Hak is divided into lifecycle stages, in order: - fetch - Download and extract the source of the dependency - link - Link the copy of the dependency into your node_modules directory -- fetchDeps - Fetch & extract any native dependencies required to build the module. - build - The Good Stuff. Configure and build any native dependencies, then the module itself. - copy - Copy the built artifact from the module build directory to the module output directory. diff --git a/scripts/hak/index.ts b/scripts/hak/index.ts index 783c683f5..9e0896bad 100644 --- a/scripts/hak/index.ts +++ b/scripts/hak/index.ts @@ -24,17 +24,17 @@ import { DependencyInfo } from "./dep"; const GENERALCOMMANDS = ["target"]; // These can only be run on specific modules -const MODULECOMMANDS = ["check", "fetch", "link", "fetchDeps", "build", "copy", "clean"]; +const MODULECOMMANDS = ["check", "fetch", "link", "build", "copy", "clean"]; // Shortcuts for multiple commands at once (useful for building universal binaries // because you can run the fetch/fetchDeps/build for each arch and then copy/link once) const METACOMMANDS: Record = { - fetchandbuild: ["check", "fetch", "fetchDeps", "build"], + fetchandbuild: ["check", "fetch", "build"], copyandlink: ["copy", "link"], }; // Scripts valid in a hak.json 'scripts' section -const HAKSCRIPTS = ["check", "fetch", "fetchDeps", "build"]; +const HAKSCRIPTS = ["check", "fetch", "build"]; async function main(): Promise { const prefix = await findNpmPrefix(process.cwd()); @@ -113,7 +113,7 @@ async function main(): Promise { let cmds: string[]; if (process.argv.length < 3) { - cmds = ["check", "fetch", "fetchDeps", "build", "copy", "link"]; + cmds = ["check", "fetch", "build", "copy", "link"]; } else if (METACOMMANDS[process.argv[2]]) { cmds = METACOMMANDS[process.argv[2]]; } else { From 055979f17238956fee04be0899b0b7f3fe605dea Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 10:44:07 +0200 Subject: [PATCH 02/12] remove prune --- hak/matrix-seshat/hak.json | 1 - 1 file changed, 1 deletion(-) diff --git a/hak/matrix-seshat/hak.json b/hak/matrix-seshat/hak.json index 9902eaa21..c0c659270 100644 --- a/hak/matrix-seshat/hak.json +++ b/hak/matrix-seshat/hak.json @@ -3,6 +3,5 @@ "check": "check.ts", "build": "build.ts" }, - "prune": ".", "copy": "index.node" } From 8e4d0a7fb0277feb92de19ea8e81a91aea30cfa6 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 14:38:47 +0200 Subject: [PATCH 03/12] tweak build --- .github/workflows/build_and_test.yaml | 5 ----- .github/workflows/build_linux.yaml | 21 +++++++++++++++------ dockerbuild/Dockerfile | 2 +- hak/matrix-seshat/build.ts | 7 ++++++- scripts/hak/hakEnv.ts | 8 ++------ 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 2f1751e6b..d547591ad 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -79,11 +79,6 @@ jobs: matrix: sqlcipher: [system, static] arch: [amd64, arm64] - exclude: - # FIXME: This combination yields a broken Seshat at this time - # Errors at launch with `undefined symbol: PKCS5_PBKDF2_HMAC - - arch: arm64 - sqlcipher: static with: config: ${{ github.event.pull_request.base.ref == 'develop' && 'element.io/nightly' || 'element.io/release' }} sqlcipher: ${{ matrix.sqlcipher }} diff --git a/.github/workflows/build_linux.yaml b/.github/workflows/build_linux.yaml index 4e8db7268..c68feb7f4 100644 --- a/.github/workflows/build_linux.yaml +++ b/.github/workflows/build_linux.yaml @@ -83,7 +83,7 @@ jobs: - name: Prepare for static sqlcipher build if: inputs.sqlcipher == 'static' run: | - echo "SQLCIPHER_STATIC=1" >> $GITHUB_ENV + echo "SQLCIPHER_BUNDLED=1" >> $GITHUB_ENV # Ideally the docker image would be ready for cross-compilation but libsqlcipher-dev is not Multi-Arch compatible # https://unix.stackexchange.com/a/349359 @@ -150,11 +150,20 @@ jobs: LIBS=$(readelf -d dist/**/*.node | grep NEEDED) echo "$LIBS" - if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then - exit 2 - fi - if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then - exit 3 + if [ "$SQLCIPHER_BUNDLED" == "1" ]; then + if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then + exit 2 + fi + if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then + exit 3 + fi + else + if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then + exit 4 + fi + if ! grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then + exit 5 + fi fi env: ARCH: ${{ steps.config.outputs.arch }} diff --git a/dockerbuild/Dockerfile b/dockerbuild/Dockerfile index 98e098eba..9bcc11c3d 100644 --- a/dockerbuild/Dockerfile +++ b/dockerbuild/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get -qq update && apt-get -qq dist-upgrade && \ libopenjp2-tools \ # Used by github actions \ jq grep file \ - # Used by seshat (when not SQLCIPHER_STATIC) \ + # Used by seshat (when not SQLCIPHER_BUNDLED) \ libsqlcipher-dev && \ # git-lfs git lfs install && \ diff --git a/hak/matrix-seshat/build.ts b/hak/matrix-seshat/build.ts index 19204ef38..c7fddb1ec 100644 --- a/hak/matrix-seshat/build.ts +++ b/hak/matrix-seshat/build.ts @@ -46,11 +46,16 @@ export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Prom }); }); + var buildTarget = "build-bundled"; + if (!hakEnv.wantsDynamicSqlCipherLinux()) { + buildTarget = "build"; + } + console.log("Running yarn build"); await new Promise((resolve, reject) => { const proc = childProcess.spawn( "yarn" + (hakEnv.isWin() ? ".cmd" : ""), - ["run", "build-bundled"], + ["run", buildTarget], { cwd: moduleInfo.moduleBuildDir, env, diff --git a/scripts/hak/hakEnv.ts b/scripts/hak/hakEnv.ts index c63ea3740..e9383b064 100644 --- a/scripts/hak/hakEnv.ts +++ b/scripts/hak/hakEnv.ts @@ -101,11 +101,7 @@ export default class HakEnv { }); } - public wantsStaticSqlCipherUnix(): boolean { - return this.isMac() || process.env.SQLCIPHER_STATIC == "1"; - } - - public wantsStaticSqlCipher(): boolean { - return this.isWin() || this.wantsStaticSqlCipherUnix(); + public wantsDynamicSqlCipherLinux(): boolean { + return this.isLinux() || process.env.SQLCIPHER_BUNDLED == "1"; } } From bf4a515c2db3d57831ba298504a352080fdb27df Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 14:42:25 +0200 Subject: [PATCH 04/12] fix lint --- hak/matrix-seshat/build.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hak/matrix-seshat/build.ts b/hak/matrix-seshat/build.ts index c7fddb1ec..2a72901e2 100644 --- a/hak/matrix-seshat/build.ts +++ b/hak/matrix-seshat/build.ts @@ -14,10 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import path from "path"; import childProcess from "child_process"; -import { mkdirp } from "mkdirp"; -import fsExtra from "fs-extra"; import HakEnv from "../../scripts/hak/hakEnv"; import { DependencyInfo } from "../../scripts/hak/dep"; @@ -46,7 +43,7 @@ export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Prom }); }); - var buildTarget = "build-bundled"; + let buildTarget = "build-bundled"; if (!hakEnv.wantsDynamicSqlCipherLinux()) { buildTarget = "build"; } From bfca1ded0fd5f7f654ebce4557cca4043ec94bb6 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 14:53:06 +0200 Subject: [PATCH 05/12] Remove comment update --- dockerbuild/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerbuild/Dockerfile b/dockerbuild/Dockerfile index 9bcc11c3d..98e098eba 100644 --- a/dockerbuild/Dockerfile +++ b/dockerbuild/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get -qq update && apt-get -qq dist-upgrade && \ libopenjp2-tools \ # Used by github actions \ jq grep file \ - # Used by seshat (when not SQLCIPHER_BUNDLED) \ + # Used by seshat (when not SQLCIPHER_STATIC) \ libsqlcipher-dev && \ # git-lfs git lfs install && \ From e64f56a9e93945d779947c3c4033a31c87c410c1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 Apr 2023 14:53:35 +0100 Subject: [PATCH 06/12] Update build_linux.yaml --- .github/workflows/build_linux.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_linux.yaml b/.github/workflows/build_linux.yaml index c68feb7f4..14f05ae1e 100644 --- a/.github/workflows/build_linux.yaml +++ b/.github/workflows/build_linux.yaml @@ -151,7 +151,7 @@ jobs: echo "$LIBS" if [ "$SQLCIPHER_BUNDLED" == "1" ]; then - if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then + if ! grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then exit 2 fi if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then From 422730c3bd44d944f6746ae9f059878e81887c23 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 16:21:55 +0200 Subject: [PATCH 07/12] mistakes --- .github/workflows/build_linux.yaml | 2 +- hak/matrix-seshat/sqlcipher-4.3.0-win.patch | 14 -------------- scripts/hak/hakEnv.ts | 2 +- 3 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 hak/matrix-seshat/sqlcipher-4.3.0-win.patch diff --git a/.github/workflows/build_linux.yaml b/.github/workflows/build_linux.yaml index 14f05ae1e..c68feb7f4 100644 --- a/.github/workflows/build_linux.yaml +++ b/.github/workflows/build_linux.yaml @@ -151,7 +151,7 @@ jobs: echo "$LIBS" if [ "$SQLCIPHER_BUNDLED" == "1" ]; then - if ! grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then + if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then exit 2 fi if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then diff --git a/hak/matrix-seshat/sqlcipher-4.3.0-win.patch b/hak/matrix-seshat/sqlcipher-4.3.0-win.patch deleted file mode 100644 index d1c61e8b4..000000000 --- a/hak/matrix-seshat/sqlcipher-4.3.0-win.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff -Nur sqlcipher-4.3.0-orig/Makefile.msc sqlcipher-4.3.0-mod/Makefile.msc ---- sqlcipher-4.3.0-orig/Makefile.msc 2019-12-20 16:40:26.000000000 +0000 -+++ sqlcipher-4.3.0-mod/Makefile.msc 2020-02-14 11:31:39.000000000 +0000 -@@ -985,8 +985,8 @@ - # default to file, 2 to default to memory, and 3 to force temporary - # tables to always be in memory. - # --TCC = $(TCC) -DSQLITE_TEMP_STORE=1 --RCC = $(RCC) -DSQLITE_TEMP_STORE=1 -+TCC = $(TCC) -DSQLITE_TEMP_STORE=2 -+RCC = $(RCC) -DSQLITE_TEMP_STORE=2 - - # Enable/disable loadable extensions, and other optional features - # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). diff --git a/scripts/hak/hakEnv.ts b/scripts/hak/hakEnv.ts index e9383b064..131a7101a 100644 --- a/scripts/hak/hakEnv.ts +++ b/scripts/hak/hakEnv.ts @@ -102,6 +102,6 @@ export default class HakEnv { } public wantsDynamicSqlCipherLinux(): boolean { - return this.isLinux() || process.env.SQLCIPHER_BUNDLED == "1"; + return this.isLinux() && process.env.SQLCIPHER_BUNDLED != "1"; } } From 97bc5e9e2f4b08f78d96ec7530636dbf2e0c775b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 Apr 2023 15:45:32 +0100 Subject: [PATCH 08/12] Update build_linux.yaml --- .github/workflows/build_linux.yaml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_linux.yaml b/.github/workflows/build_linux.yaml index c68feb7f4..f09450cc8 100644 --- a/.github/workflows/build_linux.yaml +++ b/.github/workflows/build_linux.yaml @@ -150,20 +150,13 @@ jobs: LIBS=$(readelf -d dist/**/*.node | grep NEEDED) echo "$LIBS" + set +x + assert_contains_string() { [[ "$1" == *"$2"* ]]; } + ! assert_contains_string "$LIBS" "libcrypto.so.1.1" if [ "$SQLCIPHER_BUNDLED" == "1" ]; then - if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then - exit 2 - fi - if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then - exit 3 - fi + ! assert_contains_string "$LIBS" "libsqlcipher.so.0" else - if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then - exit 4 - fi - if ! grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then - exit 5 - fi + assert_contains_string "$LIBS" "libsqlcipher.so.0" fi env: ARCH: ${{ steps.config.outputs.arch }} From 2937f0c7b222f164c825451573013072bb3da44e Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 21 Apr 2023 17:04:59 +0200 Subject: [PATCH 09/12] stupid mistake again --- hak/matrix-seshat/build.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hak/matrix-seshat/build.ts b/hak/matrix-seshat/build.ts index 2a72901e2..204c92bf4 100644 --- a/hak/matrix-seshat/build.ts +++ b/hak/matrix-seshat/build.ts @@ -44,7 +44,7 @@ export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Prom }); let buildTarget = "build-bundled"; - if (!hakEnv.wantsDynamicSqlCipherLinux()) { + if (hakEnv.wantsDynamicSqlCipherLinux()) { buildTarget = "build"; } From 7b8a9d57e94a7ceabcc806d8a5e55e62bbb603b9 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 24 Apr 2023 09:48:20 +0200 Subject: [PATCH 10/12] Do not build arm64 static for now --- .github/workflows/build_and_test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index d547591ad..2f1751e6b 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -79,6 +79,11 @@ jobs: matrix: sqlcipher: [system, static] arch: [amd64, arm64] + exclude: + # FIXME: This combination yields a broken Seshat at this time + # Errors at launch with `undefined symbol: PKCS5_PBKDF2_HMAC + - arch: arm64 + sqlcipher: static with: config: ${{ github.event.pull_request.base.ref == 'develop' && 'element.io/nightly' || 'element.io/release' }} sqlcipher: ${{ matrix.sqlcipher }} From 7f1b7652e60b73d394fe609feb73ba7520259ab3 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 24 Apr 2023 13:27:34 +0200 Subject: [PATCH 11/12] Update doc --- docs/native-node-modules.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/native-node-modules.md b/docs/native-node-modules.md index a5d7f19d5..87efc36a3 100644 --- a/docs/native-node-modules.md +++ b/docs/native-node-modules.md @@ -71,9 +71,9 @@ as usual using: On Windows & macOS we always statically link libsqlcipher for it is not generally available. On Linux by default we will use a system package, on debian & ubuntu this is `libsqlcipher0`, -but this is problematic for some other packages. -By including `SQLCIPHER_STATIC=1` in the build environment, the build scripts will statically link sqlcipher, -note that this will want a `libcrypto1.1` shared library available in the system. +but this is problematic for some other packages, and we found that it may crashes for unknown reasons. +By including `SQLCIPHER_BUNDLED=1` in the build environment, the build scripts will fully statically +link sqlcipher, including a static build of OpenSSL. More info can be found at https://github.com/matrix-org/seshat/issues/102 and https://github.com/vector-im/element-web/issues/20926. From 7e7b587972765d6af1b3dc58290df76a07bd2783 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 24 Apr 2023 13:35:42 +0200 Subject: [PATCH 12/12] Apply suggestions from code review Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --- hak/matrix-seshat/build.ts | 5 +---- scripts/hak/hakEnv.ts | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hak/matrix-seshat/build.ts b/hak/matrix-seshat/build.ts index 204c92bf4..cc389025a 100644 --- a/hak/matrix-seshat/build.ts +++ b/hak/matrix-seshat/build.ts @@ -43,10 +43,7 @@ export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Prom }); }); - let buildTarget = "build-bundled"; - if (hakEnv.wantsDynamicSqlCipherLinux()) { - buildTarget = "build"; - } + const buildTarget = hakEnv.wantsStaticSqlCipher() ? "build-bundled" : "build"; console.log("Running yarn build"); await new Promise((resolve, reject) => { diff --git a/scripts/hak/hakEnv.ts b/scripts/hak/hakEnv.ts index 131a7101a..959786242 100644 --- a/scripts/hak/hakEnv.ts +++ b/scripts/hak/hakEnv.ts @@ -101,7 +101,7 @@ export default class HakEnv { }); } - public wantsDynamicSqlCipherLinux(): boolean { - return this.isLinux() && process.env.SQLCIPHER_BUNDLED != "1"; + public wantsStaticSqlCipher(): boolean { + return !this.isLinux() || process.env.SQLCIPHER_BUNDLED == "1"; } }