(core) updates from grist-core

.github/workflows/docker.yml

@@ -19,7 +19,7 @@ env:
 jobs:
   push_to_registry:
     name: Push Docker images to Docker Hub
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         image:

.github/workflows/docker_latest.yml

@@ -45,12 +45,12 @@ env:
 jobs:
   push_to_registry:
     name: Push latest Docker image to Docker Hub
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: ${{ vars.RUN_DAILY_BUILD }}
     strategy:
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         image:
           # We build two images, `grist-oss` and `grist`.
           # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images
@@ -169,7 +169,7 @@ jobs:
 
   update_latest_branch:
     name: Update latest branch
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: push_to_registry
     steps:
       - name: Check out the repo

.github/workflows/fly-build.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   build:
     name: Build Docker image
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     # Build when the 'preview' label is added, or when PR is updated with this label present.
     if: >
       github.event_name == 'workflow_dispatch' ||

.github/workflows/fly-cleanup.yml

@@ -13,7 +13,7 @@ env:
 jobs:
   clean:
     name: Clean stale deployed apps
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: github.repository_owner == 'gristlabs'
     steps:
       - uses: actions/checkout@v3

.github/workflows/fly-deploy.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   deploy:
     name: Deploy app to fly.io
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: |
       github.event.workflow_run.event == 'pull_request' &&
       github.event.workflow_run.conclusion == 'success'

.github/workflows/fly-destroy.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   destroy:
     name: Remove app from fly.io
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     # Remove the deployment when 'preview' label is removed, or the PR is closed.
     if: |
       github.event_name == 'workflow_dispatch' ||

.github/workflows/main.yml

@@ -11,26 +11,26 @@ on:
 
 jobs:
   build_and_test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       # it is helpful to know which sets of tests would have succeeded,
       # even when there is a failure.
       fail-fast: false
       matrix:
         python-version: [3.11]
-        node-version: [18.x]
+        node-version: [22.x]
         tests:
           - ':lint:python:client:common:smoke:stubs:'
           - ':server-1-of-2:'
           - ':server-2-of-2:'
-          - ':nbrowser-^[A-G]:'
-          - ':nbrowser-^[H-L]:'
-          - ':nbrowser-^[M-O]:'
-          - ':nbrowser-^[P-S]:'
-          - ':nbrowser-^[^A-S]:'
+          - ':nbrowser-^[A-D]:'
+          - ':nbrowser-^[E-L]:'
+          - ':nbrowser-^[M-N]:'
+          - ':nbrowser-^[O-R]:'
+          - ':nbrowser-^[^A-R]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 18.x
+            node-version: 22.x
             python-version: '3.10'
     steps:
       - uses: actions/checkout@v3
@@ -125,6 +125,7 @@ jobs:
           ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g')
           echo "Artifact name is '$ARTIFACT_NAME'"
           echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV
+          mkdir -p $TESTDIR
           find $TESTDIR -iname "*.socket" -exec rm {} \;
         env:
           TESTS: ${{ matrix.tests }}
@@ -166,7 +167,7 @@ jobs:
   candidate:
     needs: build_and_test
     if: ${{ success() && github.event_name == 'push' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Fetch new candidate branch
         uses: actions/checkout@v3

.github/workflows/self-hosted.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   add-to-project:
     name: Add issue to project
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/[email protected]
         with:

.github/workflows/translation_keys.yml

@@ -12,7 +12,7 @@ permissions:
 jobs:
   build:
     if: github.repository_owner == 'gristlabs'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
         with:

Dockerfile

@@ -45,19 +45,30 @@ RUN \
 ## Python collection stage
 ################################################################################
 
-# Fetch python3.11 and python2.7
-FROM python:3.11-slim-buster AS collector
+# Fetch python3.11
+FROM python:3.11-slim-buster AS collector-py3
+ADD sandbox/requirements3.txt requirements3.txt
+RUN \
+  pip3 install -r requirements3.txt
 
-# Install all python dependencies.
+# Fetch <shame>python2.7</shame>
+# This is to support users with old documents.
+# If you have documents with python2.7 formulas, try switching
+# to python3 in the document settings. It'll probably work fine!
+# And we'll be forced to turn off python2 support eventually,
+# the workarounds needed to keep it are getting silly.
+# It doesn't exist in recent Debian, so we need to reach back
+# to buster.
+FROM python:2.7-slim-buster AS collector-py2
 ADD sandbox/requirements.txt requirements.txt
-ADD sandbox/requirements3.txt requirements3.txt
 RUN \
   apt update && \
   apt install -y --no-install-recommends python2 python-pip python-setuptools \
   build-essential libxml2-dev libxslt-dev python-dev zlib1g-dev && \
   pip2 install wheel && \
   pip2 install -r requirements.txt && \
-  pip3 install -r requirements3.txt
+  pip2 install six && \
+  find /usr/lib -iname "libffi.so.6*" -exec cp {} /usr/local/lib \;
 
 ################################################################################
 ## Sandbox collection stage
@@ -66,6 +77,8 @@ RUN \
 # Fetch gvisor-based sandbox. Note, to enable it to run within default
 # unprivileged docker, layers of protection that require privilege have
 # been stripped away, see https://github.com/google/gvisor/issues/4371
+# The sandbox binary is built on buster, but remains compatible with recent
+# Debian.
 FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 
 ################################################################################
@@ -91,13 +104,23 @@ COPY --from=builder /grist/node_modules /grist/node_modules
 COPY --from=builder /grist/_build /grist/_build
 COPY --from=builder /grist/static /grist/static-built
 
-# Copy python files.
-COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7
-COPY --from=collector /usr/lib/python2.7 /usr/lib/python2.7
-COPY --from=collector /usr/local/lib/python2.7 /usr/local/lib/python2.7
-COPY --from=collector /usr/local/bin/python3.11 /usr/bin/python3.11
-COPY --from=collector /usr/local/lib/python3.11 /usr/local/lib/python3.11
-COPY --from=collector /usr/local/lib/libpython3.11.* /usr/local/lib/
+# Copy python2 files.
+COPY --from=collector-py2 /usr/bin/python2.7 /usr/bin/python2.7
+COPY --from=collector-py2 /usr/lib/python2.7 /usr/lib/python2.7
+COPY --from=collector-py2 /usr/local/lib/python2.7 /usr/local/lib/python2.7
+# Make a small python2 tweak so that material in /usr/local/lib is found.
+RUN \
+  mkdir /etc/python2.7 && \
+  echo "import sys\nsys.path.append('/usr/local/lib/python2.7/site-packages')" > /etc/python2.7/sitecustomize.py
+# Copy across an older libffi library binary needed by python2.
+# We moved it a bit sleazily to a predictable location to avoid awkward
+# architecture-dependent logic.
+COPY --from=collector-py2 /usr/local/lib/libffi.so.6* /usr/local/lib
+
+# Copy python3 files.
+COPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11
+COPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11
+COPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/
 # Set default to python3
 RUN \
   ln -s /usr/bin/python3.11 /usr/bin/python && \

README.md

@@ -262,6 +262,8 @@ Grist can be configured in many ways. Here are the main environment variables it
 | APP_STATIC_URL                     | url prefix for static resources                                                                                                                                                                                                                                                                                                                               |
 | APP_STATIC_INCLUDE_CUSTOM_CSS      | set to "true" to include custom.css (from APP_STATIC_URL) in static pages                                                                                                                                                                                                                                                                                     |
 | APP_UNTRUSTED_URL                  | URL at which to serve/expect plugin content.                                                                                                                                                                                                                                                                                                                  |
+| GRIST_ACTION_HISTORY_MAX_ROWS | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1000. ⚠️ A too low value may make the "[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) |
+| GRIST_ACTION_HISTORY_MAX_BYTES | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1Gb. ⚠️ A too low value may make the "[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) |
 | GRIST_ADAPT_DOMAIN                 | set to "true" to support multiple base domains (careful, host header should be trustworthy)                                                                                                                                                                                                                                                                   |
 | GRIST_APP_ROOT                     | directory containing Grist sandbox and assets (specifically the sandbox and static subdirectories).                                                                                                                                                                                                                                                           |
 | GRIST_BACKUP_DELAY_SECS            | wait this long after a doc change before making a backup                                                                                                                                                                                                                                                                                                      |

app/client/aclui/AccessRules.ts

@@ -1084,6 +1084,16 @@ abstract class ObsRuleSet extends Disposable {
   public getCustomRules(): ObsRulePart[] {
     return this._body.get().filter(rule => !rule.isBuiltInOrEmpty());
   }
+
+  /**
+   * If the set applies to a special column, return its name.
+   */
+  public getSpecialColumn(): string|undefined {
+    if (this._ruleSet?.tableId === SPECIAL_RULES_TABLE_ID &&
+        this._ruleSet.colIds.length === 1) {
+      return this._ruleSet.colIds[0];
+    }
+  }
 }
 
 class ColumnObsRuleSet extends ObsRuleSet {
@@ -1635,6 +1645,14 @@ class ObsRulePart extends Disposable {
         !isEqual(use(this._permissions), this._rulePart?.permissions ?? emptyPerms)
       );
     });
+    // The formula may be invalid from the beginning. Make sure we show errors in this
+    // case.
+    const text = this._aclFormula.get();
+    if (text) {
+      this._setAclFormula(text, true).catch(e => {
+        console.error(e);
+      });
+    }
   }
 
   public getRulePart(): RuleRec {
@@ -1790,8 +1808,8 @@ class ObsRulePart extends Disposable {
     return this.isBuiltIn() && this._ruleSet.getFirstBuiltIn() !== this;
   }
 
-  private async _setAclFormula(text: string) {
-    if (text === this._aclFormula.get()) { return; }
+  private async _setAclFormula(text: string, initial: boolean = false) {
+    if (text === this._aclFormula.get() && !initial) { return; }
     this._aclFormula.set(text);
     this._checkPending.set(true);
     this._formulaProperties.set({});
@@ -1809,6 +1827,12 @@ class ObsRulePart extends Disposable {
   private _warnInvalidColIds(colIds?: string[]) {
     if (!colIds || !colIds.length) { return false; }
     const allValid = new Set(this._ruleSet.getValidColIds());
+    const specialColumn = this._ruleSet.getSpecialColumn();
+    if (specialColumn === 'SeedRule') {
+      // We allow seed rules to refer to columns without checking
+      // them (until the seed rules are used).
+      return false;
+    }
     const invalid = colIds.filter(c => !allValid.has(c));
     if (invalid.length > 0) {
       return `Invalid columns: ${invalid.join(', ')}`;

app/client/components/GristClientSocket.ts

@@ -71,7 +71,13 @@ export class GristClientSocket {
   }
 
   private _createWSSocket() {
-    if (typeof WebSocket !== 'undefined') {
+    // We used to check if WebSocket was defined here, and use it
+    // if so, secure in the fact that we were in the browser and
+    // the browser would pass along cookie information. But recent
+    // node defines WebSocket, so we narrow down this path to when
+    // a global document is defined (window doesn't work because
+    // some tests mock it).
+    if (typeof document !== 'undefined') {
       this._wsSocket = new WebSocket(this._url);
     } else {
       this._wsSocket = new WS(this._url, undefined, this._options);

app/server/lib/ActionHistoryImpl.ts

@@ -12,14 +12,28 @@ import mapValues = require('lodash/mapValues');
 import {ActionGroupOptions, ActionHistory, ActionHistoryUndoInfo, asActionGroup,
         asMinimalActionGroup} from './ActionHistory';
 import {ISQLiteDB, ResultRow} from './SQLiteDB';
+import { appSettings } from './AppSettings';
+
+const section = appSettings.section('history').section('action');
 
 // History will from time to time be pruned back to within these limits
 // on rows and the maximum total number of bytes in the "body" column.
 // Pruning is done when the history has grown above these limits, to
 // the specified factor.
-const ACTION_HISTORY_MAX_ROWS = 1000;
-const ACTION_HISTORY_MAX_BYTES = 1000 * 1000 * 1000;  // 1 GB.
-const ACTION_HISTORY_GRACE_FACTOR = 1.25;  // allow growth to 1250 rows / 1.25 GB.
+const ACTION_HISTORY_MAX_ROWS = section.flag('maxRows').requireInt({
+  envVar: 'GRIST_ACTION_HISTORY_MAX_ROWS',
+  defaultValue: 1000,
+
+  minValue: 1,
+});
+
+const ACTION_HISTORY_MAX_BYTES = section.flag('maxBytes').requireInt({
+  envVar: 'GRIST_ACTION_HISTORY_MAX_BYTES',
+  defaultValue: 1e9, // 1 GB.
+  minValue: 1,  // 1 B.
+});
+
+const ACTION_HISTORY_GRACE_FACTOR = 1.25;  // allow growth to 1.25 times the above limits.
 const ACTION_HISTORY_CHECK_PERIOD = 10;    // number of actions between size checks.
 
 /**