Merge remote-tracking branch 'upstream/master'

* upstream/master: (145 commits)
  Sync and remove all non-FOSS parts
  feat(surveys): Add UI for enabling/disabling surveys popup (#17602)
  fix: selecting from cohort people (#17610)
  refactor(hogql): remove persons related params from insight filters (#17612)
  chore(deps): Update posthog-js to 1.81.0 (#17604)
  feat(hogql): add backend side filter to query conversion method (#17542)
  feat: onboarding invite members, other products, verification steps (#17615)
  feat: new onboarding navigation (#17582)
  chore(deps): Update posthog-js to 1.80.0 (#17603)
  feat(surveys): Add opt in migration (#17600)
  chore: App metrics no hub no promise manager (#17581)
  fix: Redis server for replay (#17597)
  fix: Wait for ingester shutdown before redis shutdown (#17594)
  feat: Added option for parallel processing of replay ingestion (#17585)
  feat: Handle potential races in revoking replay partitions (#17578)
  chore: migrate plugins sortable (#17587)
  feat: hogql trends (#17519)
  fix(batch-exports): Do not export site url (#17588)
  feat: bubble menu for link / text editing (#17550)
  feat: Record S3 BatchExport errors (#17535)
  ...

.dockerignore

@@ -33,3 +33,4 @@
 !plugin-server/.prettierrc
 !share/GeoLite2-City.mmdb
 !hogvm/python
+!unit.json

.eslintrc.js

@@ -26,6 +26,7 @@ module.exports = {
         'plugin:eslint-comments/recommended',
         'plugin:storybook/recommended',
         'prettier',
+        'plugin:compat/recommended',
     ],
     globals,
     parser: '@typescript-eslint/parser',
@@ -36,7 +37,7 @@ module.exports = {
         ecmaVersion: 2018,
         sourceType: 'module',
     },
-    plugins: ['prettier', 'react', 'cypress', '@typescript-eslint', 'no-only-tests', 'jest'],
+    plugins: ['prettier', 'react', 'cypress', '@typescript-eslint', 'no-only-tests', 'jest', 'compat'],
     rules: {
         'no-console': ['error', { allow: ['warn', 'error'] }],
         'no-debugger': 'error',
@@ -147,6 +148,10 @@ module.exports = {
                         element: 'a',
                         message: 'use <Link> instead',
                     },
+                    {
+                        element: 'ReactMarkdown',
+                        message: 'use <LemonMarkdown> instead',
+                    },
                 ],
             },
         ],
@@ -175,9 +180,9 @@ module.exports = {
                         message: 'use <LemonCollapse> instead',
                     },
                     {
-                        element:'MonacoEditor',
+                        element: 'MonacoEditor',
                         message: 'use <CodeEditor> instead',
-                    }
+                    },
                 ],
             },
         ],

.github/actions/build-n-cache-image/action.yml

@@ -35,3 +35,15 @@ runs:
               platforms: linux/amd64,linux/arm64
           env:
               ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }}
+
+        - name: Build unit image
+          id: build-unit
+          uses: depot/build-push-action@v1
+          with:
+              buildx-fallback: false # buildx is so slow it's better to just fail
+              load: ${{ inputs.load }}
+              file: production-unit.Dockerfile
+              tags: ${{ steps.emit.outputs.tag }}
+              platforms: linux/amd64
+          env:
+              ACTIONS_ID_TOKEN_REQUEST_URL: ${{ inputs.actions-id-token-request-url }}

.husky/pre-commit

@@ -1,16 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-# Check if staged files contain any added or modified PNGs - skip when merging
-if \
-  git rev-parse -q --verify MERGE_HEAD \
-  && git diff --cached --name-status | grep '^[AM]' | grep -q '.png$'
-then
-  # Error if OptiPNG is not installed
-    if ! command -v optipng >/dev/null; then
-        echo "PNG files must be optimized before being committed, but OptiPNG is not installed! Fix this with \`brew/apt install optipng\`."
-        exit 1
-    fi
-fi
-
 pnpm lint-staged

.storybook/preview.tsx

@@ -15,6 +15,17 @@ const setupMsw = () => {
     // Make sure the msw worker is started
     worker.start({
         quiet: true,
+        onUnhandledRequest(request, print) {
+            // MSW warns on all unhandled requests, but we don't necessarily care
+            const pathAllowList = ['/images/']
+
+            if (pathAllowList.some((path) => request.url.pathname.startsWith(path))) {
+                return
+            }
+
+            // Otherwise, default MSW warning behavior
+            print.warning()
+        },
     })
     ;(window as any).__mockServiceWorker = worker
     ;(window as any).POSTHOG_APP_CONTEXT = getStorybookAppContext()

.storybook/test-runner.ts

@@ -56,6 +56,7 @@ const LOADER_SELECTORS = [
     '.LemonSkeleton',
     '.LemonTableLoader',
     '[aria-busy="true"]',
+    '[aria-label="Content is loading..."]',
     '.SessionRecordingPlayer--buffering',
     '.Lettermark--unknown',
 ]
@@ -72,7 +73,7 @@ module.exports = {
         const storyContext = (await getStoryContext(page, context)) as StoryContext
         const { skip = false, snapshotBrowsers = ['chromium'] } = storyContext.parameters?.testOptions ?? {}
 
-        browserContext.setDefaultTimeout(3000) // Reduce the default timeout from 30 s to 3 s to pre-empt Jest timeouts
+        browserContext.setDefaultTimeout(5000) // Reduce the default timeout from 30 s to 5 s to pre-empt Jest timeouts
         if (!skip) {
             const currentBrowser = browserContext.browser()!.browserType().name() as SupportedBrowserName
             if (snapshotBrowsers.includes(currentBrowser)) {
@@ -208,7 +209,8 @@ async function expectLocatorToMatchStorySnapshot(
         // Compare structural similarity instead of raw pixels - reducing false positives
         // See https://github.com/americanexpress/jest-image-snapshot#recommendations-when-using-ssim-comparison
         comparisonMethod: 'ssim',
-        failureThreshold: 0.0003,
+        // 0.01 would be a 1% difference
+        failureThreshold: 0.01,
         failureThresholdType: 'percent',
     })
 }

bin/copy-posthog-js

@@ -8,3 +8,4 @@ cp node_modules/posthog-js/dist/array.js* frontend/dist/
 cp node_modules/posthog-js/dist/array.full.js* frontend/dist/
 cp node_modules/posthog-js/dist/recorder.js* frontend/dist/
 cp node_modules/posthog-js/dist/recorder-v2.js* frontend/dist/
+cp node_modules/posthog-js/dist/surveys.js* frontend/dist/

bin/docker-server

@@ -11,6 +11,13 @@ trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT
 export PROMETHEUS_METRICS_EXPORT_PORT=8001
 export STATSD_PORT=${STATSD_PORT:-8125}
 
+if [[ -n $INJECT_EC2_CLIENT_RACK ]]; then
+  # To avoid cross-AZ Kafka traffic, set KAFKA_CLIENT_RACK from the EC2 metadata endpoint.
+  # TODO: switch to the downwards API when https://github.com/kubernetes/kubernetes/issues/40610 is released
+  TOKEN=$(curl --max-time 0.1 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+  export KAFKA_CLIENT_RACK=$(curl --max-time 0.1 -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/placement/availability-zone-id)
+fi
+
 exec gunicorn posthog.wsgi \
     --config gunicorn.config.py \
     --bind 0.0.0.0:8000 \
@@ -20,6 +27,7 @@ exec gunicorn posthog.wsgi \
     --worker-tmp-dir /dev/shm \
     --workers=2 \
     --threads=8 \
+    --backlog=${GUNICORN_BACKLOG:-1000} \
     --worker-class=gthread \
     ${STATSD_HOST:+--statsd-host $STATSD_HOST:$STATSD_PORT} \
     --limit-request-line=16384 $@

bin/docker-server-unit

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# To ensure we are able to expose metrics from multiple processes, we need to
+# provide a directory for `prometheus_client` to store a shared registry.
+export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d)
+chmod -R 777 $PROMETHEUS_MULTIPROC_DIR
+trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT
+
+export PROMETHEUS_METRICS_EXPORT_PORT=8001
+export STATSD_PORT=${STATSD_PORT:-8125}
+
+exec /usr/local/bin/docker-entrypoint.sh unitd --no-daemon

bin/plugin-server

@@ -46,7 +46,15 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
-[[ -n $DEBUG ]] && cmd="pnpm start:dev" || cmd="node dist/index.js"
+if [[ -n $DEBUG ]]; then
+  if [[ -n $NO_WATCH ]]; then
+    cmd="pnpm start:devNoWatch"
+  else
+    cmd="pnpm start:dev"
+  fi
+else
+  cmd="node dist/index.js"
+fi
 
 if [[ -n $NO_RESTART_LOOP ]]; then
   echo "▶️ Starting plugin server..."

cypress/e2e/auth-password-reset.cy.ts

@@ -0,0 +1,54 @@
+describe('Password Reset', () => {
+    beforeEach(() => {
+        cy.get('[data-attr=top-menu-toggle]').click()
+        cy.get('[data-attr=top-menu-item-logout]').click()
+        cy.location('pathname').should('eq', '/login')
+    })
+
+    it('Can request password reset', () => {
+        cy.get('[data-attr=login-email]').type('[email protected]').should('have.value', '[email protected]').blur()
+        cy.get('[data-attr=forgot-password]', { timeout: 5000 }).should('be.visible') // Wait for login precheck (note blur above)
+        cy.get('[data-attr="forgot-password"]').click()
+        cy.location('pathname').should('eq', '/reset')
+        cy.get('[data-attr="reset-email"]').type('[email protected]')
+        cy.get('button[type=submit]').click()
+        cy.get('div').should('contain', 'Request received successfully!')
+        cy.get('b').should('contain', '[email protected]')
+    })
+
+    it('Cannot reset with invalid token', () => {
+        cy.visit('/reset/user_id/token')
+        cy.get('div').should('contain', 'The provided link is invalid or has expired. ')
+    })
+
+    it('Shows validation error if passwords do not match', () => {
+        cy.visit('/reset/e2e_test_user/e2e_test_token')
+        cy.get('[data-attr="password"]').type('12345678')
+        cy.get('.ant-progress-bg').should('be.visible')
+        cy.get('[data-attr="password-confirm"]').type('1234567A')
+        cy.get('button[type=submit]').click()
+        cy.get('.text-danger').should('contain', 'Passwords do not match')
+        cy.location('pathname').should('eq', '/reset/e2e_test_user/e2e_test_token') // not going anywhere
+    })
+
+    it('Shows validation error if password is too short', () => {
+        cy.visit('/reset/e2e_test_user/e2e_test_token')
+        cy.get('[data-attr="password"]').type('123')
+        cy.get('[data-attr="password-confirm"]').type('123')
+        cy.get('button[type=submit]').click()
+        cy.get('.text-danger').should('be.visible')
+        cy.get('.text-danger').should('contain', 'must be at least 8 characters')
+        cy.location('pathname').should('eq', '/reset/e2e_test_user/e2e_test_token') // not going anywhere
+    })
+
+    it('Can reset password with valid token', () => {
+        cy.visit('/reset/e2e_test_user/e2e_test_token')
+        cy.get('[data-attr="password"]').type('NEW123456789')
+        cy.get('[data-attr="password-confirm"]').type('NEW123456789')
+        cy.get('button[type=submit]').click()
+        cy.get('.Toastify__toast--success').should('be.visible')
+
+        // assert the user was redirected; can't test actual redirection to /insights because the test handler doesn't actually log in the user
+        cy.location('pathname').should('not.contain', '/reset/e2e_test_user/e2e_test_token')
+    })
+})

cypress/e2e/auth.cy.ts

@@ -87,58 +87,3 @@ describe('Auth', () => {
         cy.location('pathname').should('eq', urls.projectHomepage())
     })
 })
-
-describe('Password Reset', () => {
-    beforeEach(() => {
-        cy.get('[data-attr=top-menu-toggle]').click()
-        cy.get('[data-attr=top-menu-item-logout]').click()
-        cy.location('pathname').should('eq', '/login')
-    })
-
-    it('Can request password reset', () => {
-        cy.get('[data-attr=login-email]').type('[email protected]').should('have.value', '[email protected]').blur()
-        cy.get('[data-attr=forgot-password]', { timeout: 5000 }).should('be.visible') // Wait for login precheck (note blur above)
-        cy.get('[data-attr="forgot-password"]').click()
-        cy.location('pathname').should('eq', '/reset')
-        cy.get('[data-attr="reset-email"]').type('[email protected]')
-        cy.get('button[type=submit]').click()
-        cy.get('div').should('contain', 'Request received successfully!')
-        cy.get('b').should('contain', '[email protected]')
-    })
-
-    it('Cannot reset with invalid token', () => {
-        cy.visit('/reset/user_id/token')
-        cy.get('div').should('contain', 'The provided link is invalid or has expired. ')
-    })
-
-    it('Shows validation error if passwords do not match', () => {
-        cy.visit('/reset/e2e_test_user/e2e_test_token')
-        cy.get('[data-attr="password"]').type('12345678')
-        cy.get('.ant-progress-bg').should('be.visible')
-        cy.get('[data-attr="password-confirm"]').type('1234567A')
-        cy.get('button[type=submit]').click()
-        cy.get('.text-danger').should('contain', 'Passwords do not match')
-        cy.location('pathname').should('eq', '/reset/e2e_test_user/e2e_test_token') // not going anywhere
-    })
-
-    it('Shows validation error if password is too short', () => {
-        cy.visit('/reset/e2e_test_user/e2e_test_token')
-        cy.get('[data-attr="password"]').type('123')
-        cy.get('[data-attr="password-confirm"]').type('123')
-        cy.get('button[type=submit]').click()
-        cy.get('.text-danger').should('be.visible')
-        cy.get('.text-danger').should('contain', 'must be at least 8 characters')
-        cy.location('pathname').should('eq', '/reset/e2e_test_user/e2e_test_token') // not going anywhere
-    })
-
-    it('Can reset password with valid token', () => {
-        cy.visit('/reset/e2e_test_user/e2e_test_token')
-        cy.get('[data-attr="password"]').type('NEW123456789')
-        cy.get('[data-attr="password-confirm"]').type('NEW123456789')
-        cy.get('button[type=submit]').click()
-        cy.get('.Toastify__toast--success').should('be.visible')
-
-        // assert the user was redirected; can't test actual redirection to /insights because the test handler doesn't actually log in the user
-        cy.location('pathname').should('not.contain', '/reset/e2e_test_user/e2e_test_token')
-    })
-})