fix: Handle 429s in the client (#1830)

Fixes #1770

With #1787 we've managed to
return 429s whenever there's too many concurrent shape creations that
cause the database connection pool to be exhausted.

This PR just ensures that the client does indeed retry on 429s - for now
just with our regular exponential backoff, as there is no standard for
retry headers to respect.

P.S. additional changes to the openapi spec done by my formatter 👀 I can
roll them back if you think they are worse than before

.changeset/curly-ravens-film.md

@@ -0,0 +1,5 @@
+---
+"@electric-sql/client": patch
+---
+
+Handle 429 responses with retries.

packages/typescript-client/src/fetch.ts

@@ -8,6 +8,10 @@ import {
 } from './constants'
 import { FetchError, FetchBackoffAbortError } from './error'
 
+// Some specific 4xx and 5xx HTTP status codes that we definitely
+// want to retry
+const HTTP_RETRY_STATUS_CODES = [429]
+
 export interface BackoffOptions {
   /**
    * Initial delay before retrying in milliseconds
@@ -63,6 +67,7 @@ export function createFetchWithBackoff(
           throw new FetchBackoffAbortError()
         } else if (
           e instanceof FetchError &&
+          !HTTP_RETRY_STATUS_CODES.includes(e.status) &&
           e.status >= 400 &&
           e.status < 500
         ) {

packages/typescript-client/test/fetch.test.ts

@@ -51,6 +51,27 @@ describe(`createFetchWithBackoff`, () => {
     expect(result.ok).toBe(true)
   })
 
+  it(`should retry the request on a 429 response and succeed after a retry`, async () => {
+    const mockErrorResponse = new Response(null, { status: 429 })
+    const mockSuccessResponse = new Response(null, {
+      status: 200,
+      statusText: `OK`,
+    })
+    mockFetchClient
+      .mockResolvedValueOnce(mockErrorResponse)
+      .mockResolvedValueOnce(mockSuccessResponse)
+
+    const fetchWithBackoff = createFetchWithBackoff(mockFetchClient, {
+      ...BackoffDefaults,
+      initialDelay,
+    })
+
+    const result = await fetchWithBackoff(`https://example.com`)
+
+    expect(mockFetchClient).toHaveBeenCalledTimes(2)
+    expect(result.ok).toBe(true)
+  })
+
   it(`should apply exponential backoff and retry until maxDelay is reached`, async () => {
     const mockErrorResponse = new Response(null, { status: 500 })
     const mockSuccessResponse = new Response(null, {

website/electric-api.yaml

@@ -108,10 +108,10 @@ paths:
             This should be a valid PostgreSQL WHERE clause using SQL syntax.
           examples:
             title_filter:
-              value: "\"title='Electric'\""
+              value: '"title=''Electric''"'
               summary: Only include rows where the title is 'Electric'.
             status_filter:
-              value: "\"status IN ('backlog', 'todo')\""
+              value: '"status IN (''backlog'', ''todo'')"'
               summary: Only include rows whose status is either 'backlog' or 'todo'.
         # Headers
         - name: If-None-Match
@@ -121,7 +121,7 @@ paths:
           # TODO: is this description below correct?
           description: Re-validate the shape if the etag doesn't match.
       responses:
-        '200':
+        "200":
           description: The shape request was successful.
           headers:
             cache-control:
@@ -163,7 +163,7 @@ paths:
             x-electric-schema:
               schema:
                 type: string
-                example: "{\"id\":{\"type\":\"int4\",\"dimensions\":0},\"title\":{\"type\":\"text\",\"dimensions\":0},\"status\":{\"type\":\"text\",\"dimensions\":0,\"max_length\":8}}"
+                example: '{"id":{"type":"int4","dimensions":0},"title":{"type":"text","dimensions":0},"status":{"type":"text","dimensions":0,"max_length":8}}'
                 description: |-
                   A JSON string of an object that maps column names to the corresponding schema object.
                   The schema object contains the type of the column, the number of dimensions, and possibly additional properties.
@@ -206,8 +206,7 @@ paths:
                             - insert
                             - update
                             - delete
-                          description:
-                            The type of operation that is performed on the row of the shape that is identified by the `key`.
+                          description: The type of operation that is performed on the row of the shape that is identified by the `key`.
                     offset:
                       type: string
                       description: |-
@@ -234,10 +233,10 @@ paths:
                         - for inserts it will contain the whole row
                         - for updates it will contain the primary key and the changed values
                         - for deletes it will contain just the primary key
-                        
+
                         The values are strings that are formatted according to Postgres' display settings.
                         Some Postgres types support several display settings, we format values consistently according to the following display settings:
-                        
+
                         - `bytea_output = 'hex'`
                         - `DateStyle = 'ISO, DMY'`
                         - `TimeZone = 'UTC'`
@@ -261,13 +260,13 @@ paths:
                       id: issue-2
                       title: Hello
                       status: backlog
-        '204':
+        "204":
           description: >-
             No content. The `live=true` polling request timed out without
             any new content to process.
-        '400':
+        "400":
           description: Bad request.
-        '409':
+        "409":
           description:
             The requested offset for the given shape no longer exists.
             Client should sync the shape using the relative path from the location header.
@@ -294,6 +293,21 @@ paths:
                   message: "The shape associated with this shape_id and offset was not found. Resync to fetch the latest shape"
                   shape_id: "2494_84241"
                   offset: "-1"
+        "429":
+          description:
+            Too many requests. The server is busy with other requests, potentially
+            because of high contention on the underlying database. Retry after a little time.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    description: Error message
+                example:
+                  message: "Could not establish connection to database - try again later"
+
     delete:
       summary: Delete Shape
       description: |-
@@ -331,11 +345,11 @@ paths:
             Optional, deletes the current shape if it matches the shape_id.
             If not provided, deletes the current shape.
       responses:
-        '202':
+        "202":
           description: |-
             Accepted. The shape has been deleted (or to be more precise: the shape ID
             has been invalidated and the storage will be cleaned up eventually).
-        '400':
+        "400":
           description: Bad request.
-        '404':
+        "404":
           description: Not found (or shape deletion is not enabled).