From c0c9af69bffce4d414bfed3a0b8856462a675e59 Mon Sep 17 00:00:00 2001 From: Stefanos Mousafeiris Date: Thu, 10 Oct 2024 18:04:28 +0300 Subject: [PATCH] fix: Handle 429s in the client (#1830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/electric-sql/electric/issues/1770 With https://github.com/electric-sql/electric/pull/1787 we've managed to return 429s whenever there's too many concurrent shape creations that cause the database connection pool to be exhausted. This PR just ensures that the client does indeed retry on 429s - for now just with our regular exponential backoff, as there is no standard for retry headers to respect. P.S. additional changes to the openapi spec done by my formatter 👀 I can roll them back if you think they are worse than before --- .changeset/curly-ravens-film.md | 5 +++ packages/typescript-client/src/fetch.ts | 5 +++ packages/typescript-client/test/fetch.test.ts | 21 ++++++++++ website/electric-api.yaml | 42 ++++++++++++------- 4 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 .changeset/curly-ravens-film.md diff --git a/.changeset/curly-ravens-film.md b/.changeset/curly-ravens-film.md new file mode 100644 index 0000000000..d796f9ba4e --- /dev/null +++ b/.changeset/curly-ravens-film.md @@ -0,0 +1,5 @@ +--- +"@electric-sql/client": patch +--- + +Handle 429 responses with retries. diff --git a/packages/typescript-client/src/fetch.ts b/packages/typescript-client/src/fetch.ts index 516c918d79..f14311b500 100644 --- a/packages/typescript-client/src/fetch.ts +++ b/packages/typescript-client/src/fetch.ts @@ -8,6 +8,10 @@ import { } from './constants' import { FetchError, FetchBackoffAbortError } from './error' +// Some specific 4xx and 5xx HTTP status codes that we definitely +// want to retry +const HTTP_RETRY_STATUS_CODES = [429] + export interface BackoffOptions { /** * Initial delay before retrying in milliseconds @@ -63,6 +67,7 @@ export function createFetchWithBackoff( throw new FetchBackoffAbortError() } else if ( e instanceof FetchError && + !HTTP_RETRY_STATUS_CODES.includes(e.status) && e.status >= 400 && e.status < 500 ) { diff --git a/packages/typescript-client/test/fetch.test.ts b/packages/typescript-client/test/fetch.test.ts index f160a7a5af..cb04445ab5 100644 --- a/packages/typescript-client/test/fetch.test.ts +++ b/packages/typescript-client/test/fetch.test.ts @@ -51,6 +51,27 @@ describe(`createFetchWithBackoff`, () => { expect(result.ok).toBe(true) }) + it(`should retry the request on a 429 response and succeed after a retry`, async () => { + const mockErrorResponse = new Response(null, { status: 429 }) + const mockSuccessResponse = new Response(null, { + status: 200, + statusText: `OK`, + }) + mockFetchClient + .mockResolvedValueOnce(mockErrorResponse) + .mockResolvedValueOnce(mockSuccessResponse) + + const fetchWithBackoff = createFetchWithBackoff(mockFetchClient, { + ...BackoffDefaults, + initialDelay, + }) + + const result = await fetchWithBackoff(`https://example.com`) + + expect(mockFetchClient).toHaveBeenCalledTimes(2) + expect(result.ok).toBe(true) + }) + it(`should apply exponential backoff and retry until maxDelay is reached`, async () => { const mockErrorResponse = new Response(null, { status: 500 }) const mockSuccessResponse = new Response(null, { diff --git a/website/electric-api.yaml b/website/electric-api.yaml index 4c5d924f87..3ecaaa0150 100644 --- a/website/electric-api.yaml +++ b/website/electric-api.yaml @@ -108,10 +108,10 @@ paths: This should be a valid PostgreSQL WHERE clause using SQL syntax. examples: title_filter: - value: "\"title='Electric'\"" + value: '"title=''Electric''"' summary: Only include rows where the title is 'Electric'. status_filter: - value: "\"status IN ('backlog', 'todo')\"" + value: '"status IN (''backlog'', ''todo'')"' summary: Only include rows whose status is either 'backlog' or 'todo'. # Headers - name: If-None-Match @@ -121,7 +121,7 @@ paths: # TODO: is this description below correct? description: Re-validate the shape if the etag doesn't match. responses: - '200': + "200": description: The shape request was successful. headers: cache-control: @@ -163,7 +163,7 @@ paths: x-electric-schema: schema: type: string - example: "{\"id\":{\"type\":\"int4\",\"dimensions\":0},\"title\":{\"type\":\"text\",\"dimensions\":0},\"status\":{\"type\":\"text\",\"dimensions\":0,\"max_length\":8}}" + example: '{"id":{"type":"int4","dimensions":0},"title":{"type":"text","dimensions":0},"status":{"type":"text","dimensions":0,"max_length":8}}' description: |- A JSON string of an object that maps column names to the corresponding schema object. The schema object contains the type of the column, the number of dimensions, and possibly additional properties. @@ -206,8 +206,7 @@ paths: - insert - update - delete - description: - The type of operation that is performed on the row of the shape that is identified by the `key`. + description: The type of operation that is performed on the row of the shape that is identified by the `key`. offset: type: string description: |- @@ -234,10 +233,10 @@ paths: - for inserts it will contain the whole row - for updates it will contain the primary key and the changed values - for deletes it will contain just the primary key - + The values are strings that are formatted according to Postgres' display settings. Some Postgres types support several display settings, we format values consistently according to the following display settings: - + - `bytea_output = 'hex'` - `DateStyle = 'ISO, DMY'` - `TimeZone = 'UTC'` @@ -261,13 +260,13 @@ paths: id: issue-2 title: Hello status: backlog - '204': + "204": description: >- No content. The `live=true` polling request timed out without any new content to process. - '400': + "400": description: Bad request. - '409': + "409": description: The requested offset for the given shape no longer exists. Client should sync the shape using the relative path from the location header. @@ -294,6 +293,21 @@ paths: message: "The shape associated with this shape_id and offset was not found. Resync to fetch the latest shape" shape_id: "2494_84241" offset: "-1" + "429": + description: + Too many requests. The server is busy with other requests, potentially + because of high contention on the underlying database. Retry after a little time. + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: Error message + example: + message: "Could not establish connection to database - try again later" + delete: summary: Delete Shape description: |- @@ -331,11 +345,11 @@ paths: Optional, deletes the current shape if it matches the shape_id. If not provided, deletes the current shape. responses: - '202': + "202": description: |- Accepted. The shape has been deleted (or to be more precise: the shape ID has been invalidated and the storage will be cleaned up eventually). - '400': + "400": description: Bad request. - '404': + "404": description: Not found (or shape deletion is not enabled).