Skip to content

Commit

Permalink
feat: cache 404s indefinitely for bot probes (#2668)
Browse files Browse the repository at this point in the history 
* feat: cache 404s indefinitely for bot probes

* test: update unit tests

* chore: refactor to clean up function signature with object parameterisation
  • Loading branch information
@orinokai
orinokai authored Oct 15, 2024
1 parent 0ab5748 commit 60885d2
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 85 deletions.
2 changes: 1 addition & 1 deletion src/run/handlers/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ export default async (request: Request, context: FutureContext) => {

await adjustDateHeader({ headers: response.headers, request, span, tracer, requestContext })

setCacheControlHeaders(response.headers, request, requestContext)
setCacheControlHeaders(response, request, requestContext)
setCacheTagsHeaders(response.headers, requestContext)
setVaryHeaders(response.headers, request, nextConfig)
setCacheStatusHeader(response.headers)
Expand Down
188 changes: 105 additions & 83 deletions src/run/headers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,43 +199,43 @@ describe('headers', () => {
const givenHeaders = {
'cdn-cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should not set any headers if "netlify-cdn-cache-control" is present', () => {
const givenHeaders = {
'netlify-cdn-cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should mark content as stale if "{netlify-,}cdn-cache-control" is not present and "x-nextjs-cache" is "STALE" (GET)', () => {
const givenHeaders = {
'x-nextjs-cache': 'STALE',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(1)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenCalledTimes(1)
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'netlify-cdn-cache-control',
'public, max-age=0, must-revalidate, durable',
Expand All @@ -246,107 +246,129 @@ describe('headers', () => {
const givenHeaders = {
'x-nextjs-cache': 'STALE',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(1)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenCalledTimes(1)
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'netlify-cdn-cache-control',
'public, max-age=0, must-revalidate, durable',
)
})

test('should set durable SWC=1yr with 1yr TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is `false` (HEAD)', () => {
const headers = new Headers()
const request = new Request(defaultUrl, { method: 'HEAD' })
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(1)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenCalledTimes(1)
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'netlify-cdn-cache-control',
's-maxage=31536000, stale-while-revalidate=31536000, durable',
)
})

test('should set durable SWC=1yr with given TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is a number (GET)', () => {
const headers = new Headers()
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: 7200 }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(1)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenCalledTimes(1)
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'netlify-cdn-cache-control',
's-maxage=7200, stale-while-revalidate=31536000, durable',
)
})

test('should set durable SWC=1yr with 1yr TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is a number (HEAD)', () => {
const headers = new Headers()
const request = new Request(defaultUrl, { method: 'HEAD' })
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: 7200 }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(1)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenCalledTimes(1)
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'netlify-cdn-cache-control',
's-maxage=7200, stale-while-revalidate=31536000, durable',
)
})

test('should not set any headers on POST request', () => {
const headers = new Headers()
const request = new Request(defaultUrl, { method: 'POST' })
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
setCacheControlHeaders(headers, request, ctx)
setCacheControlHeaders(response, request, ctx)

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})
})

test('should not set any headers if "cache-control" is not set and "requestContext.usedFsRead" is not truthy', () => {
const headers = new Headers()
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should set permanent, durable "netlify-cdn-cache-control" if "cache-control" is not set and "requestContext.usedFsRead" is truthy', () => {
const headers = new Headers()
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response()
vi.spyOn(response.headers, 'set')

const requestContext = createRequestContext()
requestContext.usedFsRead = true

setCacheControlHeaders(response, request, requestContext)

expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'cache-control',
'public, max-age=0, must-revalidate',
)
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'max-age=31536000, durable',
)
})

test('should set permanent, durable "netlify-cdn-cache-control" if 404 response for URl ending in .php', () => {
const request = new Request(`${defaultUrl}/admin.php`)
const response = new Response(null, { status: 404 })
vi.spyOn(response.headers, 'set')

const requestContext = createRequestContext()
requestContext.usedFsRead = true

setCacheControlHeaders(headers, request, requestContext, true)
setCacheControlHeaders(response, request, requestContext)

expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'cache-control',
'public, max-age=0, must-revalidate',
)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'max-age=31536000, durable',
Expand All @@ -358,45 +380,45 @@ describe('headers', () => {
'cache-control': 'public, max-age=0, must-revalidate',
'cdn-cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should not set any headers if "cache-control" is set and "netlify-cdn-cache-control" is present', () => {
const givenHeaders = {
'cache-control': 'public, max-age=0, must-revalidate',
'netlify-cdn-cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should set expected headers if "cache-control" is set and "cdn-cache-control" and "netlify-cdn-cache-control" are not present (GET request)', () => {
const givenHeaders = {
'cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'cache-control',
'public, max-age=0, must-revalidate',
)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'public, max-age=0, must-revalidate, durable',
Expand All @@ -407,18 +429,18 @@ describe('headers', () => {
const givenHeaders = {
'cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl, { method: 'HEAD' })
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'cache-control',
'public, max-age=0, must-revalidate',
)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'public, max-age=0, must-revalidate, durable',
Expand All @@ -429,27 +451,27 @@ describe('headers', () => {
const givenHeaders = {
'cache-control': 'public, max-age=0, must-revalidate',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl, { method: 'POST' })
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenCalledTimes(0)
expect(response.headers.set).toHaveBeenCalledTimes(0)
})

test('should remove "s-maxage" from "cache-control" header', () => {
const givenHeaders = {
'cache-control': 'public, s-maxage=604800',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'public')
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'public')
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'public, s-maxage=604800, durable',
Expand All @@ -460,14 +482,14 @@ describe('headers', () => {
const givenHeaders = {
'cache-control': 'max-age=604800, stale-while-revalidate=86400',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'max-age=604800')
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'max-age=604800')
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
'max-age=604800, stale-while-revalidate=86400, durable',
Expand All @@ -478,18 +500,18 @@ describe('headers', () => {
const givenHeaders = {
'cache-control': 's-maxage=604800, stale-while-revalidate=86400',
}
const headers = new Headers(givenHeaders)
const request = new Request(defaultUrl)
vi.spyOn(headers, 'set')
const response = new Response(null, { headers: givenHeaders })
vi.spyOn(response.headers, 'set')

setCacheControlHeaders(headers, request, createRequestContext())
setCacheControlHeaders(response, request, createRequestContext())

expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
1,
'cache-control',
'public, max-age=0, must-revalidate',
)
expect(headers.set).toHaveBeenNthCalledWith(
expect(response.headers.set).toHaveBeenNthCalledWith(
2,
'netlify-cdn-cache-control',
's-maxage=604800, stale-while-revalidate=86400, durable',
Expand Down
Loading

0 comments on commit 60885d2

Please sign in to comment.