From 22c012362ba15ac5f2ec6547374622c8c5e60302 Mon Sep 17 00:00:00 2001 From: Joe Spencer Date: Tue, 20 Dec 2022 09:32:09 -0700 Subject: [PATCH] feat: using denylist from edge-gateway (#126) --- .github/workflows/cid-verifier.yml | 2 +- .github/workflows/denylist.yml | 2 +- .github/workflows/edge-gateway.yml | 2 +- packages/cid-verifier/src/index.js | 4 +- packages/cid-verifier/src/utils/denylist.js | 34 ------ packages/cid-verifier/src/utils/evaluate.js | 57 ++++++++++ packages/cid-verifier/src/verification.js | 98 +++++------------ .../cid-verifier/test/verification.spec.js | 104 ++++++------------ packages/denylist/src/denylist.js | 4 +- packages/denylist/test/denylist.spec.js | 4 +- packages/edge-gateway/src/bindings.d.ts | 2 + packages/edge-gateway/src/constants.js | 3 + packages/edge-gateway/src/gateway.js | 16 +-- .../edge-gateway/src/utils/verification.js | 26 +++++ packages/edge-gateway/test/utils/miniflare.js | 9 +- .../test/utils/scripts/denylist.js | 23 ++++ packages/edge-gateway/wrangler.toml | 23 +++- 17 files changed, 216 insertions(+), 197 deletions(-) delete mode 100644 packages/cid-verifier/src/utils/denylist.js create mode 100644 packages/cid-verifier/src/utils/evaluate.js create mode 100644 packages/edge-gateway/src/utils/verification.js create mode 100644 packages/edge-gateway/test/utils/scripts/denylist.js diff --git a/.github/workflows/cid-verifier.yml b/.github/workflows/cid-verifier.yml index 5211ed3..e5fdb7b 100644 --- a/.github/workflows/cid-verifier.yml +++ b/.github/workflows/cid-verifier.yml @@ -15,7 +15,7 @@ on: jobs: check: runs-on: ubuntu-latest - name: Test + name: Lint steps: - uses: actions/checkout@v2 - uses: pnpm/action-setup@v2.0.1 diff --git a/.github/workflows/denylist.yml b/.github/workflows/denylist.yml index 9ffd4d7..453f9dc 100644 --- a/.github/workflows/denylist.yml +++ b/.github/workflows/denylist.yml @@ -15,7 +15,7 @@ on: jobs: check: runs-on: ubuntu-latest - name: Test + name: Lint steps: - uses: actions/checkout@v2 - uses: pnpm/action-setup@v2.0.1 diff --git a/.github/workflows/edge-gateway.yml b/.github/workflows/edge-gateway.yml index 9242d25..1ae8c1c 100644 --- a/.github/workflows/edge-gateway.yml +++ b/.github/workflows/edge-gateway.yml @@ -15,7 +15,7 @@ on: jobs: check: runs-on: ubuntu-latest - name: Test + name: Lint steps: - uses: actions/checkout@v2 - uses: pnpm/action-setup@v2.0.1 diff --git a/packages/cid-verifier/src/index.js b/packages/cid-verifier/src/index.js index 1e2a391..364eac5 100644 --- a/packages/cid-verifier/src/index.js +++ b/packages/cid-verifier/src/index.js @@ -18,8 +18,8 @@ const router = Router() router .all('*', envAll) .get('/version', withCorsHeaders(versionGet)) - .get('/denylist', withCorsHeaders(withAuthToken(verificationGet))) - .post('/', withCorsHeaders(withAuthToken(verificationPost))) + .get('/:cid', withCorsHeaders(withAuthToken(verificationGet))) + .post('/:cid', withCorsHeaders(withAuthToken(verificationPost))) /** * @param {Error} error diff --git a/packages/cid-verifier/src/utils/denylist.js b/packages/cid-verifier/src/utils/denylist.js deleted file mode 100644 index 186b8b8..0000000 --- a/packages/cid-verifier/src/utils/denylist.js +++ /dev/null @@ -1,34 +0,0 @@ -import pRetry from 'p-retry' -import * as uint8arrays from 'uint8arrays' -import { sha256 } from 'multiformats/hashes/sha2' - -/** - * Get denylist anchor with badbits format. - * - * @param {string} cid - */ -export async function toDenyListAnchor (cid) { - const multihash = await sha256.digest(uint8arrays.fromString(`${cid}/`)) - const digest = multihash.bytes.subarray(2) - return uint8arrays.toString(digest, 'hex') -} - -/** - * Get a given entry from the deny list if CID exists. - * - * @param {string} cid - * @param {import('../env').Env} env - */ -export async function getFromDenyList (cid, env) { - const datastore = env.DENYLIST - if (!datastore) { - throw new Error('db not ready') - } - - const anchor = await toDenyListAnchor(cid) - // TODO: Remove once https://github.com/nftstorage/nftstorage.link/issues/51 is fixed - return await pRetry( - () => datastore.get(anchor), - { retries: 5 } - ) -} diff --git a/packages/cid-verifier/src/utils/evaluate.js b/packages/cid-verifier/src/utils/evaluate.js new file mode 100644 index 0000000..c6faa14 --- /dev/null +++ b/packages/cid-verifier/src/utils/evaluate.js @@ -0,0 +1,57 @@ +import pRetry from 'p-retry' +export const GOOGLE_EVALUATE = 'google-evaluate' + +/** + * Get the threats we have stored in KV. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +async function getEvaluateResult (cid, env) { + const resultKey = `${cid}/${GOOGLE_EVALUATE}` + const datastore = env.CID_VERIFIER_RESULTS + if (!datastore) { + throw new Error('db not ready') + } + + return await pRetry( + () => datastore.get(resultKey), + { retries: 5 } + ) +} + +/** + * Get the threats we have stored in KV. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +export async function getStoredThreats (cid, env) { + const evaluateResult = await getEvaluateResult(cid, env) + if (evaluateResult) { + // @ts-ignore + return JSON.parse(evaluateResult)?.scores?.filter(score => !env.GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS.includes(score.confidenceLevel)).map(score => score.threatType) + } + return [] +} + +/** + * Get verification results from Google Evaluate API. + * + * Also returns any lock results if we have in progress evaluate requests. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +export async function getResults (cid, env) { + const datastore = env.CID_VERIFIER_RESULTS + if (!datastore) { + throw new Error('CID_VERIFIER_RESULTS db not ready') + } + + return (await datastore.list({ prefix: cid }))?.keys?.reduce((acc, key) => { + // @ts-ignore + acc[key?.name] = key?.metadata?.value + return acc + }, {}) +} diff --git a/packages/cid-verifier/src/verification.js b/packages/cid-verifier/src/verification.js index 421f3e6..93eaf74 100644 --- a/packages/cid-verifier/src/verification.js +++ b/packages/cid-verifier/src/verification.js @@ -1,80 +1,50 @@ /* eslint-env serviceworker, browser */ /* global Response */ import pRetry from 'p-retry' - import { normalizeCid } from './utils/cid' -import { getFromDenyList, toDenyListAnchor } from './utils/denylist' +import { + GOOGLE_EVALUATE, + getStoredThreats, + getResults +} from './utils/evaluate' import { ServiceUnavailableError } from './errors' -const GOOGLE_EVALUATE = 'google-evaluate' - -/** - * Get verification results from 3rd parties stored in KV. - * - * @param {string} cid - * @param {import('./env').Env} env - */ -async function getResults (cid, env) { - const datastore = env.CID_VERIFIER_RESULTS - if (!datastore) { - throw new Error('CID_VERIFIER_RESULTS db not ready') - } - - return (await datastore.list({ prefix: cid }))?.keys?.reduce((acc, key) => { - // @ts-ignore - acc[key?.name] = key?.metadata?.value - return acc - }, {}) -} - /** - * @param {Array} params - * @param {(params: Array, request: Request, env: import('./env').Env) => Promise} fn + * @param {(cid: string, request: import('itty-router').Request, env: import('./env').Env) => Promise} fn * @returns {import('itty-router').RouteHandler} */ -function withRequiredQueryParams (params, fn) { +function withCidPathParam (fn) { /** - * @param {Request} request + * @param {import('itty-router').Request} request * @param {import('./env').Env} env */ return async function (request, env) { - const searchParams = (new URL(request.url)).searchParams + const cid = request?.params?.cid - for (const paramName of params) { - const paramValue = searchParams.get(paramName) - if (!paramValue) { - return new Response(`${paramName} is a required query param`, { status: 400 }) - } + if (!cid) { + return new Response('cid is a required path param', { status: 400 }) + } - if (paramName === 'cid') { - try { - await normalizeCid(paramValue) - } catch (e) { - return new Response('cid query param is invalid', { status: 400 }) - } - } + try { + await normalizeCid(cid) + } catch (e) { + return new Response('cid path param is invalid', { status: 400 }) } - return await fn(params.map(param => String(searchParams.get(param))), request, env) + return await fn(cid, request, env) } } -export const verificationGet = withRequiredQueryParams(['cid'], +export const verificationGet = withCidPathParam( /** * Returns google malware result. */ - async function (params, request, env) { - const [cid] = params - - const denyListResource = await getFromDenyList(cid, env) - if (denyListResource) { - const { status } = JSON.parse(denyListResource) - if (status === 451) { - return new Response('BLOCKED FOR LEGAL REASONS', { status: 451 }) - } else { - return new Response('MALWARE DETECTED', { status: 403 }) - } + async function (cid, request, env) { + const threats = await getStoredThreats(cid, env) + if (threats?.length) { + return new Response(threats.join(', '), { status: 403 }) } + return new Response('', { status: 204 }) } ) @@ -82,9 +52,8 @@ export const verificationGet = withRequiredQueryParams(['cid'], /** * Process CID with malware verification parties. */ -export const verificationPost = withRequiredQueryParams(['cid'], - async function verificationPost (params, request, env) { - const [cid] = params +export const verificationPost = withCidPathParam( + async function verificationPost (cid, request, env) { const resultKey = `${cid}/${GOOGLE_EVALUATE}` const lockKey = `${cid}/${GOOGLE_EVALUATE}.lock` const cidVerifyResults = await getResults(cid, env) @@ -97,7 +66,7 @@ export const verificationPost = withRequiredQueryParams(['cid'], const threats = await fetchGoogleMalwareResults(cid, `https://${cid}.${env.IPFS_GATEWAY_TLD}`, env) const response = new Response('cid malware detection processed', { status: 201 }) - if (threats.length) { + if (threats?.length) { env.log.log(`MALWARE DETECTED for cid "${cid}" ${threats.join(', ')}`, 'info') env.log.end(response) } @@ -146,20 +115,7 @@ export const verificationPost = withRequiredQueryParams(['cid'], () => env.CID_VERIFIER_RESULTS.put(resultKey, stringifiedJSON, { metadata: { value: stringifiedJSON } }), { retries: 5 } ) - - // @ts-ignore - // if any score isn't what we consider to be safe we add it to the DENYLIST - threats = evaluateJson?.scores?.filter(score => !env.GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS.includes(score.confidenceLevel)).map(score => score.threatType) - if (threats.length) { - const anchor = await toDenyListAnchor(cid) - await pRetry( - () => env.DENYLIST.put(anchor, JSON.stringify({ - status: 403, - reason: threats.join(', ') - })), - { retries: 5 } - ) - } + threats = await getStoredThreats(resultKey, env) } catch (e) { // @ts-ignore env.log.log(e, 'error') diff --git a/packages/cid-verifier/test/verification.spec.js b/packages/cid-verifier/test/verification.spec.js index 050393d..2dc94be 100644 --- a/packages/cid-verifier/test/verification.spec.js +++ b/packages/cid-verifier/test/verification.spec.js @@ -1,21 +1,16 @@ // @ts-ignore import Hash from 'ipfs-only-hash' import { test, getMiniflare } from './utils/setup.js' -import { toDenyListAnchor } from '../src/utils/denylist.js' /** * @param {string} s */ const createTestCid = async (s) => await Hash.of(s, { cidVersion: 1 }) -// TODO: use valid cids and test 400 scenarios -const cidInDenyList = await createTestCid('asdfasdf') -const cidInDenyListBlockedForLeganReasons = await createTestCid('blocked for legal reasons') const pendingCid = await createTestCid('pending') const emptyCid = await createTestCid('empty') const notMalwareCid = await createTestCid('notMalware') const malwareCid = await createTestCid('malware') -const maliciousCid = await createTestCid('malicious') const safeCid = await createTestCid('safe') const errorCid = await createTestCid('error') const headers = { @@ -32,119 +27,92 @@ test.before(async (t) => { await googleMalwareResultsKv.put(`${pendingCid}/google-evaluate.lock`, 'true', { metadata: { value: 'true' } }) await googleMalwareResultsKv.put(`${notMalwareCid}/google-evaluate`, '{}', { metadata: { value: '{}' } }) await googleMalwareResultsKv.put(`${malwareCid}/google-evaluate`, JSON.stringify({ - threat: { - threatTypes: ['MALWARE'], - expireTime: '2022-08-28T07:54:04.936398042Z' - } + scores: [ + { + threatType: 'MALWARE', + confidenceLevel: 'EXTREMELY_HIGH' + } + ] }), { metadata: { value: JSON.stringify({ - threat: { - threatTypes: ['MALWARE'], - expireTime: '2022-08-28T07:54:04.936398042Z' - } + scores: [ + { + threatType: 'MALWARE', + confidenceLevel: 'EXTREMELY_HIGH' + } + ] }) } }) - const denylistKv = await mf.getKVNamespace('DENYLIST') - await denylistKv.put(await toDenyListAnchor(cidInDenyList), JSON.stringify({ status: 410, reason: 'bad' })) - await denylistKv.put(await toDenyListAnchor(cidInDenyListBlockedForLeganReasons), JSON.stringify({ status: 451, reason: 'blocked for legal reasons' })) -}) - -test('GET /denylist handles cids in DENYLIST', async (t) => { - const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${cidInDenyList}`, { headers }) - t.is(await response.text(), 'MALWARE DETECTED') - t.is(response.status, 403) }) -test('GET /denylist handles cids in DENYLIST blocked for legal reasons', async (t) => { +test('GET /:cid handles invalid cid', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${cidInDenyListBlockedForLeganReasons}`, { headers }) - t.is(await response.text(), 'BLOCKED FOR LEGAL REASONS') - t.is(response.status, 451) -}) - -test('GET /denylist handles no cid', async (t) => { - const { mf } = t.context - const response = await mf.dispatchFetch('http://localhost:8787/denylist?', { headers }) - t.is(await response.text(), 'cid is a required query param') + const response = await mf.dispatchFetch('http://localhost:8787/invalid', { headers }) + t.is(await response.text(), 'cid path param is invalid') t.is(response.status, 400) }) -test('GET /denylist handles invalid cid', async (t) => { +test('GET /:cid handles no results', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch('http://localhost:8787/denylist?cid=invalid', { headers }) - t.is(await response.text(), 'cid query param is invalid') - t.is(response.status, 400) -}) - -test('GET /denylist handles no results', async (t) => { - const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${emptyCid}`, { headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${emptyCid}`, { headers }) t.is(await response.text(), '') t.is(response.status, 204) }) -test('GET /denylist handles pending results', async (t) => { +test('GET /:cid handles pending results', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${pendingCid}`, { headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${pendingCid}`, { headers }) t.is(await response.text(), '') t.is(response.status, 204) }) -test('GET /denylist handles successful results', async (t) => { +test('GET /:cid handles successful results', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/denylist?cid=${notMalwareCid}`, { headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${notMalwareCid}`, { headers }) t.is(await response.text(), '') t.is(response.status, 204) }) -test('POST / handles no cid', async (t) => { +test('GET /:cid handles malware cid', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch('http://localhost:8787/?', { method: 'POST', headers }) - t.is(await response.text(), 'cid is a required query param') - t.is(response.status, 400) + const response = await mf.dispatchFetch(`http://localhost:8787/${malwareCid}`, { headers }) + t.is(await response.text(), 'MALWARE') + t.is(response.status, 403) }) -test('POST / handles invalid cid', async (t) => { +test('POST /:cid handles invalid cid', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch('http://localhost:8787/?cid=invalid', { method: 'POST', headers }) - t.is(await response.text(), 'cid query param is invalid') + const response = await mf.dispatchFetch('http://localhost:8787/invalid', { method: 'POST', headers }) + t.is(await response.text(), 'cid path param is invalid') t.is(response.status, 400) }) -test('POST / handles malicious urls', async (t) => { - const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${maliciousCid}`, { method: 'POST', headers }) - t.is(await response.text(), 'cid malware detection processed') - t.is(response.status, 201) -}) - -test('POST / handles safe urls', async (t) => { +test('POST /:cid handles safe urls', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${safeCid}`, { method: 'POST', headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${safeCid}`, { method: 'POST', headers }) t.is(await response.text(), 'cid malware detection processed') t.is(response.status, 201) }) -test('POST / handles invalid or error urls', async (t) => { +test('POST /:cid handles invalid or error urls', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${errorCid}`, { method: 'POST', headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${errorCid}`, { method: 'POST', headers }) t.is(await response.text(), `GOOGLE CLOUD UNABLE TO VERIFY URL "https://${errorCid}.ipfs.link.test" status code "400"`) t.is(response.status, 503) }) -test('POST / handles pending results', async (t) => { +test('POST /:cid handles pending results', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${pendingCid}`, { method: 'POST', headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${pendingCid}`, { method: 'POST', headers }) t.is(await response.text(), 'cid malware detection already processed') t.is(response.status, 202) }) -test('POST / handles overriding existing malware cid', async (t) => { +test('POST /:cid handles overriding existing malware cid', async (t) => { const { mf } = t.context - const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${malwareCid}`, { method: 'POST', headers }) + const response = await mf.dispatchFetch(`http://localhost:8787/${malwareCid}`, { method: 'POST', headers }) t.is(await response.text(), 'cid malware detection already processed') t.is(response.status, 202) }) diff --git a/packages/denylist/src/denylist.js b/packages/denylist/src/denylist.js index 949bc73..2b78cd0 100644 --- a/packages/denylist/src/denylist.js +++ b/packages/denylist/src/denylist.js @@ -13,13 +13,13 @@ export const denylistGet = async function (request, env) { const cid = request?.params?.cid if (!cid) { - return new Response('cid is a required query param', { status: 400 }) + return new Response('cid is a required path param', { status: 400 }) } try { await normalizeCid(cid) } catch (e) { - return new Response('cid query param is invalid', { status: 400 }) + return new Response('cid path param is invalid', { status: 400 }) } const denyListResource = await getFromDenyList(cid, env) diff --git a/packages/denylist/test/denylist.spec.js b/packages/denylist/test/denylist.spec.js index 104cdc6..612e3d3 100644 --- a/packages/denylist/test/denylist.spec.js +++ b/packages/denylist/test/denylist.spec.js @@ -23,10 +23,10 @@ test.before(async (t) => { await denylistKv.put(await toDenyListAnchor(cidInDenyListBlockedForLeganReasons), JSON.stringify({ status: 451, reason: 'blocked for legal reasons' })) }) -test('GET / handles invalid cid query param', async (t) => { +test('GET / handles invalid cid path param', async (t) => { const { mf } = t.context const response = await mf.dispatchFetch('http://localhost:8787/invalid') - t.is(await response.text(), 'cid query param is invalid') + t.is(await response.text(), 'cid path param is invalid') t.is(response.status, 400) }) diff --git a/packages/edge-gateway/src/bindings.d.ts b/packages/edge-gateway/src/bindings.d.ts index 0970ca4..d1973f0 100644 --- a/packages/edge-gateway/src/bindings.d.ts +++ b/packages/edge-gateway/src/bindings.d.ts @@ -22,6 +22,8 @@ export interface EnvInput { CID_VERIFIER_URL: string CID_VERIFIER: Fetcher CDN_GATEWAYS_RACE: string + DENYLIST: Fetcher + DENYLIST_URL: string IPFS_GATEWAYS_RACE_L1: string IPFS_GATEWAYS_RACE_L2: string GATEWAY_HOSTNAME: string diff --git a/packages/edge-gateway/src/constants.js b/packages/edge-gateway/src/constants.js index df90b6f..2555ddf 100644 --- a/packages/edge-gateway/src/constants.js +++ b/packages/edge-gateway/src/constants.js @@ -1,5 +1,8 @@ export const CF_CACHE_MAX_OBJECT_SIZE = 512 * Math.pow(1024, 2) // 512MB to bytes +export const ACCEPTABLE_DENYLIST_STATUS_CODES = [204, 400, 404] +export const ACCEPTABLE_CID_VERIFIER_STATUS_CODES = [204, 400, 404] + /** * @type {Record} */ diff --git a/packages/edge-gateway/src/gateway.js b/packages/edge-gateway/src/gateway.js index adabfd8..f601d84 100644 --- a/packages/edge-gateway/src/gateway.js +++ b/packages/edge-gateway/src/gateway.js @@ -12,6 +12,7 @@ import { toDenyListAnchor } from './utils/cid.js' import { getHeaders } from './utils/headers.js' +import { getCidForbiddenResponse } from './utils/verification.js' import { TimeoutError } from './errors.js' import { CF_CACHE_MAX_OBJECT_SIZE, @@ -78,9 +79,9 @@ export async function gatewayGet (request, env, ctx) { } } - const cidDenylistResponse = await env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/denylist?cid=${cid}`, { headers: { Authorization: `basic ${env.CID_VERIFIER_AUTHORIZATION_TOKEN}` } }) - if (cidDenylistResponse.status !== 204) { - return cidDenylistResponse + const cidForbiddenResponse = await getCidForbiddenResponse(cid, env) + if (cidForbiddenResponse) { + return cidForbiddenResponse } // 1st layer resolution - CDN @@ -121,10 +122,9 @@ export async function gatewayGet (request, env, ctx) { // Validation layer - resource CID const resourceCid = pathname !== '/' ? getCidFromEtag(winnerGwResponse.headers.get('etag') || cid) : cid if (winnerGwResponse && pathname !== '/' && resourceCid) { - const cidResourceDenylistResponse = await env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/denylist?cid=${resourceCid}`, { headers: { Authorization: `basic ${env.CID_VERIFIER_AUTHORIZATION_TOKEN}` } }) - // Ignore if CID received from gateway in etag header is invalid by any reason - if (cidResourceDenylistResponse.status !== 204 && cidResourceDenylistResponse.status !== 400) { - return cidResourceDenylistResponse + const resourceCidForbiddenResponse = await getCidForbiddenResponse(resourceCid, env) + if (resourceCidForbiddenResponse) { + return resourceCidForbiddenResponse } } @@ -135,7 +135,7 @@ export async function gatewayGet (request, env, ctx) { ) { // fire and forget. Let cid-verifier process this cid and url if it needs to ctx.waitUntil( - env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/?cid=${resourceCid}`, { method: 'POST', headers: { Authorization: `basic ${env.CID_VERIFIER_AUTHORIZATION_TOKEN}` } }) + env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/${resourceCid}`, { method: 'POST', headers: { Authorization: `basic ${env.CID_VERIFIER_AUTHORIZATION_TOKEN}` } }) ) } diff --git a/packages/edge-gateway/src/utils/verification.js b/packages/edge-gateway/src/utils/verification.js new file mode 100644 index 0000000..d5c9472 --- /dev/null +++ b/packages/edge-gateway/src/utils/verification.js @@ -0,0 +1,26 @@ +import { + ACCEPTABLE_CID_VERIFIER_STATUS_CODES, + ACCEPTABLE_DENYLIST_STATUS_CODES +} from '../constants.js' + +/** + * Checks to see if denylist or cid-verifier forbid this CID from being served. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +export async function getCidForbiddenResponse (cid, env) { + const [cidDenylistResponse, cidVerifierResponse] = await Promise.all([ + env.DENYLIST.fetch(`${env.DENYLIST_URL}/${cid}`), + env.isCidVerifierEnabled ? env.CID_VERIFIER.fetch(`${env.CID_VERIFIER_URL}/${cid}`, { headers: { Authorization: `basic ${env.CID_VERIFIER_AUTHORIZATION_TOKEN}` } }) : null + ]) + + if (!ACCEPTABLE_DENYLIST_STATUS_CODES.includes(cidDenylistResponse.status)) { + return cidDenylistResponse + } + + // cidVerifierResponse will be null if env.isCidVerifierEnabled is false. + if (cidVerifierResponse && !ACCEPTABLE_CID_VERIFIER_STATUS_CODES.includes(cidVerifierResponse.status)) { + return cidVerifierResponse + } +} diff --git a/packages/edge-gateway/test/utils/miniflare.js b/packages/edge-gateway/test/utils/miniflare.js index 0850fcb..a96a25d 100644 --- a/packages/edge-gateway/test/utils/miniflare.js +++ b/packages/edge-gateway/test/utils/miniflare.js @@ -36,11 +36,18 @@ export function getMiniflare (bindings = {}) { scriptPath: './test/utils/scripts/cid-verifier.js', modules: true, kvNamespaces: ['TEST_NAMESPACE'] + }, + denylist: { + scriptPath: './test/utils/scripts/denylist.js', + modules: true, + kvNamespaces: ['TEST_NAMESPACE1'] } + }, serviceBindings: { API: 'api', - CID_VERIFIER: 'cid_verifier' + CID_VERIFIER: 'cid_verifier', + DENYLIST: 'denylist' }, bindings: { PUBLIC_RACE_WINNER: createAnalyticsEngine(), diff --git a/packages/edge-gateway/test/utils/scripts/denylist.js b/packages/edge-gateway/test/utils/scripts/denylist.js new file mode 100644 index 0000000..1d5195e --- /dev/null +++ b/packages/edge-gateway/test/utils/scripts/denylist.js @@ -0,0 +1,23 @@ +/* eslint-env serviceworker, browser */ + +const invalidCid = 'invalid' + +export default { + /** + * @param {Request} request + * @param {any} env + */ + async fetch (request, env) { + if (request.method === 'GET') { + if (request.url.includes(invalidCid)) { + return new Response('invalid', { + status: 400 + }) + } + + return new Response('not found', { + status: 404 + }) + } + } +} diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml index 4f9d498..a583fa3 100644 --- a/packages/edge-gateway/wrangler.toml +++ b/packages/edge-gateway/wrangler.toml @@ -29,13 +29,11 @@ routes = [ { pattern = "*.ipns.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" }, { pattern = "*.ipns.dag.haus", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } ] -kv_namespaces = [ - { binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" } -] [env.production.vars] GATEWAY_HOSTNAME = 'ipfs.dag.haus' CID_VERIFIER_URL = 'https://cid-verifier.dag.haus' +DENYLIST_URL = 'https://denylist.dag.haus' EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' DEBUG = "false" CID_VERIFIER_ENABLED = "false" @@ -54,6 +52,12 @@ type = "service" service = "dotstorage-cid-verifier-production" environment = "production" +[[env.production.services]] +binding = "DENYLIST" +type = "service" +service = "dotstorage-denylist-production" +environment = "production" + [[env.production.unsafe.bindings]] type = "analytics_engine" dataset = "PUBLIC_RACE_WINNER_PRODUCTION" @@ -80,13 +84,13 @@ routes = [ { pattern = "*.ipns-staging.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" }, { pattern = "*.ipns-staging.dag.haus", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } ] -kv_namespaces = [ - { binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" } -] [env.staging.vars] GATEWAY_HOSTNAME = 'ipfs-staging.dag.haus' CID_VERIFIER_URL = 'https://cid-verifier-staging.dag.haus' +DENYLIST_URL = 'https://denylist-staging.dag.haus' + +# TODO: Should point to general API in the future EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' DEBUG = "true" CID_VERIFIER_ENABLED = "false" @@ -105,6 +109,12 @@ type = "service" service = "dotstorage-cid-verifier-staging" environment = "production" +[[env.staging.services]] +binding = "DENYLIST" +type = "service" +service = "dotstorage-denylist-staging" +environment = "production" + [[env.staging.unsafe.bindings]] type = "analytics_engine" dataset = "PUBLIC_RACE_WINNER_STAGING" @@ -127,6 +137,7 @@ workers_dev = true [env.test.vars] GATEWAY_HOSTNAME = 'ipfs.localhost:8787' CID_VERIFIER_URL = 'http://cid-verifier.localhost:8787' +DENYLIST_URL = 'http://denylist.localhost:8787' EDGE_GATEWAY_API_URL = 'http://localhost:8787' DEBUG = "true" CID_VERIFIER_ENABLED = "true"