feat: using denylist from edge-gateway (#126)

.github/workflows/cid-verifier.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   check:
     runs-on: ubuntu-latest
-    name: Test
+    name: Lint
     steps:
       - uses: actions/checkout@v2
       - uses: pnpm/[email protected]

.github/workflows/denylist.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   check:
     runs-on: ubuntu-latest
-    name: Test
+    name: Lint
     steps:
       - uses: actions/checkout@v2
       - uses: pnpm/[email protected]

.github/workflows/edge-gateway.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   check:
     runs-on: ubuntu-latest
-    name: Test
+    name: Lint
     steps:
       - uses: actions/checkout@v2
       - uses: pnpm/[email protected]

packages/cid-verifier/src/index.js

@@ -18,8 +18,8 @@ const router = Router()
 router
   .all('*', envAll)
   .get('/version', withCorsHeaders(versionGet))
-  .get('/denylist', withCorsHeaders(withAuthToken(verificationGet)))
-  .post('/', withCorsHeaders(withAuthToken(verificationPost)))
+  .get('/:cid', withCorsHeaders(withAuthToken(verificationGet)))
+  .post('/:cid', withCorsHeaders(withAuthToken(verificationPost)))
 
 /**
  * @param {Error} error

packages/cid-verifier/src/utils/evaluate.js

@@ -0,0 +1,57 @@
+import pRetry from 'p-retry'
+export const GOOGLE_EVALUATE = 'google-evaluate'
+
+/**
+ * Get the threats we have stored in KV.
+ *
+ * @param {string} cid
+ * @param {import('../env').Env} env
+ */
+async function getEvaluateResult (cid, env) {
+  const resultKey = `${cid}/${GOOGLE_EVALUATE}`
+  const datastore = env.CID_VERIFIER_RESULTS
+  if (!datastore) {
+    throw new Error('db not ready')
+  }
+
+  return await pRetry(
+    () => datastore.get(resultKey),
+    { retries: 5 }
+  )
+}
+
+/**
+ * Get the threats we have stored in KV.
+ *
+ * @param {string} cid
+ * @param {import('../env').Env} env
+ */
+export async function getStoredThreats (cid, env) {
+  const evaluateResult = await getEvaluateResult(cid, env)
+  if (evaluateResult) {
+    // @ts-ignore
+    return JSON.parse(evaluateResult)?.scores?.filter(score => !env.GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS.includes(score.confidenceLevel)).map(score => score.threatType)
+  }
+  return []
+}
+
+/**
+ * Get verification results from Google Evaluate API.
+ *
+ * Also returns any lock results if we have in progress evaluate requests.
+ *
+ * @param {string} cid
+ * @param {import('../env').Env} env
+ */
+export async function getResults (cid, env) {
+  const datastore = env.CID_VERIFIER_RESULTS
+  if (!datastore) {
+    throw new Error('CID_VERIFIER_RESULTS db not ready')
+  }
+
+  return (await datastore.list({ prefix: cid }))?.keys?.reduce((acc, key) => {
+    // @ts-ignore
+    acc[key?.name] = key?.metadata?.value
+    return acc
+  }, {})
+}

packages/cid-verifier/src/verification.js

@@ -1,90 +1,59 @@
 /* eslint-env serviceworker, browser */
 /* global Response */
 import pRetry from 'p-retry'
-
 import { normalizeCid } from './utils/cid'
-import { getFromDenyList, toDenyListAnchor } from './utils/denylist'
+import {
+  GOOGLE_EVALUATE,
+  getStoredThreats,
+  getResults
+} from './utils/evaluate'
 import { ServiceUnavailableError } from './errors'
 
-const GOOGLE_EVALUATE = 'google-evaluate'
-
-/**
- * Get verification results from 3rd parties stored in KV.
- *
- * @param {string} cid
- * @param {import('./env').Env} env
- */
-async function getResults (cid, env) {
-  const datastore = env.CID_VERIFIER_RESULTS
-  if (!datastore) {
-    throw new Error('CID_VERIFIER_RESULTS db not ready')
-  }
-
-  return (await datastore.list({ prefix: cid }))?.keys?.reduce((acc, key) => {
-    // @ts-ignore
-    acc[key?.name] = key?.metadata?.value
-    return acc
-  }, {})
-}
-
 /**
- * @param {Array<string>} params
- * @param {(params: Array<string>, request: Request, env: import('./env').Env) => Promise<Response>} fn
+ * @param {(cid: string, request: import('itty-router').Request, env: import('./env').Env) => Promise<Response>} fn
  * @returns {import('itty-router').RouteHandler<Request>}
  */
-function withRequiredQueryParams (params, fn) {
+function withCidPathParam (fn) {
   /**
-   * @param {Request} request
+   * @param {import('itty-router').Request} request
    * @param {import('./env').Env} env
    */
   return async function (request, env) {
-    const searchParams = (new URL(request.url)).searchParams
+    const cid = request?.params?.cid
 
-    for (const paramName of params) {
-      const paramValue = searchParams.get(paramName)
-      if (!paramValue) {
-        return new Response(`${paramName} is a required query param`, { status: 400 })
-      }
+    if (!cid) {
+      return new Response('cid is a required path param', { status: 400 })
+    }
 
-      if (paramName === 'cid') {
-        try {
-          await normalizeCid(paramValue)
-        } catch (e) {
-          return new Response('cid query param is invalid', { status: 400 })
-        }
-      }
+    try {
+      await normalizeCid(cid)
+    } catch (e) {
+      return new Response('cid path param is invalid', { status: 400 })
     }
 
-    return await fn(params.map(param => String(searchParams.get(param))), request, env)
+    return await fn(cid, request, env)
   }
 }
 
-export const verificationGet = withRequiredQueryParams(['cid'],
+export const verificationGet = withCidPathParam(
   /**
    * Returns google malware result.
    */
-  async function (params, request, env) {
-    const [cid] = params
-
-    const denyListResource = await getFromDenyList(cid, env)
-    if (denyListResource) {
-      const { status } = JSON.parse(denyListResource)
-      if (status === 451) {
-        return new Response('BLOCKED FOR LEGAL REASONS', { status: 451 })
-      } else {
-        return new Response('MALWARE DETECTED', { status: 403 })
-      }
+  async function (cid, request, env) {
+    const threats = await getStoredThreats(cid, env)
+    if (threats?.length) {
+      return new Response(threats.join(', '), { status: 403 })
     }
+
     return new Response('', { status: 204 })
   }
 )
 
 /**
  * Process CID with malware verification parties.
  */
-export const verificationPost = withRequiredQueryParams(['cid'],
-  async function verificationPost (params, request, env) {
-    const [cid] = params
+export const verificationPost = withCidPathParam(
+  async function verificationPost (cid, request, env) {
     const resultKey = `${cid}/${GOOGLE_EVALUATE}`
     const lockKey = `${cid}/${GOOGLE_EVALUATE}.lock`
     const cidVerifyResults = await getResults(cid, env)
@@ -97,7 +66,7 @@ export const verificationPost = withRequiredQueryParams(['cid'],
       const threats = await fetchGoogleMalwareResults(cid, `https://${cid}.${env.IPFS_GATEWAY_TLD}`, env)
       const response = new Response('cid malware detection processed', { status: 201 })
 
-      if (threats.length) {
+      if (threats?.length) {
         env.log.log(`MALWARE DETECTED for cid "${cid}" ${threats.join(', ')}`, 'info')
         env.log.end(response)
       }
@@ -146,20 +115,7 @@ export const verificationPost = withRequiredQueryParams(['cid'],
           () => env.CID_VERIFIER_RESULTS.put(resultKey, stringifiedJSON, { metadata: { value: stringifiedJSON } }),
           { retries: 5 }
         )
-
-        // @ts-ignore
-        // if any score isn't what we consider to be safe we add it to the DENYLIST
-        threats = evaluateJson?.scores?.filter(score => !env.GOOGLE_EVALUATE_SAFE_CONFIDENCE_LEVELS.includes(score.confidenceLevel)).map(score => score.threatType)
-        if (threats.length) {
-          const anchor = await toDenyListAnchor(cid)
-          await pRetry(
-            () => env.DENYLIST.put(anchor, JSON.stringify({
-              status: 403,
-              reason: threats.join(', ')
-            })),
-            { retries: 5 }
-          )
-        }
+        threats = await getStoredThreats(resultKey, env)
       } catch (e) {
         // @ts-ignore
         env.log.log(e, 'error')