feat: adding denylist package (closes #68)

package.json

@@ -9,18 +9,22 @@
   "scripts": {
     "lint": "run-s lint:no-fix-*",
     "lint:no-fix-cid-verifier": "pnpm --filter cid-verifier lint",
+    "lint:no-fix-denylist": "pnpm --filter denylist lint",
     "lint:no-fix-edge-gateway": "pnpm --filter edge-gateway lint",
     "lint:no-fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint",
     "lint:fix": "run-s lint:fix-*",
     "lint:fix-cid-verifier": "pnpm --filter cid-verifier lint --fix",
+    "lint:fix-denylist": "pnpm --filter denylist lint --fix",
     "lint:fix-edge-gateway": "pnpm --filter edge-gateway lint --fix",
     "lint:fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint --fix",
     "build": "run-s build:*",
     "build:cid-verifier": "pnpm --filter cid-verifier build",
+    "build:denylist": "pnpm --filter denylist build",
     "build:edge-gateway": "pnpm --filter edge-gateway build",
     "build:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race build",
     "test": "run-s test:*",
     "test:cid-verifier": "pnpm --filter cid-verifier test",
+    "test:denylist": "pnpm --filter denylist test",
     "test:edge-gateway": "pnpm --filter edge-gateway test",
     "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test",
     "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}"

packages/denylist/README.md

@@ -0,0 +1,30 @@
+# denylist
+
+> The `denylist` package serves up data from dotstorage denylists (includes [badbits](https://badbits.dwebops.pub/)).
+
+## Getting started
+
+- `pnpm install` - Install the project dependencies from the monorepo root directory.
+- `pnpm dev` - Run the worker in dev mode.
+
+## Environment setup
+
+- Add secrets
+
+  ```sh
+    wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry
+    wrangler secret put LOKI_URL --env $(whoami) # Get from Loki
+    wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki
+  ```
+
+- `pnpm run publish` - Publish the worker under desired env. An alias for `wrangler publish --env $(whoami)`
+
+## Contributing
+
+Feel free to join in. All welcome. [Open an issue](https://github.com/web3-storage/reads/issues)!
+
+If you're opening a pull request, please see the [guidelines in DEVELOPMENT.md](https://github.com/web3-storage/reads/blob/main/DEVELOPMENT.md#how-should-i-write-my-commits) on structuring your commit messages so that your PR will be compatible with our [release process](https://github.com/web3-storage/reads/blob/main/DEVELOPMENT.md#release).
+
+## License
+
+Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/reads/blob/main/LICENSE.md)

packages/denylist/ava.config.js

@@ -0,0 +1,6 @@
+export default {
+  files: ['test/*.spec.js'],
+  timeout: '5m',
+  concurrency: 1,
+  nodeArguments: ['--experimental-vm-modules']
+}

packages/denylist/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "denylist",
+  "version": "1.0.0",
+  "description": "Denylist data for dotstorage reads projects",
+  "private": true,
+  "type": "module",
+  "main": "./dist/worker.js",
+  "scripts": {
+    "lint": "standard",
+    "build": "tsc && node scripts/cli.js build",
+    "dev": "miniflare dist/worker.js --watch --debug -m",
+    "test": "npm run test:setup && npm-run-all -p test:worker",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "test:setup": "npm run build"
+  },
+  "dependencies": {
+    "@web3-storage/worker-utils": "^0.3.0-dev",
+    "ipfs-core-utils": "^0.15.0",
+    "itty-router": "^2.4.5",
+    "multiformats": "^9.6.4",
+    "p-retry": "^5.0.0",
+    "toucan-js": "^2.5.0",
+    "uint8arrays": "^3.0.0"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^3.7.1",
+    "@sentry/cli": "^1.71.0",
+    "@types/git-rev-sync": "^2.0.0",
+    "@web-std/fetch": "^4.0.0",
+    "ava": "^3.15.0",
+    "esbuild": "^0.14.2",
+    "git-rev-sync": "^3.0.1",
+    "ipfs-only-hash": "^4.0.0",
+    "miniflare": "^2.5.0",
+    "npm-run-all": "^4.1.5",
+    "sade": "^1.7.4",
+    "standard": "^17.0.0",
+    "typescript": "4.7.3"
+  },
+  "peerDependencies": {
+    "undici": "^5.8.0"
+  },
+  "standard": {
+    "ignore": [
+      "dist"
+    ]
+  },
+  "author": "jsdevel <[email protected]>",
+  "license": "Apache-2.0 OR MIT"
+}

packages/denylist/scripts/build.js

@@ -0,0 +1,60 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { build } from 'esbuild'
+import git from 'git-rev-sync'
+import Sentry from '@sentry/cli'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+export async function buildCmd (opts) {
+  const sentryRelease = `denylist-api@${pkg.version}-${opts.env}+${git.short(
+    __dirname
+  )}`
+  console.log(`Building ${sentryRelease}`)
+
+  await build({
+    entryPoints: [path.join(__dirname, '..', 'src', 'index.js')],
+    bundle: true,
+    format: 'esm',
+    outfile: path.join(__dirname, '..', 'dist', 'worker.js'),
+    legalComments: 'external',
+    define: {
+      SENTRY_RELEASE: JSON.stringify(sentryRelease),
+      VERSION: JSON.stringify(pkg.version),
+      COMMITHASH: JSON.stringify(git.long(__dirname)),
+      BRANCH: JSON.stringify(git.branch(__dirname)),
+      global: 'globalThis'
+    },
+    minify: opts.env !== 'dev',
+    sourcemap: 'external'
+  })
+
+  // Sentry release and sourcemap upload
+  if (process.env.SENTRY_UPLOAD === 'true') {
+    const cli = new Sentry(undefined, {
+      authToken: process.env.SENTRY_TOKEN,
+      org: 'protocol-labs-it',
+      project: 'denylist-api',
+      dist: git.short(__dirname)
+    })
+
+    await cli.releases.new(sentryRelease)
+    await cli.releases.setCommits(sentryRelease, {
+      auto: true,
+      ignoreEmpty: true,
+      ignoreMissing: true
+    })
+    await cli.releases.uploadSourceMaps(sentryRelease, {
+      include: [path.join(__dirname, '..', 'dist')],
+      ext: ['map', 'js']
+    })
+    await cli.releases.finalize(sentryRelease)
+    await cli.releases.newDeploy(sentryRelease, {
+      env: opts.env
+    })
+  }
+}

packages/denylist/scripts/cli.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+import sade from 'sade'
+
+import { buildCmd } from './build.js'
+
+const env = process.env.ENV || 'dev'
+const prog = sade('denylist')
+
+prog
+  .command('build')
+  .describe('Build the worker.')
+  .option('--env', 'Environment', env)
+  .action(buildCmd)
+
+prog.parse(process.argv)

packages/denylist/src/bindings.d.ts

@@ -0,0 +1,32 @@
+import Toucan from "toucan-js";
+import { Logging } from "@web3-storage/worker-utils/loki";
+
+export {};
+
+export interface EnvInput {
+  ENV: string;
+  DEBUG: string;
+  SENTRY_DSN?: string;
+  LOKI_URL?: string;
+  LOKI_TOKEN?: string;
+  DENYLIST: KVNamespace;
+}
+
+export interface EnvTransformed {
+  VERSION: string;
+  BRANCH: string;
+  COMMITHASH: string;
+  SENTRY_RELEASE: string;
+  sentry?: Toucan;
+  log: Logging;
+}
+
+export type Env = EnvInput & EnvTransformed;
+
+declare global {
+  const BRANCH: string;
+  const VERSION: string;
+  const COMMITHASH: string;
+  const SENTRY_RELEASE: string;
+  const ENV: string;
+}

packages/denylist/src/cors.js

@@ -0,0 +1,34 @@
+/* eslint-env serviceworker */
+
+/**
+ * @param {import('itty-router').RouteHandler<Request>} handler
+ */
+export function withCorsHeaders (handler) {
+  /**
+   * @param {Request} request
+   * @returns {Promise<Response>}
+   */
+  return async (request, /** @type {any} */ ...rest) => {
+    const response = await handler(request, ...rest)
+    return addCorsHeaders(request, response)
+  }
+}
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ * @returns {Response}
+ */
+export function addCorsHeaders (request, response) {
+  // Clone the response so that it's no longer immutable (like if it comes from cache or fetch)
+  response = new Response(response.body, response)
+  const origin = request.headers.get('origin')
+  if (origin) {
+    response.headers.set('Access-Control-Allow-Origin', origin)
+    response.headers.set('Vary', 'Origin')
+  } else {
+    response.headers.set('Access-Control-Allow-Origin', '*')
+  }
+  response.headers.set('Access-Control-Expose-Headers', 'Link')
+  return response
+}

packages/denylist/src/denylist.js

@@ -0,0 +1,36 @@
+/* eslint-env serviceworker, browser */
+
+import { normalizeCid } from './utils/cid'
+import { getFromDenyList } from './utils/denylist'
+import { JSONResponse } from '@web3-storage/worker-utils/response'
+
+/**
+ * Returns badbits denylist results.
+ * @param {Request} request
+ * @param {import('./env').Env} env
+ */
+export const denylistGet = async function (request, env) {
+  const cid = (new URL(request.url)).pathname?.substring(1)
+
+  if (!cid) {
+    return new Response(`cid is a required query param`, { status: 400 })
+  }
+
+  try {
+    await normalizeCid(cid)
+  } catch (e) {
+    return new Response('cid query param is invalid', { status: 400 })
+  }
+
+  const denyListResource = await getFromDenyList(cid, env)
+
+  if (denyListResource) {
+    try {
+      return new JSONResponse(JSON.parse(denyListResource), { status: 200 })
+    } catch (e) {
+      env.log.log(`ERROR WHILE PARSING DENYLIST FOR CID "${cid}" ${e}`, 'error')
+    }
+  }
+
+  return new Response('Not Found', { status: 404 })
+}

packages/denylist/src/env.js

@@ -0,0 +1,68 @@
+/* global BRANCH, VERSION, COMMITHASH, SENTRY_RELEASE */
+import Toucan from 'toucan-js'
+
+import { Logging } from '@web3-storage/worker-utils/loki'
+
+import pkg from '../package.json'
+
+/**
+ * @typedef {import('./bindings').Env} Env
+ * @typedef {import('.').Ctx} Ctx
+ */
+
+/**
+ * @param {Request} request
+ * @param {Env} env
+ * @param {Ctx} ctx
+ */
+export function envAll (request, env, ctx) {
+  // These values are replaced at build time by esbuild `define`
+  env.BRANCH = BRANCH
+  env.VERSION = VERSION
+  env.COMMITHASH = COMMITHASH
+  env.SENTRY_RELEASE = SENTRY_RELEASE
+  env.sentry = getSentry(request, env, ctx)
+
+  env.log = new Logging(request, ctx, {
+    // @ts-ignore TODO: url should be optional together with token
+    url: env.LOKI_URL,
+    token: env.LOKI_TOKEN,
+    debug: Boolean(env.DEBUG),
+    version: env.VERSION,
+    commit: env.COMMITHASH,
+    branch: env.BRANCH,
+    worker: 'cid-verifier',
+    env: env.ENV,
+    sentry: env.sentry
+  })
+  env.log.time('request')
+}
+
+/**
+ * Get sentry instance if configured
+ *
+ * @param {Request} request
+ * @param {Env} env
+ * @param {Ctx} ctx
+ */
+function getSentry (request, env, ctx) {
+  if (!env.SENTRY_DSN) {
+    return
+  }
+
+  return new Toucan({
+    request,
+    dsn: env.SENTRY_DSN,
+    context: ctx,
+    allowedHeaders: ['user-agent'],
+    allowedSearchParams: /(.*)/,
+    debug: false,
+    environment: env.ENV || 'dev',
+    rewriteFrames: {
+      // sourcemaps only work if stack filepath are absolute like `/worker.js`
+      root: '/'
+    },
+    release: env.SENTRY_RELEASE,
+    pkg
+  })
+}

packages/denylist/src/error-handler.js

@@ -0,0 +1,22 @@
+/* eslint-env serviceworker, browser */
+
+/**
+ * @param {Error & {status?: number;code?: string; contentType?: string;}} err
+ * @param {import('./env').Env} env
+ */
+export function errorHandler (err, env) {
+  console.error(err.stack)
+
+  const status = err.status || 500
+
+  if (env.sentry && status >= 500) {
+    env.log.error(err)
+  }
+
+  return new Response(err.message || 'Server Error', {
+    status,
+    headers: {
+      'content-type': err.contentType || 'text/plain'
+    }
+  })
+}