diff --git a/package.json b/package.json index 26619ee..e194cc9 100644 --- a/package.json +++ b/package.json @@ -9,18 +9,22 @@ "scripts": { "lint": "run-s lint:no-fix-*", "lint:no-fix-cid-verifier": "pnpm --filter cid-verifier lint", + "lint:no-fix-denylist": "pnpm --filter denylist lint", "lint:no-fix-edge-gateway": "pnpm --filter edge-gateway lint", "lint:no-fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint", "lint:fix": "run-s lint:fix-*", "lint:fix-cid-verifier": "pnpm --filter cid-verifier lint --fix", + "lint:fix-denylist": "pnpm --filter denylist lint --fix", "lint:fix-edge-gateway": "pnpm --filter edge-gateway lint --fix", "lint:fix-ipfs-gateway-race": "pnpm --filter ipfs-gateway-race lint --fix", "build": "run-s build:*", "build:cid-verifier": "pnpm --filter cid-verifier build", + "build:denylist": "pnpm --filter denylist build", "build:edge-gateway": "pnpm --filter edge-gateway build", "build:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race build", "test": "run-s test:*", "test:cid-verifier": "pnpm --filter cid-verifier test", + "test:denylist": "pnpm --filter denylist test", "test:edge-gateway": "pnpm --filter edge-gateway test", "test:ipfs-gateway-race": "pnpm --filter ipfs-gateway-race test", "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}" diff --git a/packages/denylist/README.md b/packages/denylist/README.md new file mode 100644 index 0000000..c68ece0 --- /dev/null +++ b/packages/denylist/README.md @@ -0,0 +1,30 @@ +# denylist + +> The `denylist` package serves up data from dotstorage denylists (includes [badbits](https://badbits.dwebops.pub/)). + +## Getting started + +- `pnpm install` - Install the project dependencies from the monorepo root directory. +- `pnpm dev` - Run the worker in dev mode. + +## Environment setup + +- Add secrets + + ```sh + wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry + wrangler secret put LOKI_URL --env $(whoami) # Get from Loki + wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki + ``` + +- `pnpm run publish` - Publish the worker under desired env. An alias for `wrangler publish --env $(whoami)` + +## Contributing + +Feel free to join in. All welcome. [Open an issue](https://github.com/web3-storage/reads/issues)! + +If you're opening a pull request, please see the [guidelines in DEVELOPMENT.md](https://github.com/web3-storage/reads/blob/main/DEVELOPMENT.md#how-should-i-write-my-commits) on structuring your commit messages so that your PR will be compatible with our [release process](https://github.com/web3-storage/reads/blob/main/DEVELOPMENT.md#release). + +## License + +Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/reads/blob/main/LICENSE.md) diff --git a/packages/denylist/ava.config.js b/packages/denylist/ava.config.js new file mode 100644 index 0000000..f620f52 --- /dev/null +++ b/packages/denylist/ava.config.js @@ -0,0 +1,6 @@ +export default { + files: ['test/*.spec.js'], + timeout: '5m', + concurrency: 1, + nodeArguments: ['--experimental-vm-modules'] +} diff --git a/packages/denylist/package.json b/packages/denylist/package.json new file mode 100644 index 0000000..a774b5d --- /dev/null +++ b/packages/denylist/package.json @@ -0,0 +1,50 @@ +{ + "name": "denylist", + "version": "1.0.0", + "description": "Denylist data for dotstorage reads projects", + "private": true, + "type": "module", + "main": "./dist/worker.js", + "scripts": { + "lint": "standard", + "build": "tsc && node scripts/cli.js build", + "dev": "miniflare dist/worker.js --watch --debug -m", + "test": "npm run test:setup && npm-run-all -p test:worker", + "test:worker": "ava --verbose test/*.spec.js", + "test:setup": "npm run build" + }, + "dependencies": { + "@web3-storage/worker-utils": "^0.3.0-dev", + "ipfs-core-utils": "^0.15.0", + "itty-router": "^2.4.5", + "multiformats": "^9.6.4", + "p-retry": "^5.0.0", + "toucan-js": "^2.5.0", + "uint8arrays": "^3.0.0" + }, + "devDependencies": { + "@cloudflare/workers-types": "^3.7.1", + "@sentry/cli": "^1.71.0", + "@types/git-rev-sync": "^2.0.0", + "@web-std/fetch": "^4.0.0", + "ava": "^3.15.0", + "esbuild": "^0.14.2", + "git-rev-sync": "^3.0.1", + "ipfs-only-hash": "^4.0.0", + "miniflare": "^2.5.0", + "npm-run-all": "^4.1.5", + "sade": "^1.7.4", + "standard": "^17.0.0", + "typescript": "4.7.3" + }, + "peerDependencies": { + "undici": "^5.8.0" + }, + "standard": { + "ignore": [ + "dist" + ] + }, + "author": "jsdevel ", + "license": "Apache-2.0 OR MIT" +} diff --git a/packages/denylist/scripts/build.js b/packages/denylist/scripts/build.js new file mode 100644 index 0000000..2fa53d1 --- /dev/null +++ b/packages/denylist/scripts/build.js @@ -0,0 +1,60 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import { build } from 'esbuild' +import git from 'git-rev-sync' +import Sentry from '@sentry/cli' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8') +) + +export async function buildCmd (opts) { + const sentryRelease = `denylist-api@${pkg.version}-${opts.env}+${git.short( + __dirname + )}` + console.log(`Building ${sentryRelease}`) + + await build({ + entryPoints: [path.join(__dirname, '..', 'src', 'index.js')], + bundle: true, + format: 'esm', + outfile: path.join(__dirname, '..', 'dist', 'worker.js'), + legalComments: 'external', + define: { + SENTRY_RELEASE: JSON.stringify(sentryRelease), + VERSION: JSON.stringify(pkg.version), + COMMITHASH: JSON.stringify(git.long(__dirname)), + BRANCH: JSON.stringify(git.branch(__dirname)), + global: 'globalThis' + }, + minify: opts.env !== 'dev', + sourcemap: 'external' + }) + + // Sentry release and sourcemap upload + if (process.env.SENTRY_UPLOAD === 'true') { + const cli = new Sentry(undefined, { + authToken: process.env.SENTRY_TOKEN, + org: 'protocol-labs-it', + project: 'denylist-api', + dist: git.short(__dirname) + }) + + await cli.releases.new(sentryRelease) + await cli.releases.setCommits(sentryRelease, { + auto: true, + ignoreEmpty: true, + ignoreMissing: true + }) + await cli.releases.uploadSourceMaps(sentryRelease, { + include: [path.join(__dirname, '..', 'dist')], + ext: ['map', 'js'] + }) + await cli.releases.finalize(sentryRelease) + await cli.releases.newDeploy(sentryRelease, { + env: opts.env + }) + } +} diff --git a/packages/denylist/scripts/cli.js b/packages/denylist/scripts/cli.js new file mode 100644 index 0000000..c2e2fd9 --- /dev/null +++ b/packages/denylist/scripts/cli.js @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +import sade from 'sade' + +import { buildCmd } from './build.js' + +const env = process.env.ENV || 'dev' +const prog = sade('cid-verifier') + +prog + .command('build') + .describe('Build the worker.') + .option('--env', 'Environment', env) + .action(buildCmd) + +prog.parse(process.argv) diff --git a/packages/denylist/src/bindings.d.ts b/packages/denylist/src/bindings.d.ts new file mode 100644 index 0000000..b585805 --- /dev/null +++ b/packages/denylist/src/bindings.d.ts @@ -0,0 +1,32 @@ +import Toucan from "toucan-js"; +import { Logging } from "@web3-storage/worker-utils/loki"; + +export {}; + +export interface EnvInput { + ENV: string; + DEBUG: string; + SENTRY_DSN?: string; + LOKI_URL?: string; + LOKI_TOKEN?: string; + DENYLIST: KVNamespace; +} + +export interface EnvTransformed { + VERSION: string; + BRANCH: string; + COMMITHASH: string; + SENTRY_RELEASE: string; + sentry?: Toucan; + log: Logging; +} + +export type Env = EnvInput & EnvTransformed; + +declare global { + const BRANCH: string; + const VERSION: string; + const COMMITHASH: string; + const SENTRY_RELEASE: string; + const ENV: string; +} diff --git a/packages/denylist/src/cors.js b/packages/denylist/src/cors.js new file mode 100644 index 0000000..cc670b4 --- /dev/null +++ b/packages/denylist/src/cors.js @@ -0,0 +1,34 @@ +/* eslint-env serviceworker */ + +/** + * @param {import('itty-router').RouteHandler} handler + */ +export function withCorsHeaders (handler) { + /** + * @param {Request} request + * @returns {Promise} + */ + return async (request, /** @type {any} */ ...rest) => { + const response = await handler(request, ...rest) + return addCorsHeaders(request, response) + } +} + +/** + * @param {Request} request + * @param {Response} response + * @returns {Response} + */ +export function addCorsHeaders (request, response) { + // Clone the response so that it's no longer immutable (like if it comes from cache or fetch) + response = new Response(response.body, response) + const origin = request.headers.get('origin') + if (origin) { + response.headers.set('Access-Control-Allow-Origin', origin) + response.headers.set('Vary', 'Origin') + } else { + response.headers.set('Access-Control-Allow-Origin', '*') + } + response.headers.set('Access-Control-Expose-Headers', 'Link') + return response +} diff --git a/packages/denylist/src/denylist.js b/packages/denylist/src/denylist.js new file mode 100644 index 0000000..c60c65f --- /dev/null +++ b/packages/denylist/src/denylist.js @@ -0,0 +1,56 @@ +/* eslint-env serviceworker, browser */ + +import { normalizeCid } from './utils/cid' +import { getFromDenyList } from './utils/denylist' +import { JSONResponse } from '@web3-storage/worker-utils/response' + +/** + * @param {Array} params + * @param {(params: Array, request: Request, env: import('./env').Env) => Promise} fn + * @returns {import('itty-router').RouteHandler} + */ +function withRequiredQueryParams (params, fn) { + /** + * @param {Request} request + * @param {import('./env').Env} env + */ + return async function (request, env) { + const searchParams = (new URL(request.url)).searchParams + + for (const paramName of params) { + const paramValue = searchParams.get(paramName) + if (!paramValue) { + return new Response(`${paramName} is a required query param`, { status: 400 }) + } + + if (paramName === 'cid') { + try { + await normalizeCid(paramValue) + } catch (e) { + return new Response('cid query param is invalid', { status: 400 }) + } + } + } + + return await fn(params.map(param => String(searchParams.get(param))), request, env) + } +} + +export const denylistGet = withRequiredQueryParams(['cid'], + /** + * Returns badbits denylist results. + */ + async function (params, request, env) { + const [cid] = params + + const denyListResource = await getFromDenyList(cid, env) + if (denyListResource) { + try { + return new JSONResponse(JSON.parse(denyListResource), { status: 200 }) + } catch (e) { + env.log.log(`ERROR WHILE PARSING DENYLIST FOR CID "${cid}" ${e}`, 'error') + } + } + return new Response('Not Found', { status: 404 }) + } +) diff --git a/packages/denylist/src/env.js b/packages/denylist/src/env.js new file mode 100644 index 0000000..3c34af4 --- /dev/null +++ b/packages/denylist/src/env.js @@ -0,0 +1,68 @@ +/* global BRANCH, VERSION, COMMITHASH, SENTRY_RELEASE */ +import Toucan from 'toucan-js' + +import { Logging } from '@web3-storage/worker-utils/loki' + +import pkg from '../package.json' + +/** + * @typedef {import('./bindings').Env} Env + * @typedef {import('.').Ctx} Ctx + */ + +/** + * @param {Request} request + * @param {Env} env + * @param {Ctx} ctx + */ +export function envAll (request, env, ctx) { + // These values are replaced at build time by esbuild `define` + env.BRANCH = BRANCH + env.VERSION = VERSION + env.COMMITHASH = COMMITHASH + env.SENTRY_RELEASE = SENTRY_RELEASE + env.sentry = getSentry(request, env, ctx) + + env.log = new Logging(request, ctx, { + // @ts-ignore TODO: url should be optional together with token + url: env.LOKI_URL, + token: env.LOKI_TOKEN, + debug: Boolean(env.DEBUG), + version: env.VERSION, + commit: env.COMMITHASH, + branch: env.BRANCH, + worker: 'cid-verifier', + env: env.ENV, + sentry: env.sentry + }) + env.log.time('request') +} + +/** + * Get sentry instance if configured + * + * @param {Request} request + * @param {Env} env + * @param {Ctx} ctx + */ +function getSentry (request, env, ctx) { + if (!env.SENTRY_DSN) { + return + } + + return new Toucan({ + request, + dsn: env.SENTRY_DSN, + context: ctx, + allowedHeaders: ['user-agent'], + allowedSearchParams: /(.*)/, + debug: false, + environment: env.ENV || 'dev', + rewriteFrames: { + // sourcemaps only work if stack filepath are absolute like `/worker.js` + root: '/' + }, + release: env.SENTRY_RELEASE, + pkg + }) +} diff --git a/packages/denylist/src/error-handler.js b/packages/denylist/src/error-handler.js new file mode 100644 index 0000000..10da5ec --- /dev/null +++ b/packages/denylist/src/error-handler.js @@ -0,0 +1,22 @@ +/* eslint-env serviceworker, browser */ + +/** + * @param {Error & {status?: number;code?: string; contentType?: string;}} err + * @param {import('./env').Env} env + */ +export function errorHandler (err, env) { + console.error(err.stack) + + const status = err.status || 500 + + if (env.sentry && status >= 500) { + env.log.error(err) + } + + return new Response(err.message || 'Server Error', { + status, + headers: { + 'content-type': err.contentType || 'text/plain' + } + }) +} diff --git a/packages/denylist/src/errors.js b/packages/denylist/src/errors.js new file mode 100644 index 0000000..06d0bcf --- /dev/null +++ b/packages/denylist/src/errors.js @@ -0,0 +1,14 @@ +export class ServiceUnavailableError extends Error { + /** + * @param {string} message + */ + constructor (message = 'Service Unavailable') { + const status = 503 + super(message) + this.name = 'ServiceUnavailableError' + this.status = status + this.code = ServiceUnavailableError.CODE + this.contentType = 'text/html' + } +} +ServiceUnavailableError.CODE = 'ERROR_SERVICE_UNAVAILABLE' diff --git a/packages/denylist/src/index.js b/packages/denylist/src/index.js new file mode 100644 index 0000000..0d267ca --- /dev/null +++ b/packages/denylist/src/index.js @@ -0,0 +1,53 @@ +/* eslint-env serviceworker */ + +import { Router } from 'itty-router' + +import { denylistGet } from './denylist.js' +import { versionGet } from './version.js' + +import { addCorsHeaders, withCorsHeaders } from './cors.js' +import { errorHandler } from './error-handler.js' +import { envAll } from './env.js' + +const router = Router() + +// https://developer.mozilla.org/en-US/docs/Web/API/FetchEvent +/** @typedef {ExecutionContext} Ctx */ + +router + .all('*', envAll) + .get('/version', withCorsHeaders(versionGet)) + .get('/', withCorsHeaders(denylistGet)) + +/** + * @param {Error} error + * @param {Request} request + * @param {import('./env').Env} env + */ +function serverError (error, request, env) { + return addCorsHeaders(request, errorHandler(error, env)) +} + +export default { + /** + * + * @param {Request} request + * @param {import("./bindings").Env} env + * @param {Ctx} ctx + */ + async fetch (request, env, ctx) { + // Needs request cloned to avoid worker bindings to have request mutated on follow up requests + const req = request.clone() + try { + const res = await router.handle(req, env, ctx) + env.log.timeEnd('request') + return env.log.end(res) + } catch (/** @type {any} */ error) { + if (env.log) { + env.log.timeEnd('request') + return env.log.end(serverError(error, req, env)) + } + return serverError(error, req, env) + } + } +} diff --git a/packages/denylist/src/utils/cid.js b/packages/denylist/src/utils/cid.js new file mode 100644 index 0000000..4e9923a --- /dev/null +++ b/packages/denylist/src/utils/cid.js @@ -0,0 +1,31 @@ +import { Multibases } from 'ipfs-core-utils/multibases' +import { bases } from 'multiformats/basics' +import { CID } from 'multiformats/cid' + +/** + * Parse CID and return normalized b32 v1. + * + * @param {string} cid + */ +export async function normalizeCid (cid) { + const baseDecoder = await getMultibaseDecoder(cid) + const c = CID.parse(cid, baseDecoder) + return c.toV1().toString() +} + +/** + * Get multibase to decode CID + * + * @param {string} cid + */ +async function getMultibaseDecoder (cid) { + const multibaseCodecs = Object.values(bases) + const basicBases = new Multibases({ + bases: multibaseCodecs + }) + + const multibasePrefix = cid[0] + const base = await basicBases.getBase(multibasePrefix) + + return base.decoder +} diff --git a/packages/denylist/src/utils/denylist.js b/packages/denylist/src/utils/denylist.js new file mode 100644 index 0000000..186b8b8 --- /dev/null +++ b/packages/denylist/src/utils/denylist.js @@ -0,0 +1,34 @@ +import pRetry from 'p-retry' +import * as uint8arrays from 'uint8arrays' +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * Get denylist anchor with badbits format. + * + * @param {string} cid + */ +export async function toDenyListAnchor (cid) { + const multihash = await sha256.digest(uint8arrays.fromString(`${cid}/`)) + const digest = multihash.bytes.subarray(2) + return uint8arrays.toString(digest, 'hex') +} + +/** + * Get a given entry from the deny list if CID exists. + * + * @param {string} cid + * @param {import('../env').Env} env + */ +export async function getFromDenyList (cid, env) { + const datastore = env.DENYLIST + if (!datastore) { + throw new Error('db not ready') + } + + const anchor = await toDenyListAnchor(cid) + // TODO: Remove once https://github.com/nftstorage/nftstorage.link/issues/51 is fixed + return await pRetry( + () => datastore.get(anchor), + { retries: 5 } + ) +} diff --git a/packages/denylist/src/version.js b/packages/denylist/src/version.js new file mode 100644 index 0000000..deb9544 --- /dev/null +++ b/packages/denylist/src/version.js @@ -0,0 +1,15 @@ +import { JSONResponse } from '@web3-storage/worker-utils/response' + +/** + * Get edge gateway API version information. + * + * @param {Request} request + * @param {import('./env').Env} env + */ +export async function versionGet (request, env) { + return new JSONResponse({ + version: env.VERSION, + commit: env.COMMITHASH, + branch: env.BRANCH + }) +} diff --git a/packages/denylist/test/denylist.spec.js b/packages/denylist/test/denylist.spec.js new file mode 100644 index 0000000..59d1f56 --- /dev/null +++ b/packages/denylist/test/denylist.spec.js @@ -0,0 +1,60 @@ +// @ts-ignore +import Hash from 'ipfs-only-hash' +import { test, getMiniflare } from './utils/setup.js' +import { toDenyListAnchor } from '../src/utils/denylist.js' + +/** + * @param {string} s + */ +const createTestCid = async (s) => await Hash.of(s, { cidVersion: 1 }) + +const cidNotInDenyList = await createTestCid('not in denylist') +const cidInDenyList = await createTestCid('asdfasdf') +const cidInDenyListBlockedForLeganReasons = await createTestCid('blocked for legal reasons') + +// Create a new Miniflare environment for each test +test.before(async (t) => { + const mf = getMiniflare() + t.context = { + mf + } + const denylistKv = await mf.getKVNamespace('DENYLIST') + await denylistKv.put(await toDenyListAnchor(cidInDenyList), JSON.stringify({ status: 410, reason: 'bad' })) + await denylistKv.put(await toDenyListAnchor(cidInDenyListBlockedForLeganReasons), JSON.stringify({ status: 451, reason: 'blocked for legal reasons' })) +}) + +test('GET / handles no matching cid in DENYLIST', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${cidNotInDenyList}`) + t.is(await response.text(), 'Not Found') + t.is(response.status, 404) +}) + +test('GET / handles cids in DENYLIST', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${cidInDenyList}`) + const json = await response.json() + t.deepEqual(json, { status: 410, reason: 'bad' }) + t.is(response.status, 200) +}) + +test('GET / handles cids in DENYLIST blocked for legal reasons', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch(`http://localhost:8787/?cid=${cidInDenyListBlockedForLeganReasons}`) + t.deepEqual(await response.json(), { status: 451, reason: 'blocked for legal reasons' }) + t.is(response.status, 200) +}) + +test('GET / handles no cid query param', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch('http://localhost:8787/?') + t.is(await response.text(), 'cid is a required query param') + t.is(response.status, 400) +}) + +test('GET / handles invalid cid query param', async (t) => { + const { mf } = t.context + const response = await mf.dispatchFetch('http://localhost:8787/?cid=invalid') + t.is(await response.text(), 'cid query param is invalid') + t.is(response.status, 400) +}) diff --git a/packages/denylist/test/utils/miniflare.js b/packages/denylist/test/utils/miniflare.js new file mode 100644 index 0000000..15ff5fb --- /dev/null +++ b/packages/denylist/test/utils/miniflare.js @@ -0,0 +1,25 @@ +import fs from 'fs' +import path from 'path' +import { Miniflare } from 'miniflare' + +export function getMiniflare () { + let envPath = path.join(process.cwd(), '../../.env') + if (!fs.statSync(envPath, { throwIfNoEntry: false })) { + // @ts-ignore + envPath = true + } + + return new Miniflare({ + envPath, + scriptPath: 'dist/worker.js', + port: 8788, + packagePath: true, + wranglerConfigPath: true, + // We don't want to rebuild our worker for each test, we're already doing + // it once before we run all tests in package.json, so disable it here. + // This will override the option in wrangler.toml. + buildCommand: undefined, + wranglerConfigEnv: 'test', + modules: true + }) +} diff --git a/packages/denylist/test/utils/setup.js b/packages/denylist/test/utils/setup.js new file mode 100644 index 0000000..ef33e9f --- /dev/null +++ b/packages/denylist/test/utils/setup.js @@ -0,0 +1,13 @@ +import anyTest from 'ava' +export * from './miniflare.js' + +/** + * @typedef {import('miniflare').Miniflare} Miniflare + * + * @typedef {Object} Context + * @property {Miniflare} mf + * + * @typedef {import('ava').TestInterface} TestFn + */ + +export const test = /** @type {TestFn} */ (anyTest) diff --git a/packages/denylist/test/version.spec.js b/packages/denylist/test/version.spec.js new file mode 100644 index 0000000..9766243 --- /dev/null +++ b/packages/denylist/test/version.spec.js @@ -0,0 +1,29 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import git from 'git-rev-sync' + +import { test, getMiniflare } from './utils/setup.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8') +) + +// Create a new Miniflare environment for each test +test.before((t) => { + t.context = { + mf: getMiniflare() + } +}) + +test('Gets Version', async (t) => { + const { mf } = t.context + + const response = await mf.dispatchFetch('http://localhost:8787/version') + const { version, commit, branch } = await response.json() + + t.is(version, pkg.version) + t.is(commit, git.long(__dirname)) + t.is(branch, git.branch(__dirname)) +}) diff --git a/packages/denylist/tsconfig.json b/packages/denylist/tsconfig.json new file mode 100644 index 0000000..fdc333f --- /dev/null +++ b/packages/denylist/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "noEmit": true, + "lib": ["ESNext"], + "types": ["@cloudflare/workers-types"] + }, + "include": ["src", "test", "package.json"], + "exclude": ["**/node_modules/**"] +} diff --git a/packages/denylist/wrangler.toml b/packages/denylist/wrangler.toml new file mode 100644 index 0000000..abc00cf --- /dev/null +++ b/packages/denylist/wrangler.toml @@ -0,0 +1,54 @@ +# dotstorage denylist wrangler config. +name = "dotstorage-denylist" +main = "./dist/worker.js" +compatibility_date = "2022-09-27" +compatibility_flags = [ "url_standard" ] +no_bundle = true + +[build] +command = "npm run build" + +[vars] +DEBUG = "true" +ENV = "dev" + +# PROD! +[env.production] +# name = "dotstorage-denylist-production" +account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account +routes = [ + { pattern = "denylist.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } +] +kv_namespaces = [ + { binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" } +] + +[env.production.vars] +DEBUG = "false" +ENV = "production" + +# Staging! +[env.staging] +# name = "dotstorage-denylist-staging" +account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account +routes = [ + { pattern = "denylist-staging.dag.haus/*", zone_id = "f2f8a5b1c557202c6e3d0ce0e98e4c8e" } +] +kv_namespaces = [ + { binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" } +] + +[env.staging.vars] +DEBUG = "true" +ENV = "staging" + +# Test! +[env.test] +workers_dev = true +kv_namespaces = [ + { binding = "DENYLIST" } +] + +[env.test.vars] +DEBUG = "true" +ENV = "test" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8a841c9..fcbf15c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,53 @@ importers: wrangler: 2.0.23 packages/cid-verifier: + specifiers: + '@cloudflare/workers-types': ^3.7.1 + '@sentry/cli': ^1.71.0 + '@types/git-rev-sync': ^2.0.0 + '@web-std/fetch': ^4.0.0 + '@web3-storage/worker-utils': ^0.3.0-dev + ava: ^3.15.0 + esbuild: ^0.14.2 + git-rev-sync: ^3.0.1 + ipfs-core-utils: ^0.15.0 + ipfs-only-hash: ^4.0.0 + itty-router: ^2.4.5 + miniflare: ^2.5.0 + multiformats: ^9.6.4 + npm-run-all: ^4.1.5 + p-retry: ^5.0.0 + sade: ^1.7.4 + smoke: ^3.1.1 + standard: ^17.0.0 + toucan-js: ^2.5.0 + typescript: 4.7.3 + uint8arrays: ^3.0.0 + dependencies: + '@web3-storage/worker-utils': 0.3.0-dev + ipfs-core-utils: 0.15.1 + itty-router: 2.6.1 + multiformats: 9.7.0 + p-retry: 5.1.1 + toucan-js: 2.6.1 + uint8arrays: 3.0.0 + devDependencies: + '@cloudflare/workers-types': 3.14.0 + '@sentry/cli': 1.74.4 + '@types/git-rev-sync': 2.0.0 + '@web-std/fetch': 4.1.0 + ava: 3.15.0 + esbuild: 0.14.48 + git-rev-sync: 3.0.2 + ipfs-only-hash: 4.0.0 + miniflare: 2.6.0 + npm-run-all: 4.1.5 + sade: 1.8.1 + smoke: 3.1.1 + standard: 17.0.0 + typescript: 4.7.3 + + packages/denylist: specifiers: '@cloudflare/workers-types': ^3.7.1 '@sentry/cli': ^1.71.0 @@ -101,7 +148,7 @@ importers: uint8arrays: ^3.0.0 dependencies: '@web3-storage/worker-utils': 0.3.0-dev - ipfs-core-utils: 0.15.1_undici@5.5.1 + ipfs-core-utils: 0.15.1 ipfs-gateway-race: link:../ipfs-gateway-race itty-router: 2.6.1 multiformats: 9.7.0 @@ -310,6 +357,21 @@ packages: dependencies: multiformats: 9.7.0 + /@libp2p/interfaces/2.0.4: + resolution: {integrity: sha512-MfwkTFyHJtvwNxkjOjzkXyIVvKFtEW2Q3IGRJPyPQMrtB6ll0rGMTlyJ3BQS1bcD0YkNhggFm+8XiU2/0LCBhQ==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + '@multiformats/multiaddr': 10.3.3 + err-code: 3.0.1 + interface-datastore: 6.1.1 + it-pushable: 2.0.2 + it-stream-types: 1.0.4 + multiformats: 9.7.0 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /@libp2p/interfaces/2.0.4_undici@5.5.1: resolution: {integrity: sha512-MfwkTFyHJtvwNxkjOjzkXyIVvKFtEW2Q3IGRJPyPQMrtB6ll0rGMTlyJ3BQS1bcD0YkNhggFm+8XiU2/0LCBhQ==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -325,6 +387,19 @@ packages: - undici dev: false + /@libp2p/logger/1.1.6: + resolution: {integrity: sha512-ZKoRUt7cyHlbxHYDZ1Fn3A+ByqGABdmd4z07+1TfVvpEQSpn2IVcV0mt6ff5kUUtGuVeSrqK1/ZDzWqhgg56vg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + '@libp2p/interfaces': 2.0.4 + debug: 4.3.4 + interface-datastore: 6.1.1 + multiformats: 9.7.0 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /@libp2p/logger/1.1.6_undici@5.5.1: resolution: {integrity: sha512-ZKoRUt7cyHlbxHYDZ1Fn3A+ByqGABdmd4z07+1TfVvpEQSpn2IVcV0mt6ff5kUUtGuVeSrqK1/ZDzWqhgg56vg==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -640,6 +715,15 @@ packages: resolution: {integrity: sha512-eMk0b9ReBbV23xXU693TAIrLyeO5iTgBZGSJfpqriG8UkYvr/hC9u9pyMlAakDNHWmbhMZCDs6KQO0jzKD8OTw==} dev: true + /@multiformats/multiaddr-to-uri/9.0.1: + resolution: {integrity: sha512-kSyHZ2lKjoEzHu/TM4ZVwFj4AWV1B9qFBFJjYb/fK1NqrnrNb/M3uhoyckJvP7WZvpDsnEc7fUCpmPipDY6LMw==} + dependencies: + '@multiformats/multiaddr': 10.3.3 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /@multiformats/multiaddr-to-uri/9.0.1_undici@5.5.1: resolution: {integrity: sha512-kSyHZ2lKjoEzHu/TM4ZVwFj4AWV1B9qFBFJjYb/fK1NqrnrNb/M3uhoyckJvP7WZvpDsnEc7fUCpmPipDY6LMw==} dependencies: @@ -649,6 +733,21 @@ packages: - undici dev: false + /@multiformats/multiaddr/10.3.3: + resolution: {integrity: sha512-+LX9RovG7DJsANb+U6VchV/tApcdJzeafbi5+MPUam90oL91BbEh6ozNZOz4Qf5ZEeilexc12oomatmODJh1/w==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + dns-over-http-resolver: 2.1.0 + err-code: 3.0.1 + is-ip: 4.0.0 + multiformats: 9.7.0 + uint8arrays: 3.0.0 + varint: 6.0.0 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /@multiformats/multiaddr/10.3.3_undici@5.5.1: resolution: {integrity: sha512-+LX9RovG7DJsANb+U6VchV/tApcdJzeafbi5+MPUam90oL91BbEh6ozNZOz4Qf5ZEeilexc12oomatmODJh1/w==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -2116,6 +2215,18 @@ packages: - supports-color dev: true + /dns-over-http-resolver/2.1.0: + resolution: {integrity: sha512-eb8RGy6k54JdD7Rjw8g65y1MyA4z3m3IIYh7uazkgZuKIdFn8gYt8dydMm3op+2UshDdk9EexrXcDluKNY/CDg==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + dependencies: + debug: 4.3.4 + native-fetch: 4.0.2 + receptacle: 1.3.2 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /dns-over-http-resolver/2.1.0_undici@5.5.1: resolution: {integrity: sha512-eb8RGy6k54JdD7Rjw8g65y1MyA4z3m3IIYh7uazkgZuKIdFn8gYt8dydMm3op+2UshDdk9EexrXcDluKNY/CDg==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -3769,6 +3880,19 @@ packages: engines: {node: '>= 0.10'} dev: true + /ipfs-core-types/0.11.1: + resolution: {integrity: sha512-K7ZSx9EPEvT4At2kExKyMCqY4Jo3Nb/VnC/iSWqFKRaqb0MTB4IJgvWrwwDuN541tn+dvUnTfOp2wWQSov1UAw==} + dependencies: + '@ipld/dag-pb': 2.1.17 + '@multiformats/multiaddr': 10.3.3 + interface-datastore: 6.1.1 + ipfs-unixfs: 6.0.9 + multiformats: 9.7.0 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /ipfs-core-types/0.11.1_undici@5.5.1: resolution: {integrity: sha512-K7ZSx9EPEvT4At2kExKyMCqY4Jo3Nb/VnC/iSWqFKRaqb0MTB4IJgvWrwwDuN541tn+dvUnTfOp2wWQSov1UAw==} dependencies: @@ -3821,6 +3945,34 @@ packages: - supports-color dev: true + /ipfs-core-utils/0.15.1: + resolution: {integrity: sha512-nZmUiLctJWMFrEciVkKdDUO2xLpXWy7Ilt0VMJ35Y5+OJznCXxMHUQo1WUALATlo9ziHgDdHFrAUuyW0yB2rww==} + dependencies: + '@libp2p/logger': 1.1.6 + '@multiformats/multiaddr': 10.3.3 + '@multiformats/multiaddr-to-uri': 9.0.1 + any-signal: 3.0.1 + blob-to-it: 1.0.4 + browser-readablestream-to-it: 1.0.3 + err-code: 3.0.1 + ipfs-core-types: 0.11.1 + ipfs-unixfs: 6.0.9 + ipfs-utils: 9.0.7 + it-all: 1.0.6 + it-map: 1.0.6 + it-peekable: 1.0.3 + it-to-stream: 1.0.0 + merge-options: 3.0.4 + multiformats: 9.7.0 + nanoid: 3.3.4 + parse-duration: 1.0.2 + timeout-abort-controller: 3.0.0 + uint8arrays: 3.0.0 + transitivePeerDependencies: + - supports-color + - undici + dev: false + /ipfs-core-utils/0.15.1_undici@5.5.1: resolution: {integrity: sha512-nZmUiLctJWMFrEciVkKdDUO2xLpXWy7Ilt0VMJ35Y5+OJznCXxMHUQo1WUALATlo9ziHgDdHFrAUuyW0yB2rww==} dependencies: @@ -5081,6 +5233,12 @@ packages: node-fetch: 3.2.6 dev: true + /native-fetch/4.0.2: + resolution: {integrity: sha512-4QcVlKFtv2EYVS5MBgsGX5+NWKtbDbIECdUXDBGDMAZXq3Jkv9zf+y8iS7Ub8fEdga3GpYeazp9gauNqXHJOCg==} + peerDependencies: + undici: '*' + dev: false + /native-fetch/4.0.2_undici@5.5.1: resolution: {integrity: sha512-4QcVlKFtv2EYVS5MBgsGX5+NWKtbDbIECdUXDBGDMAZXq3Jkv9zf+y8iS7Ub8fEdga3GpYeazp9gauNqXHJOCg==} peerDependencies: