feat: raw metadata chain config + rename chain

Readme.md

@@ -99,9 +99,9 @@ capi sync --import-map import_map.json
 Retrieve the first 10 entries from a storage map of Polkadot.
 
 ```ts
-import { chain } from "@capi/polkadot"
+import { Polkadot } from "@capi/polkadot"
 
-const accounts = await chain.System.Account
+const accounts = await Polkadot.System.Account
   .entryPage(10, null)
   .run()
 ```
@@ -111,8 +111,13 @@ const accounts = await chain.System.Account
 Let's modify the usage above to target our configured devnet.
 
 ```diff
-- import { chain } from "@capi/polkadot"
-+ import { chain } from "@capi/polkadot-dev"
+- import { Polkadot } from "@capi/polkadot"
++ import { PolkadotDev } from "@capi/polkadot-dev"
+
+- const accounts = await Polkadot.System.Account
++ const accounts = await PolkadotDev.System.Account
+    .entryPage(10, null)
+    .run()
 ```
 
 To run code that depends on a devnet, use the `serve` command, followed by a

capi.config.ts

@@ -1,10 +1,10 @@
-import { binary, CapiConfig } from "./mod.ts"
+import { binary, Config } from "./mod.ts"
 
 const polkadot = binary("polkadot", "v0.9.38")
 const polkadotParachain = binary("polkadot-parachain", "v0.9.380")
 const substrateContractsNode = binary("substrate-contracts-node", "v0.24.0")
 
-export const config: CapiConfig = {
+export const config: Config = {
   server: "http://localhost:4646/",
   chains: {
     polkadot: {
@@ -74,5 +74,8 @@ export const config: CapiConfig = {
         },
       },
     },
+    polkadotFromMetadata: {
+      metadata: await Deno.readFile("examples/raw_rpc/metadata"),
+    },
   },
 }

cli/resolveConfig.ts

@@ -1,6 +1,6 @@
 import * as flags from "../deps/std/flags.ts"
 import * as path from "../deps/std/path.ts"
-import { CapiConfig } from "../devnets/mod.ts"
+import { Config } from "../devnets/mod.ts"
 
 export async function resolveConfig(...args: string[]) {
   const { config: rawConfigPath } = flags.parse(args, {
@@ -15,5 +15,5 @@ export async function resolveConfig(...args: string[]) {
   const config = configModule.config
   if (typeof config !== "object") throw new Error("config file must have a config export")
 
-  return config as CapiConfig
+  return config as Config
 }

cli/sync.ts

@@ -1,4 +1,6 @@
 import * as flags from "../deps/std/flags.ts"
+import { blue } from "../deps/std/fmt/colors.ts"
+import * as path from "../deps/std/path.ts"
 import { assertEquals } from "../deps/std/testing/asserts.ts"
 import { createTempDir } from "../devnets/createTempDir.ts"
 import { syncConfig } from "../devnets/mod.ts"
@@ -20,7 +22,6 @@ export default async function(...args: string[]) {
   const tempDir = await createTempDir()
 
   const baseUrl = await syncConfig(tempDir, config)
-  console.log(baseUrl)
 
   if (importMapFile) {
     const importMapText = await Deno.readTextFile(importMapFile)
@@ -31,6 +32,7 @@ export default async function(...args: string[]) {
     } else {
       await Deno.writeTextFile(importMapFile, JSON.stringify(importMap, null, 2) + "\n")
     }
+    logSynced(importMapFile)
   }
 
   if (packageJsonFile) {
@@ -53,5 +55,10 @@ export default async function(...args: string[]) {
     } else {
       await Deno.writeTextFile(packageJsonFile, JSON.stringify(packageJson, null, 2) + "\n")
     }
+    logSynced(packageJsonFile)
   }
 }
+
+function logSynced(manifestPath: string) {
+  console.log(blue("Synced"), "and updated", path.relative(Deno.cwd(), path.resolve(manifestPath)))
+}

codegen/frameCodegen.ts

@@ -50,7 +50,7 @@ export function frameCodegen(
 
     for (const storage of Object.values(pallet.storage)) {
       palletDeclarationStatements.push(
-        `${storage.name}: C.StorageRune<${chainIdent}, "${pallet.name}", "${storage.name}", never>`,
+        `${storage.name}: C.StorageRune<${chainIdent}Chain, "${pallet.name}", "${storage.name}", never>`,
       )
       palletStatements.push(`${storage.name} = this.storage("${storage.name}")`)
     }
@@ -77,7 +77,7 @@ export function frameCodegen(
                   `
                     ${variant.tag}: <X>(
                       props: C.RunicArgs<X, Omit<${factory}.${variant.tag}, "type">>
-                    ) => C.ExtrinsicRune<${chainIdent}, C.RunicArgs.U<X>>
+                    ) => C.ExtrinsicRune<${chainIdent}Chain, C.RunicArgs.U<X>>
                   `,
                 )
                 palletStatements.push(extrinsicFactory(variant.tag))
@@ -90,7 +90,7 @@ export function frameCodegen(
               throw new Error("pallet call non-string literalUnion is unsupported")
             }
             palletDeclarationStatements.push(
-              `${value}: () => C.ExtrinsicRune<${chainIdent}, never>`,
+              `${value}: () => C.ExtrinsicRune<${chainIdent}Chain, never>`,
             )
             palletStatements.push(extrinsicFactory(value))
           }))
@@ -109,7 +109,7 @@ export function frameCodegen(
     }
 
     palletDeclarations.push(`
-      export class ${pallet.name}PalletRune<out U> extends C.PalletRune<${chainIdent}, "${pallet.name}", U> {
+      export class ${pallet.name}PalletRune<out U> extends C.PalletRune<${chainIdent}Chain, "${pallet.name}", U> {
         ${palletDeclarationStatements.join("\n")}
       }
     `)
@@ -141,9 +141,9 @@ export function frameCodegen(
       ${importsCommon}
       import { metadata } from "./metadata.js"
 
-      export interface ${chainIdent} extends C.Chain<typeof metadata> {}
+      export interface ${chainIdent}Chain extends C.Chain<typeof metadata> {}
 
-      export class ${chainIdent}ChainRune<out U> extends C.ChainRune<${chainIdent}, U> {
+      export class ${chainIdent}ChainRune<out U> extends C.ChainRune<${chainIdent}Chain, U> {
         static override from(connect: (signal: AbortSignal) => C.Connection): ${chainIdent}ChainRune<never>
 
         override with(connection: (signal: AbortSignal) => C.Connection): ${chainIdent}ChainRune<U>
@@ -152,16 +152,13 @@ export function frameCodegen(
       }
 
       ${palletDeclarations.join("\n")}
-
-      export const chain: ${chainIdent}ChainRune<never>
     `,
   )
 
   files.set(
     "chain.js",
     `
       ${importsCommon}
-      import { connect } from "./connection.js"
       import { metadata } from "./metadata.js"
 
       export class ${chainIdent}ChainRune extends C.ChainRune {
@@ -177,8 +174,6 @@ export function frameCodegen(
       }
 
       ${palletDefinitions.join("\n")}
-
-      export const chain = ${chainIdent}ChainRune.from(connect)
     `,
   )
 }

cspell.json

@@ -11,14 +11,14 @@
   "enableGlobDot": true,
   "dictionaries": ["project-words"],
   "ignorePaths": [
-    ".git",
-    "**/*.wasm",
-    "target",
-    "**/__snapshots__/*.snap",
     "**/*.contract",
-    "examples/ink/erc20.json",
     "**/*.scale",
     "**/*.svg",
+    "**/__snapshots__/*.snap",
+    ".git",
+    "examples/ink/erc20.json",
+    "examples/raw_rpc/metadata",
+    "target",
     "util/_artifacts"
   ]
 }

deps/std/fmt/colors.ts

@@ -0,0 +1 @@
+export * from "https://deno.land/[email protected]/fmt/colors.ts"

devnets/CapiConfig.ts

@@ -1,14 +1,28 @@
 import { Binary } from "./binary.ts"
 
-export interface WsChain {
+export interface Config {
+  server: string
+  chains?: Record<string, ChainConfig>
+}
+
+export type ChainConfig =
+  | WsChainConfig
+  | DevChainConfig
+  | RawMetadataChainConfig
+
+export interface WsChainConfig {
   url: string
   binary?: never
+  metadata?: never
+
   version: string
 }
 
-export interface NetworkConfig {
+export interface DevChainConfig {
   url?: never
   binary: Binary
+  metadata?: never
+
   chain: string
   nodes?: number
   parachains?: Record<string, {
@@ -19,7 +33,8 @@ export interface NetworkConfig {
   }>
 }
 
-export interface CapiConfig {
-  server: string
-  chains?: Record<string, WsChain | NetworkConfig>
+export interface RawMetadataChainConfig {
+  url?: never
+  binary?: never
+  metadata: Uint8Array
 }

devnets/devnetsHandler.ts

@@ -2,14 +2,14 @@ import * as path from "../deps/std/path.ts"
 import { $ } from "../mod.ts"
 import * as f from "../server/factories.ts"
 import { PermanentMemo } from "../util/memo.ts"
-import { CapiConfig } from "./CapiConfig.ts"
+import { Config } from "./CapiConfig.ts"
 import { devUserPublicKeys } from "./dev_users.ts"
 import { proxyWebSocket } from "./proxyWebSocket.ts"
 import { Network, startNetwork } from "./startNetwork.ts"
 
 const rDevnetsApi = /^\/devnets\/([\w-]+)(?:\/([\w-]+))?$/
 
-export function createDevnetsHandler(tempDir: string, config: CapiConfig, signal: AbortSignal) {
+export function createDevnetsHandler(tempDir: string, config: Config, signal: AbortSignal) {
   const networkMemo = new PermanentMemo<string, Network>()
   let devUserIndex = 0
   return async (request: Request) => {

devnets/startNetwork.ts

@@ -6,7 +6,7 @@ import * as path from "../deps/std/path.ts"
 import { writableStreamFromWriter } from "../deps/std/streams.ts"
 import { getFreePort, portReady } from "../util/port.ts"
 import { resolveBinary } from "./binary.ts"
-import { NetworkConfig } from "./CapiConfig.ts"
+import { DevChainConfig } from "./CapiConfig.ts"
 import {
   createCustomChainSpec,
   createRawChainSpec,
@@ -27,7 +27,7 @@ export interface NetworkChain {
 
 export async function startNetworkForMetadata(
   tempDir: string,
-  config: NetworkConfig,
+  config: DevChainConfig,
   signal: AbortSignal,
 ): Promise<Network> {
   const relayBinary = await resolveBinary(config.binary, signal)
@@ -70,7 +70,7 @@ export async function startNetworkForMetadata(
 
 export async function startNetwork(
   tempDir: string,
-  config: NetworkConfig,
+  config: DevChainConfig,
   signal: AbortSignal,
 ): Promise<Network> {
   const paras = await Promise.all(

devnets/syncConfig.ts

@@ -2,56 +2,73 @@ export * from "./binary.ts"
 
 import { blake2_512, blake2_64, Hasher } from "../crypto/hashers.ts"
 import { hex } from "../crypto/mod.ts"
+import { gray, green } from "../deps/std/fmt/colors.ts"
 import * as path from "../deps/std/path.ts"
 import { WsConnection } from "../mod.ts"
 import { $codegenSpec, CodegenEntry, CodegenSpec } from "../server/codegenSpec.ts"
 import { normalizePackageName, withSignal } from "../util/mod.ts"
 import { normalizeTypeName } from "../util/normalize.ts"
-import { CapiConfig } from "./CapiConfig.ts"
+import { Config } from "./CapiConfig.ts"
 import { startNetworkForMetadata } from "./startNetwork.ts"
 
-export async function syncConfig(tempDir: string, config: CapiConfig) {
+export async function syncConfig(tempDir: string, config: Config) {
   return withSignal(async (signal) => {
     const { server } = config
     const entries = new Map<string, CodegenEntry>()
+    const chainConfigEntries = Object.entries(config.chains ?? {})
+    let n = chainConfigEntries.length
+    chainConfigEntries.forEach(([_, entry]) => {
+      n += entry.binary && entry.parachains ? Object.values(entry.parachains).length : 0
+    })
+    let synced = 0
     await Promise.all(
-      Object.entries(config.chains ?? {}).map(async ([name, chain]) => {
+      chainConfigEntries.map(async ([name, chain]) => {
+        const relayPackageName = normalizePackageName(name)
+        console.log(gray("Syncing"), relayPackageName)
         if (chain.url != null) {
           const metadata = await uploadMetadata(server, chain.url)
-          entries.set(normalizePackageName(name), {
+          entries.set(relayPackageName, {
             type: "frame",
             metadata,
             chainName: normalizeTypeName(name),
             connection: { type: "WsConnection", discovery: chain.url },
           })
-          return
-        }
-        const network = await startNetworkForMetadata(path.join(tempDir, name), chain, signal)
-        await Promise.all(
-          [
-            [undefined, network.relay] as const,
-            ...Object.entries(network.paras),
-          ].map(
-            async ([paraName, chain]) => {
-              const metadata = await uploadMetadata(
-                server,
-                `ws://127.0.0.1:${chain.ports[0]}`,
-              )
-              entries.set(
-                normalizePackageName(name) + (paraName ? `/${normalizePackageName(paraName)}` : ""),
-                {
+          logSynced(relayPackageName)
+        } else if (chain.metadata) {
+          const metadata = await _upload(server, "metadata", chain.metadata, blake2_512)
+          entries.set(relayPackageName, {
+            type: "frame",
+            metadata,
+            chainName: normalizeTypeName(name),
+            connection: { type: "NoConnection" },
+          })
+          logSynced(relayPackageName)
+        } else {
+          const network = await startNetworkForMetadata(path.join(tempDir, name), chain, signal)
+          await Promise.all(
+            [[undefined, network.relay] as const, ...Object.entries(network.paras)].map(
+              async ([paraName, chain]) => {
+                const metadata = await uploadMetadata(server, `ws://127.0.0.1:${chain.ports[0]}`)
+                const maybeParaPackageName = relayPackageName
+                  + (paraName ? `/${normalizePackageName(paraName)}` : "")
+                entries.set(maybeParaPackageName, {
                   type: "frame",
                   metadata: metadata,
                   chainName: normalizeTypeName(paraName ?? name),
                   connection: {
                     type: "DevnetConnection",
                     discovery: name + (paraName ? `/${paraName}` : ""),
                   },
-                },
-              )
-            },
-          ),
-        )
+                })
+                logSynced(maybeParaPackageName)
+              },
+            ),
+          )
+        }
+
+        function logSynced(packageName: string) {
+          console.log(green("Synced"), gray(`(${++synced}/${n})`), `@capi/${packageName}`)
+        }
       }),
     )
     const sortedEntries = new Map([...entries].sort((a, b) => a[0] < b[0] ? 1 : -1))

dprint.json

@@ -11,13 +11,12 @@
     "lineWidth": 80,
     "textWrap": "always"
   },
-  "includes": [
-    "**.{dockerfile,json,md,toml,ts,tsx}"
-  ],
+  "includes": ["**.{dockerfile,json,md,toml,ts,tsx}"],
   "excludes": [
+    "examples/ink/erc20.json",
+    "examples/raw_rpc/metadata",
     "target",
-    "util/_artifacts",
-    "examples/ink/erc20.json"
+    "util/_artifacts"
   ],
   "plugins": [
     "https://plugins.dprint.dev/dockerfile-0.3.0.wasm",