Skip to content

Commit

Permalink
Fixes run moonbeam fork
Browse files Browse the repository at this point in the history
  • Loading branch information
@crystalin
crystalin committed Oct 6, 2023
1 parent 64ee7fb commit 0aa0b78
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 55 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@
"build:bins:export-state": "node_modules/.bin/esbuild src/tools/export-state.ts --bundle --outfile=dist/export-state.cjs --format=cjs --platform=node --loader:.html=text && node_modules/.bin/pkg -t node18-linux-x64 dist/export-state.cjs -o dist/export-state",
"build:readme": "./scripts/update-readme.sh",
"lint": "npx prettier --write .",
"fork": "npx ts-node ./src/tools/run-moonbeam-fork.ts --base-path /tmp/fork-data/ -n moonbeam"
"fork": "npx ts-node ./src/tools/run-moonbeam-fork.ts --base-path /tmp/fork-data/ --network moonbeam --smaller-state"
},
"repository": {
"type": "git",
Expand Down
118 changes: 82 additions & 36 deletions src/libs/helpers/state-manipulator/state-manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,26 +34,54 @@ export const STORAGE_NAMES: { [name in NetworkName]: string } = {
stagenet: "stagenet",
};

export interface StateInfo {
"file": string,
"cleanFile"?: string,
"name": string,
"chainId": string,
"blockHash": string,
"blockNumber": number,
"runtime": {
"specName": string,
"implName": string,
"authoringVersion": number,
"specVersion": number,
"implVersion": number,
"apis": [string, number][],
"transactionVersion": 2,
"stateVersion": 0
},
}

export interface DownloadOptions {
network: string;
outPath: string;

// Download new state if available
checkLatest?: boolean;

// Prefers clean state (without heavy contract, 80% smaller on moonbeam)
useCleanState?: boolean;
}

// Downloads the exported state from s3. Only if the xxx-chain-info.json file hasn't changed
// 2 files are created:
export async function downloadExportedState(
network: NetworkName,
outPath: string,
checkLatest = true,
options: DownloadOptions,
onStart?: (size: number) => void,
onProgress?: (bytes: number) => void,
onComplete?: () => void
): Promise<{ file: string; blockNumber: number }> {
): Promise<{ stateFile: string; stateInfo: StateInfo }> {
const { network, outPath, checkLatest, useCleanState } = options;

if (!STORAGE_NAMES[network]) {
throw new Error(
`Invalid network ${network}, expecting ${Object.keys(STORAGE_NAMES).join(", ")}`
);
}

const stateInfoFileName = `${network}-chain-info.json`;
const stateInfoFileName = `${network}-state.info.json`;
const stateInfoFile = path.join(outPath, stateInfoFileName);
const stateFileName = `${network}-state.json`;
const stateFile = path.join(outPath, stateFileName);

debug(`Checking ${STORAGE_NAMES[network]} in ${stateInfoFile}`);

Expand All @@ -63,53 +91,71 @@ export async function downloadExportedState(
.access(stateInfoFile)
.then(() => true)
.catch(() => null);
const stateExist = await fs
.access(stateInfoFile)
.then(() => true)
.catch(() => false);

const stateInfo = await fs
const stateInfo: StateInfo = await fs
.readFile(stateInfoFile)
.then((d) => JSON.parse(d.toString()))
.catch(() => null);

// No check for latest, skip if files already exists
if (stateInfoExists && stateExist && !checkLatest) {
return { file: stateFile, blockNumber: parseInt(stateInfo.best_number) };
if (stateInfoExists && !checkLatest) {
const stateFileName = (useCleanState && stateInfo.cleanFile) ? stateInfo.cleanFile : stateInfo.file;
const stateFile = path.join(outPath, stateFileName);

const stateExist = await fs
.access(stateFile)
.then(() => true)
.catch(() => false);

if (stateExist) {
return { stateFile, stateInfo };
}
}
const client = new Client(`https://s3.us-east-2.amazonaws.com`);
const downloadedStateInfo = await (
const client = new Client(`https://states.kaki.dev`);
const downloadedStateInfo: StateInfo = await (
await client.request({
path:
`/chain-state.moonbeam.network/${STORAGE_NAMES[network]}/` +
`latest/${STORAGE_NAMES[network]}-chain-info.json`,
`/${network}-state.info.json`,
method: "GET",
})
).body.json();

// Already latest version
if (stateInfo && stateInfo.best_hash == downloadedStateInfo.best_hash) {
client.close();
return { file: stateFile, blockNumber: parseInt(stateInfo.best_number) };
if (stateInfo && stateInfo.blockHash == downloadedStateInfo.blockHash) {

const stateFileName = (useCleanState && stateInfo.cleanFile) ? stateInfo.cleanFile : stateInfo.file;
const stateFile = path.join(outPath, stateFileName);

const stateExist = await fs
.access(stateFile)
.then(() => true)
.catch(() => false);

if (stateExist) {
client.close();
return { stateFile, stateInfo };
}
}

const stateFileName = (useCleanState && downloadedStateInfo.cleanFile) ? downloadedStateInfo.cleanFile : downloadedStateInfo.file;
const stateFile = path.join(outPath, stateFileName);

const fileStream = (await fs.open(stateFile, "w")).createWriteStream();

debug(
`Preparing to download ${stateFileName} (best-hash: ${downloadedStateInfo.best_hash}) to ${stateFile}`
`Preparing to download ${stateFileName} (best-hash: ${downloadedStateInfo.blockHash}) to ${stateFile}`
);

let transferredBytes = 0;
await new Promise<void>((resolve, reject) => {
client.dispatch(
{
path:
`/chain-state.moonbeam.network/${STORAGE_NAMES[network]}/` +
`latest/${STORAGE_NAMES[network]}-state.json`,
`/${stateFileName}`,
method: "GET",
},
{
onConnect: () => {},
onConnect: () => { },
onError: (error) => {
reject(error);
},
Expand Down Expand Up @@ -142,7 +188,7 @@ export async function downloadExportedState(
await fs.writeFile(stateInfoFile, JSON.stringify(downloadedStateInfo));
debug(`Downloaded ${stateFileName} to ${stateFile}`);

return { file: stateFile, blockNumber: parseInt(downloadedStateInfo.best_number) };
return { stateFile, stateInfo: downloadedStateInfo };
}

// Customize a Moonbeam exported state spec to make it usable locally
Expand All @@ -168,17 +214,17 @@ export async function neutralizeExportedState(
new HRMPManipulator(),
dev
? new SpecManipulator({
name: `Forked Dev Network`,
chainType: `Development`,
relayChain: `dev-service`,
devService: true,
paraId: 0,
protocolId: "",
})
name: `Forked Dev Network`,
chainType: `Development`,
relayChain: `dev-service`,
devService: true,
paraId: 0,
protocolId: "",
})
: new SpecManipulator({
name: `Fork Network`,
relayChain: `rococo-local`,
}),
name: `Fork Network`,
relayChain: `rococo-local`,
}),
new CollectiveManipulator("TechCommitteeCollective", [ALITH_ADDRESS]),
new CollectiveManipulator("CouncilCollective", [ALITH_ADDRESS]),
new ValidationManipulator(),
Expand Down
42 changes: 24 additions & 18 deletions src/tools/run-moonbeam-fork.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ const argv = yargs(process.argv.slice(2))
description: "Removes ALL files at the base-path directory, use with CAUTION.",
default: false,
},
"smaller-state": {
type: "boolean",
description: "Downloads the smaller state version (without super heavy contracts)",
default: true,
},
sealing: {
type: "string",
alias: "s",
Expand Down Expand Up @@ -282,10 +287,13 @@ const main = async () => {

process.stdout.write(`\t - Checking exported state...`);
let progressBar: SingleBar;
const { file: stateFile, blockNumber } = await downloadExportedState(
argv.network as NetworkName,
argv["base-path"],
argv.latest,
const { stateFile, stateInfo } = await downloadExportedState(
{
network: argv.network as NetworkName,
outPath: argv["base-path"],
checkLatest: argv.latest,
useCleanState: argv["smaller-state"]
},
(length) => {
process.stdout.write(`${chalk.yellow(`Downloading`)}\n`);
progressBar = new SingleBar({
Expand All @@ -310,7 +318,7 @@ const main = async () => {
process.stdout.write(`\t - ${chalk.yellow(`Saving`)} ${argv.network} exported state...`);
}
);
process.stdout.write(` ${chalk.green(stateFile)} (#${chalk.yellow(blockNumber)}) ✓\n`);
process.stdout.write(` ${chalk.green(stateFile)} (#${chalk.yellow(stateInfo.blockNumber)}) ✓\n`);

process.stdout.write(`\t - Checking parachain id...`);
const paraId = parseInt(
Expand Down Expand Up @@ -369,7 +377,7 @@ const main = async () => {
hasChanged = true;
process.stdout.write(` ${chalk.yellow(`extracting`)}...`);
const grepLine = await runTask(
`grep ' "0x3a636f6465"' ${stateFile} | cut -c 26- | rev | cut -c 3- | rev | tr -d '\n' | tee ${codeFile} | wc -c`
`grep -m 1 '"0x3a636f6465"' ${stateFile} | head -1 | sed 's/[ \",]//g' | cut -d ':' -f 2 | tr -d '\n' | tee ${codeFile} | wc -c`
);
process.stdout.write(` ${prettyBytes(parseInt(grepLine) / 2)} ✓\n`);
process.stdout.write(`\t - ${chalk.yellow(`Saving`)} wasm code...`);
Expand Down Expand Up @@ -486,8 +494,7 @@ const main = async () => {
await fs.mkdir(aliceFolder, { recursive: true });
aliceLogHandler = await fs.open(aliceLogs, "w");
aliceProcess = await spawnTask(
`${polkadotBinaryPath} --database paritydb --base-path ${aliceFolder} --alice --chain ${relayRawSpecFile} --rpc-port 12001 --port 10001 --node-key ${
Object.keys(NODE_KEYS)[0]
`${polkadotBinaryPath} --database paritydb --base-path ${aliceFolder} --log=debug,parachain=trace,netlink=info,sync=info,lib=info,multi=info,trie=info,grandpa=info,wasm_overrides=info,wasmtime_cranelift=info,parity-db=info --alice --chain ${relayRawSpecFile} --rpc-port 12001 --port 10001 --node-key ${Object.keys(NODE_KEYS)[0]
} --validator`
);
process.stdout.write(` ✓\n`);
Expand All @@ -498,8 +505,7 @@ const main = async () => {
await fs.mkdir(bobFolder, { recursive: true });
bobLogHandler = await fs.open(bobLogs, "w");
bobProcess = await spawnTask(
`${polkadotBinaryPath} --database paritydb --base-path ${bobFolder} --bob --chain ${relayRawSpecFile} --rpc-port 12002 --port 10002 --node-key ${
Object.keys(NODE_KEYS)[1]
`${polkadotBinaryPath} --database paritydb --base-path ${bobFolder} --bob --chain ${relayRawSpecFile} --rpc-port 12002 --port 10002 --node-key ${Object.keys(NODE_KEYS)[1]
} --validator`
);
process.stdout.write(` ✓\n`);
Expand All @@ -512,15 +518,15 @@ const main = async () => {
process.stdout.write(`\t\t - ${chalk.yellow(`Logs`)}: ${alithLogs}`);
await fs.mkdir(alithFolder, { recursive: true });
const alithLogHandler = await fs.open(alithLogs, "w");
// const logs="--log=trace,netlink=trace,sync=trace,lib=trace,sub=trace,multi=trace,evm=debug,parity-db=info,trie=info,wasmtime_cranelift=info";
const logs="";
const alithProcess = argv.dev
? await spawnTask(
`${moonbeamBinaryPath} --database paritydb --base-path ${alithFolder} --execution native --log=info,netlink=info,sync=info,lib=info,multi=info,evm=debug --alice --collator --db-cache 4096 --trie-cache-size ${argv["trie-cache-size"]} --chain ${modFile} --no-hardware-benchmarks --no-prometheus --no-telemetry --sealing=${argv.sealing}`
`${moonbeamBinaryPath} --database paritydb --base-path ${alithFolder} --execution native ${logs} --alice --collator --db-cache 4096 --trie-cache-size ${argv["trie-cache-size"]} --chain ${modFile} --no-hardware-benchmarks --no-prometheus --no-telemetry --sealing=${argv.sealing}`
)
: await spawnTask(
`${moonbeamBinaryPath} --database paritydb --base-path ${alithFolder} --execution native --log=debug,netlink=info,sync=info,lib=info,multi=info,evm=debug --alice --collator --db-cache 4096 --trie-cache-size ${
argv["trie-cache-size"]
} --chain ${modFile} -- --chain ${relayRawSpecFile} --rpc-port 12003 --port 10003 --node-key ${
Object.keys(NODE_KEYS)[2]
`${moonbeamBinaryPath} --database paritydb --base-path ${alithFolder} --execution native ${logs} --alice --collator --db-cache 4096 --trie-cache-size ${argv["trie-cache-size"]
} --chain ${modFile} -- --chain ${relayRawSpecFile} --rpc-port 12003 --port 10003 --node-key ${Object.keys(NODE_KEYS)[2]
}`
);
process.stdout.write(` ✓\n`);
Expand All @@ -535,7 +541,7 @@ const main = async () => {
process.on("exit", () => {
try {
alithProcess.kill();
} catch (e) {}
} catch (e) { }
});
}),
];
Expand All @@ -552,7 +558,7 @@ const main = async () => {
process.on("exit", () => {
try {
aliceProcess.kill();
} catch (e) {}
} catch (e) { }
});
}),
new Promise<void>((resolve) => {
Expand All @@ -565,7 +571,7 @@ const main = async () => {
process.on("exit", () => {
try {
bobProcess.kill();
} catch (e) {}
} catch (e) { }
});
})
);
Expand Down

0 comments on commit 0aa0b78

Please sign in to comment.