From 19155d02a1248c85e94f14a2a0bb383a4edeb16f Mon Sep 17 00:00:00 2001 From: kevaundray Date: Sat, 16 Dec 2023 18:11:07 +0000 Subject: [PATCH] feat: Add context-centric based API for noir_wasm (#3798) # Description Currently this PR is not breaking This adds an API that maps closer to the API we'd have in the CLI or atleast one that differs less with the native version. One can imagine a shared Context object and in nargo_cli the main difference is the PathToSourceMap struct that gets passed in for example. ## Problem\* Resolves ## Summary\* ## Additional Context ## Documentation\* Check one: - [ ] No documentation needed. - [ ] Documentation included in this PR. - [ ] **[Exceptional Case]** Documentation to be submitted in a separate PR. # PR Checklist\* - [ ] I have tested the changes locally. - [ ] I have formatted the changes with [Prettier](https://prettier.io/) and/or `cargo fmt` on default settings. --------- Co-authored-by: Tom French --- compiler/wasm/src/compile.rs | 18 +- compiler/wasm/src/compile_new.rs | 336 ++++++++++++++++++++++++++ compiler/wasm/src/lib.rs | 4 + compiler/wasm/test/node/index.test.ts | 78 +++++- 4 files changed, 426 insertions(+), 10 deletions(-) create mode 100644 compiler/wasm/src/compile_new.rs diff --git a/compiler/wasm/src/compile.rs b/compiler/wasm/src/compile.rs index 2d99963ea1d..6171267cb03 100644 --- a/compiler/wasm/src/compile.rs +++ b/compiler/wasm/src/compile.rs @@ -120,12 +120,11 @@ impl JsCompileResult { } } -#[derive(Deserialize)] -struct DependencyGraph { - root_dependencies: Vec, - library_dependencies: HashMap>, +#[derive(Deserialize, Default)] +pub(crate) struct DependencyGraph { + pub(crate) root_dependencies: Vec, + pub(crate) library_dependencies: HashMap>, } - #[wasm_bindgen] // This is a map containing the paths of all of the files in the entry-point crate and // the transitive dependencies of the entry-point crate. @@ -133,7 +132,7 @@ struct DependencyGraph { // This is for all intents and purposes the file system that the compiler will use to resolve/compile // files in the crate being compiled and its dependencies. #[derive(Deserialize, Default)] -pub struct PathToFileSourceMap(HashMap); +pub struct PathToFileSourceMap(pub(crate) HashMap); #[wasm_bindgen] impl PathToFileSourceMap { @@ -228,7 +227,7 @@ pub fn compile( // // For all intents and purposes, the file manager being returned // should be considered as immutable. -fn file_manager_with_source_map(source_map: PathToFileSourceMap) -> FileManager { +pub(crate) fn file_manager_with_source_map(source_map: PathToFileSourceMap) -> FileManager { let root = Path::new(""); let mut fm = FileManager::new(root); @@ -277,7 +276,7 @@ fn add_noir_lib(context: &mut Context, library_name: &CrateName) -> CrateId { prepare_dependency(context, &path_to_lib) } -fn preprocess_program(program: CompiledProgram) -> CompileResult { +pub(crate) fn preprocess_program(program: CompiledProgram) -> CompileResult { let debug_artifact = DebugArtifact { debug_symbols: vec![program.debug], file_map: program.file_map, @@ -295,7 +294,8 @@ fn preprocess_program(program: CompiledProgram) -> CompileResult { CompileResult::Program { program: preprocessed_program, debug: debug_artifact } } -fn preprocess_contract(contract: CompiledContract) -> CompileResult { +// TODO: This method should not be doing so much, most of this should be done in nargo or the driver +pub(crate) fn preprocess_contract(contract: CompiledContract) -> CompileResult { let debug_artifact = DebugArtifact { debug_symbols: contract.functions.iter().map(|function| function.debug.clone()).collect(), file_map: contract.file_map, diff --git a/compiler/wasm/src/compile_new.rs b/compiler/wasm/src/compile_new.rs new file mode 100644 index 00000000000..5c0a06b18f0 --- /dev/null +++ b/compiler/wasm/src/compile_new.rs @@ -0,0 +1,336 @@ +use crate::compile::{ + file_manager_with_source_map, preprocess_contract, preprocess_program, JsCompileResult, + PathToFileSourceMap, +}; +use crate::errors::{CompileError, JsCompileError}; +use noirc_driver::{ + add_dep, compile_contract, compile_main, prepare_crate, prepare_dependency, CompileOptions, +}; +use noirc_frontend::{ + graph::{CrateGraph, CrateId, CrateName}, + hir::Context, +}; +use std::path::Path; +use wasm_bindgen::prelude::wasm_bindgen; + +/// This is a wrapper class that is wasm-bindgen compatible +/// We do not use js_name and rename it like CrateId because +/// then the impl block is not picked up in javascript. +#[wasm_bindgen] +pub struct CompilerContext { + context: Context, +} + +#[wasm_bindgen(js_name = "CrateId")] +#[derive(Debug, Copy, Clone)] +pub struct CrateIDWrapper(CrateId); + +#[wasm_bindgen] +impl CompilerContext { + #[wasm_bindgen(constructor)] + pub fn new(source_map: PathToFileSourceMap) -> CompilerContext { + console_error_panic_hook::set_once(); + + let fm = file_manager_with_source_map(source_map); + let graph = CrateGraph::default(); + CompilerContext { context: Context::new(fm, graph) } + } + + #[cfg(test)] + pub(crate) fn crate_graph(&self) -> &CrateGraph { + &self.context.crate_graph + } + #[cfg(test)] + pub(crate) fn root_crate_id(&self) -> CrateIDWrapper { + CrateIDWrapper(*self.context.root_crate_id()) + } + + // Processes the root crate by adding it to the package graph and automatically + // importing the stdlib as a dependency for it. + // + // Its ID in the package graph is returned + pub fn process_root_crate(&mut self, path_to_crate: String) -> CrateIDWrapper { + let path_to_crate = Path::new(&path_to_crate); + + // Adds the root crate to the crate graph and returns its crate id + CrateIDWrapper(prepare_crate(&mut self.context, path_to_crate)) + } + + pub fn process_dependency_crate(&mut self, path_to_crate: String) -> CrateIDWrapper { + let path_to_crate = Path::new(&path_to_crate); + + // Adds the root crate to the crate graph and returns its crate id + CrateIDWrapper(prepare_dependency(&mut self.context, path_to_crate)) + } + + // Adds a named edge from one crate to the other. + // + // For example, lets say we have two crates CrateId1 and CrateId2 + // This function will add an edge from CrateId1 to CrateId2 and the edge will be named `crate_name` + // + // This essentially says that CrateId1 depends on CrateId2 and the dependency is named `crate_name` + // + // We pass references to &CrateIdWrapper even though it is a copy because Rust's move semantics are + // not respected once we use javascript. ie it will actually allocated a new object in javascript + // then deallocate that object if we do not pass as a reference. + pub fn add_dependency_edge( + &mut self, + crate_name: String, + from: &CrateIDWrapper, + to: &CrateIDWrapper, + ) { + let parsed_crate_name: CrateName = crate_name + .parse() + .unwrap_or_else(|_| panic!("Failed to parse crate name {}", crate_name)); + add_dep(&mut self.context, from.0, to.0, parsed_crate_name); + } + + pub fn compile_program( + mut self, + program_width: usize, + ) -> Result { + let compile_options = CompileOptions::default(); + let np_language = acvm::Language::PLONKCSat { width: program_width }; + + let root_crate_id = *self.context.root_crate_id(); + + let compiled_program = + compile_main(&mut self.context, root_crate_id, &compile_options, None, true) + .map_err(|errs| { + CompileError::with_file_diagnostics( + "Failed to compile program", + errs, + &self.context.file_manager, + ) + })? + .0; + + let optimized_program = nargo::ops::optimize_program(compiled_program, np_language); + + let compile_output = preprocess_program(optimized_program); + Ok(JsCompileResult::new(compile_output)) + } + + pub fn compile_contract( + mut self, + program_width: usize, + ) -> Result { + let compile_options = CompileOptions::default(); + let np_language = acvm::Language::PLONKCSat { width: program_width }; + let root_crate_id = *self.context.root_crate_id(); + + let compiled_contract = + compile_contract(&mut self.context, root_crate_id, &compile_options) + .map_err(|errs| { + CompileError::with_file_diagnostics( + "Failed to compile contract", + errs, + &self.context.file_manager, + ) + })? + .0; + + let optimized_contract = nargo::ops::optimize_contract(compiled_contract, np_language); + + let compile_output = preprocess_contract(optimized_contract); + Ok(JsCompileResult::new(compile_output)) + } +} + +/// This is a method that exposes the same API as `compile` +/// But uses the Context based APi internally +#[wasm_bindgen] +pub fn compile_( + entry_point: String, + contracts: Option, + dependency_graph: Option, + file_source_map: PathToFileSourceMap, +) -> Result { + use std::collections::HashMap; + + console_error_panic_hook::set_once(); + + let dependency_graph: crate::compile::DependencyGraph = + if let Some(dependency_graph) = dependency_graph { + ::into_serde( + &wasm_bindgen::JsValue::from(dependency_graph), + ) + .map_err(|err| err.to_string())? + } else { + crate::compile::DependencyGraph::default() + }; + + let mut compiler_context = CompilerContext::new(file_source_map); + + // Set the root crate + let root_id = compiler_context.process_root_crate(entry_point.clone()); + + let add_noir_lib = |context: &mut CompilerContext, lib_name: &CrateName| -> CrateIDWrapper { + let lib_name_string = lib_name.to_string(); + let path_to_lib = Path::new(&lib_name_string) + .join("lib.nr") + .to_str() + .expect("paths are expected to be valid utf-8") + .to_string(); + context.process_dependency_crate(path_to_lib) + }; + + // Add the dependency graph + let mut crate_names: HashMap = HashMap::new(); + // + // Process the direct dependencies of the root + for lib_name in dependency_graph.root_dependencies { + let lib_name_string = lib_name.to_string(); + + let crate_id = add_noir_lib(&mut compiler_context, &lib_name); + + crate_names.insert(lib_name.clone(), crate_id); + + // Add the dependency edges + compiler_context.add_dependency_edge(lib_name_string, &root_id, &crate_id); + } + + // Process the transitive dependencies of the root + for (lib_name, dependencies) in &dependency_graph.library_dependencies { + // first create the library crate if needed + // this crate might not have been registered yet because of the order of the HashMap + // e.g. {root: [lib1], libs: { lib2 -> [lib3], lib1 -> [lib2] }} + let crate_id = *crate_names + .entry(lib_name.clone()) + .or_insert_with(|| add_noir_lib(&mut compiler_context, lib_name)); + + for dependency_name in dependencies { + let dependency_name_string = dependency_name.to_string(); + + let dep_crate_id = crate_names + .entry(dependency_name.clone()) + .or_insert_with(|| add_noir_lib(&mut compiler_context, dependency_name)); + + compiler_context.add_dependency_edge(dependency_name_string, &crate_id, dep_crate_id); + } + } + + let is_contract = contracts.unwrap_or(false); + let program_width = 3; + + if is_contract { + compiler_context.compile_contract(program_width) + } else { + compiler_context.compile_program(program_width) + } +} + +#[cfg(test)] +mod test { + use noirc_driver::prepare_crate; + use noirc_frontend::{graph::CrateGraph, hir::Context}; + + use crate::compile::{file_manager_with_source_map, PathToFileSourceMap}; + + use std::path::Path; + + use super::CompilerContext; + + fn setup_test_context(source_map: PathToFileSourceMap) -> CompilerContext { + let mut fm = file_manager_with_source_map(source_map); + // Add this due to us calling prepare_crate on "/main.nr" below + fm.add_file_with_source(Path::new("/main.nr"), "fn foo() {}".to_string()); + + let graph = CrateGraph::default(); + let mut context = Context::new(fm, graph); + prepare_crate(&mut context, Path::new("/main.nr")); + + CompilerContext { context } + } + + #[test] + fn test_works_with_empty_dependency_graph() { + let source_map = PathToFileSourceMap::default(); + let context = setup_test_context(source_map); + + // one stdlib + one root crate + assert_eq!(context.crate_graph().number_of_crates(), 2); + } + + #[test] + fn test_works_with_root_dependencies() { + let source_map = PathToFileSourceMap( + vec![(Path::new("lib1/lib.nr").to_path_buf(), "fn foo() {}".to_string())] + .into_iter() + .collect(), + ); + + let mut context = setup_test_context(source_map); + context.process_dependency_crate("lib1/lib.nr".to_string()); + + assert_eq!(context.crate_graph().number_of_crates(), 3); + } + + #[test] + fn test_works_with_duplicate_root_dependencies() { + let source_map = PathToFileSourceMap( + vec![(Path::new("lib1/lib.nr").to_path_buf(), "fn foo() {}".to_string())] + .into_iter() + .collect(), + ); + let mut context = setup_test_context(source_map); + + let lib1_crate_id = context.process_dependency_crate("lib1/lib.nr".to_string()); + let root_crate_id = context.root_crate_id(); + + context.add_dependency_edge("lib1".to_string(), &root_crate_id, &lib1_crate_id); + context.add_dependency_edge("lib1".to_string(), &root_crate_id, &lib1_crate_id); + + assert_eq!(context.crate_graph().number_of_crates(), 3); + } + + #[test] + fn test_works_with_transitive_dependencies() { + let source_map = PathToFileSourceMap( + vec![ + (Path::new("lib1/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + (Path::new("lib2/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + (Path::new("lib3/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + ] + .into_iter() + .collect(), + ); + + let mut context = setup_test_context(source_map); + + let lib1_crate_id = context.process_dependency_crate("lib1/lib.nr".to_string()); + let lib2_crate_id = context.process_dependency_crate("lib2/lib.nr".to_string()); + let lib3_crate_id = context.process_dependency_crate("lib3/lib.nr".to_string()); + let root_crate_id = context.root_crate_id(); + + context.add_dependency_edge("lib1".to_string(), &root_crate_id, &lib1_crate_id); + context.add_dependency_edge("lib2".to_string(), &lib1_crate_id, &lib2_crate_id); + context.add_dependency_edge("lib3".to_string(), &lib2_crate_id, &lib3_crate_id); + + assert_eq!(context.crate_graph().number_of_crates(), 5); + } + + #[test] + fn test_works_with_missing_dependencies() { + let source_map = PathToFileSourceMap( + vec![ + (Path::new("lib1/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + (Path::new("lib2/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + (Path::new("lib3/lib.nr").to_path_buf(), "fn foo() {}".to_string()), + ] + .into_iter() + .collect(), + ); + let mut context = setup_test_context(source_map); + + let lib1_crate_id = context.process_dependency_crate("lib1/lib.nr".to_string()); + let lib2_crate_id = context.process_dependency_crate("lib2/lib.nr".to_string()); + let lib3_crate_id = context.process_dependency_crate("lib3/lib.nr".to_string()); + let root_crate_id = context.root_crate_id(); + + context.add_dependency_edge("lib1".to_string(), &root_crate_id, &lib1_crate_id); + context.add_dependency_edge("lib3".to_string(), &lib2_crate_id, &lib3_crate_id); + + assert_eq!(context.crate_graph().number_of_crates(), 5); + } +} diff --git a/compiler/wasm/src/lib.rs b/compiler/wasm/src/lib.rs index 9f2f558f85c..43095fee4d4 100644 --- a/compiler/wasm/src/lib.rs +++ b/compiler/wasm/src/lib.rs @@ -14,11 +14,15 @@ use wasm_bindgen::prelude::*; mod circuit; mod compile; +mod compile_new; mod errors; pub use circuit::{acir_read_bytes, acir_write_bytes}; pub use compile::compile; +// Expose the new Context-Centric API +pub use compile_new::{compile_, CompilerContext, CrateIDWrapper}; + #[derive(Serialize, Deserialize)] pub struct BuildInfo { git_hash: &'static str, diff --git a/compiler/wasm/test/node/index.test.ts b/compiler/wasm/test/node/index.test.ts index 5cf9e3be2df..390cc940361 100644 --- a/compiler/wasm/test/node/index.test.ts +++ b/compiler/wasm/test/node/index.test.ts @@ -9,7 +9,7 @@ import { } from '../shared'; import { readFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; -import { compile, PathToFileSourceMap } from '@noir-lang/noir_wasm'; +import { compile, compile_, CompilerContext, PathToFileSourceMap } from '@noir-lang/noir_wasm'; // eslint-disable-next-line @typescript-eslint/no-explicit-any async function getPrecompiledSource(path: string): Promise { @@ -72,4 +72,80 @@ describe('noir wasm compilation', () => { expect(wasmCircuit.program.backend).to.eq(cliCircuit.backend); }).timeout(10e3); }); + + describe('can compile scripts with dependencies -- context-api', () => { + let sourceMap: PathToFileSourceMap; + beforeEach(() => { + sourceMap = new PathToFileSourceMap(); + sourceMap.add_source_code('script/main.nr', readFileSync(join(__dirname, depsScriptSourcePath), 'utf-8')); + sourceMap.add_source_code('lib_a/lib.nr', readFileSync(join(__dirname, libASourcePath), 'utf-8')); + sourceMap.add_source_code('lib_b/lib.nr', readFileSync(join(__dirname, libBSourcePath), 'utf-8')); + }); + + it('matching nargos compilation - context-api', async () => { + const compilerContext = new CompilerContext(sourceMap); + + // Process root crate + const root_crate_id = compilerContext.process_root_crate('script/main.nr'); + // Process dependencies + // + // This can be direct dependencies or transitive dependencies + // I have named these crate_id_1 and crate_id_2 instead of `lib_a_crate_id` and `lib_b_crate_id` + // because the names of crates in a dependency graph are not determined by the actual package. + // + // It is true that each package is given a name, but if I include a `lib_a` as a dependency + // in my library, I do not need to refer to it as `lib_a` in my dependency graph. + // See https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml + // + // If you have looked at graphs before, then you can think of the dependency graph as a directed acyclic graph (DAG) + const crate_id_1 = compilerContext.process_dependency_crate('lib_a/lib.nr'); + const crate_id_2 = compilerContext.process_dependency_crate('lib_b/lib.nr'); + + // Root crate depends on `crate_id_1` and this edge is called `lib_a` + compilerContext.add_dependency_edge('lib_a', root_crate_id, crate_id_1); + // `crate_id_1` depends on `crate_id_2` and this edge is called `lib_b` + compilerContext.add_dependency_edge('lib_b', crate_id_1, crate_id_2); + + const program_width = 3; + const wasmCircuit = await compilerContext.compile_program(program_width); + + const cliCircuit = await getPrecompiledSource(depsScriptExpectedArtifact); + + if (!('program' in wasmCircuit)) { + throw Error('Expected program to be present'); + } + + // We don't expect the hashes to match due to how `noir_wasm` handles dependencies + expect(wasmCircuit.program.noir_version).to.eq(cliCircuit.noir_version); + expect(wasmCircuit.program.bytecode).to.eq(cliCircuit.bytecode); + expect(wasmCircuit.program.abi).to.deep.eq(cliCircuit.abi); + expect(wasmCircuit.program.backend).to.eq(cliCircuit.backend); + }).timeout(10e3); + + it('matching nargos compilation - context-implementation-compile-api', async () => { + const wasmCircuit = await compile_( + 'script/main.nr', + false, + { + root_dependencies: ['lib_a'], + library_dependencies: { + lib_a: ['lib_b'], + }, + }, + sourceMap, + ); + + const cliCircuit = await getPrecompiledSource(depsScriptExpectedArtifact); + + if (!('program' in wasmCircuit)) { + throw Error('Expected program to be present'); + } + + // We don't expect the hashes to match due to how `noir_wasm` handles dependencies + expect(wasmCircuit.program.noir_version).to.eq(cliCircuit.noir_version); + expect(wasmCircuit.program.bytecode).to.eq(cliCircuit.bytecode); + expect(wasmCircuit.program.abi).to.deep.eq(cliCircuit.abi); + expect(wasmCircuit.program.backend).to.eq(cliCircuit.backend); + }).timeout(10e3); + }); });