From 4524c40371c0928e364bd0eb1efe5b4d9c34b04f Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Tue, 18 Jun 2024 16:14:25 -0500 Subject: [PATCH 01/12] Finally get this parser compiling --- compiler/noirc_frontend/src/ast/expression.rs | 2 +- .../src/elaborator/expressions.rs | 2 +- .../noirc_frontend/src/hir/type_check/expr.rs | 2 +- compiler/noirc_frontend/src/hir_def/types.rs | 6 +- compiler/noirc_frontend/src/lexer/token.rs | 3 + compiler/noirc_frontend/src/parser/parser.rs | 58 ++++++++++++++++--- .../noirc_frontend/src/parser/parser/types.rs | 9 +++ 7 files changed, 68 insertions(+), 14 deletions(-) diff --git a/compiler/noirc_frontend/src/ast/expression.rs b/compiler/noirc_frontend/src/ast/expression.rs index 075df4ee4e3..793911d855d 100644 --- a/compiler/noirc_frontend/src/ast/expression.rs +++ b/compiler/noirc_frontend/src/ast/expression.rs @@ -33,7 +33,7 @@ pub enum ExpressionKind { Tuple(Vec), Lambda(Box), Parenthesized(Box), - Quote(BlockExpression, Span), + Quote(Vec, Span), Unquote(Box), Comptime(BlockExpression, Span), diff --git a/compiler/noirc_frontend/src/elaborator/expressions.rs b/compiler/noirc_frontend/src/elaborator/expressions.rs index d6ad752b67a..c4a750543ba 100644 --- a/compiler/noirc_frontend/src/elaborator/expressions.rs +++ b/compiler/noirc_frontend/src/elaborator/expressions.rs @@ -650,7 +650,7 @@ impl<'context> Elaborator<'context> { let mut unquoted_exprs = Vec::new(); self.find_unquoted_exprs_in_block(&mut block, &mut unquoted_exprs); let quoted = HirQuoted { quoted_block: block, unquoted_exprs }; - (HirExpression::Quote(quoted), Type::Quoted(QuotedType::Expr)) + (HirExpression::Quote(quoted), Type::Slice(Box::new(Type::Quoted(QuotedType::Symbol)))) } fn elaborate_comptime_block(&mut self, block: BlockExpression, span: Span) -> (ExprId, Type) { diff --git a/compiler/noirc_frontend/src/hir/type_check/expr.rs b/compiler/noirc_frontend/src/hir/type_check/expr.rs index 46e8db8f5ff..844d793bcc6 100644 --- a/compiler/noirc_frontend/src/hir/type_check/expr.rs +++ b/compiler/noirc_frontend/src/hir/type_check/expr.rs @@ -307,7 +307,7 @@ impl<'interner> TypeChecker<'interner> { Type::Function(params, Box::new(lambda.return_type), Box::new(env_type)) } - HirExpression::Quote(_) => Type::Quoted(crate::QuotedType::Expr), + HirExpression::Quote(_) => Type::Slice(Box::new(Type::Quoted(crate::QuotedType::Expr))), HirExpression::Comptime(block) => self.check_block(block), // Unquote should be inserted & removed by the comptime interpreter. diff --git a/compiler/noirc_frontend/src/hir_def/types.rs b/compiler/noirc_frontend/src/hir_def/types.rs index ce36a22cf88..d24720ec152 100644 --- a/compiler/noirc_frontend/src/hir_def/types.rs +++ b/compiler/noirc_frontend/src/hir_def/types.rs @@ -190,9 +190,10 @@ impl Type { #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)] pub enum QuotedType { Expr, - TypeDefinition, + Symbol, TopLevelItem, Type, + TypeDefinition, } /// A list of TypeVariableIds to bind to a type. Storing the @@ -966,9 +967,10 @@ impl std::fmt::Display for QuotedType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { QuotedType::Expr => write!(f, "Expr"), - QuotedType::TypeDefinition => write!(f, "TypeDefinition"), + QuotedType::Symbol => write!(f, "Symbol"), QuotedType::TopLevelItem => write!(f, "TopLevelItem"), QuotedType::Type => write!(f, "Type"), + QuotedType::TypeDefinition => write!(f, "TypeDefinition"), } } } diff --git a/compiler/noirc_frontend/src/lexer/token.rs b/compiler/noirc_frontend/src/lexer/token.rs index ec589a816f6..18b58aa2acc 100644 --- a/compiler/noirc_frontend/src/lexer/token.rs +++ b/compiler/noirc_frontend/src/lexer/token.rs @@ -865,6 +865,7 @@ pub enum Keyword { String, Struct, Super, + Symbol, TopLevelItem, Trait, Type, @@ -913,6 +914,7 @@ impl fmt::Display for Keyword { Keyword::String => write!(f, "str"), Keyword::Struct => write!(f, "struct"), Keyword::Super => write!(f, "super"), + Keyword::Symbol => write!(f, "Symbol"), Keyword::TopLevelItem => write!(f, "TopLevelItem"), Keyword::Trait => write!(f, "trait"), Keyword::Type => write!(f, "type"), @@ -964,6 +966,7 @@ impl Keyword { "str" => Keyword::String, "struct" => Keyword::Struct, "super" => Keyword::Super, + "Symbol" => Keyword::Symbol, "TopLevelItem" => Keyword::TopLevelItem, "trait" => Keyword::Trait, "type" => Keyword::Type, diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index e8838c58772..eb0b8e870dc 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -1074,8 +1074,8 @@ where }, lambdas::lambda(expr_parser.clone()), block(statement.clone()).map(ExpressionKind::Block), - comptime_expr(statement.clone()), - quote(statement), + comptime_expr(statement), + quote(), unquote(expr_parser.clone()), variable(), literal(), @@ -1101,21 +1101,61 @@ where .labelled(ParsingRuleLabel::Atom) } -fn quote<'a, P>(statement: P) -> impl NoirParser + 'a -where - P: NoirParser + 'a, -{ - keyword(Keyword::Quote).ignore_then(spanned(block(statement))).validate( - |(block, block_span), span, emit| { +fn quote() -> impl NoirParser { + keyword(Keyword::Quote).ignore_then(spanned(token_stream_block(false, true))).validate( + |(tokens, block_span), span, emit| { emit(ParserError::with_reason( ParserErrorReason::ExperimentalFeature("quoted expressions"), span, )); - ExpressionKind::Quote(block, block_span) + ExpressionKind::Quote(tokens, block_span) }, ) } +/// Parses a stream of tokens terminated by '{' or '}'. +/// - parse_braces: if true, parses '{' and '}' surrounding the token stream. +/// - include_braces: if true, include the surrounding braces in the returned tokens vec +fn token_stream_block(include_braces: bool, parse_braces: bool) -> impl NoirParser> { + let append_vecs = |(mut vec1, mut vec2): (Vec, _)| { + vec1.append(&mut vec2); + vec1 + }; + + // Parse a stream of tokens ending in '{' or '}'. + // - If we ended with a '}': end + // - If we ended with a '{': recursively parse another token stream block + let inner_stream = none_of([Token::LeftBrace, Token::RightBrace]) + .repeated() + .then(one_of([Token::LeftBrace, Token::RightBrace]).rewind().then_with(move |end| { + match end { + Token::LeftBrace => token_stream_block(true, true) + .then(token_stream_block(false, false)) + .map(append_vecs) + .boxed(), + _ => empty().map(|_| Vec::new()).boxed(), + } + })) + .map(append_vecs); + + if parse_braces { + just(Token::LeftBrace) + .ignore_then(inner_stream) + .then_ignore(just(Token::RightBrace)) + .map(move |mut stream| { + let mut ret = if include_braces { vec![Token::LeftBrace] } else { vec![] }; + ret.append(&mut stream); + if include_braces { + ret.push(Token::LeftBrace); + } + ret + }) + .boxed() + } else { + inner_stream.boxed() + } +} + /// unquote: '$' variable /// | '$' '(' expression ')' fn unquote<'a, P>(expr_parser: P) -> impl NoirParser + 'a diff --git a/compiler/noirc_frontend/src/parser/parser/types.rs b/compiler/noirc_frontend/src/parser/parser/types.rs index a961b32f9d7..fabc6fbcab7 100644 --- a/compiler/noirc_frontend/src/parser/parser/types.rs +++ b/compiler/noirc_frontend/src/parser/parser/types.rs @@ -27,6 +27,7 @@ pub(super) fn parse_type_inner<'a>( type_definition_type(), top_level_item_type(), quoted_type(), + symbol_type(), format_string_type(recursive_type_parser.clone()), named_type(recursive_type_parser.clone()), named_trait(recursive_type_parser.clone()), @@ -98,6 +99,14 @@ fn quoted_type() -> impl NoirParser { .map_with_span(|_, span| UnresolvedTypeData::Quoted(QuotedType::Type).with_span(span)) } +/// This is the type of a quoted token, most often quoted as part of a token stream. +fn symbol_type() -> impl NoirParser { + keyword(Keyword::Symbol).map_with_span(|_, span| { + let symbol = UnresolvedTypeData::Quoted(QuotedType::Symbol).with_span(span); + UnresolvedTypeData::Slice(Box::new(symbol)).with_span(span) + }) +} + pub(super) fn string_type() -> impl NoirParser { keyword(Keyword::String) .ignore_then(type_expression().delimited_by(just(Token::Less), just(Token::Greater))) From 56f940538a6a4a0a537436da98217de611e3c3bb Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Thu, 20 Jun 2024 15:11:50 -0500 Subject: [PATCH 02/12] Use token streams instead of ASTs for macros --- compiler/noirc_frontend/src/ast/expression.rs | 18 +- .../src/elaborator/expressions.rs | 16 +- .../noirc_frontend/src/elaborator/unquote.rs | 323 ++---------------- .../noirc_frontend/src/hir/comptime/errors.rs | 200 +++++++++-- .../src/hir/comptime/interpreter.rs | 19 +- .../src/hir/comptime/interpreter/unquote.rs | 301 ++-------------- .../noirc_frontend/src/hir/comptime/scan.rs | 2 +- .../noirc_frontend/src/hir/comptime/value.rs | 38 ++- .../src/hir/resolution/resolver.rs | 10 +- .../src/hir/type_check/errors.rs | 6 +- .../noirc_frontend/src/hir/type_check/expr.rs | 4 +- compiler/noirc_frontend/src/hir_def/expr.rs | 15 +- compiler/noirc_frontend/src/hir_def/types.rs | 4 +- compiler/noirc_frontend/src/lexer/token.rs | 21 +- compiler/noirc_frontend/src/node_interner.rs | 2 +- compiler/noirc_frontend/src/parser/mod.rs | 4 +- compiler/noirc_frontend/src/parser/parser.rs | 32 +- .../noirc_frontend/src/parser/parser/types.rs | 14 +- compiler/noirc_frontend/src/tests.rs | 24 ++ tooling/nargo_fmt/src/rewrite/expr.rs | 5 +- 20 files changed, 358 insertions(+), 700 deletions(-) diff --git a/compiler/noirc_frontend/src/ast/expression.rs b/compiler/noirc_frontend/src/ast/expression.rs index 793911d855d..e65328c6aac 100644 --- a/compiler/noirc_frontend/src/ast/expression.rs +++ b/compiler/noirc_frontend/src/ast/expression.rs @@ -7,7 +7,7 @@ use crate::ast::{ }; use crate::macros_api::StructId; use crate::node_interner::ExprId; -use crate::token::{Attributes, Token}; +use crate::token::{Attributes, Token, Tokens}; use acvm::{acir::AcirField, FieldElement}; use iter_extended::vecmap; use noirc_errors::{Span, Spanned}; @@ -33,18 +33,10 @@ pub enum ExpressionKind { Tuple(Vec), Lambda(Box), Parenthesized(Box), - Quote(Vec, Span), + Quote(Tokens, Span), Unquote(Box), Comptime(BlockExpression, Span), - /// Unquote expressions are replaced with UnquoteMarkers when Quoted - /// expressions are resolved. Since the expression being quoted remains an - /// ExpressionKind (rather than a resolved ExprId), the UnquoteMarker must be - /// here in the AST even though it is technically HIR-only. - /// Each usize in an UnquoteMarker is an index which corresponds to the index of the - /// expression in the Hir::Quote expression list to replace it with. - UnquoteMarker(usize), - // This variant is only emitted when inlining the result of comptime // code. It is used to translate function values back into the AST while // guaranteeing they have the same instantiated type and definition id without resolving again. @@ -557,12 +549,14 @@ impl Display for ExpressionKind { } Lambda(lambda) => lambda.fmt(f), Parenthesized(sub_expr) => write!(f, "({sub_expr})"), - Quote(block, _) => write!(f, "quote {block}"), Comptime(block, _) => write!(f, "comptime {block}"), Error => write!(f, "Error"), Resolved(_) => write!(f, "?Resolved"), Unquote(expr) => write!(f, "$({expr})"), - UnquoteMarker(index) => write!(f, "${index}"), + Quote(tokens, _) => { + let tokens = vecmap(&tokens.0, ToString::to_string); + write!(f, "quote {{ {} }}", tokens.join(" ")) + } } } } diff --git a/compiler/noirc_frontend/src/elaborator/expressions.rs b/compiler/noirc_frontend/src/elaborator/expressions.rs index c4a750543ba..24f525f2f8c 100644 --- a/compiler/noirc_frontend/src/elaborator/expressions.rs +++ b/compiler/noirc_frontend/src/elaborator/expressions.rs @@ -18,7 +18,7 @@ use crate::{ HirArrayLiteral, HirBinaryOp, HirBlockExpression, HirCallExpression, HirCastExpression, HirConstructorExpression, HirIfExpression, HirIndexExpression, HirInfixExpression, HirLambda, HirMemberAccess, HirMethodCallExpression, HirMethodReference, - HirPrefixExpression, HirQuoted, + HirPrefixExpression, }, traits::TraitConstraint, }, @@ -28,6 +28,7 @@ use crate::{ MethodCallExpression, PrefixExpression, }, node_interner::{DefinitionKind, ExprId, FuncId}, + token::Tokens, QuotedType, Shared, StructType, Type, }; @@ -68,9 +69,6 @@ impl<'context> Elaborator<'context> { self.push_err(ResolverError::UnquoteUsedOutsideQuote { span: expr.span }); (HirExpression::Error, Type::Error) } - ExpressionKind::UnquoteMarker(index) => { - unreachable!("UnquoteMarker({index}) remaining in runtime code") - } }; let id = self.interner.push_expr(hir_expr); self.interner.push_expr_location(id, expr.span, self.file); @@ -646,11 +644,9 @@ impl<'context> Elaborator<'context> { (expr, Type::Function(arg_types, Box::new(body_type), Box::new(env_type))) } - fn elaborate_quote(&mut self, mut block: BlockExpression) -> (HirExpression, Type) { - let mut unquoted_exprs = Vec::new(); - self.find_unquoted_exprs_in_block(&mut block, &mut unquoted_exprs); - let quoted = HirQuoted { quoted_block: block, unquoted_exprs }; - (HirExpression::Quote(quoted), Type::Slice(Box::new(Type::Quoted(QuotedType::Symbol)))) + fn elaborate_quote(&mut self, mut tokens: Tokens) -> (HirExpression, Type) { + tokens = self.find_unquoted_exprs_tokens(tokens); + (HirExpression::Quote(tokens), Type::Quoted(QuotedType::Quoted)) } fn elaborate_comptime_block(&mut self, block: BlockExpression, span: Span) -> (ExprId, Type) { @@ -716,7 +712,7 @@ impl<'context> Elaborator<'context> { location: Location, return_type: Type, ) -> Option<(HirExpression, Type)> { - self.unify(&return_type, &Type::Quoted(QuotedType::Expr), || { + self.unify(&return_type, &Type::Quoted(QuotedType::Quoted), || { TypeCheckError::MacroReturningNonExpr { typ: return_type.clone(), span: location.span } }); diff --git a/compiler/noirc_frontend/src/elaborator/unquote.rs b/compiler/noirc_frontend/src/elaborator/unquote.rs index 6767f471f16..ed12ba21398 100644 --- a/compiler/noirc_frontend/src/elaborator/unquote.rs +++ b/compiler/noirc_frontend/src/elaborator/unquote.rs @@ -1,306 +1,41 @@ use crate::{ - ast::{ - ArrayLiteral, AssignStatement, ConstrainStatement, ConstructorExpression, IfExpression, - InfixExpression, Lambda, - }, - macros_api::{ - BlockExpression, CallExpression, CastExpression, Expression, ExpressionKind, - ForLoopStatement, ForRange, IndexExpression, LetStatement, Literal, MemberAccessExpression, - MethodCallExpression, PrefixExpression, Statement, StatementKind, - }, - node_interner::ExprId, + macros_api::Path, + token::{SpannedToken, Token, Tokens}, }; use super::Elaborator; impl<'a> Elaborator<'a> { - pub fn find_unquoted_exprs_in_block( - &mut self, - block: &mut BlockExpression, - unquoted_exprs: &mut Vec, - ) { - for statement in &mut block.statements { - self.find_unquoted_exprs_in_statement(statement, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_statement( - &mut self, - statement: &mut Statement, - unquoted_exprs: &mut Vec, - ) { - match &mut statement.kind { - StatementKind::Let(let_) => self.find_unquoted_exprs_in_let(let_, unquoted_exprs), - StatementKind::Constrain(constrain) => { - self.find_unquoted_exprs_in_constrain(constrain, unquoted_exprs); - } - StatementKind::Expression(expr) => { - self.find_unquoted_exprs_in_expr(expr, unquoted_exprs); - } - StatementKind::Assign(assign) => { - self.find_unquoted_exprs_in_assign(assign, unquoted_exprs); - } - StatementKind::For(for_) => self.find_unquoted_exprs_in_for(for_, unquoted_exprs), - StatementKind::Break => (), - StatementKind::Continue => (), - StatementKind::Comptime(comptime) => { - self.find_unquoted_exprs_in_statement(comptime, unquoted_exprs); - } - StatementKind::Semi(expr) => self.find_unquoted_exprs_in_expr(expr, unquoted_exprs), - StatementKind::Error => (), - } - } - - fn find_unquoted_exprs_in_constrain( - &mut self, - constrain: &mut ConstrainStatement, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut constrain.0, unquoted_exprs); - if let Some(msg) = constrain.1.as_mut() { - self.find_unquoted_exprs_in_expr(msg, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_let( - &mut self, - let_: &mut LetStatement, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut let_.expression, unquoted_exprs); - } - - fn find_unquoted_exprs_in_assign( - &mut self, - assign: &mut AssignStatement, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut assign.expression, unquoted_exprs); - } - - fn find_unquoted_exprs_in_for( - &mut self, - for_: &mut ForLoopStatement, - unquoted_exprs: &mut Vec, - ) { - match &mut for_.range { - ForRange::Range(start, end) => { - self.find_unquoted_exprs_in_expr(start, unquoted_exprs); - self.find_unquoted_exprs_in_expr(end, unquoted_exprs); - } - ForRange::Array(array) => { - self.find_unquoted_exprs_in_expr(array, unquoted_exprs); - } - }; - self.find_unquoted_exprs_in_expr(&mut for_.block, unquoted_exprs); - } - - fn find_unquoted_exprs_in_expr( - &mut self, - expr: &mut Expression, - unquoted_exprs: &mut Vec, - ) { - match &mut expr.kind { - ExpressionKind::Literal(literal) => { - self.find_unquoted_exprs_in_literal(literal, unquoted_exprs); - } - ExpressionKind::Block(block) => { - self.find_unquoted_exprs_in_block(block, unquoted_exprs); - } - ExpressionKind::Prefix(prefix) => { - self.find_unquoted_exprs_in_prefix(prefix, unquoted_exprs); - } - ExpressionKind::Index(index) => { - self.find_unquoted_exprs_in_index(index, unquoted_exprs); - } - ExpressionKind::Call(call) => self.find_unquoted_exprs_in_call(call, unquoted_exprs), - ExpressionKind::MethodCall(call) => { - self.find_unquoted_exprs_in_method_call(call, unquoted_exprs); - } - ExpressionKind::Constructor(constructor) => { - self.find_unquoted_exprs_in_constructor(constructor, unquoted_exprs); - } - ExpressionKind::MemberAccess(access) => { - self.find_unquoted_exprs_in_access(access, unquoted_exprs); - } - ExpressionKind::Cast(cast) => self.find_unquoted_exprs_in_cast(cast, unquoted_exprs), - ExpressionKind::Infix(infix) => { - self.find_unquoted_exprs_in_infix(infix, unquoted_exprs); - } - ExpressionKind::If(if_) => self.find_unquoted_exprs_in_if(if_, unquoted_exprs), - ExpressionKind::Variable(_, _) => (), - ExpressionKind::Tuple(tuple) => { - self.find_unquoted_exprs_in_tuple(tuple, unquoted_exprs); - } - ExpressionKind::Lambda(lambda) => { - self.find_unquoted_exprs_in_lambda(lambda, unquoted_exprs); - } - ExpressionKind::Parenthesized(expr) => { - self.find_unquoted_exprs_in_expr(expr, unquoted_exprs); - } - ExpressionKind::Quote(quote, _) => { - self.find_unquoted_exprs_in_block(quote, unquoted_exprs); - } - ExpressionKind::Comptime(block, _) => { - self.find_unquoted_exprs_in_block(block, unquoted_exprs); - } - ExpressionKind::Resolved(_) => (), - ExpressionKind::Error => (), - ExpressionKind::UnquoteMarker(_) => (), - ExpressionKind::Unquote(unquoted) => { - // Avoid an expensive clone for unquoted - let empty_expr = Expression::new(ExpressionKind::Error, unquoted.span); - let unquote = std::mem::replace(unquoted.as_mut(), empty_expr); - self.replace_unquote(expr, unquote, unquoted_exprs); - } - } - } - - fn find_unquoted_exprs_in_literal( - &mut self, - literal: &mut Literal, - unquoted_exprs: &mut Vec, - ) { - match literal { - Literal::Array(array) | Literal::Slice(array) => match array { - ArrayLiteral::Standard(elements) => { - for element in elements { - self.find_unquoted_exprs_in_expr(element, unquoted_exprs); + /// Go through the given tokens looking for a '$' token followed by a variable to unquote. + /// Each time these two tokens are found, they are replaced by a new UnquoteMarker token + /// containing the ExprId of the resolved variable to unquote. + pub fn find_unquoted_exprs_tokens(&mut self, tokens: Tokens) -> Tokens { + let token_count = tokens.0.len(); + let mut new_tokens = Vec::with_capacity(token_count); + let mut tokens = tokens.0.into_iter(); + + while let Some(token) = tokens.next() { + let is_unquote = matches!(token.token(), Token::DollarSign); + new_tokens.push(token); + + if is_unquote { + if let Some(next) = tokens.next() { + let span = next.to_span(); + + match next.into_token() { + Token::Ident(name) => { + // Don't want the leading `$` anymore + new_tokens.pop(); + let path = Path::from_single(name, span); + let (expr_id, _) = self.elaborate_variable(path, None); + new_tokens.push(SpannedToken::new(Token::UnquoteMarker(expr_id), span)); + } + other_next => new_tokens.push(SpannedToken::new(other_next, span)), } } - ArrayLiteral::Repeated { repeated_element, length } => { - self.find_unquoted_exprs_in_expr(repeated_element, unquoted_exprs); - self.find_unquoted_exprs_in_expr(length, unquoted_exprs); - } - }, - Literal::Bool(_) - | Literal::Integer(_, _) - | Literal::Str(_) - | Literal::RawStr(_, _) - | Literal::FmtStr(_) - | Literal::Unit => (), - } - } - - fn find_unquoted_exprs_in_prefix( - &mut self, - prefix: &mut PrefixExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut prefix.rhs, unquoted_exprs); - } - - fn find_unquoted_exprs_in_index( - &mut self, - index: &mut IndexExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut index.collection, unquoted_exprs); - self.find_unquoted_exprs_in_expr(&mut index.index, unquoted_exprs); - } - - fn find_unquoted_exprs_in_call( - &mut self, - call: &mut CallExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut call.func, unquoted_exprs); - - for arg in &mut call.arguments { - self.find_unquoted_exprs_in_expr(arg, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_method_call( - &mut self, - call: &mut MethodCallExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut call.object, unquoted_exprs); - - for arg in &mut call.arguments { - self.find_unquoted_exprs_in_expr(arg, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_constructor( - &mut self, - constructor: &mut ConstructorExpression, - unquoted_exprs: &mut Vec, - ) { - for (_, field) in &mut constructor.fields { - self.find_unquoted_exprs_in_expr(field, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_access( - &mut self, - member_access: &mut MemberAccessExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut member_access.lhs, unquoted_exprs); - } - - fn find_unquoted_exprs_in_cast( - &mut self, - cast: &mut CastExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut cast.lhs, unquoted_exprs); - } - - fn find_unquoted_exprs_in_infix( - &mut self, - infix: &mut InfixExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut infix.lhs, unquoted_exprs); - self.find_unquoted_exprs_in_expr(&mut infix.rhs, unquoted_exprs); - } - - fn find_unquoted_exprs_in_if( - &mut self, - if_: &mut IfExpression, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut if_.condition, unquoted_exprs); - self.find_unquoted_exprs_in_expr(&mut if_.consequence, unquoted_exprs); - - if let Some(alternate) = if_.alternative.as_mut() { - self.find_unquoted_exprs_in_expr(alternate, unquoted_exprs); - } - } - - fn find_unquoted_exprs_in_tuple( - &mut self, - tuple: &mut [Expression], - unquoted_exprs: &mut Vec, - ) { - for field in tuple { - self.find_unquoted_exprs_in_expr(field, unquoted_exprs); + } } - } - - fn find_unquoted_exprs_in_lambda( - &mut self, - lambda: &mut Lambda, - unquoted_exprs: &mut Vec, - ) { - self.find_unquoted_exprs_in_expr(&mut lambda.body, unquoted_exprs); - } - /// Elaborate and store the unquoted expression in the given vector, then - /// replace it with an unquote expression with an UnquoteMarker expression to mark the position - /// to replace it with later. - fn replace_unquote( - &mut self, - expr: &mut Expression, - unquoted: Expression, - unquoted_exprs: &mut Vec, - ) { - let (expr_id, _) = self.elaborate_expression(unquoted); - let unquote_marker_id = unquoted_exprs.len(); - unquoted_exprs.push(expr_id); - expr.kind = ExpressionKind::UnquoteMarker(unquote_marker_id); + Tokens(new_tokens) } } diff --git a/compiler/noirc_frontend/src/hir/comptime/errors.rs b/compiler/noirc_frontend/src/hir/comptime/errors.rs index 05962420f8a..0e64d3c7941 100644 --- a/compiler/noirc_frontend/src/hir/comptime/errors.rs +++ b/compiler/noirc_frontend/src/hir/comptime/errors.rs @@ -1,5 +1,10 @@ -use crate::{hir::def_collector::dc_crate::CompilationError, Type}; +use std::rc::Rc; + +use crate::{ + hir::def_collector::dc_crate::CompilationError, parser::ParserError, token::Tokens, Type, +}; use acvm::{acir::AcirField, FieldElement}; +use iter_extended::vecmap; use noirc_errors::{CustomDiagnostic, Location}; use super::value::Value; @@ -7,41 +12,144 @@ use super::value::Value; /// The possible errors that can halt the interpreter. #[derive(Debug, Clone, PartialEq, Eq)] pub enum InterpreterError { - ArgumentCountMismatch { expected: usize, actual: usize, location: Location }, - TypeMismatch { expected: Type, value: Value, location: Location }, - NonComptimeVarReferenced { name: String, location: Location }, - IntegerOutOfRangeForType { value: FieldElement, typ: Type, location: Location }, - ErrorNodeEncountered { location: Location }, - NonFunctionCalled { value: Value, location: Location }, - NonBoolUsedInIf { value: Value, location: Location }, - NonBoolUsedInConstrain { value: Value, location: Location }, - FailingConstraint { message: Option, location: Location }, - NoMethodFound { name: String, typ: Type, location: Location }, - NonIntegerUsedInLoop { value: Value, location: Location }, - NonPointerDereferenced { value: Value, location: Location }, - NonTupleOrStructInMemberAccess { value: Value, location: Location }, - NonArrayIndexed { value: Value, location: Location }, - NonIntegerUsedAsIndex { value: Value, location: Location }, - NonIntegerIntegerLiteral { typ: Type, location: Location }, - NonIntegerArrayLength { typ: Type, location: Location }, - NonNumericCasted { value: Value, location: Location }, - IndexOutOfBounds { index: usize, length: usize, location: Location }, - ExpectedStructToHaveField { value: Value, field_name: String, location: Location }, - TypeUnsupported { typ: Type, location: Location }, - InvalidValueForUnary { value: Value, operator: &'static str, location: Location }, - InvalidValuesForBinary { lhs: Value, rhs: Value, operator: &'static str, location: Location }, - CastToNonNumericType { typ: Type, location: Location }, - QuoteInRuntimeCode { location: Location }, - NonStructInConstructor { typ: Type, location: Location }, - CannotInlineMacro { value: Value, location: Location }, - UnquoteFoundDuringEvaluation { location: Location }, + ArgumentCountMismatch { + expected: usize, + actual: usize, + location: Location, + }, + TypeMismatch { + expected: Type, + value: Value, + location: Location, + }, + NonComptimeVarReferenced { + name: String, + location: Location, + }, + IntegerOutOfRangeForType { + value: FieldElement, + typ: Type, + location: Location, + }, + ErrorNodeEncountered { + location: Location, + }, + NonFunctionCalled { + value: Value, + location: Location, + }, + NonBoolUsedInIf { + value: Value, + location: Location, + }, + NonBoolUsedInConstrain { + value: Value, + location: Location, + }, + FailingConstraint { + message: Option, + location: Location, + }, + NoMethodFound { + name: String, + typ: Type, + location: Location, + }, + NonIntegerUsedInLoop { + value: Value, + location: Location, + }, + NonPointerDereferenced { + value: Value, + location: Location, + }, + NonTupleOrStructInMemberAccess { + value: Value, + location: Location, + }, + NonArrayIndexed { + value: Value, + location: Location, + }, + NonIntegerUsedAsIndex { + value: Value, + location: Location, + }, + NonIntegerIntegerLiteral { + typ: Type, + location: Location, + }, + NonIntegerArrayLength { + typ: Type, + location: Location, + }, + NonNumericCasted { + value: Value, + location: Location, + }, + IndexOutOfBounds { + index: usize, + length: usize, + location: Location, + }, + ExpectedStructToHaveField { + value: Value, + field_name: String, + location: Location, + }, + TypeUnsupported { + typ: Type, + location: Location, + }, + InvalidValueForUnary { + value: Value, + operator: &'static str, + location: Location, + }, + InvalidValuesForBinary { + lhs: Value, + rhs: Value, + operator: &'static str, + location: Location, + }, + CastToNonNumericType { + typ: Type, + location: Location, + }, + QuoteInRuntimeCode { + location: Location, + }, + NonStructInConstructor { + typ: Type, + location: Location, + }, + CannotInlineMacro { + value: Value, + location: Location, + }, + UnquoteFoundDuringEvaluation { + location: Location, + }, + FailedToParseMacro { + error: ParserError, + tokens: Rc, + parse_rule: &'static str, + file: fm::FileId, + }, - Unimplemented { item: String, location: Location }, + Unimplemented { + item: String, + location: Location, + }, // Perhaps this should be unreachable! due to type checking also preventing this error? // Currently it and the Continue variant are the only interpreter errors without a Location field - BreakNotInLoop { location: Location }, - ContinueNotInLoop { location: Location }, + BreakNotInLoop { + location: Location, + }, + ContinueNotInLoop { + location: Location, + }, // These cases are not errors, they are just used to prevent us from running more code // until the loop can be resumed properly. These cases will never be displayed to users. @@ -97,6 +205,9 @@ impl InterpreterError { | InterpreterError::Unimplemented { location, .. } | InterpreterError::BreakNotInLoop { location, .. } | InterpreterError::ContinueNotInLoop { location, .. } => *location, + InterpreterError::FailedToParseMacro { error, file, .. } => { + Location::new(error.span(), *file) + } InterpreterError::Break | InterpreterError::Continue => { panic!("Tried to get the location of Break/Continue error!") } @@ -258,6 +369,31 @@ impl<'a> From<&'a InterpreterError> for CustomDiagnostic { let secondary = "This is a bug".into(); CustomDiagnostic::simple_error(msg, secondary, location.span) } + InterpreterError::FailedToParseMacro { error, tokens, parse_rule, file: _ } => { + let message = format!("Failed to parse macro's token stream into {parse_rule}"); + let tokens = vecmap(&tokens.0, ToString::to_string).join(" "); + + // 10 is an aribtrary number of tokens here chosen to fit roughly onto one line + let token_stream = if tokens.len() > 10 { + format!("The resulting token stream was: {tokens}") + } else { + format!( + "The resulting token stream was: (stream starts on next line)\n {tokens}" + ) + }; + + let push_the_problem_on_the_library_author = "To avoid this error in the future, try adding input validation to your macro. Erroring out early with an `assert` can be a good way to provide a user-friendly error message".into(); + + let mut diagnostic = CustomDiagnostic::from(error); + // Swap the parser's primary note to become the secondary note so that it is + // more clear this error originates from failing to parse a macro. + let secondary = std::mem::take(&mut diagnostic.message); + diagnostic.add_secondary(secondary, error.span()); + diagnostic.message = message; + diagnostic.add_note(token_stream); + diagnostic.add_note(push_the_problem_on_the_library_author); + diagnostic + } InterpreterError::Unimplemented { item, location } => { let msg = format!("{item} is currently unimplemented"); CustomDiagnostic::simple_error(msg, String::new(), location.span) diff --git a/compiler/noirc_frontend/src/hir/comptime/interpreter.rs b/compiler/noirc_frontend/src/hir/comptime/interpreter.rs index 21b6897f5dd..5d710884db1 100644 --- a/compiler/noirc_frontend/src/hir/comptime/interpreter.rs +++ b/compiler/noirc_frontend/src/hir/comptime/interpreter.rs @@ -7,7 +7,7 @@ use noirc_errors::Location; use rustc_hash::FxHashMap as HashMap; use crate::ast::{BinaryOpKind, FunctionKind, IntegerBitSize, Signedness}; -use crate::hir_def::expr::HirQuoted; +use crate::token::Tokens; use crate::{ hir_def::{ expr::{ @@ -26,8 +26,6 @@ use crate::{ Shared, Type, TypeBinding, TypeBindings, TypeVariableKind, }; -use self::unquote::UnquoteArgs; - use super::errors::{IResult, InterpreterError}; use super::value::Value; @@ -343,9 +341,9 @@ impl<'a> Interpreter<'a> { HirExpression::If(if_) => self.evaluate_if(if_, id), HirExpression::Tuple(tuple) => self.evaluate_tuple(tuple), HirExpression::Lambda(lambda) => self.evaluate_lambda(lambda, id), - HirExpression::Quote(block) => self.evaluate_quote(block, id), + HirExpression::Quote(tokens) => self.evaluate_quote(tokens, id), HirExpression::Comptime(block) => self.evaluate_block(block), - HirExpression::Unquote(block) => { + HirExpression::Unquote(tokens) => { // An Unquote expression being found is indicative of a macro being // expanded within another comptime fn which we don't currently support. let location = self.interner.expr_location(&id); @@ -1136,13 +1134,10 @@ impl<'a> Interpreter<'a> { Ok(Value::Closure(lambda, environment, typ)) } - fn evaluate_quote(&mut self, mut quoted: HirQuoted, expr_id: ExprId) -> IResult { - let file = self.interner.expr_location(&expr_id).file; - let values = try_vecmap(quoted.unquoted_exprs, |value| self.evaluate(value))?; - let args = UnquoteArgs { values, file }; - - self.substitute_unquoted_values_into_block(&mut quoted.quoted_block, &args); - Ok(Value::Code(Rc::new(quoted.quoted_block))) + fn evaluate_quote(&mut self, mut tokens: Tokens, expr_id: ExprId) -> IResult { + let location = self.interner.expr_location(&expr_id); + tokens = self.substitute_unquoted_values_into_tokens(tokens, location)?; + Ok(Value::Code(Rc::new(tokens))) } pub fn evaluate_statement(&mut self, statement: StmtId) -> IResult { diff --git a/compiler/noirc_frontend/src/hir/comptime/interpreter/unquote.rs b/compiler/noirc_frontend/src/hir/comptime/interpreter/unquote.rs index 58ca345ca90..a1ceb27afb2 100644 --- a/compiler/noirc_frontend/src/hir/comptime/interpreter/unquote.rs +++ b/compiler/noirc_frontend/src/hir/comptime/interpreter/unquote.rs @@ -1,287 +1,42 @@ -use fm::FileId; use noirc_errors::Location; use crate::{ - ast::{ - ArrayLiteral, AssignStatement, ConstrainStatement, ConstructorExpression, IfExpression, - InfixExpression, Lambda, - }, - hir::comptime::{errors::IResult, Value}, - macros_api::{ - BlockExpression, CallExpression, CastExpression, Expression, ExpressionKind, - ForLoopStatement, ForRange, IndexExpression, LetStatement, Literal, MemberAccessExpression, - MethodCallExpression, PrefixExpression, Statement, StatementKind, - }, + hir::comptime::{errors::IResult, value::unwrap_rc, Value}, + token::{SpannedToken, Token, Tokens}, }; use super::Interpreter; -pub(super) struct UnquoteArgs { - pub(super) values: Vec, - pub(super) file: FileId, -} - impl<'a> Interpreter<'a> { - pub(super) fn substitute_unquoted_values_into_block( - &mut self, - block: &mut BlockExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - for statement in &mut block.statements { - self.substitute_unquoted_into_statement(statement, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_statement( - &mut self, - statement: &mut Statement, - args: &UnquoteArgs, - ) -> IResult<()> { - match &mut statement.kind { - StatementKind::Let(let_) => self.substitute_unquoted_into_let(let_, args), - StatementKind::Constrain(constrain) => { - self.substitute_unquoted_into_constrain(constrain, args) - } - StatementKind::Expression(expr) => self.substitute_unquoted_into_expr(expr, args), - StatementKind::Assign(assign) => self.substitute_unquoted_into_assign(assign, args), - StatementKind::For(for_) => self.substitute_unquoted_into_for(for_, args), - StatementKind::Break => Ok(()), - StatementKind::Continue => Ok(()), - StatementKind::Comptime(comptime) => { - self.substitute_unquoted_into_statement(comptime, args) - } - StatementKind::Semi(expr) => self.substitute_unquoted_into_expr(expr, args), - StatementKind::Error => Ok(()), - } - } - - fn substitute_unquoted_into_constrain( - &mut self, - constrain: &mut ConstrainStatement, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut constrain.0, args)?; - if let Some(msg) = constrain.1.as_mut() { - self.substitute_unquoted_into_expr(msg, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_let( - &mut self, - let_: &mut LetStatement, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut let_.expression, args) - } - - fn substitute_unquoted_into_assign( - &mut self, - assign: &mut AssignStatement, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut assign.expression, args) - } - - fn substitute_unquoted_into_for( - &mut self, - for_: &mut ForLoopStatement, - args: &UnquoteArgs, - ) -> IResult<()> { - match &mut for_.range { - ForRange::Range(start, end) => { - self.substitute_unquoted_into_expr(start, args)?; - self.substitute_unquoted_into_expr(end, args)?; - } - ForRange::Array(array) => { - self.substitute_unquoted_into_expr(array, args)?; - } - }; - self.substitute_unquoted_into_expr(&mut for_.block, args) - } - - fn substitute_unquoted_into_expr( - &mut self, - expr: &mut Expression, - args: &UnquoteArgs, - ) -> IResult<()> { - match &mut expr.kind { - ExpressionKind::Literal(literal) => { - self.substitute_unquoted_into_literal(literal, args) - } - ExpressionKind::Block(block) => self.substitute_unquoted_values_into_block(block, args), - ExpressionKind::Prefix(prefix) => self.substitute_unquoted_into_prefix(prefix, args), - ExpressionKind::Index(index) => self.substitute_unquoted_into_index(index, args), - ExpressionKind::Call(call) => self.substitute_unquoted_into_call(call, args), - ExpressionKind::MethodCall(call) => { - self.substitute_unquoted_into_method_call(call, args) - } - ExpressionKind::Constructor(constructor) => { - self.substitute_unquoted_into_constructor(constructor, args) - } - ExpressionKind::MemberAccess(access) => { - self.substitute_unquoted_into_access(access, args) - } - ExpressionKind::Cast(cast) => self.substitute_unquoted_into_cast(cast, args), - ExpressionKind::Infix(infix) => self.substitute_unquoted_into_infix(infix, args), - ExpressionKind::If(if_) => self.substitute_unquoted_into_if(if_, args), - ExpressionKind::Variable(_, _) => Ok(()), - ExpressionKind::Tuple(tuple) => self.substitute_unquoted_into_tuple(tuple, args), - ExpressionKind::Lambda(lambda) => self.substitute_unquoted_into_lambda(lambda, args), - ExpressionKind::Parenthesized(expr) => self.substitute_unquoted_into_expr(expr, args), - ExpressionKind::Quote(quote, _) => { - self.substitute_unquoted_values_into_block(quote, args) - } - ExpressionKind::Unquote(unquote) => self.substitute_unquoted_into_expr(unquote, args), - ExpressionKind::Comptime(comptime, _) => { - self.substitute_unquoted_values_into_block(comptime, args) - } - ExpressionKind::Resolved(_) => Ok(()), - ExpressionKind::Error => Ok(()), - ExpressionKind::UnquoteMarker(index) => { - let location = Location::new(expr.span, args.file); - *expr = args.values[*index].clone().into_expression(self.interner, location)?; - Ok(()) - } - } - } - - fn substitute_unquoted_into_literal( - &mut self, - literal: &mut Literal, - args: &UnquoteArgs, - ) -> IResult<()> { - match literal { - Literal::Array(array) | Literal::Slice(array) => match array { - ArrayLiteral::Standard(elements) => { - for element in elements { - self.substitute_unquoted_into_expr(element, args)?; + /// Evaluates any expressions within UnquoteMarkers in the given token list + /// and replaces the expression held by the marker with the evaluated value + /// in expression form. + pub(super) fn substitute_unquoted_values_into_tokens( + &mut self, + tokens: Tokens, + location: Location, + ) -> IResult { + let mut new_tokens = Vec::with_capacity(tokens.0.len()); + + for token in tokens.0 { + let span = token.to_span(); + match token.token() { + Token::UnquoteMarker(id) => { + match self.evaluate(*id)? { + // If the value is already quoted we don't want to change the token stream by + // turning it into a Quoted block (which would add `quote`, `{`, and `}` tokens). + Value::Code(stream) => new_tokens.extend(unwrap_rc(stream).0), + value => { + let new_id = value.into_hir_expression(self.interner, location)?; + let new_token = Token::UnquoteMarker(new_id); + new_tokens.push(SpannedToken::new(new_token, span)); + } } - Ok(()) - } - ArrayLiteral::Repeated { repeated_element, length } => { - self.substitute_unquoted_into_expr(repeated_element, args)?; - self.substitute_unquoted_into_expr(length, args)?; - Ok(()) } - }, - Literal::Bool(_) - | Literal::Integer(_, _) - | Literal::Str(_) - | Literal::RawStr(_, _) - | Literal::FmtStr(_) - | Literal::Unit => Ok(()), - } - } - - fn substitute_unquoted_into_prefix( - &mut self, - prefix: &mut PrefixExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut prefix.rhs, args) - } - - fn substitute_unquoted_into_index( - &mut self, - index: &mut IndexExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut index.collection, args)?; - self.substitute_unquoted_into_expr(&mut index.index, args) - } - - fn substitute_unquoted_into_call( - &mut self, - call: &mut CallExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut call.func, args)?; - for arg in &mut call.arguments { - self.substitute_unquoted_into_expr(arg, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_method_call( - &mut self, - call: &mut MethodCallExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut call.object, args)?; - for arg in &mut call.arguments { - self.substitute_unquoted_into_expr(arg, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_constructor( - &mut self, - constructor: &mut ConstructorExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - for (_, field) in &mut constructor.fields { - self.substitute_unquoted_into_expr(field, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_access( - &mut self, - access: &mut MemberAccessExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut access.lhs, args) - } - - fn substitute_unquoted_into_cast( - &mut self, - cast: &mut CastExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut cast.lhs, args) - } - - fn substitute_unquoted_into_infix( - &mut self, - infix: &mut InfixExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut infix.lhs, args)?; - self.substitute_unquoted_into_expr(&mut infix.rhs, args) - } - - fn substitute_unquoted_into_if( - &mut self, - if_: &mut IfExpression, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut if_.condition, args)?; - self.substitute_unquoted_into_expr(&mut if_.consequence, args)?; - - if let Some(alternative) = if_.alternative.as_mut() { - self.substitute_unquoted_into_expr(alternative, args)?; - } - Ok(()) - } - - fn substitute_unquoted_into_tuple( - &mut self, - tuple: &mut [Expression], - args: &UnquoteArgs, - ) -> IResult<()> { - for field in tuple { - self.substitute_unquoted_into_expr(field, args)?; + _ => new_tokens.push(token), + } } - Ok(()) - } - fn substitute_unquoted_into_lambda( - &mut self, - lambda: &mut Lambda, - args: &UnquoteArgs, - ) -> IResult<()> { - self.substitute_unquoted_into_expr(&mut lambda.body, args) + Ok(Tokens(new_tokens)) } } diff --git a/compiler/noirc_frontend/src/hir/comptime/scan.rs b/compiler/noirc_frontend/src/hir/comptime/scan.rs index f2e21e608ea..770c523bd7f 100644 --- a/compiler/noirc_frontend/src/hir/comptime/scan.rs +++ b/compiler/noirc_frontend/src/hir/comptime/scan.rs @@ -100,7 +100,7 @@ impl<'interner> Interpreter<'interner> { // missed it somehow. In the future we may allow users to manually write unquote // expressions in their code but for now this is unreachable. HirExpression::Unquote(block) => { - unreachable!("Found unquote block while scanning: {block}") + unreachable!("Found unquote block while scanning: {block:?}") } } } diff --git a/compiler/noirc_frontend/src/hir/comptime/value.rs b/compiler/noirc_frontend/src/hir/comptime/value.rs index a75d4fd2b24..3bb53912888 100644 --- a/compiler/noirc_frontend/src/hir/comptime/value.rs +++ b/compiler/noirc_frontend/src/hir/comptime/value.rs @@ -1,20 +1,21 @@ use std::{borrow::Cow, fmt::Display, rc::Rc}; use acvm::{AcirField, FieldElement}; +use chumsky::Parser; use im::Vector; use iter_extended::{try_vecmap, vecmap}; use noirc_errors::Location; use crate::{ - ast::{ - ArrayLiteral, BlockExpression, ConstructorExpression, Ident, IntegerBitSize, Signedness, - }, + ast::{ArrayLiteral, ConstructorExpression, Ident, IntegerBitSize, Signedness}, hir_def::expr::{HirArrayLiteral, HirConstructorExpression, HirIdent, HirLambda, ImplKind}, macros_api::{ Expression, ExpressionKind, HirExpression, HirLiteral, Literal, NodeInterner, Path, StructId, }, node_interner::{ExprId, FuncId}, + parser, + token::{SpannedToken, Token, Tokens}, QuotedType, Shared, Type, }; use rustc_hash::FxHashMap as HashMap; @@ -42,7 +43,7 @@ pub enum Value { Pointer(Shared), Array(Vector, Type), Slice(Vector, Type), - Code(Rc), + Code(Rc), TypeDefinition(StructId), } @@ -72,7 +73,7 @@ impl Value { Value::Struct(_, typ) => return Cow::Borrowed(typ), Value::Array(_, typ) => return Cow::Borrowed(typ), Value::Slice(_, typ) => return Cow::Borrowed(typ), - Value::Code(_) => Type::Quoted(QuotedType::Expr), + Value::Code(_) => Type::Quoted(QuotedType::Quoted), Value::TypeDefinition(_) => Type::Quoted(QuotedType::TypeDefinition), Value::Pointer(element) => { let element = element.borrow().get_type().into_owned(); @@ -174,7 +175,22 @@ impl Value { try_vecmap(elements, |element| element.into_expression(interner, location))?; ExpressionKind::Literal(Literal::Slice(ArrayLiteral::Standard(elements))) } - Value::Code(block) => ExpressionKind::Block(unwrap_rc(block)), + Value::Code(tokens) => { + // Wrap the tokens in '{' and '}' so that we can parse statements as well. + let mut tokens_to_parse = tokens.as_ref().clone(); + tokens_to_parse.0.insert(0, SpannedToken::new(Token::LeftBrace, location.span)); + tokens_to_parse.0.push(SpannedToken::new(Token::RightBrace, location.span)); + + return match parser::expression().parse(tokens_to_parse) { + Ok(expr) => Ok(expr), + Err(mut errors) => Err(InterpreterError::FailedToParseMacro { + error: errors.swap_remove(0), + file: location.file, + tokens, + parse_rule: "an expression", + }), + }; + } Value::Pointer(_) | Value::TypeDefinition(_) => { return Err(InterpreterError::CannotInlineMacro { value: self, location }) } @@ -306,7 +322,7 @@ impl Value { } /// Unwraps an Rc value without cloning the inner value if the reference count is 1. Clones otherwise. -fn unwrap_rc(rc: Rc) -> T { +pub(crate) fn unwrap_rc(rc: Rc) -> T { Rc::try_unwrap(rc).unwrap_or_else(|rc| (*rc).clone()) } @@ -351,7 +367,13 @@ impl Display for Value { let values = vecmap(values, ToString::to_string); write!(f, "&[{}]", values.join(", ")) } - Value::Code(block) => write!(f, "quote {block}"), + Value::Code(tokens) => { + write!(f, "quote {{")?; + for token in tokens.0.iter() { + write!(f, " {token}")?; + } + write!(f, " }}") + } Value::TypeDefinition(_) => write!(f, "(type definition)"), } } diff --git a/compiler/noirc_frontend/src/hir/resolution/resolver.rs b/compiler/noirc_frontend/src/hir/resolution/resolver.rs index 97de66be817..8ff0a92bf1d 100644 --- a/compiler/noirc_frontend/src/hir/resolution/resolver.rs +++ b/compiler/noirc_frontend/src/hir/resolution/resolver.rs @@ -17,7 +17,7 @@ use crate::hir_def::expr::{ HirArrayLiteral, HirBinaryOp, HirBlockExpression, HirCallExpression, HirCapturedVar, HirCastExpression, HirConstructorExpression, HirExpression, HirIdent, HirIfExpression, HirIndexExpression, HirInfixExpression, HirLambda, HirLiteral, HirMemberAccess, - HirMethodCallExpression, HirPrefixExpression, HirQuoted, ImplKind, + HirMethodCallExpression, HirPrefixExpression, ImplKind, }; use crate::hir_def::function::FunctionBody; @@ -1634,10 +1634,7 @@ impl<'a> Resolver<'a> { ExpressionKind::Parenthesized(sub_expr) => return self.resolve_expression(*sub_expr), // The quoted expression isn't resolved since we don't want errors if variables aren't defined - ExpressionKind::Quote(block, _) => { - let quoted = HirQuoted { quoted_block: block, unquoted_exprs: Vec::new() }; - HirExpression::Quote(quoted) - } + ExpressionKind::Quote(block, _) => HirExpression::Quote(block), ExpressionKind::Comptime(block, _) => { HirExpression::Comptime(self.resolve_block(block)) } @@ -1648,9 +1645,6 @@ impl<'a> Resolver<'a> { self.push_err(ResolverError::UnquoteUsedOutsideQuote { span: expr.span }); HirExpression::Literal(HirLiteral::Unit) } - ExpressionKind::UnquoteMarker(index) => { - unreachable!("UnquoteMarker({index}) remaining in runtime code") - } }; // If these lines are ever changed, make sure to change the early return diff --git a/compiler/noirc_frontend/src/hir/type_check/errors.rs b/compiler/noirc_frontend/src/hir/type_check/errors.rs index bd32ba2fce5..d9d021aee3f 100644 --- a/compiler/noirc_frontend/src/hir/type_check/errors.rs +++ b/compiler/noirc_frontend/src/hir/type_check/errors.rs @@ -143,7 +143,7 @@ pub enum TypeCheckError { }, #[error("Strings do not support indexed assignment")] StringIndexAssign { span: Span }, - #[error("Macro calls may only return Expr values")] + #[error("Macro calls may only return `Quoted` values")] MacroReturningNonExpr { typ: Type, span: Span }, } @@ -338,8 +338,8 @@ impl<'a> From<&'a TypeCheckError> for Diagnostic { Diagnostic::simple_error(msg, "".into(), *span) }, TypeCheckError::MacroReturningNonExpr { typ, span } => Diagnostic::simple_error( - format!("Expected macro call to return an `Expr` but found a(n) {typ}"), - "Macro calls must return quoted expressions, otherwise there is no code to insert".into(), + format!("Expected macro call to return a `Quoted` but found a(n) `{typ}`"), + "Macro calls must return quoted values, otherwise there is no code to insert".into(), *span, ), } diff --git a/compiler/noirc_frontend/src/hir/type_check/expr.rs b/compiler/noirc_frontend/src/hir/type_check/expr.rs index 80b988dbbe1..4ded04ec2a4 100644 --- a/compiler/noirc_frontend/src/hir/type_check/expr.rs +++ b/compiler/noirc_frontend/src/hir/type_check/expr.rs @@ -307,13 +307,13 @@ impl<'interner> TypeChecker<'interner> { Type::Function(params, Box::new(lambda.return_type), Box::new(env_type)) } - HirExpression::Quote(_) => Type::Slice(Box::new(Type::Quoted(crate::QuotedType::Expr))), + HirExpression::Quote(_) => Type::Quoted(crate::QuotedType::Quoted), HirExpression::Comptime(block) => self.check_block(block), // Unquote should be inserted & removed by the comptime interpreter. // Even if we allowed it here, we wouldn't know what type to give to the result. HirExpression::Unquote(block) => { - unreachable!("Unquote remaining during type checking {block}") + unreachable!("Unquote remaining during type checking {block:?}") } }; diff --git a/compiler/noirc_frontend/src/hir_def/expr.rs b/compiler/noirc_frontend/src/hir_def/expr.rs index 9547aef866b..8de4f118774 100644 --- a/compiler/noirc_frontend/src/hir_def/expr.rs +++ b/compiler/noirc_frontend/src/hir_def/expr.rs @@ -4,6 +4,7 @@ use noirc_errors::Location; use crate::ast::{BinaryOp, BinaryOpKind, Ident, UnaryOp}; use crate::node_interner::{DefinitionId, ExprId, FuncId, NodeInterner, StmtId, TraitMethodId}; +use crate::token::Tokens; use crate::Shared; use super::stmt::HirPattern; @@ -33,8 +34,8 @@ pub enum HirExpression { If(HirIfExpression), Tuple(Vec), Lambda(HirLambda), - Quote(HirQuoted), - Unquote(crate::ast::BlockExpression), + Quote(Tokens), + Unquote(Tokens), Comptime(HirBlockExpression), Error, } @@ -291,13 +292,3 @@ pub struct HirLambda { pub body: ExprId, pub captures: Vec, } - -#[derive(Debug, Clone)] -pub struct HirQuoted { - pub quoted_block: crate::ast::BlockExpression, - - /// Each expression here corresponds to a `ExpressionKind::UnquoteMarker(index)` in `quoted_block`. - /// The values of these expressions after evaluation will be inlined into the position - /// indicated by their corresponding UnquoteMarker with that index. - pub unquoted_exprs: Vec, -} diff --git a/compiler/noirc_frontend/src/hir_def/types.rs b/compiler/noirc_frontend/src/hir_def/types.rs index b9746dede8d..19ab3f8853e 100644 --- a/compiler/noirc_frontend/src/hir_def/types.rs +++ b/compiler/noirc_frontend/src/hir_def/types.rs @@ -190,7 +190,7 @@ impl Type { #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)] pub enum QuotedType { Expr, - Symbol, + Quoted, TopLevelItem, Type, TypeDefinition, @@ -1009,7 +1009,7 @@ impl std::fmt::Display for QuotedType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { QuotedType::Expr => write!(f, "Expr"), - QuotedType::Symbol => write!(f, "Symbol"), + QuotedType::Quoted => write!(f, "Quoted"), QuotedType::TopLevelItem => write!(f, "TopLevelItem"), QuotedType::Type => write!(f, "Type"), QuotedType::TypeDefinition => write!(f, "TypeDefinition"), diff --git a/compiler/noirc_frontend/src/lexer/token.rs b/compiler/noirc_frontend/src/lexer/token.rs index 18b58aa2acc..8d40d81db3b 100644 --- a/compiler/noirc_frontend/src/lexer/token.rs +++ b/compiler/noirc_frontend/src/lexer/token.rs @@ -2,7 +2,7 @@ use acvm::{acir::AcirField, FieldElement}; use noirc_errors::{Position, Span, Spanned}; use std::{fmt, iter::Map, vec::IntoIter}; -use crate::lexer::errors::LexerErrorKind; +use crate::{lexer::errors::LexerErrorKind, node_interner::ExprId}; /// Represents a token in noir's grammar - a word, number, /// or symbol that can be used in noir's syntax. This is the @@ -94,6 +94,11 @@ pub enum BorrowedToken<'input> { Whitespace(&'input str), + /// This is an implementation detail on how macros are implemented by quoting token streams. + /// This token marks where an unquote operation is performed. The ExprId argument is the + /// resolved variable which is being unquoted at this position in the token stream. + UnquoteMarker(ExprId), + /// An invalid character is one that is not in noir's language or grammar. /// /// We don't report invalid tokens in the source as errors until parsing to @@ -188,6 +193,11 @@ pub enum Token { Whitespace(String), + /// This is an implementation detail on how macros are implemented by quoting token streams. + /// This token marks where an unquote operation is performed. The ExprId argument is the + /// resolved variable which is being unquoted at this position in the token stream. + UnquoteMarker(ExprId), + /// An invalid character is one that is not in noir's language or grammar. /// /// We don't report invalid tokens in the source as errors until parsing to @@ -246,6 +256,7 @@ pub fn token_to_borrowed_token(token: &Token) -> BorrowedToken<'_> { Token::EOF => BorrowedToken::EOF, Token::Invalid(c) => BorrowedToken::Invalid(*c), Token::Whitespace(ref s) => BorrowedToken::Whitespace(s), + Token::UnquoteMarker(id) => BorrowedToken::UnquoteMarker(*id), } } @@ -358,6 +369,7 @@ impl fmt::Display for Token { Token::EOF => write!(f, "end of input"), Token::Invalid(c) => write!(f, "{c}"), Token::Whitespace(ref s) => write!(f, "{s}"), + Token::UnquoteMarker(_) => write!(f, "(UnquoteMarker)"), } } } @@ -860,12 +872,12 @@ pub enum Keyword { Mut, Pub, Quote, + Quoted, Return, ReturnData, String, Struct, Super, - Symbol, TopLevelItem, Trait, Type, @@ -909,12 +921,12 @@ impl fmt::Display for Keyword { Keyword::Mut => write!(f, "mut"), Keyword::Pub => write!(f, "pub"), Keyword::Quote => write!(f, "quote"), + Keyword::Quoted => write!(f, "Quoted"), Keyword::Return => write!(f, "return"), Keyword::ReturnData => write!(f, "return_data"), Keyword::String => write!(f, "str"), Keyword::Struct => write!(f, "struct"), Keyword::Super => write!(f, "super"), - Keyword::Symbol => write!(f, "Symbol"), Keyword::TopLevelItem => write!(f, "TopLevelItem"), Keyword::Trait => write!(f, "trait"), Keyword::Type => write!(f, "type"), @@ -961,12 +973,12 @@ impl Keyword { "mut" => Keyword::Mut, "pub" => Keyword::Pub, "quote" => Keyword::Quote, + "Quoted" => Keyword::Quoted, "return" => Keyword::Return, "return_data" => Keyword::ReturnData, "str" => Keyword::String, "struct" => Keyword::Struct, "super" => Keyword::Super, - "Symbol" => Keyword::Symbol, "TopLevelItem" => Keyword::TopLevelItem, "trait" => Keyword::Trait, "type" => Keyword::Type, @@ -987,6 +999,7 @@ impl Keyword { } } +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Tokens(pub Vec); type TokenMapIter = Map, fn(SpannedToken) -> (Token, Span)>; diff --git a/compiler/noirc_frontend/src/node_interner.rs b/compiler/noirc_frontend/src/node_interner.rs index 915093df514..78b2c056dfd 100644 --- a/compiler/noirc_frontend/src/node_interner.rs +++ b/compiler/noirc_frontend/src/node_interner.rs @@ -303,7 +303,7 @@ impl StmtId { } } -#[derive(Debug, Eq, PartialEq, Hash, Copy, Clone)] +#[derive(Debug, Eq, PartialEq, Hash, Copy, Clone, PartialOrd, Ord)] pub struct ExprId(Index); impl ExprId { diff --git a/compiler/noirc_frontend/src/parser/mod.rs b/compiler/noirc_frontend/src/parser/mod.rs index 9e60383afee..72b1ea05ec2 100644 --- a/compiler/noirc_frontend/src/parser/mod.rs +++ b/compiler/noirc_frontend/src/parser/mod.rs @@ -22,7 +22,7 @@ use chumsky::primitive::Container; pub use errors::ParserError; pub use errors::ParserErrorReason; use noirc_errors::Span; -pub use parser::parse_program; +pub use parser::{expression, parse_program}; #[derive(Debug, Clone)] pub(crate) enum TopLevelStatement { @@ -45,7 +45,7 @@ pub trait NoirParser: Parser + Sized + Clone { impl NoirParser for P where P: Parser + Clone {} // ExprParser just serves as a type alias for NoirParser + Clone -trait ExprParser: NoirParser {} +pub trait ExprParser: NoirParser {} impl

ExprParser for P where P: NoirParser {} fn parenthesized(parser: P) -> impl NoirParser diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index eb0b8e870dc..f70b54ce698 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -44,7 +44,7 @@ use crate::ast::{ }; use crate::lexer::{lexer::from_spanned_token_result, Lexer}; use crate::parser::{force, ignore_then_commit, statement_recovery}; -use crate::token::{Keyword, Token, TokenKind}; +use crate::token::{Keyword, SpannedToken, Token, TokenKind, Tokens}; use chumsky::prelude::*; use iter_extended::vecmap; @@ -691,7 +691,7 @@ fn optional_visibility() -> impl NoirParser { }) } -fn expression() -> impl ExprParser { +pub fn expression() -> impl ExprParser { recursive(|expr| { expression_with_precedence( Precedence::Lowest, @@ -1108,7 +1108,7 @@ fn quote() -> impl NoirParser { ParserErrorReason::ExperimentalFeature("quoted expressions"), span, )); - ExpressionKind::Quote(tokens, block_span) + ExpressionKind::Quote(Tokens(tokens), block_span) }, ) } @@ -1116,8 +1116,11 @@ fn quote() -> impl NoirParser { /// Parses a stream of tokens terminated by '{' or '}'. /// - parse_braces: if true, parses '{' and '}' surrounding the token stream. /// - include_braces: if true, include the surrounding braces in the returned tokens vec -fn token_stream_block(include_braces: bool, parse_braces: bool) -> impl NoirParser> { - let append_vecs = |(mut vec1, mut vec2): (Vec, _)| { +fn token_stream_block( + include_braces: bool, + parse_braces: bool, +) -> impl NoirParser> { + let append_vecs = |(mut vec1, mut vec2): (Vec<_>, _)| { vec1.append(&mut vec2); vec1 }; @@ -1125,7 +1128,8 @@ fn token_stream_block(include_braces: bool, parse_braces: bool) -> impl NoirPars // Parse a stream of tokens ending in '{' or '}'. // - If we ended with a '}': end // - If we ended with a '{': recursively parse another token stream block - let inner_stream = none_of([Token::LeftBrace, Token::RightBrace]) + let inner_stream = spanned(none_of([Token::LeftBrace, Token::RightBrace])) + .map(|(token, span)| SpannedToken::new(token, span)) .repeated() .then(one_of([Token::LeftBrace, Token::RightBrace]).rewind().then_with(move |end| { match end { @@ -1139,14 +1143,18 @@ fn token_stream_block(include_braces: bool, parse_braces: bool) -> impl NoirPars .map(append_vecs); if parse_braces { - just(Token::LeftBrace) - .ignore_then(inner_stream) - .then_ignore(just(Token::RightBrace)) - .map(move |mut stream| { - let mut ret = if include_braces { vec![Token::LeftBrace] } else { vec![] }; + spanned(just(Token::LeftBrace)) + .then(inner_stream) + .then(spanned(just(Token::RightBrace))) + .map(move |((left_brace, mut stream), right_brace)| { + let mut ret = if include_braces { + vec![SpannedToken::new(Token::LeftBrace, left_brace.1)] + } else { + vec![] + }; ret.append(&mut stream); if include_braces { - ret.push(Token::LeftBrace); + ret.push(SpannedToken::new(Token::RightBrace, right_brace.1)); } ret }) diff --git a/compiler/noirc_frontend/src/parser/parser/types.rs b/compiler/noirc_frontend/src/parser/parser/types.rs index fabc6fbcab7..14840bafa04 100644 --- a/compiler/noirc_frontend/src/parser/parser/types.rs +++ b/compiler/noirc_frontend/src/parser/parser/types.rs @@ -26,8 +26,8 @@ pub(super) fn parse_type_inner<'a>( expr_type(), type_definition_type(), top_level_item_type(), + type_of_quoted_types(), quoted_type(), - symbol_type(), format_string_type(recursive_type_parser.clone()), named_type(recursive_type_parser.clone()), named_trait(recursive_type_parser.clone()), @@ -94,17 +94,15 @@ fn top_level_item_type() -> impl NoirParser { } /// This is the type `Type` - the type of a quoted noir type. -fn quoted_type() -> impl NoirParser { +fn type_of_quoted_types() -> impl NoirParser { keyword(Keyword::TypeType) .map_with_span(|_, span| UnresolvedTypeData::Quoted(QuotedType::Type).with_span(span)) } -/// This is the type of a quoted token, most often quoted as part of a token stream. -fn symbol_type() -> impl NoirParser { - keyword(Keyword::Symbol).map_with_span(|_, span| { - let symbol = UnresolvedTypeData::Quoted(QuotedType::Symbol).with_span(span); - UnresolvedTypeData::Slice(Box::new(symbol)).with_span(span) - }) +/// This is the type of a quoted, unparsed token stream. +fn quoted_type() -> impl NoirParser { + keyword(Keyword::Quoted) + .map_with_span(|_, span| UnresolvedTypeData::Quoted(QuotedType::Quoted).with_span(span)) } pub(super) fn string_type() -> impl NoirParser { diff --git a/compiler/noirc_frontend/src/tests.rs b/compiler/noirc_frontend/src/tests.rs index 1acf1fbf3f2..f4845625b87 100644 --- a/compiler/noirc_frontend/src/tests.rs +++ b/compiler/noirc_frontend/src/tests.rs @@ -14,6 +14,7 @@ use fm::FileId; use iter_extended::vecmap; use noirc_errors::Location; +use crate::hir::comptime::InterpreterError; use crate::hir::def_collector::dc_crate::CompilationError; use crate::hir::def_collector::errors::{DefCollectorErrorKind, DuplicateType}; use crate::hir::def_map::ModuleData; @@ -1444,3 +1445,26 @@ fn specify_method_types_with_turbofish() { let errors = get_program_errors(src); assert_eq!(errors.len(), 0); } + +#[test] +fn quote_code_fragments() { + // This test ensures we can quote (and unquote/splice) code fragments + // which by themselves are not valid code. They only need to be valid + // by the time they are unquoted into the macro's call site. + let src = r#" + fn main() { + comptime { + concat!(quote { assert( }, quote { false); }); + } + } + + comptime fn concat(a: Quoted, b: Quoted) -> Quoted { + quote { $a $b } + } + "#; + let errors = get_program_errors(src); + assert_eq!(errors.len(), 1); + + use InterpreterError::FailingConstraint; + assert!(matches!(&errors[0].0, CompilationError::InterpreterError(FailingConstraint { .. }))); +} diff --git a/tooling/nargo_fmt/src/rewrite/expr.rs b/tooling/nargo_fmt/src/rewrite/expr.rs index ec8ac4abec7..24bcc426983 100644 --- a/tooling/nargo_fmt/src/rewrite/expr.rs +++ b/tooling/nargo_fmt/src/rewrite/expr.rs @@ -168,9 +168,7 @@ pub(crate) fn rewrite( format!("{path_string}{turbofish}") } ExpressionKind::Lambda(_) => visitor.slice(span).to_string(), - ExpressionKind::Quote(block, block_span) => { - format!("quote {}", rewrite_block(visitor, block, block_span)) - } + ExpressionKind::Quote(_, block_span) => format!("quote {}", visitor.slice(block_span)), ExpressionKind::Comptime(block, block_span) => { format!("comptime {}", rewrite_block(visitor, block, block_span)) } @@ -185,7 +183,6 @@ pub(crate) fn rewrite( format!("$({})", rewrite_sub_expr(visitor, shape, *expr)) } } - ExpressionKind::UnquoteMarker(_) => unreachable!("UnquoteMarker in runtime code"), } } From 25698d630578c11a4855c5910b8a8290441d484b Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Thu, 20 Jun 2024 15:13:16 -0500 Subject: [PATCH 03/12] Formatting --- .../compile_success_empty/comptime_array_len/src/main.nr | 3 +-- .../compile_success_empty/comptime_as_slice/src/main.nr | 3 +-- .../compile_success_empty/comptime_mut_global/src/main.nr | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test_programs/compile_success_empty/comptime_array_len/src/main.nr b/test_programs/compile_success_empty/comptime_array_len/src/main.nr index c98a3de01dd..283f3ea8d3e 100644 --- a/test_programs/compile_success_empty/comptime_array_len/src/main.nr +++ b/test_programs/compile_success_empty/comptime_array_len/src/main.nr @@ -1,6 +1,5 @@ fn main() { - comptime - { + comptime { assert_eq([1, 2, 3].len(), 3); } } diff --git a/test_programs/compile_success_empty/comptime_as_slice/src/main.nr b/test_programs/compile_success_empty/comptime_as_slice/src/main.nr index 07c5e344cc2..bc3f649e52a 100644 --- a/test_programs/compile_success_empty/comptime_as_slice/src/main.nr +++ b/test_programs/compile_success_empty/comptime_as_slice/src/main.nr @@ -1,6 +1,5 @@ fn main() { - comptime - { + comptime { let ws: [Field; 3] = [1; 3]; let ws_as_slice: [Field] = ws.as_slice(); diff --git a/test_programs/compile_success_empty/comptime_mut_global/src/main.nr b/test_programs/compile_success_empty/comptime_mut_global/src/main.nr index fa739289d37..111968a8a41 100644 --- a/test_programs/compile_success_empty/comptime_mut_global/src/main.nr +++ b/test_programs/compile_success_empty/comptime_mut_global/src/main.nr @@ -7,15 +7,13 @@ comptime fn get_unique_id() -> Field { } fn id1() -> Field { - comptime - { + comptime { get_unique_id() } } fn id2() -> Field { - comptime - { + comptime { get_unique_id() } } From 6143d4e70ed0c6083d29a1aeec4277fd224b128e Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Thu, 20 Jun 2024 15:43:48 -0500 Subject: [PATCH 04/12] Parse UnquoteMarker as an expression --- compiler/noirc_frontend/src/lexer/token.rs | 7 +++++-- compiler/noirc_frontend/src/parser/parser.rs | 3 ++- compiler/noirc_frontend/src/parser/parser/primitives.rs | 7 +++++++ test_programs/compile_success_empty/macros/src/main.nr | 6 ++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/compiler/noirc_frontend/src/lexer/token.rs b/compiler/noirc_frontend/src/lexer/token.rs index 8d40d81db3b..9f72752c306 100644 --- a/compiler/noirc_frontend/src/lexer/token.rs +++ b/compiler/noirc_frontend/src/lexer/token.rs @@ -382,6 +382,7 @@ pub enum TokenKind { Literal, Keyword, Attribute, + UnquoteMarker, } impl fmt::Display for TokenKind { @@ -392,13 +393,14 @@ impl fmt::Display for TokenKind { TokenKind::Literal => write!(f, "literal"), TokenKind::Keyword => write!(f, "keyword"), TokenKind::Attribute => write!(f, "attribute"), + TokenKind::UnquoteMarker => write!(f, "macro result"), } } } impl Token { pub fn kind(&self) -> TokenKind { - match *self { + match self { Token::Ident(_) => TokenKind::Ident, Token::Int(_) | Token::Bool(_) @@ -407,7 +409,8 @@ impl Token { | Token::FmtStr(_) => TokenKind::Literal, Token::Keyword(_) => TokenKind::Keyword, Token::Attribute(_) => TokenKind::Attribute, - ref tok => TokenKind::Token(tok.clone()), + Token::UnquoteMarker(_) => TokenKind::UnquoteMarker, + tok => TokenKind::Token(tok.clone()), } } diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index f70b54ce698..2c35cfc5168 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -23,7 +23,7 @@ //! prevent other parsers from being tried afterward since there is no longer an error. Thus, they should //! be limited to cases like the above `fn` example where it is clear we shouldn't back out of the //! current parser to try alternative parsers in a `choice` expression. -use self::primitives::{keyword, mutable_reference, variable}; +use self::primitives::{keyword, macro_quote_marker, mutable_reference, variable}; use self::types::{generic_type_args, maybe_comp_time, parse_type}; use super::{ @@ -1079,6 +1079,7 @@ where unquote(expr_parser.clone()), variable(), literal(), + macro_quote_marker(), )) .map_with_span(Expression::new) .or(parenthesized(expr_parser.clone()).map_with_span(|sub_expr, span| { diff --git a/compiler/noirc_frontend/src/parser/parser/primitives.rs b/compiler/noirc_frontend/src/parser/parser/primitives.rs index 9da19c0a185..88f9e591aba 100644 --- a/compiler/noirc_frontend/src/parser/parser/primitives.rs +++ b/compiler/noirc_frontend/src/parser/parser/primitives.rs @@ -94,6 +94,13 @@ pub(super) fn variable_no_turbofish() -> impl NoirParser { path().map(|path| ExpressionKind::Variable(path, None)) } +pub(super) fn macro_quote_marker() -> impl NoirParser { + token_kind(TokenKind::UnquoteMarker).map(|token| match token { + Token::UnquoteMarker(expr_id) => ExpressionKind::Resolved(expr_id), + other => unreachable!("Non-unquote-marker parsed as an unquote marker: {other:?}"), + }) +} + #[cfg(test)] mod test { use crate::parser::parser::{ diff --git a/test_programs/compile_success_empty/macros/src/main.nr b/test_programs/compile_success_empty/macros/src/main.nr index f466226d575..1b00a084c61 100644 --- a/test_programs/compile_success_empty/macros/src/main.nr +++ b/test_programs/compile_success_empty/macros/src/main.nr @@ -1,10 +1,8 @@ -comptime fn my_macro(x: Field, y: Field) -> Expr { +comptime fn my_macro(x: Field, y: Field) -> Quoted { // Current version of macros in Noir are not hygienic // so we can quote a and b here and expect them to resolve // to the a and b in main at the callsite of my_macro. - quote { - $x + $y + a + b - } + quote { $x + $y + a + b } } fn main() { From ca009addc63fac7dd42f189e87e002e9e069deb4 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Thu, 20 Jun 2024 16:16:05 -0500 Subject: [PATCH 05/12] Undo formatting changes --- .../compile_success_empty/comptime_array_len/src/main.nr | 3 ++- .../compile_success_empty/comptime_as_slice/src/main.nr | 3 ++- .../compile_success_empty/comptime_mut_global/src/main.nr | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test_programs/compile_success_empty/comptime_array_len/src/main.nr b/test_programs/compile_success_empty/comptime_array_len/src/main.nr index 283f3ea8d3e..c98a3de01dd 100644 --- a/test_programs/compile_success_empty/comptime_array_len/src/main.nr +++ b/test_programs/compile_success_empty/comptime_array_len/src/main.nr @@ -1,5 +1,6 @@ fn main() { - comptime { + comptime + { assert_eq([1, 2, 3].len(), 3); } } diff --git a/test_programs/compile_success_empty/comptime_as_slice/src/main.nr b/test_programs/compile_success_empty/comptime_as_slice/src/main.nr index bc3f649e52a..07c5e344cc2 100644 --- a/test_programs/compile_success_empty/comptime_as_slice/src/main.nr +++ b/test_programs/compile_success_empty/comptime_as_slice/src/main.nr @@ -1,5 +1,6 @@ fn main() { - comptime { + comptime + { let ws: [Field; 3] = [1; 3]; let ws_as_slice: [Field] = ws.as_slice(); diff --git a/test_programs/compile_success_empty/comptime_mut_global/src/main.nr b/test_programs/compile_success_empty/comptime_mut_global/src/main.nr index 111968a8a41..fa739289d37 100644 --- a/test_programs/compile_success_empty/comptime_mut_global/src/main.nr +++ b/test_programs/compile_success_empty/comptime_mut_global/src/main.nr @@ -7,13 +7,15 @@ comptime fn get_unique_id() -> Field { } fn id1() -> Field { - comptime { + comptime + { get_unique_id() } } fn id2() -> Field { - comptime { + comptime + { get_unique_id() } } From 3152fe442dcbd3f0c2131300b5d6995c793b5666 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Fri, 21 Jun 2024 11:18:04 -0500 Subject: [PATCH 06/12] Add parser test for quote --- compiler/noirc_frontend/src/parser/parser.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index 2c35cfc5168..18251810ed6 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -1691,4 +1691,23 @@ mod test { check_cases_with_errors(&cases[..], block(fresh_statement())); } + + #[test] + fn test_quote() { + let cases = vec![ + "quote {}", + "quote { a.b }", + "quote { ) ( }", // invalid syntax is fine in a quote + "quote { { } }", // Nested `{` and `}` shouldn't close the quote as long as they are matched. + "quote { 1 { 2 { 3 { 4 { 5 } 4 4 } 3 3 } 2 2 } 1 1 }", + ]; + parse_all(quote(), cases); + + let failing = vec![ + "quote {}}", + "quote a", + "quote { { { } } } }" + ]; + parse_all_failing(quote(), failing); + } } From 21b4658ccc80f16be8e30224a6366bb000ac39c4 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Fri, 21 Jun 2024 11:18:57 -0500 Subject: [PATCH 07/12] fmt --- compiler/noirc_frontend/src/parser/parser.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index 18251810ed6..0d37923427b 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -1703,11 +1703,7 @@ mod test { ]; parse_all(quote(), cases); - let failing = vec![ - "quote {}}", - "quote a", - "quote { { { } } } }" - ]; + let failing = vec!["quote {}}", "quote a", "quote { { { } } } }"]; parse_all_failing(quote(), failing); } } From 2de85dd27bb2bd4c0b911fd6ff61a8c1db56f8e8 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Fri, 21 Jun 2024 11:23:50 -0500 Subject: [PATCH 08/12] Fix InterpreterError formatting --- .../noirc_frontend/src/hir/comptime/errors.rs | 168 ++++-------------- 1 file changed, 33 insertions(+), 135 deletions(-) diff --git a/compiler/noirc_frontend/src/hir/comptime/errors.rs b/compiler/noirc_frontend/src/hir/comptime/errors.rs index 0e64d3c7941..659888f100f 100644 --- a/compiler/noirc_frontend/src/hir/comptime/errors.rs +++ b/compiler/noirc_frontend/src/hir/comptime/errors.rs @@ -4,6 +4,7 @@ use crate::{ hir::def_collector::dc_crate::CompilationError, parser::ParserError, token::Tokens, Type, }; use acvm::{acir::AcirField, FieldElement}; +use fm::FileId; use iter_extended::vecmap; use noirc_errors::{CustomDiagnostic, Location}; @@ -12,144 +13,41 @@ use super::value::Value; /// The possible errors that can halt the interpreter. #[derive(Debug, Clone, PartialEq, Eq)] pub enum InterpreterError { - ArgumentCountMismatch { - expected: usize, - actual: usize, - location: Location, - }, - TypeMismatch { - expected: Type, - value: Value, - location: Location, - }, - NonComptimeVarReferenced { - name: String, - location: Location, - }, - IntegerOutOfRangeForType { - value: FieldElement, - typ: Type, - location: Location, - }, - ErrorNodeEncountered { - location: Location, - }, - NonFunctionCalled { - value: Value, - location: Location, - }, - NonBoolUsedInIf { - value: Value, - location: Location, - }, - NonBoolUsedInConstrain { - value: Value, - location: Location, - }, - FailingConstraint { - message: Option, - location: Location, - }, - NoMethodFound { - name: String, - typ: Type, - location: Location, - }, - NonIntegerUsedInLoop { - value: Value, - location: Location, - }, - NonPointerDereferenced { - value: Value, - location: Location, - }, - NonTupleOrStructInMemberAccess { - value: Value, - location: Location, - }, - NonArrayIndexed { - value: Value, - location: Location, - }, - NonIntegerUsedAsIndex { - value: Value, - location: Location, - }, - NonIntegerIntegerLiteral { - typ: Type, - location: Location, - }, - NonIntegerArrayLength { - typ: Type, - location: Location, - }, - NonNumericCasted { - value: Value, - location: Location, - }, - IndexOutOfBounds { - index: usize, - length: usize, - location: Location, - }, - ExpectedStructToHaveField { - value: Value, - field_name: String, - location: Location, - }, - TypeUnsupported { - typ: Type, - location: Location, - }, - InvalidValueForUnary { - value: Value, - operator: &'static str, - location: Location, - }, - InvalidValuesForBinary { - lhs: Value, - rhs: Value, - operator: &'static str, - location: Location, - }, - CastToNonNumericType { - typ: Type, - location: Location, - }, - QuoteInRuntimeCode { - location: Location, - }, - NonStructInConstructor { - typ: Type, - location: Location, - }, - CannotInlineMacro { - value: Value, - location: Location, - }, - UnquoteFoundDuringEvaluation { - location: Location, - }, - FailedToParseMacro { - error: ParserError, - tokens: Rc, - parse_rule: &'static str, - file: fm::FileId, - }, - - Unimplemented { - item: String, - location: Location, - }, + ArgumentCountMismatch { expected: usize, actual: usize, location: Location }, + TypeMismatch { expected: Type, value: Value, location: Location }, + NonComptimeVarReferenced { name: String, location: Location }, + IntegerOutOfRangeForType { value: FieldElement, typ: Type, location: Location }, + ErrorNodeEncountered { location: Location }, + NonFunctionCalled { value: Value, location: Location }, + NonBoolUsedInIf { value: Value, location: Location }, + NonBoolUsedInConstrain { value: Value, location: Location }, + FailingConstraint { message: Option, location: Location }, + NoMethodFound { name: String, typ: Type, location: Location }, + NonIntegerUsedInLoop { value: Value, location: Location }, + NonPointerDereferenced { value: Value, location: Location }, + NonTupleOrStructInMemberAccess { value: Value, location: Location }, + NonArrayIndexed { value: Value, location: Location }, + NonIntegerUsedAsIndex { value: Value, location: Location }, + NonIntegerIntegerLiteral { typ: Type, location: Location }, + NonIntegerArrayLength { typ: Type, location: Location }, + NonNumericCasted { value: Value, location: Location }, + IndexOutOfBounds { index: usize, length: usize, location: Location }, + ExpectedStructToHaveField { value: Value, field_name: String, location: Location }, + TypeUnsupported { typ: Type, location: Location }, + InvalidValueForUnary { value: Value, operator: &'static str, location: Location }, + InvalidValuesForBinary { lhs: Value, rhs: Value, operator: &'static str, location: Location }, + CastToNonNumericType { typ: Type, location: Location }, + QuoteInRuntimeCode { location: Location }, + NonStructInConstructor { typ: Type, location: Location }, + CannotInlineMacro { value: Value, location: Location }, + UnquoteFoundDuringEvaluation { location: Location }, + FailedToParseMacro { error: ParserError, tokens: Rc, parse_rule: String, file: FileId }, + Unimplemented { item: String, location: Location }, // Perhaps this should be unreachable! due to type checking also preventing this error? // Currently it and the Continue variant are the only interpreter errors without a Location field - BreakNotInLoop { - location: Location, - }, - ContinueNotInLoop { - location: Location, - }, + BreakNotInLoop { location: Location }, + ContinueNotInLoop { location: Location }, // These cases are not errors, they are just used to prevent us from running more code // until the loop can be resumed properly. These cases will never be displayed to users. From c75749e017b8e78816a391f6ade7f977946a7e24 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Fri, 21 Jun 2024 11:29:03 -0500 Subject: [PATCH 09/12] Fix formatting --- compiler/noirc_frontend/src/hir/comptime/errors.rs | 7 ++++--- compiler/noirc_frontend/src/hir/comptime/value.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/noirc_frontend/src/hir/comptime/errors.rs b/compiler/noirc_frontend/src/hir/comptime/errors.rs index 659888f100f..4eab12af308 100644 --- a/compiler/noirc_frontend/src/hir/comptime/errors.rs +++ b/compiler/noirc_frontend/src/hir/comptime/errors.rs @@ -41,7 +41,8 @@ pub enum InterpreterError { NonStructInConstructor { typ: Type, location: Location }, CannotInlineMacro { value: Value, location: Location }, UnquoteFoundDuringEvaluation { location: Location }, - FailedToParseMacro { error: ParserError, tokens: Rc, parse_rule: String, file: FileId }, + FailedToParseMacro { error: ParserError, tokens: Rc, rule: &'static str, file: FileId }, + Unimplemented { item: String, location: Location }, // Perhaps this should be unreachable! due to type checking also preventing this error? @@ -267,8 +268,8 @@ impl<'a> From<&'a InterpreterError> for CustomDiagnostic { let secondary = "This is a bug".into(); CustomDiagnostic::simple_error(msg, secondary, location.span) } - InterpreterError::FailedToParseMacro { error, tokens, parse_rule, file: _ } => { - let message = format!("Failed to parse macro's token stream into {parse_rule}"); + InterpreterError::FailedToParseMacro { error, tokens, rule, file: _ } => { + let message = format!("Failed to parse macro's token stream into {rule}"); let tokens = vecmap(&tokens.0, ToString::to_string).join(" "); // 10 is an aribtrary number of tokens here chosen to fit roughly onto one line diff --git a/compiler/noirc_frontend/src/hir/comptime/value.rs b/compiler/noirc_frontend/src/hir/comptime/value.rs index 3bb53912888..d51d69f9226 100644 --- a/compiler/noirc_frontend/src/hir/comptime/value.rs +++ b/compiler/noirc_frontend/src/hir/comptime/value.rs @@ -183,12 +183,12 @@ impl Value { return match parser::expression().parse(tokens_to_parse) { Ok(expr) => Ok(expr), - Err(mut errors) => Err(InterpreterError::FailedToParseMacro { - error: errors.swap_remove(0), - file: location.file, - tokens, - parse_rule: "an expression", - }), + Err(mut errors) => { + let error = errors.swap_remove(0); + let file = location.file; + let rule = "an expression"; + Err(InterpreterError::FailedToParseMacro { error, file, tokens, rule }) + } }; } Value::Pointer(_) | Value::TypeDefinition(_) => { From 8ecae64574e8f5094f79d9cfdbe8c7b31d8044bb Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Mon, 24 Jun 2024 12:13:12 -0500 Subject: [PATCH 10/12] Move to 'parsing' token streams in the lexer --- compiler/noirc_frontend/src/ast/expression.rs | 4 +- .../src/elaborator/expressions.rs | 2 +- .../src/hir/resolution/resolver.rs | 2 +- compiler/noirc_frontend/src/lexer/errors.rs | 6 ++ compiler/noirc_frontend/src/lexer/lexer.rs | 86 ++++++++++++++++++- compiler/noirc_frontend/src/lexer/token.rs | 21 +++-- .../noirc_frontend/src/noir_parser.lalrpop | 1 - compiler/noirc_frontend/src/parser/parser.rs | 73 +++------------- tooling/nargo_fmt/src/rewrite/expr.rs | 2 +- 9 files changed, 122 insertions(+), 75 deletions(-) diff --git a/compiler/noirc_frontend/src/ast/expression.rs b/compiler/noirc_frontend/src/ast/expression.rs index e65328c6aac..2657869a9d7 100644 --- a/compiler/noirc_frontend/src/ast/expression.rs +++ b/compiler/noirc_frontend/src/ast/expression.rs @@ -33,7 +33,7 @@ pub enum ExpressionKind { Tuple(Vec), Lambda(Box), Parenthesized(Box), - Quote(Tokens, Span), + Quote(Tokens), Unquote(Box), Comptime(BlockExpression, Span), @@ -553,7 +553,7 @@ impl Display for ExpressionKind { Error => write!(f, "Error"), Resolved(_) => write!(f, "?Resolved"), Unquote(expr) => write!(f, "$({expr})"), - Quote(tokens, _) => { + Quote(tokens) => { let tokens = vecmap(&tokens.0, ToString::to_string); write!(f, "quote {{ {} }}", tokens.join(" ")) } diff --git a/compiler/noirc_frontend/src/elaborator/expressions.rs b/compiler/noirc_frontend/src/elaborator/expressions.rs index 24f525f2f8c..9d864a0de91 100644 --- a/compiler/noirc_frontend/src/elaborator/expressions.rs +++ b/compiler/noirc_frontend/src/elaborator/expressions.rs @@ -59,7 +59,7 @@ impl<'context> Elaborator<'context> { ExpressionKind::Tuple(tuple) => self.elaborate_tuple(tuple), ExpressionKind::Lambda(lambda) => self.elaborate_lambda(*lambda), ExpressionKind::Parenthesized(expr) => return self.elaborate_expression(*expr), - ExpressionKind::Quote(quote, _) => self.elaborate_quote(quote), + ExpressionKind::Quote(quote) => self.elaborate_quote(quote), ExpressionKind::Comptime(comptime, _) => { return self.elaborate_comptime_block(comptime, expr.span) } diff --git a/compiler/noirc_frontend/src/hir/resolution/resolver.rs b/compiler/noirc_frontend/src/hir/resolution/resolver.rs index 8ff0a92bf1d..5706e62e193 100644 --- a/compiler/noirc_frontend/src/hir/resolution/resolver.rs +++ b/compiler/noirc_frontend/src/hir/resolution/resolver.rs @@ -1634,7 +1634,7 @@ impl<'a> Resolver<'a> { ExpressionKind::Parenthesized(sub_expr) => return self.resolve_expression(*sub_expr), // The quoted expression isn't resolved since we don't want errors if variables aren't defined - ExpressionKind::Quote(block, _) => HirExpression::Quote(block), + ExpressionKind::Quote(block) => HirExpression::Quote(block), ExpressionKind::Comptime(block, _) => { HirExpression::Comptime(self.resolve_block(block)) } diff --git a/compiler/noirc_frontend/src/lexer/errors.rs b/compiler/noirc_frontend/src/lexer/errors.rs index 73c75af4cd7..dbaf2f3a81e 100644 --- a/compiler/noirc_frontend/src/lexer/errors.rs +++ b/compiler/noirc_frontend/src/lexer/errors.rs @@ -27,6 +27,8 @@ pub enum LexerErrorKind { "'\\{escaped}' is not a valid escape sequence. Use '\\' for a literal backslash character." )] InvalidEscape { escaped: char, span: Span }, + #[error("Invalid quote delimiter `{delimiter}`, valid delimiters are `{{`, `[`, and `(`")] + InvalidQuoteDelimiter { delimiter: SpannedToken }, } impl From for ParserError { @@ -47,6 +49,7 @@ impl LexerErrorKind { LexerErrorKind::UnterminatedBlockComment { span } => *span, LexerErrorKind::UnterminatedStringLiteral { span } => *span, LexerErrorKind::InvalidEscape { span, .. } => *span, + LexerErrorKind::InvalidQuoteDelimiter { delimiter } => delimiter.to_span(), } } @@ -92,6 +95,9 @@ impl LexerErrorKind { ("Unterminated string literal".to_string(), "Unterminated string literal".to_string(), *span), LexerErrorKind::InvalidEscape { escaped, span } => (format!("'\\{escaped}' is not a valid escape sequence. Use '\\' for a literal backslash character."), "Invalid escape sequence".to_string(), *span), + LexerErrorKind::InvalidQuoteDelimiter { delimiter } => { + ("Invalid quote delimiter `{delimiter}`".to_string(), "Valid delimiters are `{`, `[`, and `(`".to_string(), delimiter.to_span()) + }, } } } diff --git a/compiler/noirc_frontend/src/lexer/lexer.rs b/compiler/noirc_frontend/src/lexer/lexer.rs index 3d052e22e36..249103b3d15 100644 --- a/compiler/noirc_frontend/src/lexer/lexer.rs +++ b/compiler/noirc_frontend/src/lexer/lexer.rs @@ -145,6 +145,7 @@ impl<'a> Lexer<'a> { Some('"') => self.eat_string_literal(), Some('f') => self.eat_format_string_or_alpha_numeric(), Some('r') => self.eat_raw_string_or_alpha_numeric(), + Some('q') => self.eat_quote_or_alpha_numeric(), Some('#') => self.eat_attribute(), Some(ch) if ch.is_ascii_alphanumeric() || ch == '_' => self.eat_alpha_numeric(ch), Some(ch) => { @@ -310,14 +311,25 @@ impl<'a> Lexer<'a> { //XXX(low): Can increase performance if we use iterator semantic and utilize some of the methods on String. See below // https://doc.rust-lang.org/stable/std/primitive.str.html#method.rsplit fn eat_word(&mut self, initial_char: char) -> SpannedTokenResult { - let start = self.position; + let (start, word, end) = self.lex_word(initial_char); + self.lookup_word_token(word, start, end) + } + /// Lex the next word in the input stream. Returns (start position, word, end position) + fn lex_word(&mut self, initial_char: char) -> (Position, String, Position) { + let start = self.position; let word = self.eat_while(Some(initial_char), |ch| { ch.is_ascii_alphabetic() || ch.is_numeric() || ch == '_' }); + (start, word, self.position) + } - let end = self.position; - + fn lookup_word_token( + &self, + word: String, + start: Position, + end: Position, + ) -> SpannedTokenResult { // Check if word either an identifier or a keyword if let Some(keyword_token) = Keyword::lookup_keyword(&word) { return Ok(keyword_token.into_span(start, end)); @@ -509,6 +521,44 @@ impl<'a> Lexer<'a> { } } + fn eat_quote_or_alpha_numeric(&mut self) -> SpannedTokenResult { + let (start, word, end) = self.lex_word('q'); + if word != "quote" { + return self.lookup_word_token(word, start, end); + } + + let delimiter = self.next_token()?; + let (start_delim, end_delim) = match delimiter.token() { + Token::LeftBrace => (Token::LeftBrace, Token::RightBrace), + Token::LeftBracket => (Token::LeftBracket, Token::RightBracket), + Token::LeftParen => (Token::LeftParen, Token::RightParen), + _ => return Err(LexerErrorKind::InvalidQuoteDelimiter { delimiter }), + }; + + let mut tokens = Vec::new(); + let mut nested_delimiters = 1; + + while nested_delimiters != 0 { + let token = self.next_token()?; + + if *token.token() == start_delim { + nested_delimiters += 1; + } else if *token.token() == end_delim { + nested_delimiters -= 1; + } + + tokens.push(token); + } + + // Pop the closing delimiter from the token stream + if !tokens.is_empty() { + tokens.pop(); + } + + let end = self.position; + Ok(Token::Quote(Tokens(tokens)).into_span(start, end)) + } + fn parse_comment(&mut self, start: u32) -> SpannedTokenResult { let doc_style = match self.peek_char() { Some('!') => { @@ -604,6 +654,8 @@ impl<'a> Iterator for Lexer<'a> { #[cfg(test)] mod tests { + use iter_extended::vecmap; + use super::*; use crate::token::{FunctionAttribute, SecondaryAttribute, TestScope}; @@ -1232,4 +1284,32 @@ mod tests { } } } + + #[test] + fn test_quote() { + // cases is a vector of pairs of (test string, expected # of tokens in token stream) + let cases = vec![ + ("quote {}", 0), + ("quote { a.b }", 3), + ("quote { ) ( }", 2), // invalid syntax is fine in a quote + ("quote { { } }", 2), // Nested `{` and `}` shouldn't close the quote as long as they are matched. + ("quote { 1 { 2 { 3 { 4 { 5 } 4 4 } 3 3 } 2 2 } 1 1 }", 21), + ("quote [ } } ]", 2), // In addition to `{}`, `[]`, and `()` can also be used as delimiters. + ("quote [ } foo[] } ]", 5), + ("quote ( } () } )", 4), + ]; + + for (source, expected_stream_length) in cases { + let mut tokens = vecmap(Lexer::new(source), |result| result.unwrap().into_token()); + + // All examples should be a single TokenStream token followed by an EOF token. + assert_eq!(tokens.len(), 2, "Unexpected token count: {tokens:?}"); + + tokens.pop(); + match tokens.pop().unwrap() { + Token::Quote(stream) => assert_eq!(stream.0.len(), expected_stream_length), + other => panic!("test_quote test failure! Expected a single TokenStream token, got {other} for input `{source}`") + } + } + } } diff --git a/compiler/noirc_frontend/src/lexer/token.rs b/compiler/noirc_frontend/src/lexer/token.rs index 9f72752c306..6830ee528d6 100644 --- a/compiler/noirc_frontend/src/lexer/token.rs +++ b/compiler/noirc_frontend/src/lexer/token.rs @@ -23,6 +23,7 @@ pub enum BorrowedToken<'input> { Attribute(Attribute), LineComment(&'input str, Option), BlockComment(&'input str, Option), + Quote(&'input Tokens), /// < Less, /// <= @@ -122,6 +123,8 @@ pub enum Token { Attribute(Attribute), LineComment(String, Option), BlockComment(String, Option), + // A `quote { ... }` along with the tokens in its token stream. + Quote(Tokens), /// < Less, /// <= @@ -219,6 +222,7 @@ pub fn token_to_borrowed_token(token: &Token) -> BorrowedToken<'_> { Token::Attribute(ref a) => BorrowedToken::Attribute(a.clone()), Token::LineComment(ref s, _style) => BorrowedToken::LineComment(s, *_style), Token::BlockComment(ref s, _style) => BorrowedToken::BlockComment(s, *_style), + Token::Quote(stream) => BorrowedToken::Quote(stream), Token::IntType(ref i) => BorrowedToken::IntType(i.clone()), Token::Less => BorrowedToken::Less, Token::LessEqual => BorrowedToken::LessEqual, @@ -266,7 +270,7 @@ pub enum DocStyle { Inner, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct SpannedToken(Spanned); impl PartialEq for Token { @@ -332,6 +336,13 @@ impl fmt::Display for Token { Token::Attribute(ref a) => write!(f, "{a}"), Token::LineComment(ref s, _style) => write!(f, "//{s}"), Token::BlockComment(ref s, _style) => write!(f, "/*{s}*/"), + Token::Quote(ref stream) => { + write!(f, "quote {{")?; + for token in stream.0.iter() { + write!(f, " {token}")?; + } + write!(f, "}}") + } Token::IntType(ref i) => write!(f, "{i}"), Token::Less => write!(f, "<"), Token::LessEqual => write!(f, "<="), @@ -382,6 +393,7 @@ pub enum TokenKind { Literal, Keyword, Attribute, + Quote, UnquoteMarker, } @@ -393,6 +405,7 @@ impl fmt::Display for TokenKind { TokenKind::Literal => write!(f, "literal"), TokenKind::Keyword => write!(f, "keyword"), TokenKind::Attribute => write!(f, "attribute"), + TokenKind::Quote => write!(f, "quote"), TokenKind::UnquoteMarker => write!(f, "macro result"), } } @@ -410,6 +423,7 @@ impl Token { Token::Keyword(_) => TokenKind::Keyword, Token::Attribute(_) => TokenKind::Attribute, Token::UnquoteMarker(_) => TokenKind::UnquoteMarker, + Token::Quote(_) => TokenKind::Quote, tok => TokenKind::Token(tok.clone()), } } @@ -874,7 +888,6 @@ pub enum Keyword { Mod, Mut, Pub, - Quote, Quoted, Return, ReturnData, @@ -923,7 +936,6 @@ impl fmt::Display for Keyword { Keyword::Mod => write!(f, "mod"), Keyword::Mut => write!(f, "mut"), Keyword::Pub => write!(f, "pub"), - Keyword::Quote => write!(f, "quote"), Keyword::Quoted => write!(f, "Quoted"), Keyword::Return => write!(f, "return"), Keyword::ReturnData => write!(f, "return_data"), @@ -975,7 +987,6 @@ impl Keyword { "mod" => Keyword::Mod, "mut" => Keyword::Mut, "pub" => Keyword::Pub, - "quote" => Keyword::Quote, "Quoted" => Keyword::Quoted, "return" => Keyword::Return, "return_data" => Keyword::ReturnData, @@ -1002,7 +1013,7 @@ impl Keyword { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Tokens(pub Vec); type TokenMapIter = Map, fn(SpannedToken) -> (Token, Span)>; diff --git a/compiler/noirc_frontend/src/noir_parser.lalrpop b/compiler/noirc_frontend/src/noir_parser.lalrpop index 5768e3c1d17..5bf48a764d6 100644 --- a/compiler/noirc_frontend/src/noir_parser.lalrpop +++ b/compiler/noirc_frontend/src/noir_parser.lalrpop @@ -80,7 +80,6 @@ extern { "mod" => BorrowedToken::Keyword(noir_token::Keyword::Mod), "mut" => BorrowedToken::Keyword(noir_token::Keyword::Mut), "pub" => BorrowedToken::Keyword(noir_token::Keyword::Pub), - "quote" => BorrowedToken::Keyword(noir_token::Keyword::Quote), "return" => BorrowedToken::Keyword(noir_token::Keyword::Return), "return_data" => BorrowedToken::Keyword(noir_token::Keyword::ReturnData), "str" => BorrowedToken::Keyword(noir_token::Keyword::String), diff --git a/compiler/noirc_frontend/src/parser/parser.rs b/compiler/noirc_frontend/src/parser/parser.rs index 0d37923427b..0ae810fe4d9 100644 --- a/compiler/noirc_frontend/src/parser/parser.rs +++ b/compiler/noirc_frontend/src/parser/parser.rs @@ -44,7 +44,7 @@ use crate::ast::{ }; use crate::lexer::{lexer::from_spanned_token_result, Lexer}; use crate::parser::{force, ignore_then_commit, statement_recovery}; -use crate::token::{Keyword, SpannedToken, Token, TokenKind, Tokens}; +use crate::token::{Keyword, Token, TokenKind}; use chumsky::prelude::*; use iter_extended::vecmap; @@ -1103,66 +1103,17 @@ where } fn quote() -> impl NoirParser { - keyword(Keyword::Quote).ignore_then(spanned(token_stream_block(false, true))).validate( - |(tokens, block_span), span, emit| { - emit(ParserError::with_reason( - ParserErrorReason::ExperimentalFeature("quoted expressions"), - span, - )); - ExpressionKind::Quote(Tokens(tokens), block_span) - }, - ) -} - -/// Parses a stream of tokens terminated by '{' or '}'. -/// - parse_braces: if true, parses '{' and '}' surrounding the token stream. -/// - include_braces: if true, include the surrounding braces in the returned tokens vec -fn token_stream_block( - include_braces: bool, - parse_braces: bool, -) -> impl NoirParser> { - let append_vecs = |(mut vec1, mut vec2): (Vec<_>, _)| { - vec1.append(&mut vec2); - vec1 - }; - - // Parse a stream of tokens ending in '{' or '}'. - // - If we ended with a '}': end - // - If we ended with a '{': recursively parse another token stream block - let inner_stream = spanned(none_of([Token::LeftBrace, Token::RightBrace])) - .map(|(token, span)| SpannedToken::new(token, span)) - .repeated() - .then(one_of([Token::LeftBrace, Token::RightBrace]).rewind().then_with(move |end| { - match end { - Token::LeftBrace => token_stream_block(true, true) - .then(token_stream_block(false, false)) - .map(append_vecs) - .boxed(), - _ => empty().map(|_| Vec::new()).boxed(), - } - })) - .map(append_vecs); - - if parse_braces { - spanned(just(Token::LeftBrace)) - .then(inner_stream) - .then(spanned(just(Token::RightBrace))) - .map(move |((left_brace, mut stream), right_brace)| { - let mut ret = if include_braces { - vec![SpannedToken::new(Token::LeftBrace, left_brace.1)] - } else { - vec![] - }; - ret.append(&mut stream); - if include_braces { - ret.push(SpannedToken::new(Token::RightBrace, right_brace.1)); - } - ret - }) - .boxed() - } else { - inner_stream.boxed() - } + token_kind(TokenKind::Quote).validate(|token, span, emit| { + let tokens = match token { + Token::Quote(tokens) => tokens, + _ => unreachable!("token_kind(Quote) should guarantee parsing only a quote token"), + }; + emit(ParserError::with_reason( + ParserErrorReason::ExperimentalFeature("quoted expressions"), + span, + )); + ExpressionKind::Quote(tokens) + }) } /// unquote: '$' variable diff --git a/tooling/nargo_fmt/src/rewrite/expr.rs b/tooling/nargo_fmt/src/rewrite/expr.rs index 24bcc426983..015644c15cb 100644 --- a/tooling/nargo_fmt/src/rewrite/expr.rs +++ b/tooling/nargo_fmt/src/rewrite/expr.rs @@ -168,7 +168,7 @@ pub(crate) fn rewrite( format!("{path_string}{turbofish}") } ExpressionKind::Lambda(_) => visitor.slice(span).to_string(), - ExpressionKind::Quote(_, block_span) => format!("quote {}", visitor.slice(block_span)), + ExpressionKind::Quote(_) => visitor.slice(span).to_string(), ExpressionKind::Comptime(block, block_span) => { format!("comptime {}", rewrite_block(visitor, block, block_span)) } From 9b59f6bf1973fe3f502f66c92650ffe517805e66 Mon Sep 17 00:00:00 2001 From: Jake Fecher Date: Mon, 24 Jun 2024 12:41:16 -0500 Subject: [PATCH 11/12] Add unclosed quote error --- compiler/noirc_frontend/src/lexer/errors.rs | 8 +++++- compiler/noirc_frontend/src/lexer/lexer.rs | 27 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/compiler/noirc_frontend/src/lexer/errors.rs b/compiler/noirc_frontend/src/lexer/errors.rs index dbaf2f3a81e..2452e034c1c 100644 --- a/compiler/noirc_frontend/src/lexer/errors.rs +++ b/compiler/noirc_frontend/src/lexer/errors.rs @@ -29,6 +29,8 @@ pub enum LexerErrorKind { InvalidEscape { escaped: char, span: Span }, #[error("Invalid quote delimiter `{delimiter}`, valid delimiters are `{{`, `[`, and `(`")] InvalidQuoteDelimiter { delimiter: SpannedToken }, + #[error("Expected `{end_delim}` to close this {start_delim}")] + UnclosedQuote { start_delim: SpannedToken, end_delim: Token }, } impl From for ParserError { @@ -50,6 +52,7 @@ impl LexerErrorKind { LexerErrorKind::UnterminatedStringLiteral { span } => *span, LexerErrorKind::InvalidEscape { span, .. } => *span, LexerErrorKind::InvalidQuoteDelimiter { delimiter } => delimiter.to_span(), + LexerErrorKind::UnclosedQuote { start_delim, .. } => start_delim.to_span(), } } @@ -96,8 +99,11 @@ impl LexerErrorKind { LexerErrorKind::InvalidEscape { escaped, span } => (format!("'\\{escaped}' is not a valid escape sequence. Use '\\' for a literal backslash character."), "Invalid escape sequence".to_string(), *span), LexerErrorKind::InvalidQuoteDelimiter { delimiter } => { - ("Invalid quote delimiter `{delimiter}`".to_string(), "Valid delimiters are `{`, `[`, and `(`".to_string(), delimiter.to_span()) + (format!("Invalid quote delimiter `{delimiter}`"), "Valid delimiters are `{`, `[`, and `(`".to_string(), delimiter.to_span()) }, + LexerErrorKind::UnclosedQuote { start_delim, end_delim } => { + ("Unclosed `quote` expression".to_string(), format!("Expected a `{end_delim}` to close this `{start_delim}`"), start_delim.to_span()) + } } } } diff --git a/compiler/noirc_frontend/src/lexer/lexer.rs b/compiler/noirc_frontend/src/lexer/lexer.rs index 249103b3d15..d2bc3d0f519 100644 --- a/compiler/noirc_frontend/src/lexer/lexer.rs +++ b/compiler/noirc_frontend/src/lexer/lexer.rs @@ -536,15 +536,21 @@ impl<'a> Lexer<'a> { }; let mut tokens = Vec::new(); - let mut nested_delimiters = 1; - while nested_delimiters != 0 { + // Keep track of each nested delimiter we need to close. + let mut nested_delimiters = vec![delimiter]; + + while !nested_delimiters.is_empty() { let token = self.next_token()?; if *token.token() == start_delim { - nested_delimiters += 1; + nested_delimiters.push(token.clone()); } else if *token.token() == end_delim { - nested_delimiters -= 1; + nested_delimiters.pop(); + } else if *token.token() == Token::EOF { + let start_delim = + nested_delimiters.pop().expect("If this were empty, we wouldn't be looping"); + return Err(LexerErrorKind::UnclosedQuote { start_delim, end_delim }); } tokens.push(token); @@ -1312,4 +1318,17 @@ mod tests { } } } + + #[test] + fn test_unclosed_quote() { + let cases = vec!["quote {", "quote { { }", "quote [ []", "quote (((((((())))"]; + + for source in cases { + // `quote` is not itself a keyword so if the token stream fails to + // parse we don't expect any valid tokens from the quote construct + for token in Lexer::new(source) { + assert!(token.is_err(), "Expected Err, found {token:?}"); + } + } + } } From a818d950574b95404cae7b1cf49b124fe9b8b39a Mon Sep 17 00:00:00 2001 From: jfecher Date: Mon, 24 Jun 2024 13:21:50 -0500 Subject: [PATCH 12/12] feat: Add some metaprogramming methods on `TypeDefinition` (#5310) # Description ## Problem\* Resolves #5285 ## Summary\* Implements `as_type`, `generics`, and `fields` on the `TypeDefinition` type. The `Type` type isn't really supported yet so `as_type` and `fields` return `Quoted` token streams instead of `Type`s for now. ## Additional Context A few bugs still need to be fixed after this PR for the vertical slice to work. ## Documentation\* Check one: - [ ] No documentation needed. - [ ] Documentation included in this PR. - [x] **[For Experimental Features]** Documentation to be submitted in a separate PR. # PR Checklist\* - [x] I have tested the changes locally. - [x] I have formatted the changes with [Prettier](https://prettier.io/) and/or `cargo fmt` on default settings. --------- Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> --- .../src/hir/comptime/interpreter.rs | 21 ++- .../src/hir/comptime/interpreter/builtin.rs | 159 +++++++++++++++++- compiler/noirc_frontend/src/hir_def/types.rs | 19 ++- noir_stdlib/src/lib.nr | 1 + noir_stdlib/src/meta.nr | 1 + noir_stdlib/src/meta/type_def.nr | 16 ++ .../comptime_type_definition/Nargo.toml | 7 + .../comptime_type_definition/src/main.nr | 13 ++ .../derive_impl/Nargo.toml | 7 + .../derive_impl/src/main.nr | 44 +++++ 10 files changed, 272 insertions(+), 16 deletions(-) create mode 100644 noir_stdlib/src/meta.nr create mode 100644 noir_stdlib/src/meta/type_def.nr create mode 100644 test_programs/compile_success_empty/comptime_type_definition/Nargo.toml create mode 100644 test_programs/compile_success_empty/comptime_type_definition/src/main.nr create mode 100644 test_programs/compile_success_empty/derive_impl/Nargo.toml create mode 100644 test_programs/compile_success_empty/derive_impl/src/main.nr diff --git a/compiler/noirc_frontend/src/hir/comptime/interpreter.rs b/compiler/noirc_frontend/src/hir/comptime/interpreter.rs index 5d710884db1..5e236a2b980 100644 --- a/compiler/noirc_frontend/src/hir/comptime/interpreter.rs +++ b/compiler/noirc_frontend/src/hir/comptime/interpreter.rs @@ -98,14 +98,7 @@ impl<'a> Interpreter<'a> { .expect("all builtin functions must contain a function attribute which contains the opcode which it links to"); if let Some(builtin) = func_attrs.builtin() { - match builtin.as_str() { - "array_len" => builtin::array_len(&arguments), - "as_slice" => builtin::as_slice(arguments), - _ => { - let item = format!("Comptime evaluation for builtin function {builtin}"); - Err(InterpreterError::Unimplemented { item, location }) - } - } + builtin::call_builtin(self.interner, builtin, arguments, location) } else if let Some(foreign) = func_attrs.foreign() { let item = format!("Comptime evaluation for foreign functions like {foreign}"); Err(InterpreterError::Unimplemented { item, location }) @@ -934,6 +927,18 @@ impl<'a> Interpreter<'a> { fn evaluate_access(&mut self, access: HirMemberAccess, id: ExprId) -> IResult { let (fields, struct_type) = match self.evaluate(access.lhs)? { Value::Struct(fields, typ) => (fields, typ), + Value::Tuple(fields) => { + let (fields, field_types): (HashMap, Value>, Vec) = fields + .into_iter() + .enumerate() + .map(|(i, field)| { + let field_type = field.get_type().into_owned(); + let key_val_pair = (Rc::new(i.to_string()), field); + (key_val_pair, field_type) + }) + .unzip(); + (fields, Type::Tuple(field_types)) + } value => { let location = self.interner.expr_location(&id); return Err(InterpreterError::NonTupleOrStructInMemberAccess { value, location }); diff --git a/compiler/noirc_frontend/src/hir/comptime/interpreter/builtin.rs b/compiler/noirc_frontend/src/hir/comptime/interpreter/builtin.rs index 9216ba271c1..cccc9c6d545 100644 --- a/compiler/noirc_frontend/src/hir/comptime/interpreter/builtin.rs +++ b/compiler/noirc_frontend/src/hir/comptime/interpreter/builtin.rs @@ -1,11 +1,36 @@ +use std::rc::Rc; + use noirc_errors::Location; use crate::{ - hir::comptime::{errors::IResult, Value}, - Type, + hir::comptime::{errors::IResult, InterpreterError, Value}, + lexer::Lexer, + macros_api::NodeInterner, + token::{SpannedToken, Token, Tokens}, + QuotedType, Type, }; -pub(super) fn array_len(arguments: &[(Value, Location)]) -> IResult { +pub(super) fn call_builtin( + interner: &NodeInterner, + name: &str, + arguments: Vec<(Value, Location)>, + location: Location, +) -> IResult { + match name { + "array_len" => array_len(&arguments), + "as_slice" => as_slice(arguments), + "slice_push_back" => slice_push_back(arguments), + "type_def_as_type" => type_def_as_type(interner, arguments), + "type_def_generics" => type_def_generics(interner, arguments), + "type_def_fields" => type_def_fields(interner, arguments), + _ => { + let item = format!("Comptime evaluation for builtin function {name}"); + Err(InterpreterError::Unimplemented { item, location }) + } + } +} + +fn array_len(arguments: &[(Value, Location)]) -> IResult { assert_eq!(arguments.len(), 1, "ICE: `array_len` should only receive a single argument"); match &arguments[0].0 { Value::Array(values, _) | Value::Slice(values, _) => Ok(Value::U32(values.len() as u32)), @@ -14,7 +39,7 @@ pub(super) fn array_len(arguments: &[(Value, Location)]) -> IResult { } } -pub(super) fn as_slice(mut arguments: Vec<(Value, Location)>) -> IResult { +fn as_slice(mut arguments: Vec<(Value, Location)>) -> IResult { assert_eq!(arguments.len(), 1, "ICE: `as_slice` should only receive a single argument"); let (array, _) = arguments.pop().unwrap(); match array { @@ -23,3 +48,129 @@ pub(super) fn as_slice(mut arguments: Vec<(Value, Location)>) -> IResult _ => unreachable!("ICE: Cannot convert types other than arrays into slices"), } } + +fn slice_push_back(mut arguments: Vec<(Value, Location)>) -> IResult { + assert_eq!(arguments.len(), 2, "ICE: `slice_push_back` should only receive two arguments"); + let (element, _) = arguments.pop().unwrap(); + let (slice, _) = arguments.pop().unwrap(); + match slice { + Value::Slice(mut values, typ) => { + values.push_back(element); + Ok(Value::Slice(values, typ)) + } + // Type checking should prevent this branch being taken. + _ => unreachable!("ICE: `slice_push_back` expects a slice as its first argument"), + } +} + +/// fn as_type(self) -> Quoted +fn type_def_as_type( + interner: &NodeInterner, + mut arguments: Vec<(Value, Location)>, +) -> IResult { + assert_eq!(arguments.len(), 1, "ICE: `generics` should only receive a single argument"); + let (type_def, span) = match arguments.pop() { + Some((Value::TypeDefinition(id), location)) => (id, location.span), + other => { + unreachable!("ICE: `as_type` expected a `TypeDefinition` argument, found {other:?}") + } + }; + + let struct_def = interner.get_struct(type_def); + let struct_def = struct_def.borrow(); + let make_token = |name| SpannedToken::new(Token::Str(name), span); + + let mut tokens = vec![make_token(struct_def.name.to_string())]; + + for (i, generic) in struct_def.generics.iter().enumerate() { + if i != 0 { + tokens.push(SpannedToken::new(Token::Comma, span)); + } + tokens.push(make_token(generic.borrow().to_string())); + } + + Ok(Value::Code(Rc::new(Tokens(tokens)))) +} + +/// fn generics(self) -> [Quoted] +fn type_def_generics( + interner: &NodeInterner, + mut arguments: Vec<(Value, Location)>, +) -> IResult { + assert_eq!(arguments.len(), 1, "ICE: `generics` should only receive a single argument"); + let (type_def, span) = match arguments.pop() { + Some((Value::TypeDefinition(id), location)) => (id, location.span), + other => { + unreachable!("ICE: `as_type` expected a `TypeDefinition` argument, found {other:?}") + } + }; + + let struct_def = interner.get_struct(type_def); + + let generics = struct_def + .borrow() + .generics + .iter() + .map(|generic| { + let name = SpannedToken::new(Token::Str(generic.borrow().to_string()), span); + Value::Code(Rc::new(Tokens(vec![name]))) + }) + .collect(); + + let typ = Type::Slice(Box::new(Type::Quoted(QuotedType::Quoted))); + Ok(Value::Slice(generics, typ)) +} + +/// fn fields(self) -> [(Quoted, Quoted)] +/// Returns (name, type) pairs of each field of this TypeDefinition +fn type_def_fields( + interner: &NodeInterner, + mut arguments: Vec<(Value, Location)>, +) -> IResult { + assert_eq!(arguments.len(), 1, "ICE: `generics` should only receive a single argument"); + let (type_def, span) = match arguments.pop() { + Some((Value::TypeDefinition(id), location)) => (id, location.span), + other => { + unreachable!("ICE: `as_type` expected a `TypeDefinition` argument, found {other:?}") + } + }; + + let struct_def = interner.get_struct(type_def); + let struct_def = struct_def.borrow(); + + let make_token = |name| SpannedToken::new(Token::Str(name), span); + let make_quoted = |tokens| Value::Code(Rc::new(Tokens(tokens))); + + let mut fields = im::Vector::new(); + + for (name, typ) in struct_def.get_fields_as_written() { + let name = make_quoted(vec![make_token(name)]); + let typ = Value::Code(Rc::new(type_to_tokens(&typ)?)); + fields.push_back(Value::Tuple(vec![name, typ])); + } + + let typ = Type::Slice(Box::new(Type::Tuple(vec![ + Type::Quoted(QuotedType::Quoted), + Type::Quoted(QuotedType::Quoted), + ]))); + Ok(Value::Slice(fields, typ)) +} + +/// FIXME(https://github.com/noir-lang/noir/issues/5309): This code is temporary. +/// It will produce poor results for type variables and will result in incorrect +/// spans on the returned tokens. +fn type_to_tokens(typ: &Type) -> IResult { + let (mut tokens, mut errors) = Lexer::lex(&typ.to_string()); + + if let Some(last) = tokens.0.last() { + if matches!(last.token(), Token::EOF) { + tokens.0.pop(); + } + } + + if !errors.is_empty() { + let error = errors.swap_remove(0); + todo!("Got lexer error: {error}") + } + Ok(tokens) +} diff --git a/compiler/noirc_frontend/src/hir_def/types.rs b/compiler/noirc_frontend/src/hir_def/types.rs index 19ab3f8853e..86d1fafd502 100644 --- a/compiler/noirc_frontend/src/hir_def/types.rs +++ b/compiler/noirc_frontend/src/hir_def/types.rs @@ -296,6 +296,16 @@ impl StructType { }) } + /// Returns the name and raw types of each field of this type. + /// This will not substitute any generic arguments so a generic field like `x` + /// in `struct Foo { x: T }` will return a `("x", T)` pair. + /// + /// This method is almost never what is wanted for type checking or monomorphization, + /// prefer to use `get_fields` whenever possible. + pub fn get_fields_as_written(&self) -> Vec<(String, Type)> { + vecmap(&self.fields, |(name, typ)| (name.0.contents.clone(), typ.clone())) + } + pub fn field_names(&self) -> BTreeSet { self.fields.iter().map(|(name, _)| name.clone()).collect() } @@ -805,10 +815,11 @@ impl Type { | Type::FmtString(_, _) | Type::Error => true, - Type::MutableReference(_) - | Type::Forall(_, _) - | Type::Quoted(_) - | Type::TraitAsType(..) => false, + // Quoted objects only exist at compile-time where the only execution + // environment is the interpreter. In this environment, they are valid. + Type::Quoted(_) => true, + + Type::MutableReference(_) | Type::Forall(_, _) | Type::TraitAsType(..) => false, Type::Alias(alias, generics) => { let alias = alias.borrow(); diff --git a/noir_stdlib/src/lib.nr b/noir_stdlib/src/lib.nr index ad47171fa46..1a756f441ba 100644 --- a/noir_stdlib/src/lib.nr +++ b/noir_stdlib/src/lib.nr @@ -26,6 +26,7 @@ mod prelude; mod uint128; mod bigint; mod runtime; +mod meta; // Oracle calls are required to be wrapped in an unconstrained function // Thus, the only argument to the `println` oracle is expected to always be an ident diff --git a/noir_stdlib/src/meta.nr b/noir_stdlib/src/meta.nr new file mode 100644 index 00000000000..1825888130b --- /dev/null +++ b/noir_stdlib/src/meta.nr @@ -0,0 +1 @@ +mod type_def; diff --git a/noir_stdlib/src/meta/type_def.nr b/noir_stdlib/src/meta/type_def.nr new file mode 100644 index 00000000000..b9354485921 --- /dev/null +++ b/noir_stdlib/src/meta/type_def.nr @@ -0,0 +1,16 @@ +impl TypeDefinition { + /// Return a syntactic version of this type definition as a type. + /// For example, `as_type(quote { type Foo { ... } })` would return `Foo` + #[builtin(type_def_as_type)] + fn as_type(self) -> Quoted {} + + /// Return each generic on this type. The names of these generics are unchanged + /// so users may need to keep name collisions in mind if this is used directly in a macro. + #[builtin(type_def_generics)] + fn generics(self) -> [Quoted] {} + + /// Returns (name, type) pairs of each field in this type. Each type is as-is + /// with any generic arguments unchanged. + #[builtin(type_def_fields)] + fn fields(self) -> [(Quoted, Quoted)] {} +} diff --git a/test_programs/compile_success_empty/comptime_type_definition/Nargo.toml b/test_programs/compile_success_empty/comptime_type_definition/Nargo.toml new file mode 100644 index 00000000000..099545a9e71 --- /dev/null +++ b/test_programs/compile_success_empty/comptime_type_definition/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "comptime_type_definition" +type = "bin" +authors = [""] +compiler_version = ">=0.31.0" + +[dependencies] \ No newline at end of file diff --git a/test_programs/compile_success_empty/comptime_type_definition/src/main.nr b/test_programs/compile_success_empty/comptime_type_definition/src/main.nr new file mode 100644 index 00000000000..025f6a0b0bf --- /dev/null +++ b/test_programs/compile_success_empty/comptime_type_definition/src/main.nr @@ -0,0 +1,13 @@ +fn main() {} + +#[my_comptime_fn] +struct MyType { + field1: [A; 10], + field2: (B, C), +} + +comptime fn my_comptime_fn(typ: TypeDefinition) { + let _ = typ.as_type(); + assert_eq(typ.generics().len(), 3); + assert_eq(typ.fields().len(), 2); +} diff --git a/test_programs/compile_success_empty/derive_impl/Nargo.toml b/test_programs/compile_success_empty/derive_impl/Nargo.toml new file mode 100644 index 00000000000..26a6020a6b1 --- /dev/null +++ b/test_programs/compile_success_empty/derive_impl/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "derive_impl" +type = "bin" +authors = [""] +compiler_version = ">=0.30.0" + +[dependencies] diff --git a/test_programs/compile_success_empty/derive_impl/src/main.nr b/test_programs/compile_success_empty/derive_impl/src/main.nr new file mode 100644 index 00000000000..abad6d4f8e1 --- /dev/null +++ b/test_programs/compile_success_empty/derive_impl/src/main.nr @@ -0,0 +1,44 @@ +comptime fn derive_default(typ: TypeDefinition) -> Quoted { + let generics: [Quoted] = typ.generics(); + assert_eq( + generics.len(), 0, "derive_default: Deriving Default on generic types is currently unimplemented" + ); + + let type_name = typ.as_type(); + let fields = typ.fields(); + + let fields = join(make_field_exprs(fields)); + + quote { + impl Default for $type_name { + fn default() -> Self { + Self { $fields } + } + } + } +} + +#[derive_default] +struct Foo { + x: Field, + y: u32, +} + +comptime fn make_field_exprs(fields: [(Quoted, Quoted)]) -> [Quoted] { + let mut result = &[]; + for my_field in fields { + let name = my_field.0; + result = result.push_back(quote { $name: Default::default(), }); + } + result +} + +comptime fn join(slice: [Quoted]) -> Quoted { + let mut result = quote {}; + for elem in slice { + result = quote { $result $elem }; + } + result +} + +fn main() {}