Support optional for all symbols (#144)

src/main/resources/manuals/d711ba960cd12b76/tycheck-ignore.json

@@ -33,7 +33,6 @@
   "GetSubstitution",
   "GetThisValue",
   "GetValue",
-  "HostEnqueuePromiseJob",
   "InnerModuleEvaluation",
   "IntegerIndexedObjectCreate",
   "LabelledItem[1,0].LabelledEvaluation",
@@ -62,4 +61,4 @@
   "StringCreate",
   "ToBigInt",
   "ToNumber"
-]
+]

src/main/scala/esmeta/analyzer/AbsTransfer.scala

@@ -164,7 +164,7 @@ class AbsTransfer(sem: AbsSemantics) extends Optimized with PruneHelper {
     case Syntactic(name, _, rhsIdx, children) =>
       val rhs = cfg.grammar.nameMap(name).rhsList(rhsIdx)
       val optionals = (for {
-        (opt, child) <- rhs.nts.map(_.optional) zip children if opt
+        ((_, opt), child) <- rhs.ntsWithOptional zip children if opt
       } yield !child.isEmpty)
       optionals.reverse.zipWithIndex.foldLeft(0) {
         case (acc, (true, idx)) => acc + scala.math.pow(2, idx).toInt

src/main/scala/esmeta/analyzer/domain/value/TypeDomain.scala

@@ -510,7 +510,7 @@ object TypeDomain extends value.Domain {
       case Production(lhs, _, _, rhsList) =>
         val name = lhs.name
         val subs = rhsList.collect {
-          case Rhs(_, List(Nonterminal(name, _, _)), _) => name
+          case Rhs(_, List(Nonterminal(name, _)), _) => name
         }.toSet
         name -> subs
     }).toMap

src/main/scala/esmeta/interpreter/Interpreter.scala

@@ -610,7 +610,7 @@ class Interpreter(
     case Syntactic(name, _, rhsIdx, children) =>
       val rhs = cfg.grammar.nameMap(name).rhsList(rhsIdx)
       val optionals = (for {
-        (opt, child) <- rhs.nts.map(_.optional) zip children if opt
+        ((_, opt), child) <- rhs.ntsWithOptional zip children if opt
       } yield !child.isEmpty)
       optionals.reverse.zipWithIndex.foldLeft(0) {
         case (acc, (true, idx)) => acc + scala.math.pow(2, idx).toInt

src/main/scala/esmeta/mutator/synthesizer/RandomSynthesizer.scala

@@ -36,10 +36,11 @@ class RandomSynthesizer(
     symbol: Symbol,
   ): Option[Option[Ast]] = symbol match
     case ButNot(nt, _) => synSymbol(argsMap)(nt)
-    case Nonterminal(name, args, optional) =>
+    case Optional(symbol) =>
+      if (randBool) Some(None) else synSymbol(argsMap)(symbol)
+    case Nonterminal(name, args) =>
       if (reservedLexicals contains name)
         Some(Some(Lexical(name, reservedLexicals(name))))
-      else if (optional && randBool) Some(None)
       else {
         import NonterminalArgumentKind.*
         val newArgs = for (arg <- args) yield arg.kind match

src/main/scala/esmeta/mutator/synthesizer/SimpleSynthesizer.scala

@@ -42,11 +42,11 @@ class SimpleSynthesizer(
   private def synSymbol(argsMap: Map[String, Boolean])(
     symbol: Symbol,
   ): Option[Option[Ast]] = symbol match
-    case ButNot(nt, _) => synSymbol(argsMap)(nt)
-    case Nonterminal(name, args, optional) =>
+    case ButNot(nt, _)    => synSymbol(argsMap)(nt)
+    case Optional(symbol) => Some(None)
+    case Nonterminal(name, args) =>
       if (reservedLexicals contains name)
         Some(Some(Lexical(name, reservedLexicals(name))))
-      else if (optional) Some(None)
       else {
         import NonterminalArgumentKind.*
         val newArgs = for (arg <- args) yield arg.kind match

src/main/scala/esmeta/parser/ESParser.scala

@@ -148,37 +148,37 @@ case class ESParser(
     prev: LAParser[List[Option[Ast]]],
     symbol: Symbol,
     argsSet: Set[String],
-  ): LAParser[List[Option[Ast]]] =
-    symbol match {
-      case Terminal(")") if name == "DoWhileStatement" => prev <~ doWhileCloseT
-      case Terminal(term)                              => prev <~ t(term)
-      case Nonterminal(name, args, optional) =>
-        lazy val parser =
+    optional: Boolean = false,
+  ): LAParser[List[Option[Ast]]] = symbol match
+    case Terminal(")") if name == "DoWhileStatement" => prev <~ doWhileCloseT
+    case Terminal(term)                              => prev <~ t(term)
+    case symbol: NtBase =>
+      lazy val parser = symbol match
+        case Nonterminal(name, args) =>
           if (lexNames contains name) nt(name, lexers(name, 0))
           else parsers(name)(toBools(argsSet, args))
-        if (optional) prev ~ opt(parser) ^^ { case l ~ s => s :: l }
-        else prev ~ parser ^^ { case l ~ s => Some(s) :: l }
-      case ButNot(base, cases) =>
-        val name = s"$base \\ ${cases.mkString("(", ", ", ")")}"
-        lazy val parser = nt(name, getSymbolParser(symbol, argsSet))
-        prev ~ parser ^^ { case l ~ s => Some(s) :: l }
-      case ButOnlyIf(base, methodName, cond) => ???
-      case Lookahead(contains, cases) =>
-        lazy val parser = cases
-          .map(
-            _.map(_ match {
-              case Terminal(t) if t.matches("[a-z]+") => t <~ not(IDContinue)
-              case symbol => getSymbolParser(symbol, argsSet)
-            }).reduce(_ %% _),
-          )
-          .reduce(_ | _)
-        lazy val lookahead = ntl(symbol.toString, parser)
-        prev <~ (if (contains) +lookahead else -lookahead)
-      case Empty               => prev
-      case NoLineTerminator    => prev <~ noLineTerminator
-      case CodePointAbbr(abbr) => ???
-      case UnicodeSet(cond)    => ???
-    }
+        case ButNot(base, cases) =>
+          val name = s"$base \\ ${cases.mkString("(", ", ", ")")}"
+          nt(name, getSymbolParser(symbol, argsSet))
+        case ButOnlyIf(base, methodName, cond) => ???
+      if (optional) prev ~ opt(parser) ^^ { case l ~ s => s :: l }
+      else prev ~ parser ^^ { case l ~ s => Some(s) :: l }
+    case Optional(symbol) => appendParser(name, prev, symbol, argsSet, true)
+    case Lookahead(contains, cases) =>
+      lazy val parser = cases
+        .map(
+          _.map(_ match {
+            case Terminal(t) if t.matches("[a-z]+") => t <~ not(IDContinue)
+            case symbol => getSymbolParser(symbol, argsSet)
+          }).reduce(_ %% _),
+        )
+        .reduce(_ | _)
+      lazy val lookahead = ntl(symbol.toString, parser)
+      prev <~ (if (contains) +lookahead else -lookahead)
+    case Empty               => prev
+    case NoLineTerminator    => prev <~ noLineTerminator
+    case CodePointAbbr(abbr) => ???
+    case UnicodeSet(cond)    => ???
 
   // a terminal lexer
   protected val TERMINAL: Lexer = grammar.prods.foldLeft[Parser[String]]("") {
@@ -428,10 +428,10 @@ case class ESParser(
   }
 
   // check whether it is a left-recursive production
-  private def isLR(name: String, rhs: Rhs): Boolean = rhs.symbols match {
-    case Nonterminal(`name`, _, _) :: _ => true
-    case _                              => false
-  }
+  private def isLR(name: String, rhs: Rhs): Boolean = (for {
+    symbol <- rhs.symbols.headOption
+    nt <- symbol.getNt
+  } yield nt.name == name).getOrElse(false)
 
   // TODO more general rule
   // handle indirect left-recursive case

src/main/scala/esmeta/parser/Lexer.scala

@@ -94,10 +94,10 @@ trait Lexer extends UnicodeParsers {
     argsSet: Set[String],
   ): Parser[String] = symbol match
     case Terminal(term) => term
-    case Nonterminal(name, args, optional) =>
-      val parser = lexers((name, toBit(toBools(argsSet, args))))
-      if (optional) opt(parser) ^^ { _.getOrElse("") }
-      else parser
+    case Nonterminal(name, args) =>
+      lexers((name, toBit(toBools(argsSet, args))))
+    case Optional(symbol) =>
+      opt(getSymbolParser(symbol, argsSet)) ^^ { _.getOrElse("") }
     case ButNot(base, cases) =>
       val parser = getSymbolParser(base, argsSet)
       val exclude = cases.map(getSymbolParser(_, argsSet)).reduce(_ ||| _)

src/main/scala/esmeta/spec/Production.scala

@@ -19,12 +19,18 @@ case class Production(
     (name, j) <- rhs.allNames.zipWithIndex
   } yield lhs.name + ":" + name -> (i, j)).toMap
 
-  /** get non-terminals in RHSs */
+  /** get non-terminals with whether it is optional in an RHS */
   lazy val nts: List[Nonterminal] = for {
     rhs <- rhsList
     nt <- rhs.nts
   } yield nt
 
+  /** get non-terminals with whether it is optional in an RHS */
+  lazy val ntsWithOptional: List[(Nonterminal, Boolean)] = for {
+    rhs <- rhsList
+    pair <- rhs.ntsWithOptional
+  } yield pair
+
   /** get terminals in RHSs */
   lazy val ts: List[Terminal] = for {
     rhs <- rhsList

src/main/scala/esmeta/spec/Rhs.scala

@@ -15,36 +15,49 @@ case class Rhs(
   /** get RHS all names */
   def allNames: List[String] =
     symbols.foldLeft(List[String]("")) {
-      case (names, Terminal(term)) => names.map(_ + term)
-      case (names, Nonterminal(name, _, optional)) =>
-        names.flatMap(x => {
-          if (optional) List(x, x + name) else List(x + name)
-        })
-      case (names, ButNot(base, _)) =>
-        names.map(_ + base.name)
-      case (names, ButOnlyIf(base, _, _)) =>
-        names.map(_ + base.name)
-      case (names, _) => names
+      case (names, Terminal(term)) =>
+        names.map(_ + term)
+      case (names, s: Optional) =>
+        s.getName.fold(names)(name => names.flatMap(x => List(x, x + name)))
+      case (names, s) =>
+        s.getName.fold(names)(name => names.map(_ + name))
     }
 
-  /** get non-terminals in an RHS */
-  lazy val nts: List[Nonterminal] = symbols.flatMap(_.getNt)
+  /** optional symbols */
+  lazy val optionals = symbols.collect { case opt: Optional => opt }
+
+  lazy val nts: List[Nonterminal] = for {
+    symbol <- symbols
+    nt <- symbol.getNt
+  } yield nt
+
+  /** get non-terminals with whether it is optional in an RHS */
+  lazy val ntsWithOptional: List[(Nonterminal, Boolean)] = for {
+    symbol <- symbols
+    nt <- symbol.getNt
+  } yield (nt, symbol.isInstanceOf[Optional])
+
+  /** get terminals in an RHS */
   lazy val ts: List[Terminal] = symbols.flatMap(_.getT)
+
+  /** get terminals in an RHS */
   lazy val getNts = cached[Int, List[Option[String]]] { subIdx =>
     val binStr = subIdx.toBinaryString
-    val optCount = nts.count(_.optional)
+    val optCount = optionals.size
     var flags = (("0" * (optCount - binStr.length)) + binStr).map(_ == '1')
-    nts.map(nt =>
-      if (nt.optional) {
+    for {
+      symbol <- symbols
+      nt <- symbol.getNt
+    } yield symbol match
+      case _: Optional =>
         val present = flags.head
         flags = flags.tail
         if (present) Some(nt.name) else None
-      } else Some(nt.name),
-    )
+      case _ => Some(nt.name)
   }
 
   /** count sub production */
-  lazy val countSubs: Int = scala.math.pow(2, nts.count(_.optional)).toInt
+  lazy val countSubs: Int = scala.math.pow(2, optionals.size).toInt
 
   /** check if empty */
   def isEmpty: Boolean = symbols match
@@ -53,14 +66,10 @@ case class Rhs(
 
   /** get index of non-terminal */
   def getNtIndex(ntName: String): Option[Int] =
-    val filtered = nts.zipWithIndex.filter { case (nt, _) => nt.name == ntName }
-    filtered match
-      case (_, idx) :: rest => Some(idx)
-      case _                => None
+    nts.zipWithIndex.find(_.head.name == ntName).map(_.last)
 
   /** get parameters from RHSs */
-  def params: List[Param] =
-    nts.map(nt => Param(nt.name, Type(AstT(nt.name))))
+  def params: List[Param] = nts.map(nt => Param(nt.name, Type(AstT(nt.name))))
 
   /** check whether the RHS is available */
   def available(argsSet: Set[String]): Boolean = conditions.forall {

src/main/scala/esmeta/spec/Symbol.scala

@@ -8,26 +8,36 @@ sealed trait Symbol extends SpecElem {
   /** get an non-terminal or nothing from a symbol */
   def getNt: Option[Nonterminal] = this match
     case (nt: Nonterminal)     => Some(nt)
+    case Optional(symbol)      => symbol.getNt
     case ButNot(base, _)       => Some(base)
     case ButOnlyIf(base, _, _) => Some(base)
     case _                     => None
 
   /** get a terminal or nothing from a symbol */
-  def getT: Option[Terminal] = this match {
+  def getT: Option[Terminal] = this match
     case t: Terminal => Some(t)
     case _           => None
-  }
+
+  /** get the name of a symbol */
+  def getName: Option[String] = this match
+    case Nonterminal(name, _)  => Some(name)
+    case Optional(symbol)      => symbol.getName
+    case ButNot(base, _)       => Some(base.name)
+    case ButOnlyIf(base, _, _) => Some(base.name)
+    case _                     => None
 }
 object Symbol extends Parser.From(Parser.symbol)
 
 /** terminal symbols */
 case class Terminal(term: String) extends Symbol
 
+/** nonterminal symbols as base symbols */
+type NtBase = Nonterminal | ButNot | ButOnlyIf
+
 /** nonterminal symbols */
 case class Nonterminal(
   name: String,
   args: List[NonterminalArgument],
-  optional: Boolean,
 ) extends Symbol
 case class NonterminalArgument(
   kind: NonterminalArgumentKind,
@@ -37,6 +47,9 @@ object NonterminalArgument extends Parser.From(Parser.ntArg)
 enum NonterminalArgumentKind extends SpecElem { case True, False, Pass }
 object NonterminalArgumentKind extends Parser.From(Parser.ntArgKind)
 
+/** optional symbols */
+case class Optional(symbol: Symbol) extends Symbol
+
 /** butnot symbols */
 case class ButNot(base: Nonterminal, notCases: List[Symbol]) extends Symbol
 

src/main/scala/esmeta/spec/util/JsonProtocol.scala

@@ -44,6 +44,7 @@ object JsonProtocol extends BasicJsonProtocol {
     : List[(String, HCursor => Decoder.Result[Symbol])] = List(
     "term" -> (_.as[Terminal]),
     "name" -> (_.as[Nonterminal]),
+    "symbol" -> (_.as[Optional]),
     "notCases" -> (_.as[ButNot]),
     "methodName" -> (_.as[ButOnlyIf]),
     "cases" -> (_.as[Lookahead]),
@@ -55,6 +56,7 @@ object JsonProtocol extends BasicJsonProtocol {
   given Encoder[Symbol] = Encoder.instance {
     case symbol: Terminal      => symbol.asJson
     case symbol: Nonterminal   => symbol.asJson
+    case symbol: Optional      => symbol.asJson
     case symbol: ButNot        => symbol.asJson
     case symbol: ButOnlyIf     => symbol.asJson
     case symbol: Lookahead     => symbol.asJson
@@ -71,6 +73,8 @@ object JsonProtocol extends BasicJsonProtocol {
   given Encoder[NonterminalArgument] = deriveEncoder
   given Decoder[NonterminalArgumentKind] = deriveDecoder
   given Encoder[NonterminalArgumentKind] = deriveEncoder
+  given Decoder[Optional] = deriveDecoder
+  given Encoder[Optional] = deriveEncoder
   given Decoder[ButNot] = deriveDecoder
   given Encoder[ButNot] = deriveEncoder
   given Decoder[ButOnlyIf] = deriveDecoder

src/main/scala/esmeta/spec/util/Parser.scala

@@ -80,7 +80,10 @@ trait Parsers extends LangParsers {
 
   /** grammar symbols */
   given symbol: Parser[Symbol] = {
-    term | butnot | lookahead | butOnlyIf | nt | abbr | unicodeSet | empty | nlt
+    {
+      term | butnot | lookahead | butOnlyIf | nt | abbr | unicodeSet |
+      empty | nlt
+    } ~ opt("?") ^^ { case s ~ o => if (o.isDefined) Optional(s) else s }
   }.named("spec.Symbol")
 
   /** terminals */
@@ -93,9 +96,8 @@ trait Parsers extends LangParsers {
 
   /** nonterminals */
   lazy val nt: Parser[Nonterminal] = {
-    word ~ opt("[" ~> rep1sep(ntArg, ",") <~ "]") ~ opt("?") ^^ {
-      case name ~ args ~ opt =>
-        Nonterminal(name, args.getOrElse(Nil), opt.isDefined)
+    word ~ opt("[" ~> rep1sep(ntArg, ",") <~ "]") ^^ {
+      case name ~ args => Nonterminal(name, args.getOrElse(Nil))
     }
   }.named("spec.Nonterminal")
 

src/main/scala/esmeta/spec/util/Stringifier.scala

@@ -129,10 +129,11 @@ object Stringifier {
       case Empty               => app >> "[empty]"
       case NoLineTerminator    => app >> "[no LineTerminator here]"
       case CodePointAbbr(abbr) => app >> "<" >> abbr >> ">"
-      case Nonterminal(name, args, opt) =>
+      case Nonterminal(name, args) =>
         app >> name
-        if (!args.isEmpty) app >> args
-        if (opt) app >> "?" else app
+        if (!args.isEmpty) app >> args else app
+      case Optional(symbol) =>
+        app >> symbol >> "?"
       case Lookahead(b, cases) =>
         app >> "[lookahead " >> (if (b) "<" else "<!") >> " " >> cases >> "]"
       case ButOnlyIf(base, name, cond) =>