[UI] Adds CodeMirror as editor

build-tools/heroku/generate-grammars-and-js

@@ -9,5 +9,8 @@ python3 ../../content/yaml_to_lark_utils.py
 echo '-----> Creating python prefixes for TypeScript'
 ./generate-prefixes-ts
 
+echo '-----> Generating the lezer parsers'
+./generate-lezer-parsers
+
 echo '-----> Compiling TypeScript'
 ./generate-typescript

build-tools/heroku/generate-lezer-parsers

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eu
+
+# Generate the lezer parsers from the grammars specified in  highlighting/lezer-grammars/
+for grammarFile in ../../highlighting/lezer-grammars/level*.grammar ; do
+    baseName=$(basename "$grammarFile")
+    tsFile=${baseName%.grammar}-parser.ts
+    npx lezer-generator --typeScript "$grammarFile" -o ../../static/js/lezer-parsers/$tsFile
+done

highlighting/lezer-grammars/level1.grammar

@@ -0,0 +1,34 @@
+@top Program { eol* (Command eol+)* Command? }
+Command {
+    Print | Ask | Echo | Turtle | ErrorInvalid
+}
+Print { print+ Text+ }
+Ask { ask+ Text+ }
+Echo { echo+ Text* }
+
+Turtle[@isGroup=turtle] {
+    Forward { forward+ Text } |
+    Turn { turn+ Text } |
+    Color { color+ Text }
+}
+
+ErrorInvalid[@dynamicPrecedence=-10] { Text+ }
+
+@external extend { Text } extendKeyword from "./tokens" {
+    print[@name="print"],
+    forward[@name="forward"],
+    turn[@name="turn"],
+    echo[@name="echo"],
+    ask[@name="ask"],
+    color[@name="color"]
+}
+
+
+@tokens {
+    Comment { "#" ![\n]* }
+    eol { "\n" }
+    space { " " }
+    Text { ![\n #]+ }
+}
+
+@skip { Comment | space }

highlighting/lezer-grammars/level10.grammar

@@ -0,0 +1,118 @@
+@top Program { eol* (Command eol+)* Command? }
+Command {
+    Assign | AssignList | Ask | Clear |Print | Turtle | Sleep | Add | Remove| If | Else | Repeat | For | ErrorInvalid
+}
+
+@local tokens {    
+    singleQuoteStringEnd { "'" | "\n" }
+    @else singleQuoteStringContent
+  }
+
+  @skip {} {
+    singleQuotedString { "'" (singleQuoteStringContent)* singleQuoteStringEnd }
+}
+
+@local tokens {    
+    doubleQuoteStringEnd { '"' | "\n" }
+    @else doubleQuoteStringContent
+  }
+
+  @skip {} {
+    doubleQuotedString { '"' (doubleQuoteStringContent)* doubleQuoteStringEnd }
+}
+
+String { doubleQuotedString | singleQuotedString }
+
+Print { print ( String | ListAccess | Expression )+ }
+Ask { Text (is+ | Op<"=">) ask+ ( String | ListAccess | Expression )+ }
+
+AssignList { Text (is+ | Op<"=">) (Text+ | Int ) ~ambig  (Comma (Text+ | Int ))+ } // comma doesn't really parse well if it is not separated by spaces
+Assign { Text (is+ | Op<"=">) (Expression ~ambig  | ListAccess)+ }
+Sleep { sleep+ (Expression | ListAccess )? }
+ListAccess { Text at+ (random+ | Text) }
+Add { add+ Text toList+ Text }
+Remove { remove+ Text from+ Text }
+Clear { clear+ }
+
+Turtle[@isGroup=turtle] {
+    Forward { forward+ (Expression | ListAccess ) } |
+    Turn { turn+ (Expression | ListAccess ) } |
+    Color { color+ (Expression | ListAccess)  }
+}   
+
+If { ifs+ Condition }
+Else { elses+ }
+
+Condition { 
+    EqualityCheck { Text is+ (String | Expression | pressed+) } |
+    InListCheck { Text ins+  Text } |
+    NotInListCheck { Text not_in+  Text }
+}
+
+Repeat { repeat+ (Int | Text) times+ }
+
+For { fors+ Text ins+ Text }
+
+ErrorInvalid[@dynamicPrecedence=-10] { Text+ }
+
+Op<expr> { expr }
+
+binaryExpression {
+    Expression !times Op<"*" | "/"> Expression |
+    Expression !plus Op<"+" | "-"> Expression
+}
+
+Expression {
+    Int |
+    Text |
+    binaryExpression
+}
+
+@precedence {
+    times @left,
+    plus @right
+}
+
+@external specialize { Text } specializeKeyword from "./tokens" {
+    ask[@name="ask"],
+    at[@name="at"],
+    random[@name="random"],
+    ifs[@name="if"],
+    pressed[@name="pressed"],
+    elses[@name="else"]
+}
+
+@external extend { Text } extendKeyword from "./tokens" {
+    print[@name="print"],
+    forward[@name="forward"],
+    turn[@name="turn"],
+    color[@name="color"],
+    sleep[@name="sleep"],
+    is[@name="is"],
+    add[@name="add"],
+    from[@name="from"],
+    remove[@name="remove"],
+    toList[@name="toList"],
+    clear[@name="clear"],    
+    ins[@name="in"],
+    not_in[@name="not_in"],
+    repeat[@name="repeat"],
+    times[@name="times"],
+    fors[@name="for"]
+}
+
+@tokens {
+    @precedence {
+        Int,
+        Text
+    }
+    Comment { "#" ![\n]* }
+    eol { "\n" }
+    Comma { $[,،，、] } // every language comma is valid here
+    Text { ![^\n,،，、 '"#+\-*/=]+ } // a little bit of a hack not to parse commas never, but lezer adds them as error nodes, so it still
+    space { " " }
+    digit { $[٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹0123456789] } // latin, arab and pa_PK (Panjabi) scripts
+    Int { digit+ }
+}
+
+@skip { Comment | space }

highlighting/lezer-grammars/level11.grammar

@@ -0,0 +1,120 @@
+@top Program { eol* (Command eol+)* Command? }
+Command {
+    Assign | AssignList | Ask | Clear |Print | Turtle | Sleep | Add | Remove| If | Else | Repeat | For | ErrorInvalid
+}
+
+@local tokens {    
+    singleQuoteStringEnd { "'" | "\n" }
+    @else singleQuoteStringContent
+  }
+
+  @skip {} {
+    singleQuotedString { "'" (singleQuoteStringContent)* singleQuoteStringEnd }
+}
+
+@local tokens {    
+    doubleQuoteStringEnd { '"' | "\n" }
+    @else doubleQuoteStringContent
+  }
+
+  @skip {} {
+    doubleQuotedString { '"' (doubleQuoteStringContent)* doubleQuoteStringEnd }
+}
+
+String { doubleQuotedString | singleQuotedString }
+
+Print { print+ ( String | ListAccess | Expression )+ }
+Ask { Text (is+ | Op<"=">) ask+ ( String | ListAccess | Expression )+ }
+
+AssignList { Text (is+ | Op<"=">) (Text+ | Int ) ~ambig  (Comma (Text+ | Int ))+ } // comma doesn't really parse well if it is not separated by spaces
+Assign { Text (is+ | Op<"=">) (Expression ~ambig  | ListAccess)+ }
+Sleep { sleep+ (Expression | ListAccess )? }
+ListAccess { Text at+ (random+ | Text) }
+Add { add+ Text toList+ Text }
+Remove { remove+ Text from+ Text }
+Clear { clear+ }
+
+Turtle[@isGroup=turtle] {
+    Forward { forward+ (Expression | ListAccess ) } |
+    Turn { turn+ (Expression | ListAccess ) } |
+    Color { color+ (Expression | ListAccess)  }
+}   
+
+If { ifs+ Condition }
+Else { elses+ }
+
+Condition { 
+    EqualityCheck { Text is+ (String | Expression | pressed+) } |
+    InListCheck { Text ins+  Text } |
+    NotInListCheck { Text not_in+  Text }
+}
+
+Repeat { repeat+ (Int | Text) times+ }
+
+For { fors+ Text ins+ (Text | range+ (Text | Int) to+ (Text | Int)) }
+
+ErrorInvalid[@dynamicPrecedence=-10] { Text+ }
+
+Op<expr> { expr }
+
+binaryExpression {
+    Expression !times Op<"*" | "/"> Expression |
+    Expression !plus Op<"+" | "-"> Expression
+}
+
+Expression {
+    Int |
+    Text |
+    binaryExpression
+}
+
+@precedence {
+    times @left,
+    plus @right
+}
+
+@external specialize { Text } specializeKeyword from "./tokens" {
+    ask[@name="ask"],
+    at[@name="at"],
+    random[@name="random"],
+    ifs[@name="if"],
+    pressed[@name="pressed"],
+    elses[@name="else"]
+}
+
+@external extend { Text } extendKeyword from "./tokens" {
+    print[@name="print"],
+    forward[@name="forward"],
+    turn[@name="turn"],
+    color[@name="color"],
+    sleep[@name="sleep"],
+    is[@name="is"],
+    add[@name="add"],
+    from[@name="from"],
+    remove[@name="remove"],
+    toList[@name="toList"],
+    clear[@name="clear"],    
+    ins[@name="in"],
+    not_in[@name="not_in"],
+    repeat[@name="repeat"],
+    times[@name="times"],
+    fors[@name="for"],
+    to[@name="to"],
+    range[@name="range"]
+}
+
+@tokens {
+    @precedence {
+        Int,
+        Text
+    }
+    Comment { "#" ![\n]* }
+    eol { "\n" }
+    Comma { $[,،，、] } // every language comma is valid here
+    Text { ![^\n,،，、 '"#+\-*/=]+ } // a little bit of a hack not to parse commas never, but lezer adds them as error nodes, so it still
+    space { " " }
+    digit { $[٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹0123456789] } // latin, arab and pa_PK (Panjabi) scripts
+    Int { digit+ }
+}
+
+@skip { Comment | space }