[LANGUAGE, FIX] Fix non-en colors in lists and ask (#4756)

Reverts #4754. This is a redo of #4675, which fixed #3356 but caused #4752.

This has additional commits that also fix #4752.

**Original text from #4675:**

**Description**

Allows the use of Dutch (and other langs) names in a list, fixing  #3356. This new way of processing numbers to English at runtime also allows for using non-English colors in an ask:

<img width="1608" alt="image" src="https://github.com/hedyorg/hedy/assets/1003685/fa373815-47d5-4e68-b50b-d69404082c6f">

This PR:
<img width="1592" alt="image" src="https://github.com/hedyorg/hedy/assets/1003685/07d0f3ee-e8aa-4407-ad5d-0660c651cba8">

main:
<img width="1422" alt="image" src="https://github.com/hedyorg/hedy/assets/1003685/c7a2d2e5-7c33-48a7-9b63-ce9b0182d932">

**Fixes #3356**

**How to test**

I added a test about the list access, for the ask issue, check that the images above work for you also.

grammars/level1.lark

@@ -3,9 +3,10 @@ program: _EOL* (command _EOL+)* command?
 command: print | ask | echo | turtle | error_invalid_space | error_invalid | empty_line
 
 print: _PRINT (text)?
+_color_literal : black | blue | brown | gray | green | orange | pink | purple | red | white | yellow
 ask: _ASK (text)?
 echo: _ECHO (text)?
-turtle: _FORWARD ((NUMBER | text))? -> forward | _TURN ((left | right ))? -> turn | _COLOR ((black | blue | brown | gray | green | orange | pink | purple | red | white | yellow | text))? -> color
+turtle: _FORWARD ((NUMBER | text))? -> forward | _TURN ((left | right ))? -> turn | _COLOR (_color_literal | text)? -> color
 error_invalid_space: _SPACE any
 
 // We give the error_invalid rule below a priority of -100,

grammars/level3-Additions.lark

@@ -6,7 +6,7 @@ assign: var _IS (list_access | text) -> assign
 assign_list: var _IS text_list (_COMMA text_list)+
 
 list_access: var_access _AT (INT | random)
-turtle: _FORWARD ((NUMBER | list_access | textwithoutspaces))? -> forward | _TURN ((NUMBER | list_access | textwithoutspaces))? -> turn | _COLOR ((black | blue | brown | gray | green | orange | pink | purple | red | white | yellow | list_access | textwithoutspaces))? -> color
+turtle: _FORWARD ((NUMBER | list_access | textwithoutspaces))? -> forward | _TURN ((NUMBER | list_access | textwithoutspaces))? -> turn | _COLOR (_color_literal | list_access | textwithoutspaces)? -> color
 
 sleep: _SLEEP (INT | list_access | var_access)?
 // lists are introduced and list separators (comma and arabic comma) have to excluded from text.

grammars/level4-Additions.lark

@@ -3,6 +3,7 @@ command: clear | print | ask | turtle | assign_list | add | remove | sleep | err
 
 print: _PRINT (_print_argument)? -> print
 error_print_no_quotes: _PRINT text -> error_print_nq
+
 // FH, feb 2023 Shall we rename this 'var' to declare maybe? I confused myself :)
 ask: var _IS _ASK (_print_argument)?
 error_ask_no_quotes: var _IS _ASK text  -> error_print_nq

hedy.py

@@ -292,7 +292,20 @@ def promote_types(types, rules):
 }
 
 command_turn_literals = ['right', 'left']
-command_make_color = ['black', 'blue', 'brown', 'gray', 'green', 'orange', 'pink', 'purple', 'red', 'white', 'yellow']
+english_colors = ['black', 'blue', 'brown', 'gray', 'green', 'orange', 'pink', 'purple', 'red', 'white', 'yellow']
+
+
+def color_commands_local(language):
+    colors_local = [hedy_translation.translate_keyword_from_en(k, language) for k in english_colors]
+    return colors_local
+
+
+def command_make_color_local(language):
+    if language == "en":
+        return english_colors
+    else:
+        return english_colors + color_commands_local(language)
+
 
 # Commands and their types per level (only partially filled!)
 commands_and_types_per_level = {
@@ -310,8 +323,8 @@ def promote_types(types, rules):
                    2: [HedyType.integer, HedyType.input],
                    12: [HedyType.integer, HedyType.input, HedyType.float]
                    },
-    Command.color: {1: command_make_color,
-                    2: [command_make_color, HedyType.string, HedyType.input]},
+    Command.color: {1: english_colors,
+                    2: [english_colors, HedyType.string, HedyType.input]},
     Command.forward: {1: [HedyType.integer, HedyType.input],
                       12: [HedyType.integer, HedyType.input, HedyType.float]
                       },
@@ -1015,7 +1028,7 @@ class AllCommands(Transformer):
     def __init__(self, level):
         self.level = level
 
-    def translate_keyword(self, keyword):
+    def standardize_keyword(self, keyword):
         # some keywords have names that are not a valid name for a command
         # that's why we call them differently in the grammar
         # we have to translate them to the regular names here for further communciation
@@ -1047,7 +1060,7 @@ def translate_keyword(self, keyword):
 
     def __default__(self, args, children, meta):
         # if we are matching a rule that is a command
-        production_rule_name = self.translate_keyword(args)
+        production_rule_name = self.standardize_keyword(args)
         leaves = flatten_list_of_lists_to_list(children)
         # for the achievements we want to be able to also detect which operators were used by a kid
         operators = ['addition', 'subtraction', 'multiplication', 'division']
@@ -1242,8 +1255,9 @@ def decorator(c):
 
 @v_args(meta=True)
 class ConvertToPython(Transformer):
-    def __init__(self, lookup, numerals_language="Latin", is_debug=False):
+    def __init__(self, lookup, language="en", numerals_language="Latin", is_debug=False):
         self.lookup = lookup
+        self.language = language
         self.numerals_language = numerals_language
         self.is_debug = is_debug
 
@@ -1382,8 +1396,9 @@ def indent(s, spaces_amount=2):
 @source_map_transformer(source_map)
 class ConvertToPython_1(ConvertToPython):
 
-    def __init__(self, lookup, numerals_language, is_debug):
+    def __init__(self, lookup, language, numerals_language, is_debug):
         self.numerals_language = numerals_language
+        self.language = language
         self.lookup = lookup
         self.is_debug = is_debug
         __class__.level = 1
@@ -1439,7 +1454,7 @@ def color(self, meta, args):
             return f"t.pencolor('black'){self.add_debug_breakpoint()}"  # no arguments defaults to black ink
 
         arg = args[0].data
-        if arg in command_make_color:
+        if arg in command_make_color_local(self.language):
             return f"t.pencolor('{arg}'){self.add_debug_breakpoint()}"
         else:
             # the TypeValidator should protect against reaching this line:
@@ -1466,8 +1481,8 @@ def make_turn(self, parameter):
     def make_forward(self, parameter):
         return self.make_turtle_command(parameter, Command.forward, 'forward', True, 'int')
 
-    def make_color(self, parameter):
-        return self.make_turtle_color_command(parameter, Command.color, 'pencolor')
+    def make_color(self, parameter, language):
+        return self.make_turtle_color_command(parameter, Command.color, 'pencolor', language)
 
     def make_turtle_command(self, parameter, command, command_text, add_sleep, type):
         exception = ''
@@ -1485,12 +1500,23 @@ def make_turtle_command(self, parameter, command, command_text, add_sleep, type)
             return sleep_after(transpiled, False, self.is_debug)
         return transpiled
 
-    def make_turtle_color_command(self, parameter, command, command_text):
+    def make_turtle_color_command(self, parameter, command, command_text, language):
+        both_colors = command_make_color_local(language)
         variable = self.get_fresh_var('__trtl')
+
+        # we translate the color value to English at runtime, since it might be decided at runtime
+        # coming from a random list or ask
+
+        color_dict = {hedy_translation.translate_keyword_from_en(x, language): x for x in english_colors}
+
         return textwrap.dedent(f"""\
             {variable} = f'{parameter}'
-            if {variable} not in {command_make_color}:
+            color_dict = {color_dict}
+            if {variable} not in {both_colors}:
               raise Exception(f'While running your program the command {style_command(command)} received the value {style_command('{' + variable + '}')} which is not allowed. Try using another color.')
+            else:
+              if not {variable} in {english_colors}:
+                {variable} = color_dict[{variable}]
             t.{command_text}({variable}){self.add_debug_breakpoint()}""")
 
     def make_catch_exception(self, args):
@@ -1540,7 +1566,7 @@ def color(self, meta, args):
 
         arg = self.process_variable_for_fstring(arg)
 
-        return self.make_color(arg)
+        return self.make_color(arg, self.language)
 
     def turn(self, meta, args):
         if len(args) == 0:
@@ -1738,8 +1764,8 @@ def clear(self, meta, args):  # todo not sure about it being here
 @hedy_transpiler(level=5)
 @source_map_transformer(source_map)
 class ConvertToPython_5(ConvertToPython_4):
-    def __init__(self, lookup, numerals_language, is_debug):
-        super().__init__(lookup, numerals_language, is_debug)
+    def __init__(self, lookup, language, numerals_language, is_debug):
+        super().__init__(lookup, language, numerals_language, is_debug)
 
     def ifs(self, meta, args):  # might be worth asking if we want a debug breakpoint here
         return f"""if {args[0]}:{self.add_debug_breakpoint()}
@@ -3331,7 +3357,7 @@ def transpile_inner(input_string, level, lang="en", populate_source_map=False, i
 
         # grab the right transpiler from the lookup
         convertToPython = TRANSPILER_LOOKUP[level]
-        python = convertToPython(lookup_table, numerals_language, is_debug).transform(abstract_syntax_tree)
+        python = convertToPython(lookup_table, lang, numerals_language, is_debug).transform(abstract_syntax_tree)
 
         commands = AllCommands(level).transform(program_root)
 

tests/Tester.py

@@ -350,12 +350,19 @@ def sleep_command_transpiled(val):
           raise Exception(f'While running your program the command <span class="command-highlighted">sleep</span> received the value <span class="command-highlighted">{{{val}}}</span> which is not allowed. Try changing the value to a number.')""")
 
     @staticmethod
-    def turtle_color_command_transpiled(val):
+    def turtle_color_command_transpiled(val, lang="en"):
+        color_dict = {hedy_translation.translate_keyword_from_en(x, lang): x for x in hedy.english_colors}
+        both_colors = hedy.command_make_color_local(lang)
+
         return textwrap.dedent(f"""\
-      __trtl = f'{val}'
-      if __trtl not in ['black', 'blue', 'brown', 'gray', 'green', 'orange', 'pink', 'purple', 'red', 'white', 'yellow']:
-        raise Exception(f'While running your program the command <span class="command-highlighted">color</span> received the value <span class="command-highlighted">{{__trtl}}</span> which is not allowed. Try using another color.')
-      t.pencolor(__trtl)""")
+        __trtl = f'{val}'
+        color_dict = {color_dict}
+        if __trtl not in {both_colors}:
+          raise Exception(f'While running your program the command <span class="command-highlighted">color</span> received the value <span class="command-highlighted">{{__trtl}}</span> which is not allowed. Try using another color.')
+        else:
+          if not __trtl in {hedy.english_colors}:
+            __trtl = color_dict[__trtl]
+        t.pencolor(__trtl)""")
 
     @staticmethod
     def input_transpiled(var_name, text):

tests/test_level/test_level_02.py

@@ -468,14 +468,15 @@ def test_color_with_var(self):
         )
 
     def test_color_translated(self):
+        lang = 'nl'
         code = "kleur blauw"
-        expected = HedyTester.turtle_color_command_transpiled('blue')
+        expected = HedyTester.turtle_color_command_transpiled('blue', lang)
 
         self.multi_level_tester(
             code=code,
             expected=expected,
             extra_check_function=self.is_turtle(),
-            lang='nl',
+            lang=lang,
             max_level=10
         )
 

tests/test_level/test_level_03.py

@@ -56,6 +56,40 @@ def test_print_list_random(self):
             extra_check_function=check_in_list
         )
 
+    def test_color_basic(self):
+        code = textwrap.dedent("""\
+        color red
+        forward 10""")
+
+        expected = HedyTester.dedent(HedyTester.turtle_color_command_transpiled("red", 'en'),
+                                     HedyTester.forward_transpiled('10', self.level))
+
+        self.multi_level_tester(
+            max_level=11,
+            code=code,
+            translate=False,
+            expected=expected
+        )
+
+    def test_random_turle_dutch(self):
+        lang = 'nl'
+        code = textwrap.dedent("""\
+        lijstkleuren is blauw, groen, wit
+        kleur lijstkleuren at random
+        vooruit 10""")
+
+        expected = HedyTester.dedent("lijstkleuren = ['blauw', 'groen', 'wit']",
+                                     HedyTester.turtle_color_command_transpiled("{random.choice(lijstkleuren)}", lang),
+                                     HedyTester.forward_transpiled('10', self.level))
+
+        self.multi_level_tester(
+            max_level=11,
+            code=code,
+            lang=lang,
+            translate=False,
+            expected=expected
+        )
+
     def test_print_list_random_punctuation(self):
         code = textwrap.dedent("""\
         gerechten is spaghetti, spruitjes, hamburgers