From 27ccc0108fc8f2b152ffbb9892c90ac8caddd66d Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke@waleson.com>
Date: Wed, 22 Nov 2023 14:40:11 +0100
Subject: [PATCH] take all arguments to the single test hash into account

---
 tests/Tester.py | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/tests/Tester.py b/tests/Tester.py
index 52f7072973b..522dce35e22 100644
--- a/tests/Tester.py
+++ b/tests/Tester.py
@@ -1,4 +1,5 @@
 import textwrap
+import pickle
 import hashlib
 import hedy
 import hedy_translation
@@ -89,10 +90,12 @@ class HedyTester(unittest.TestCase):
     def setUpClass(cls):
         os.environ["ENABLE_SKIP_FAULTY"] = 'True'  # Always test with skipping faulty enabled
 
-    def snippet_already_tested_with_current_hedy_version(self, snippet, level):
+    def snippet_already_tested_with_current_hedy_version(self, test_hash):
         try:
-            hash_language_plus_snippet_and_level = create_hash(get_hedy_source_hash(), snippet, level)
-            filename = get_hash_filename(hash_language_plus_snippet_and_level)
+            total_hash_incl_the_hedy_language = create_hash(get_hedy_source_hash(), test_hash)
+            if total_hash_incl_the_hedy_language is None:
+                return False
+            filename = get_hash_filename(total_hash_incl_the_hedy_language)
             already_successful = os.path.isfile(filename)
             return already_successful
         except UnicodeEncodeError:  # some tests (generated by Hypothesis) can't be hashed
@@ -230,7 +233,24 @@ def single_level_tester(
     ):
         if level is None:  # no level set (from the multi-tester)? grap current level from class
             level = self.level
-        if not self.snippet_already_tested_with_current_hedy_version(code, level):
+
+        # To speed up the test executing we calculate a hash of the test.
+        # It is important to capture all the parameters that are passed to the function
+        # as sometimes we expect a snippet to fail and sometimes we expect it to succeed.
+        # We do this with `locals`, which captures a kwargs of this function and then we hash that.
+        # This means we have to collect the locals at the beginning because else there will be
+        # other things than arguments in the `locals()` output.
+
+        all_args = locals()
+        del all_args["self"]
+        try:
+            # we use pickle instead of hash for consistency across test-runs
+            # see PYTHONHASHSEED
+            test_hash = pickle.dumps(all_args)
+        except AttributeError:
+            test_hash = None
+
+        if not self.snippet_already_tested_with_current_hedy_version(test_hash):
             if skipped_mappings is not None:
                 result = hedy.transpile(code, level, lang, skip_faulty=skip_faulty)
                 for skipped in skipped_mappings:
@@ -282,7 +302,7 @@ def single_level_tester(
                         self.assertTrue(extra_check_function(result))
 
             # all ok? -> save hash!
-            hash_of_run = create_hash(get_hedy_source_hash(), code, level)
+            hash_of_run = create_hash(get_hedy_source_hash(), test_hash)
             if hash_of_run:
                 filename = get_hash_filename(hash_of_run)
                 os.makedirs(os.path.dirname(filename), mode=0o777, exist_ok=True)
@@ -436,12 +456,11 @@ def translate_keywords_in_snippets(snippets):
         return snippets
 
 
-def create_hash(hedy_language, snippet, level):
-    try:
-        t = snippet + "|\n" + str(level) + "|\n" + hedy_language
-        return hashlib.sha1(t.encode('utf-8')).hexdigest()
-    except UnicodeEncodeError:  # some tests can't be hashed
-        return ''
+def create_hash(hedy_language, test_hash):
+    if test_hash is None:
+        return None
+    t = str(test_hash) + "|\n" + hedy_language
+    return hashlib.sha1(t.encode('utf-8')).hexdigest()
 
 
 def get_hash_filename(input_hash):