Wippersnapper preprocessor script for printing the library versions to boot outtxt

all_platforms.py

@@ -27,6 +27,7 @@
     "funhouse" : ["espressif:esp32:adafruit_funhouse_esp32s2", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "funhouse_noota" : ["espressif:esp32:adafruit_funhouse_esp32s2:PartitionScheme=tinyuf2_noota", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "metroesp32s2" : ["espressif:esp32:adafruit_metro_esp32s2", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
+    "metroesp32s2_debug" : ["espressif:esp32:adafruit_metro_esp32s2:DebugLevel=verbose", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "qtpy_esp32s2" : ["espressif:esp32:adafruit_qtpy_esp32s2", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "feather_esp32s2" : ["espressif:esp32:adafruit_feather_esp32s2", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "feather_esp32s2_debug" : ["espressif:esp32:adafruit_feather_esp32s2:DebugLevel=verbose", "0xbfdd4eee", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
@@ -46,8 +47,10 @@
     "feather_esp32s3_reverse_tft" : ["espressif:esp32:adafruit_feather_esp32s3_reversetft", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "feather_esp32s3_reverse_tft_debug" : ["espressif:esp32:adafruit_feather_esp32s3_reversetft:DebugLevel=verbose", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "wippersnapper_feather_esp32s3_reverse_tft" : ["espressif:esp32:adafruit_feather_esp32s3_reversetft:PartitionScheme=tinyuf2_noota", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
+    "wippersnapper_feather_esp32s3_reverse_tft_debug" : ["espressif:esp32:adafruit_feather_esp32s3_reversetft:DebugLevel=verbose,PartitionScheme=tinyuf2_noota", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "matrixportal_s3" : ["esp32:esp32:adafruit_matrixportal_esp32s3", "0xc47e5767", None],
     "metro_esp32s3" : ["espressif:esp32:adafruit_metro_esp32s3", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
+    "metro_esp32s3_debug" : ["espressif:esp32:adafruit_metro_esp32s3:DebugLevel=verbose", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
     "pycamera_s3" : ["esp32:esp32:adafruit_camera_esp32s3", "0xc47e5767", None],
     "qualia_s3_rgb666" : ["esp32:esp32:adafruit_qualia_s3_rgb666", "0xc47e5767", None],
     "qtpy_esp32s3" : ["espressif:esp32:adafruit_qtpy_esp32s3_nopsram", "0xc47e5767", "adafruit/wipper-bsp-3.0.5-idf-5.1.4"],
@@ -162,4 +165,4 @@
                           "cpb", "cpx_ada"),
     "wippersnapper_platforms" : ("metro_m4_airliftlite_tinyusb", "pyportal_tinyusb"),
     "rp2040_platforms" : ("pico_rp2040", "feather_rp2040")
-}
+}

build_platform.py

@@ -29,6 +29,20 @@
     sys.argv.pop(sys.argv.index("--build_timeout") + 1)
     sys.argv.remove("--build_timeout")
 
+# optional wippersnapper argument to generate a dependencies header file
+PRINT_DEPENDENCIES_AS_HEADER = False
+INCLUDE_PRINT_DEPENDENCIES_HEADER = False
+PRINT_DEPENDENCIES_AS_HEADER_FILENAME = None
+if "--include_print_dependencies_header" in sys.argv:
+    # check argument not null and folder path exists
+    header_index = sys.argv.index("--include_print_dependencies_header") + 1
+    PRINT_DEPENDENCIES_AS_HEADER_FILENAME = sys.argv[header_index] if len(sys.argv) > header_index else None
+    if PRINT_DEPENDENCIES_AS_HEADER_FILENAME is None or not os.path.exists(PRINT_DEPENDENCIES_AS_HEADER_FILENAME):
+        raise ValueError("Header file path not found or not provided to --include_print_dependencies_header argument")
+    INCLUDE_PRINT_DEPENDENCIES_HEADER = True
+    sys.argv.pop(header_index)
+    sys.argv.remove("--include_print_dependencies_header")
+
 # add user bin to path!
 BUILD_DIR = ''
 # add user bin to path!
@@ -56,9 +70,6 @@
     print("Found MetroX Examples Repo")
     IS_LEARNING_SYS = True
 
-#os.system('pwd')
-#os.system('ls -lA')
-
 CROSS = u'\N{cross mark}'
 CHECK = u'\N{check mark}'
 
@@ -104,7 +115,13 @@ def manually_install_esp32_bsp(repo_info):
     # Assemble git url
     repo_url = "git clone -b {0} https://github.com/{1}/arduino-esp32.git esp32".format(repo_info.split("/")[1], repo_info.split("/")[0])
     # Locally clone repo (https://docs.espressif.com/projects/arduino-esp32/en/latest/installing.html#linux)
-    os.system("mkdir -p /home/runner/Arduino/hardware/espressif")
+    subprocess_result = subprocess.run("mkdir -p /home/runner/Arduino/hardware/espressif", shell=True)
+    if subprocess_result.returncode != 0:
+        ColorPrint.print_fail("Failed to create ESP32 Arduino BSP directory!\n" +
+                              "Maybe the runner work location changed?\n" +
+                              "Tried to create /home/runner/Arduino/hardware/espressif but failed: {}".format(subprocess_result.returncode))
+        ColorPrint.print_fail(subprocess_result.stderr)
+        exit(-1)
     print("Cloning %s"%repo_url)
     cmd = "cd /home/runner/Arduino/hardware/espressif && " + repo_url
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
@@ -135,10 +152,11 @@ def manually_install_esp32_bsp(repo_info):
 
 
 def install_platform(fqbn, full_platform_name=None):
+    print("Checking for drazzy.json (before moving)")
     if os.path.exists("/home/runner/.arduino15/package_drazzy.json"):
         print("Moving drazzy.json")
         shutil.move("/home/runner/.arduino15/package_drazzy.json", "/home/runner/.arduino15/package_drazzy.com_index.json")
-    print("Installing", fqbn, end=" ")
+    print("Installing", fqbn, " (", full_platform_name, ")", end=" ")
     if fqbn == "adafruit:avr":   # we have a platform dep
         install_platform("arduino:avr", full_platform_name)
     if full_platform_name[2] is not None:
@@ -147,7 +165,7 @@ def install_platform(fqbn, full_platform_name=None):
         return # bail out
     for retry in range(0, 3):
         print("arduino-cli core install "+fqbn+" --additional-urls "+BSP_URLS)
-        if os.system("arduino-cli core install "+fqbn+" --additional-urls "+BSP_URLS+" > /dev/null") == 0:
+        if subprocess.run("arduino-cli core install "+fqbn+" --additional-urls "+BSP_URLS+" > /dev/null", shell=True, check=False).returncode == 0:
             break
         print("...retrying...", end=" ")
         time.sleep(10) # wait 10 seconds then try again?
@@ -157,14 +175,16 @@ def install_platform(fqbn, full_platform_name=None):
         exit(-1)
     ColorPrint.print_pass(CHECK)
     # print installed core version
-    print(os.popen('arduino-cli core list | grep {}'.format(fqbn)).read(), end='')
+    result = subprocess.Popen('arduino-cli core list | grep {}'.format(fqbn), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    print(result.stdout.read().decode(), end='')
+
 
 
 def run_or_die(cmd, error):
     print(cmd)
     attempt = 0
     while attempt < 3:
-        if os.system(cmd) == 0:
+        if subprocess.run(cmd, shell=True, check=False).returncode == 0:
             return
         attempt += 1
         print('attempt {} failed, {} retry left'.format(attempt, 3-attempt))
@@ -175,7 +195,7 @@ def run_or_die(cmd, error):
 
 def is_library_installed(lib_name):
     try:
-        installed_libs = subprocess.check_output(["arduino-cli", "lib", "list"]).decode("utf-8")
+        installed_libs = subprocess.check_output("arduino-cli lib list", shell=True).decode("utf-8")
         return not all(not item for item in [re.match('^'+lib_name+'\\s*\\d+\\.', line) for line in installed_libs.split('\n')])
     except subprocess.CalledProcessError as e:
         print("Error checking installed libraries:", e)
@@ -290,6 +310,7 @@ def generate_uf2(platform, fqbn, example_path):
         family_id = ALL_PLATFORMS[platform][1]
         cmd = ['python3', 'uf2conv.py', hex_input_file, '-c', '-f', family_id, '-o', output_file]
     else:
+        # ColorPrint.print_info(subprocess.check_output(["tree"]).decode("utf-8"))
         cli_build_path = "build/*.*." + fqbn.split(':')[2] + "/*.ino.bin"
         input_file = glob1(os.path.join(example_path, cli_build_path))
         output_file = os.path.splitext(input_file)[0] + ".uf2"
@@ -329,27 +350,101 @@ def group_output(title):
         sys.stdout.flush()
 
 
+def extract_dependencies(output):
+    print("Extracting libraries from output:", output)
+    print(f"{"############## Done #############":-^80}")
+
+    IS_LIBS_FOUND = False
+    IS_BSP_FOUND = False
+    libraries = []
+    platforms = []
+    COLS = []
+    for i,line in enumerate(output.split('\n')):
+        if line.strip() == '':
+            continue
+        if not IS_LIBS_FOUND:
+            if re.match(r'Used library', line):
+                IS_LIBS_FOUND = True
+                print("Found libraries token using regex")
+                print("find Version:", line.find('Version'))
+                print("find Path:", line.find('Path'))
+                COLS = [0,line.find('Version'), line.find('Path')]
+                continue
+            else:
+                if line.find("Used library") != -1:
+                    print("Found libraries token using find")
+                    IS_LIBS_FOUND = True
+                    print("non regex find Version:", line.find('Version'))
+                    print("find Path:", line.find('Path'))
+                    COLS = [0,line.find('Version'), line.find('Path')]
+        else:
+            if not IS_BSP_FOUND:
+                if re.match(r'Used platform', line):
+                    print("Found platform token using regex")
+                    IS_BSP_FOUND = True
+                    COLS = [0,line.find('Version'), line.find('Path')]
+                    continue
+                elif line.find("Used platform") != -1:
+                    print("Found platform token using find")
+                    IS_BSP_FOUND = True
+                    COLS = [0,line.find('Version'), line.find('Path')]
+                    continue
+                else:
+                    libraries.append([line[:COLS[1]].strip(),line[COLS[1]:COLS[2]].strip()])
+            else:
+                platforms.append([line[:COLS[1]].strip(),line[COLS[1]:COLS[2]].strip()])
+
+    dependencies = {
+        'libraries': libraries,
+        'platforms': platforms
+    }
+    print("Extracted list of dependencies:", dependencies)
+    return dependencies
+
+def write_dependencies_to_header(dependencies, output_file):
+    # header file
+    with open(output_file, 'w') as f:
+        f.write('#ifndef PROJECT_DEPENDENCIES_H\n')
+        f.write('#define PROJECT_DEPENDENCIES_H\n\n')
+        f.write('#define PRINT_DEPENDENCIES 1\n')
+        f.write('extern const char* project_dependencies;\n\n')
+        f.write('#endif // PROJECT_DEPENDENCIES_H\n')
+    # cpp file
+    with open(re.sub(r'\.h$','.cpp', output_file), 'w') as f:
+        f.write('#include "print_dependencies.h"\n\n')
+        f.write('const char* project_dependencies = R"(\n')
+
+        f.write('Libraries and Versions:\n')
+        for lib in dependencies['libraries']:
+            f.write(f'Library: {lib[0].strip()}, Version: {lib[1].strip()}\n')
+
+        f.write('\nPlatforms and Versions:\n')
+        for plat in dependencies['platforms']:
+            f.write(f'Platform: {plat[0].strip()}, Version: {plat[1].strip()}\n')
+
+        f.write(')";\n\n')
+
 def test_examples_in_folder(platform, folderpath):
-    global success
+    global success, BUILD_TIMEOUT, popen_timeout, BUILD_WALL, BUILD_WARN, PRINT_DEPENDENCIES_AS_HEADER, INCLUDE_PRINT_DEPENDENCIES_HEADER, IS_LEARNING_SYS, BUILD_DIR
     fqbn = ALL_PLATFORMS[platform][0]
     for example in sorted(os.listdir(folderpath)):
-        examplepath = folderpath+"/"+example
+        examplepath = folderpath + "/" + example
         if os.path.isdir(examplepath):
             test_examples_in_folder(platform, examplepath)
             continue
         if not examplepath.endswith(".ino"):
             continue
 
-        print('\t'+example, end=' ')
+        print('\t' + example, end=' ')
 
         # check if we should SKIP
-        skipfilename = folderpath+"/."+platform+".test.skip"
-        onlyfilename = folderpath+"/."+platform+".test.only"
+        skipfilename = folderpath + "/." + platform + ".test.skip"
+        onlyfilename = folderpath + "/." + platform + ".test.only"
         # check if we should GENERATE UF2
-        gen_file_name = folderpath+"/."+platform+".generate"
+        gen_file_name = folderpath + "/." + platform + ".generate"
 
         # .skip txt include all skipped platforms, one per line
-        skip_txt = folderpath+"/.skip.txt"
+        skip_txt = folderpath + "/.skip.txt"
 
         is_skip = False
         if os.path.exists(skipfilename):
@@ -365,10 +460,10 @@ def test_examples_in_folder(platform, folderpath):
             ColorPrint.print_warn("skipping")
             continue
 
-        if glob.glob(folderpath+"/.*.test.only"):
-            platformname = glob.glob(folderpath+"/.*.test.only")[0].split('.')[1]
-            if platformname != "none" and not platformname in ALL_PLATFORMS:
-                # uh oh, this isnt a valid testonly!
+        if glob.glob(folderpath + "/.*.test.only"):
+            platformname = glob.glob(folderpath + "/.*.test.only")[0].split('.')[1]
+            if platformname != "none" and platformname not in ALL_PLATFORMS:
+                # uh oh, this isn't a valid testonly!
                 ColorPrint.print_fail(CROSS)
                 ColorPrint.print_fail("This example does not have a valid .platform.test.only file")
                 success = 1
@@ -386,8 +481,16 @@ def test_examples_in_folder(platform, folderpath):
                 cmd = ['arduino-cli', 'compile', '--warnings', 'all', '--fqbn', fqbn, folderpath]
         else:
             cmd = ['arduino-cli', 'compile', '--warnings', 'none', '--export-binaries', '--fqbn', fqbn, folderpath]
-        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
+
+        if PRINT_DEPENDENCIES_AS_HEADER:
+            cmd.append('--only-compilation-database')
+            cmd.append('--no-color')
+        elif INCLUDE_PRINT_DEPENDENCIES_HEADER:
+            cmd.append('--build-property')
+            cmd.append('"build.extra_flags=\'-DPRINT_DEPENDENCIES -I\"' + os.path.join(BUILD_DIR, "print_dependencies.cpp") + '\"\'"')
+            cmd.append('--verbose')
+
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         try:
             if BUILD_TIMEOUT:
                 out, err = proc.communicate(timeout=popen_timeout)
@@ -405,7 +508,11 @@ def test_examples_in_folder(platform, folderpath):
                 # also print out warning message
                 with group_output(f"{example} {fqbn} build output"):
                     ColorPrint.print_fail(err.decode("utf-8"))
-            if os.path.exists(gen_file_name):
+            if PRINT_DEPENDENCIES_AS_HEADER:
+                # Extract dependencies and write to header for the first successful example
+                dependencies = extract_dependencies(out.decode("utf-8") + err.decode("utf-8"))
+                write_dependencies_to_header(dependencies, PRINT_DEPENDENCIES_AS_HEADER_FILENAME)
+            elif os.path.exists(gen_file_name):
                 if ALL_PLATFORMS[platform][1] is None:
                     ColorPrint.print_info("Platform does not support UF2 files, skipping...")
                 else:
@@ -415,10 +522,10 @@ def test_examples_in_folder(platform, folderpath):
                         success = 1  # failure
                     if IS_LEARNING_SYS:
                         fqbnpath, uf2file = filename.split("/")[-2:]
-                        os.makedirs(BUILD_DIR+"/build", exist_ok=True)
-                        os.makedirs(BUILD_DIR+"/build/"+fqbnpath, exist_ok=True)
-                        shutil.copy(filename, BUILD_DIR+"/build/"+fqbnpath+"-"+uf2file)
-                        os.system("ls -lR "+BUILD_DIR+"/build")
+                        os.makedirs(BUILD_DIR + "/build", exist_ok=True)
+                        os.makedirs(BUILD_DIR + "/build/" + fqbnpath, exist_ok=True)
+                        shutil.copy(filename, BUILD_DIR + "/build/" + fqbnpath + "-" + uf2file)
+                        subprocess.run("ls -lR " + BUILD_DIR + "/build", shell=True, check=True)
         else:
             ColorPrint.print_fail(CROSS)
             with group_output(f"{example} {fqbn} built output"):
@@ -428,6 +535,7 @@ def test_examples_in_folder(platform, folderpath):
 
 
 def main():
+    global INCLUDE_PRINT_DEPENDENCIES_HEADER, PRINT_DEPENDENCIES_AS_HEADER
     # Test platforms
     platforms = []
 
@@ -449,15 +557,22 @@ def main():
     for platform in platforms:
         fqbn = ALL_PLATFORMS[platform][0]
         print('#'*80)
-        ColorPrint.print_info("SWITCHING TO "+fqbn)
+        ColorPrint.print_info("SWITCHING TO " + fqbn)
         install_platform(":".join(fqbn.split(':', 2)[0:2]), ALL_PLATFORMS[platform]) # take only first two elements
         print('#'*80)
         if not IS_LEARNING_SYS:
-            test_examples_in_folder(platform, BUILD_DIR+"/examples")
+            if INCLUDE_PRINT_DEPENDENCIES_HEADER:
+                PRINT_DEPENDENCIES_AS_HEADER = True
+                INCLUDE_PRINT_DEPENDENCIES_HEADER = False
+                test_examples_in_folder(platform, BUILD_DIR + "/examples")
+                PRINT_DEPENDENCIES_AS_HEADER = False
+                INCLUDE_PRINT_DEPENDENCIES_HEADER = True
+                test_examples_in_folder(platform, BUILD_DIR + "/examples")
+            else:
+                test_examples_in_folder(platform, BUILD_DIR + "/examples")
         else:
             test_examples_in_folder(platform, BUILD_DIR)
 
-
 if __name__ == "__main__":
     main()
     exit(success)