[pregen] Use pathlib in pre-generation scripts

.github/workflows/pregenerate.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Run ntcore
         run: ./ntcore/generate_topics.py
       - name: Run wpimath
-        run: ./wpimath/generate_numbers.py && ./wpimath/generate_quickbuf.py protoc protoc-gen-quickbuf-1.3.3-linux-x86_64.exe
+        run: ./wpimath/generate_numbers.py && ./wpimath/generate_quickbuf.py --quickbuf_plugin=protoc-gen-quickbuf-1.3.3-linux-x86_64.exe
       - name: Run HIDs
         run: ./wpilibj/generate_hids.py && ./wpilibc/generate_hids.py && ./wpilibNewCommands/generate_hids.py
       - name: Add untracked files to index so they count as changes

hal/generate_usage_reporting.py

@@ -3,38 +3,43 @@
 # Copyright (c) FIRST and other WPILib contributors.
 # Open Source Software; you can modify and/or share it under the terms of
 # the WPILib BSD license file in the root directory of this project.
-import pathlib
+from pathlib import Path
+import sys
+import argparse
 
 
-def main():
+def generate_usage_reporting(output_directory: Path, template_directory: Path):
     # Gets the folder this script is in (the hal/ directory)
-    HAL_ROOT = pathlib.Path(__file__).parent
     java_package = "edu/wpi/first/hal"
     # fmt: off
-    (HAL_ROOT / "src/generated/main/native/include/hal").mkdir(parents=True, exist_ok=True)
-    (HAL_ROOT / f"src/generated/main/java/{java_package}").mkdir(parents=True, exist_ok=True)
+    (output_directory / "main/native/include/hal").mkdir(parents=True, exist_ok=True)
+    (output_directory / f"main/java/{java_package}").mkdir(parents=True, exist_ok=True)
     # fmt: on
     usage_reporting_types_cpp = []
     usage_reporting_instances_cpp = []
     usage_reporting_types = []
     usage_reporting_instances = []
-    with open(HAL_ROOT / "src/generate/Instances.txt") as instances:
+    with (template_directory / "Instances.txt").open(encoding="utf-8") as instances:
         for instance in instances:
             usage_reporting_instances_cpp.append(f"    {instance.strip()},")
             usage_reporting_instances.append(
                 f"    /** {instance.strip()}. */\n"
                 f"    public static final int {instance.strip()};"
             )
 
-    with open(HAL_ROOT / "src/generate/ResourceType.txt") as resource_types:
+    with (template_directory / "ResourceType.txt").open(
+        encoding="utf-8"
+    ) as resource_types:
         for resource_type in resource_types:
             usage_reporting_types_cpp.append(f"    {resource_type.strip()},")
             usage_reporting_types.append(
                 f"    /** {resource_type.strip()}. */\n"
                 f"    public static final int {resource_type.strip()};"
             )
 
-    with open(HAL_ROOT / "src/generate/FRCNetComm.java.in") as java_usage_reporting:
+    with (template_directory / "FRCNetComm.java.in").open(
+        encoding="utf-8"
+    ) as java_usage_reporting:
         contents = (
             # fmt: off
             java_usage_reporting.read()
@@ -43,12 +48,12 @@ def main():
             # fmt: on
         )
 
-        with open(
-            HAL_ROOT / f"src/generated/main/java/{java_package}/FRCNetComm.java", "w"
-        ) as java_out:
-            java_out.write(contents)
+        frc_net_comm = output_directory / f"main/java/{java_package}/FRCNetComm.java"
+        frc_net_comm.write_text(contents, encoding="utf-8")
 
-    with open(HAL_ROOT / "src/generate/FRCUsageReporting.h.in") as cpp_usage_reporting:
+    with (template_directory / "FRCUsageReporting.h.in").open(
+        encoding="utf-8"
+    ) as cpp_usage_reporting:
         contents = (
             # fmt: off
             cpp_usage_reporting.read()
@@ -57,11 +62,33 @@ def main():
             # fmt: on
         )
 
-        with open(
-            HAL_ROOT / "src/generated/main/native/include/hal/FRCUsageReporting.h", "w"
-        ) as cpp_out:
-            cpp_out.write(contents)
+        usage_reporting_hdr = (
+            output_directory / "main/native/include/hal/FRCUsageReporting.h"
+        )
+        usage_reporting_hdr.write_text(contents, encoding="utf-8")
+
+
+def main(argv):
+
+    dirname = Path(__file__).parent
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output_directory",
+        help="Optional. If set, will output the generated files to this directory, otherwise it will use a path relative to the script",
+        default=dirname / "src/generated",
+        type=Path,
+    )
+    parser.add_argument(
+        "--template_root",
+        help="Optional. If set, will use this directory as the root for the jinja templates",
+        default=dirname / "src/generate",
+        type=Path,
+    )
+    args = parser.parse_args(argv)
+
+    generate_usage_reporting(args.output_directory, args.template_root)
 
 
 if __name__ == "__main__":
-    main()
+    main(sys.argv[1:])

ntcore/generate_topics.py

@@ -1,127 +1,156 @@
 #!/usr/bin/env python3
-
-import glob
-import os
-import sys
-from jinja2 import Environment, FileSystemLoader
+import argparse
 import json
+import sys
+from pathlib import Path
+from typing import Dict, Any
 
+from jinja2 import Environment, FileSystemLoader
+from jinja2.environment import Template
 
-def Output(outPath, outfn, contents):
-    if not os.path.exists(outPath):
-        os.makedirs(outPath)
-
-    outpathname = f"{outPath}/{outfn}"
-
-    if os.path.exists(outpathname):
-        with open(outpathname, "r") as f:
-            if f.read() == contents:
-                return
-
-    # File either doesn't exist or has different contents
-    with open(outpathname, "w", newline="\n") as f:
-        f.write(contents)
 
+def Output(output_dir: Path, controller_name: str, contents: str):
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_file = output_dir / controller_name
+    output_file.write_text(contents, encoding="utf-8")
 
-def main():
-    dirname, _ = os.path.split(os.path.abspath(__file__))
 
-    with open(f"{dirname}/src/generate/types.json") as f:
+def generate_topics(
+    output_directory: Path, template_root: Path, types_schema_file: Path
+):
+    with (types_schema_file).open(encoding="utf-8") as f:
         types = json.load(f)
 
     # Java files
+    java_template_directory = template_root / "main/java"
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/java"), autoescape=False
+        loader=FileSystemLoader(java_template_directory), autoescape=False
     )
-    rootPath = f"{dirname}/src/generated/main/java/edu/wpi/first/networktables"
-    for fn in glob.glob(f"{dirname}/src/generate/main/java/*.jinja"):
-        template = env.get_template(os.path.basename(fn))
-        outfn = os.path.basename(fn)[:-6]  # drop ".jinja"
-        if os.path.basename(fn).startswith("NetworkTable") or os.path.basename(
-            fn
-        ).startswith("Generic"):
+
+    generated_output_dir = output_directory / "main/java/edu/wpi/first/networktables"
+    for fn in java_template_directory.glob("*.jinja"):
+        template = env.get_template(fn.name)
+        outfn = fn.stem
+        if outfn.startswith("NetworkTable") or outfn.startswith("Generic"):
             output = template.render(types=types)
-            Output(rootPath, outfn, output)
+            Output(generated_output_dir, outfn, output)
         else:
             for replacements in types:
                 output = template.render(replacements)
                 if outfn == "Timestamped.java":
                     outfn2 = f"Timestamped{replacements['TypeName']}.java"
                 else:
                     outfn2 = f"{replacements['TypeName']}{outfn}"
-                Output(rootPath, outfn2, output)
+                Output(generated_output_dir, outfn2, output)
 
     # C++ classes
+    cpp_subdirectory = "main/native/include/networktables"
+    cpp_template_directory = template_root / cpp_subdirectory
     env = Environment(
-        loader=FileSystemLoader(
-            f"{dirname}/src/generate/main/native/include/networktables"
-        ),
+        loader=FileSystemLoader(cpp_template_directory),
         autoescape=False,
     )
-    rootPath = f"{dirname}/src/generated/main/native/include/networktables"
-    for fn in glob.glob(
-        f"{dirname}/src/generate/main/native/include/networktables/*.jinja"
-    ):
-        template = env.get_template(os.path.basename(fn))
-        outfn = os.path.basename(fn)[:-6]  # drop ".jinja"
+
+    generated_output_dir = output_directory / cpp_subdirectory
+    for fn in cpp_template_directory.glob("*.jinja"):
+        template = env.get_template(fn.name)
+        outfn = fn.stem  # drop ".jinja"
         for replacements in types:
             output = template.render(replacements)
             outfn2 = f"{replacements['TypeName']}{outfn}"
-            Output(rootPath, outfn2, output)
+            Output(generated_output_dir, outfn2, output)
 
     # C++ handle API (header)
+    hdr_subdirectory = "main/native/include"
+    hdr_template_directory = template_root / hdr_subdirectory
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/native/include"),
+        loader=FileSystemLoader(hdr_template_directory),
         autoescape=False,
     )
     template = env.get_template("ntcore_cpp_types.h.jinja")
     output = template.render(types=types)
     Output(
-        f"{dirname}/src/generated/main/native/include",
+        output_directory / hdr_subdirectory,
         "ntcore_cpp_types.h",
         output,
     )
 
     # C++ handle API (source)
+    cpp_subdirectory = "main/native/cpp"
+    cpp_template_directory = template_root / cpp_subdirectory
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/native/cpp"),
+        loader=FileSystemLoader(cpp_template_directory),
         autoescape=False,
     )
     template = env.get_template("ntcore_cpp_types.cpp.jinja")
     output = template.render(types=types)
-    Output(f"{dirname}/src/generated/main/native/cpp", "ntcore_cpp_types.cpp", output)
+    Output(
+        output_directory / cpp_subdirectory,
+        "ntcore_cpp_types.cpp",
+        output,
+    )
 
     # C handle API (header)
+    hdr_subdirectory = "main/native/include"
+    hdr_template_directory = template_root / hdr_subdirectory
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/native/include"),
+        loader=FileSystemLoader(hdr_template_directory),
         autoescape=False,
     )
     template = env.get_template("ntcore_c_types.h.jinja")
     output = template.render(types=types)
-    Output(
-        f"{dirname}/src/generated/main/native/include",
-        "ntcore_c_types.h",
-        output,
-    )
+    Output(output_directory / hdr_subdirectory, "ntcore_c_types.h", output)
 
     # C handle API (source)
+    c_subdirectory = "main/native/cpp"
+    c_template_directory = template_root / c_subdirectory
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/native/cpp"),
+        loader=FileSystemLoader(c_template_directory),
         autoescape=False,
     )
     template = env.get_template("ntcore_c_types.cpp.jinja")
     output = template.render(types=types)
-    Output(f"{dirname}/src/generated/main/native/cpp", "ntcore_c_types.cpp", output)
+    Output(output_directory / c_subdirectory, "ntcore_c_types.cpp", output)
 
     # JNI
+    jni_subdirectory = "main/native/cpp/jni"
+    jni_template_directory = template_root / jni_subdirectory
     env = Environment(
-        loader=FileSystemLoader(f"{dirname}/src/generate/main/native/cpp/jni"),
+        loader=FileSystemLoader(jni_template_directory),
         autoescape=False,
     )
     template = env.get_template("types_jni.cpp.jinja")
     output = template.render(types=types)
-    Output(f"{dirname}/src/generated/main/native/cpp/jni", "types_jni.cpp", output)
+    Output(output_directory / jni_subdirectory, "types_jni.cpp", output)
+
+
+def main(argv):
+    script_path = Path(__file__).resolve()
+    dirname = script_path.parent
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output_directory",
+        help="Optional. If set, will output the generated files to this directory, otherwise it will use a path relative to the script",
+        default=dirname / "src/generated",
+        type=Path,
+    )
+    parser.add_argument(
+        "--types_schema_file",
+        help="Optional. If set, this file will be used to load the types schema",
+        default=dirname / "src/generate/types.json",
+        type=Path,
+    )
+    parser.add_argument(
+        "--template_root",
+        help="Optional. If set, will use this directory as the root for the jinja templates",
+        default=dirname / "src/generate",
+        type=Path,
+    )
+    args = parser.parse_args(argv)
+
+    generate_topics(args.output_directory, args.template_root, args.types_schema_file)
 
 
 if __name__ == "__main__":
-    main()
+    main(sys.argv[1:])