Add CodeQL support

Adds a CodeQL plugin that supports CodeQL in the build system.

1. CodeQlBuildPlugin - Generates a CodeQL database for a given build.
2. CodeQlAnalyzePlugin - Analyzes a CodeQL database and interprets
   results.
3. External dependencies - Assist with downloading the CodeQL CLI and
   making it available to the CodeQL plugins.
4. MuCodeQlQueries.qls - A Project Mu CodeQL query set run against
   Project Mu code.
5. Readme.md - A comprehensive readme file to help:
     - Platform integrators understand how to configure the plugin
     - Developers understand how to modify the plugin
     - Users understand how to use the plugin

Read Readme.md for additional details.

Signed-off-by: Michael Kubacki <[email protected]>

.pytool/Plugin/CodeQL/CodeQlAnalyzePlugin.py

@@ -0,0 +1,167 @@
+# @file CodeQAnalyzePlugin.py
+#
+# A build plugin that analyzes a CodeQL database.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import json
+import logging
+import os
+import yaml
+
+from common import codeql_plugin
+
+from edk2toolext import edk2_logging
+from edk2toolext.environment.plugintypes.uefi_build_plugin import \
+    IUefiBuildPlugin
+from edk2toolext.environment.uefi_build import UefiBuilder
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
+from edk2toollib.utility_functions import RunCmd
+from pathlib import Path
+
+
+class CodeQlAnalyzePlugin(IUefiBuildPlugin):
+
+    def do_post_build(self, builder: UefiBuilder) -> int:
+        """CodeQL analysis post-build functionality.
+
+        Args:
+            builder (UefiBuilder): A UEFI builder object for this build.
+
+        Returns:
+            int: The number of CodeQL errors found. Zero indicates that
+            AuditOnly mode is enabled or no failures were found.
+        """
+
+        pp = builder.pp.split(os.pathsep)
+        edk2_path = Edk2Path(builder.ws, pp)
+
+        self.builder = builder
+        self.package = edk2_path.GetContainingPackage(
+                            builder.mws.join(builder.ws,
+                                             builder.env.GetValue(
+                                                "ACTIVE_PLATFORM")))
+        self.package_path = Path(
+            edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
+                self.package))
+        self.target = builder.env.GetValue("TARGET")
+
+        self.codeql_db_path = codeql_plugin.get_codeql_db_path(
+                                builder.ws, self.package, self.target,
+                                new_path=False)
+
+        self.codeql_path = codeql_plugin.get_codeql_cli_path()
+        if not self.codeql_path:
+            logging.critical("CodeQL build enabled but CodeQL CLI application "
+                             "not found.")
+            return -1
+
+        codeql_sarif_dir_path = self.codeql_db_path[
+                                        :self.codeql_db_path.rindex('-')]
+        codeql_sarif_dir_path = codeql_sarif_dir_path.replace(
+                                        "-db-", "-analysis-")
+        self.codeql_sarif_path = os.path.join(
+                                        codeql_sarif_dir_path,
+                                        (os.path.basename(
+                                            self.codeql_db_path) +
+                                            ".sarif"))
+
+        edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) "
+                                  f"CodeQL database at:\n"
+                                  f"           {self.codeql_db_path}")
+        edk2_logging.log_progress(f"Results will be written to:\n"
+                                  f"           {self.codeql_sarif_path}")
+
+        # Packages are allowed to specify package-specific query specifiers
+        # in the package CI YAML file that override the global query specifier.
+        audit_only = False
+        query_specifiers = None
+        package_config_file = Path(os.path.join(
+                                self.package_path, self.package + ".ci.yaml"))
+        if package_config_file.is_file():
+            with open(package_config_file, 'r') as cf:
+                package_config_file_data = yaml.safe_load(cf)
+                if "CodeQlAnalyze" in package_config_file_data:
+                    plugin_data = package_config_file_data["CodeQlAnalyze"]
+                    if "AuditOnly" in plugin_data:
+                        audit_only = plugin_data["AuditOnly"]
+                    if "QuerySpecifiers" in plugin_data:
+                        logging.debug(f"Loading CodeQL query specifiers in "
+                                      f"{str(package_config_file)}")
+                        query_specifiers = plugin_data["QuerySpecifiers"]
+
+        global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY")
+        if global_audit_only:
+            if global_audit_only.strip().lower() == "true":
+                audit_only = True
+
+        if audit_only:
+            logging.info(f"CodeQL Analyze plugin is in audit only mode for "
+                         f"{self.package} ({self.target}).")
+
+        # Builds can override the query specifiers defined in this plugin
+        # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS
+        # environment variable.
+        if not query_specifiers:
+            builder.env.GetValue("STUART_CODEQL_QUERY_SPECIFIERS")
+
+        # Use this plugins query set file as the default fallback if it is
+        # not overridden. It is possible the file is not present if modified
+        # locally. In that case, skip the plugin.
+        plugin_query_set = Path(Path(__file__).parent, "MuCodeQlQueries.qls")
+
+        if not query_specifiers and plugin_query_set.is_file():
+            query_specifiers = str(plugin_query_set.resolve())
+
+        if not query_specifiers:
+            logging.warning("Skipping CodeQL analysis since no CodeQL query "
+                            "specifiers were provided.")
+            return 0
+
+        codeql_params = (f'database analyze {self.codeql_db_path} '
+                         f'{query_specifiers} --format=sarifv2.1.0 '
+                         f'--output={self.codeql_sarif_path} --download '
+                         f'--threads=0')
+
+        # CodeQL requires the sarif file parent directory to exist already.
+        Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True)
+
+        cmd_ret = RunCmd(self.codeql_path, codeql_params)
+        if cmd_ret != 0:
+            logging.critical(f"CodeQL CLI analysis failed with return code "
+                             f"{cmd_ret}.")
+
+        if not os.path.isfile(self.codeql_sarif_path):
+            logging.critical(f"The sarif file {self.codeql_sarif_path} was "
+                             f"not created. Analysis cannot continue.")
+            return -1
+
+        with open(self.codeql_sarif_path, 'r') as sf:
+            sarif_file_data = json.load(sf)
+
+        try:
+            # Perform minimal JSON parsing to find the number of errors.
+            total_errors = 0
+            for run in sarif_file_data['runs']:
+                total_errors += len(run['results'])
+        except KeyError:
+            logging.critical("Sarif file does not contain expected data. "
+                             "Analysis cannot continue.")
+            return -1
+
+        if total_errors > 0:
+            if audit_only:
+                # Show a warning message so CodeQL analysis is not forgotten.
+                # If the repo owners truly do not want to fix CodeQL issues,
+                # analysis should be disabled entirely.
+                logging.warning(f"{self.package} ({self.target}) CodeQL "
+                                f"analysis ignored {total_errors} errors due "
+                                f"to audit mode being enabled.")
+                return 0
+            else:
+                logging.error(f"{self.package} ({self.target}) CodeQL "
+                              f"analysis failed with {total_errors} errors.")
+
+        return total_errors

.pytool/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml

@@ -0,0 +1,13 @@
+## @file CodeQlAnalyze_plug_in.py
+#
+# Build plugin used to analyze CodeQL results.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+{
+  "scope": "codeql-analyze",
+  "name": "CodeQL Analyze Plugin",
+  "module": "CodeQlAnalyzePlugin"
+}

.pytool/Plugin/CodeQL/CodeQlBuildPlugin.py

@@ -0,0 +1,152 @@
+# @file CodeQlBuildPlugin.py
+#
+# A build plugin that produces CodeQL results for the present build.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import logging
+import os
+import stat
+from common import codeql_plugin
+from pathlib import Path
+
+from edk2toolext import edk2_logging
+from edk2toolext.environment.plugintypes.uefi_build_plugin import \
+    IUefiBuildPlugin
+from edk2toolext.environment.uefi_build import UefiBuilder
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
+from edk2toollib.utility_functions import GetHostInfo
+
+
+class CodeQlBuildPlugin(IUefiBuildPlugin):
+
+    def do_pre_build(self, builder: UefiBuilder) -> int:
+        """CodeQL pre-build functionality.
+
+        Args:
+            builder (UefiBuilder): A UEFI builder object for this build.
+
+        Returns:
+            int: The plugin return code. Zero indicates the plugin ran
+            successfully. A non-zero value indicates an unexpected error
+            occurred during plugin execution.
+        """
+
+        pp = builder.pp.split(os.pathsep)
+        edk2_path = Edk2Path(builder.ws, pp)
+
+        self.builder = builder
+        self.package = edk2_path.GetContainingPackage(
+                            builder.mws.join(builder.ws,
+                                             builder.env.GetValue(
+                                                "ACTIVE_PLATFORM")))
+        self.target = builder.env.GetValue("TARGET")
+
+        self.codeql_db_path = codeql_plugin.get_codeql_db_path(
+                                builder.ws, self.package, self.target)
+
+        edk2_logging.log_progress(f"{self.package} will be built for CodeQL")
+        edk2_logging.log_progress(f"  CodeQL database will be written to "
+                                  f"{self.codeql_db_path}")
+
+        self.codeql_path = codeql_plugin.get_codeql_cli_path()
+        if not self.codeql_path:
+            logging.critical("CodeQL build enabled but CodeQL CLI application "
+                             "not found.")
+            return -1
+
+        # CodeQL can only generate a database on clean build
+        builder.CleanTree()
+
+        # A build is required to generate a database
+        builder.SkipBuild = False
+
+        # CodeQL CLI does not handle spaces passed in CLI commands well
+        # (perhaps at all) as discussed here:
+        #   1. https://github.com/github/codeql-cli-binaries/issues/73
+        #   2. https://github.com/github/codeql/issues/4910
+        #
+        # Since it's unclear how quotes are handled and may change in the
+        # future, this code is going to use the workaround to place the
+        # command in an executable file that is instead passed to CodeQL.
+        self.codeql_cmd_path = Path(builder.mws.join(
+                                    builder.ws, builder.env.GetValue(
+                                        "BUILD_OUTPUT_BASE"),
+                                    "codeql_build_command"))
+
+        build_params = self._get_build_params()
+
+        codeql_build_cmd = ""
+        if GetHostInfo().os == "Windows":
+            self.codeql_cmd_path = self.codeql_cmd_path.parent / (
+                self.codeql_cmd_path.name + '.bat')
+        elif GetHostInfo().os == "Linux":
+            self.codeql_cmd_path.suffix = self.codeql_cmd_path.parent / (
+                self.codeql_cmd_path.name + '.sh')
+            codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}"
+        codeql_build_cmd += "build " + build_params
+
+        self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True)
+        self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd)
+
+        if GetHostInfo().os == "Linux":
+            os.chmod(self.codeql_cmd_path,
+                     os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC)
+
+        codeql_params = (f'database create {self.codeql_db_path} '
+                         f'--language=cpp '
+                         f'--source-root={builder.ws} '
+                         f'--command={self.codeql_cmd_path}')
+
+        # Set environment variables so the CodeQL build command is picked up
+        # as the active build command.
+        #
+        # Note: Requires recent changes in edk2-pytool-extensions (0.20.0)
+        #       to support reading these variables.
+        builder.env.SetValue(
+            "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin")
+        builder.env.SetValue(
+            "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin")
+
+        return 0
+
+    def _get_build_params(self) -> str:
+        """Returns the build command parameters for this build.
+
+        Based on the well-defined `build` command-line parameters.
+
+        Returns:
+            str: A string representing the parameters for the build command.
+        """
+        build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}"
+        build_params += f" -b {self.target}"
+        build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}"
+
+        max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER')
+        if max_threads is not None:
+            build_params += f" -n {max_threads}"
+
+        rt = self.builder.env.GetValue("TARGET_ARCH").split(" ")
+        for t in rt:
+            build_params += " -a " + t
+
+        if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"):
+            build_params += (" -y " +
+                             self.builder.env.GetValue("BUILDREPORT_FILE"))
+            rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ")
+            for t in rt:
+                build_params += " -Y " + t
+
+        # add special processing to handle building a single module
+        mod = self.builder.env.GetValue("BUILDMODULE")
+        if (mod is not None and len(mod.strip()) > 0):
+            build_params += " -m " + mod
+            edk2_logging.log_progress("Single Module Build: " + mod)
+
+        build_vars = self.builder.env.GetAllBuildKeyValues(self.target)
+        for key, value in build_vars.items():
+            build_params += " -D " + key + "=" + value
+
+        return build_params

.pytool/Plugin/CodeQL/CodeQlBuild_plug_in.yaml

@@ -0,0 +1,13 @@
+## @file CodeQlBuild_plug_in.py
+#
+# Build plugin used to produce a CodeQL database from a build.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+{
+  "scope": "codeql-build",
+  "name": "CodeQL Build Plugin",
+  "module": "CodeQlBuildPlugin"
+}