diff --git a/.azurepipelines/Platform-Build-Job.yml b/.azurepipelines/Platform-Build-Job.yml index ba3b54cae..8fcccf1ca 100644 --- a/.azurepipelines/Platform-Build-Job.yml +++ b/.azurepipelines/Platform-Build-Job.yml @@ -91,7 +91,7 @@ jobs: - template: Steps/BuildPlatform.yml@mu_devops parameters: - tool_chain_tag: $(tool_chain_tag) + tool_chain_tag: ${{ parameters.tool_chain_tag }} build_pkg: ${{ item.Value.BuildPackage }} build_target: ${{ item.Value.BuildTarget }} build_file: ${{ item.Value.BuildFile }} @@ -100,7 +100,7 @@ jobs: run_timeout: ${{ parameters.run_timeout }} extra_install_step: ${{ parameters.extra_install_step }} install_tools: ${{ and(ne(item.Value.SelfHostAgent, true), eq(parameters.container_image, '')) }} - artifacts_identifier: '${{ item.Value.BuildPackage }} $(tool_chain_tag) ${{ item.Value.BuildTarget }} ${{ item.Value.BuildExtraTag }}' + artifacts_identifier: '${{ item.Value.BuildPackage }} ${{ parameters.tool_chain_tag }} ${{ item.Value.BuildTarget }} ${{ item.Value.BuildExtraTag }}' artifacts_binary: ${{ item.Value.BuildArtifactsBinary }} ${{ if or(eq(item.Value.Run, true), eq(parameters.os_type, 'Linux'), EndsWith(item.Key, 'CODE_COVERAGE')) }}: artifacts_other: ${{ item.Value.BuildArtifactsOther }} diff --git a/.azurepipelines/Platform-Build-VS.yml b/.azurepipelines/Platform-Build-VS.yml index 10f3b69f0..a914f2a36 100644 --- a/.azurepipelines/Platform-Build-VS.yml +++ b/.azurepipelines/Platform-Build-VS.yml @@ -122,3 +122,70 @@ jobs: displayName: Install Report Generator BuildArtifactsOther: "**/Coverage/**" Run: false + +- template: Platform-Build-Job.yml + parameters: + extra_install_step: + - template: Steps/InstallCoverageTools.yml@mu_devops + tool_chain_tag: 'CLANGPDB' + vm_image: $(vm_image) + os_type: Windows_NT + build_matrix: + QemuQ35_DEBUG_CLANGPDB: + BuildPackage: QemuQ35Pkg + BuildFile: "Platforms/QemuQ35Pkg/PlatformBuild.py" + BuildFlags: "" + BuildTarget: "DEBUG" + BuildExtraTag: "" + BuildExtraStep: + - template: Steps/RustSetupSteps.yml@mu_devops + Run: true + RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE BLD_*_QEMU_CORE_NUM=2 TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE" + BuildArtifactsBinary: | + **/QEMUQ35_*.fd + BuildArtifactsOther: "**/unit_test_results/*" + + QemuQ35_RELEASE_CLANGPDB: + BuildPackage: QemuQ35Pkg + BuildFile: "Platforms/QemuQ35Pkg/PlatformBuild.py" + BuildFlags: "" + BuildTarget: "RELEASE" + BuildExtraTag: "" + BuildExtraStep: + - template: Steps/RustSetupSteps.yml@mu_devops + Run: true + RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE BLD_*_QEMU_CORE_NUM=2 TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE" + BuildArtifactsBinary: | + **/QEMUQ35_*.fd + BuildArtifactsOther: "**/unit_test_results/*" + + QemuSbsa_DEBUG_CLANGPDB: + BuildPackage: QemuSbsaPkg + BuildFile: "Platforms/QemuSbsaPkg/PlatformBuild.py" + BuildTarget: "DEBUG" + BuildExtraTag: "" + BuildExtraStep: + - template: Steps/RustSetupSteps.yml@mu_devops + - template: Steps/SetupToolChainTagPreReqs.yml@mu_devops + Run: true + RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE" + BuildArtifactsBinary: | + **/QEMU_EFI.fd + **/SECURE_FLASH0.fd + BuildArtifactsOther: "**/unit_test_results/*" + + QemuSbsa_RELEASE_CLANGPDB: + BuildPackage: QemuSbsaPkg + BuildFile: "Platforms/QemuSbsaPkg/PlatformBuild.py" + BuildFlags: "" + BuildTarget: "RELEASE" + BuildExtraTag: "" + BuildExtraStep: + - template: Steps/RustSetupSteps.yml@mu_devops + - template: Steps/SetupToolChainTagPreReqs.yml@mu_devops + Run: true + RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE" + BuildArtifactsBinary: | + **/QEMU_EFI.fd + **/SECURE_FLASH0.fd + BuildArtifactsOther: "**/unit_test_results/*" diff --git a/.gitmodules b/.gitmodules index a9f9699ca..6f8e6827c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,6 +31,6 @@ [submodule "Features/MM_SUPV"] path = Features/MM_SUPV url = https://github.com/microsoft/mu_feature_mm_supv.git -[submodule "Features/DEBUGGER"] - path = Features/DEBUGGER - url = https://github.com/microsoft/mu_feature_debugger.git +[submodule "Features/DEBUGGER"] + path = Features/DEBUGGER + url = https://github.com/microsoft/mu_feature_debugger.git diff --git a/Common/MU b/Common/MU index 0a1ce577e..2d6746b8f 160000 --- a/Common/MU +++ b/Common/MU @@ -1 +1 @@ -Subproject commit 0a1ce577e8c8f151f33d239ff52a80d4bc652e44 +Subproject commit 2d6746b8fe04a9be3998b3e1a682a4cbffcd7345 diff --git a/Features/MM_SUPV b/Features/MM_SUPV index e67ea5450..83b2f31e3 160000 --- a/Features/MM_SUPV +++ b/Features/MM_SUPV @@ -1 +1 @@ -Subproject commit e67ea54502452430e425062546dffaf5cbf878e4 +Subproject commit 83b2f31e356ece531398737ef5d51c4134d97dcf diff --git a/MU_BASECORE b/MU_BASECORE index ad553d085..4298c31df 160000 --- a/MU_BASECORE +++ b/MU_BASECORE @@ -1 +1 @@ -Subproject commit ad553d0853e98883e30442740626d820497d6918 +Subproject commit 4298c31df7a47fc9731885292daefe8c170dd0c7 diff --git a/Platforms/QemuQ35Pkg/QemuQ35Pkg.dsc b/Platforms/QemuQ35Pkg/QemuQ35Pkg.dsc index 34627432a..b90aeabd1 100644 --- a/Platforms/QemuQ35Pkg/QemuQ35Pkg.dsc +++ b/Platforms/QemuQ35Pkg/QemuQ35Pkg.dsc @@ -1533,4 +1533,5 @@ QemuQ35Pkg/Library/ResetSystemLib/StandaloneMmResetSystemLib.inf # protection of DXE_SMM_DRIVER/SMM_CORE modules [BuildOptions.common.EDKII.DXE_SMM_DRIVER, BuildOptions.common.EDKII.DXE_RUNTIME_DRIVER, BuildOptions.common.EDKII.SMM_CORE, BuildOptions.common.EDKII.DXE_DRIVER, BuildOptions.common.EDKII.DXE_CORE, BuildOptions.common.EDKII.UEFI_DRIVER, BuildOptions.common.EDKII.UEFI_APPLICATION, BuildOptions.common.EDKII.MM_CORE_STANDALONE, BuildOptions.common.EDKII.MM_STANDALONE] MSFT:*_*_*_DLINK_FLAGS = /ALIGN:4096 - GCC:*_*_*_DLINK_FLAGS = -z common-page-size=0x1000 + GCC:*_GCC5_*_DLINK_FLAGS = -z common-page-size=0x1000 + GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:4096 diff --git a/Platforms/QemuSbsaPkg/Library/SbsaConfigDataLib/SbsaConfigDataLib.c b/Platforms/QemuSbsaPkg/Library/SbsaConfigDataLib/SbsaConfigDataLib.c index 4fe64d248..65576f53a 100644 --- a/Platforms/QemuSbsaPkg/Library/SbsaConfigDataLib/SbsaConfigDataLib.c +++ b/Platforms/QemuSbsaPkg/Library/SbsaConfigDataLib/SbsaConfigDataLib.c @@ -14,5 +14,6 @@ #include #include -PROFILE gProfileData = { 0 }; -UINTN gNumProfiles = 0; +PROFILE gProfileData = { 0 }; +UINTN gNumProfiles = 0; +CHAR8 *gProfileFlavorNames[] = { NULL }; diff --git a/Platforms/QemuSbsaPkg/PlatformBuild.py b/Platforms/QemuSbsaPkg/PlatformBuild.py index 7623ec2fb..18c9ce469 100644 --- a/Platforms/QemuSbsaPkg/PlatformBuild.py +++ b/Platforms/QemuSbsaPkg/PlatformBuild.py @@ -20,6 +20,8 @@ from edk2toolext.invocables.edk2_update import UpdateSettingsManager from edk2toolext.invocables.edk2_parse import ParseSettingsManager from edk2toollib.utility_functions import RunCmd +from edk2toollib.windows.locate_tools import QueryVcVariables +from edk2toollib.utility_functions import GetHostInfo # Declare test whose failure will not return a non-zero exit code FAILURE_EXEMPT_TESTS = { @@ -40,7 +42,7 @@ class CommonPlatform(): PackagesSupported = ("QemuSbsaPkg",) ArchSupported = ("AARCH64",) TargetsSupported = ("DEBUG", "RELEASE", "NOOPT") - Scopes = ('qemu', 'qemusbsa', 'gcc_aarch64_linux', 'edk2-build', 'cibuild', 'configdata', 'rust-ci') + Scopes = ('qemu', 'qemusbsa', 'edk2-build', 'cibuild', 'configdata', 'rust-ci') WorkspaceRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) PackagesPath = ( "Platforms", @@ -113,7 +115,13 @@ def GetWorkspaceRoot(self): def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' - return CommonPlatform.Scopes + scopes = CommonPlatform.Scopes + actual_tool_chain_tag = shell_environment.GetBuildVars().GetValue( + "TOOL_CHAIN_TAG", "" + ) + if actual_tool_chain_tag.upper().startswith("GCC"): + scopes += ("gcc_aarch64_linux",) + return scopes def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list: ''' Filter other cases that this package should be built @@ -157,13 +165,54 @@ class PlatformBuilder(UefiBuilder, BuildSettingsManager): def __init__(self): UefiBuilder.__init__(self) + # Helper function to query the VC variables of interest and inject them into the environment + def InjectVcVarsOfInterests(self, vcvars: list): + HostInfo = GetHostInfo() + + # check to see if host is configured + # HostType for VS tools should be (defined in tools_def): + # x86 == 32bit Intel + # x64 == 64bit Intel + # arm == 32bit Arm + # arm64 == 64bit Arm + # + HostType = shell_environment.GetEnvironment().get_shell_var("CLANG_VS_HOST") + if HostType is not None: + HostType = HostType.lower() + logging.info( + f"CLANG_VS_HOST defined by environment. Value is {HostType}") + else: + #figure it out based on host info + if HostInfo.arch == "x86": + if HostInfo.bit == "32": + HostType = "x86" + elif HostInfo.bit == "64": + HostType = "x64" + else: + # anything other than x86 or x64 is not supported + raise NotImplementedError() + + # CLANG_VS_HOST options are not exactly the same as QueryVcVariables. This translates. + VC_HOST_ARCH_TRANSLATOR = { + "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"} + + # now get the environment variables for the platform + shell_env = shell_environment.GetEnvironment() + # Use the tools lib to determine the correct values for the vars that interest us. + vs_vars = QueryVcVariables( + vcvars, VC_HOST_ARCH_TRANSLATOR[HostType]) + for (k, v) in vs_vars.items(): + shell_env.set_shell_var(k, v) + def CleanTree(self, RemoveConfTemplateFilesToo=False): + # If this is a Windows Clang build, we need to inject the VC variables of interest + if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB" and os.name == 'nt': + self.InjectVcVarsOfInterests(["VCToolsInstallDir", "Path", "LIB"]) + # Add a step to clean up BL31 as well, if asked cmd = "make" args = "distclean" - ret = RunCmd(cmd, args, workingdir= self.env.GetValue("ARM_TFA_PATH")) - if ret != 0: - return ret + RunCmd(cmd, args, workingdir=self.env.GetValue("ARM_TFA_PATH")) return super().CleanTree(RemoveConfTemplateFilesToo) @@ -182,7 +231,13 @@ def GetPackagesPath(self): def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' - return CommonPlatform.Scopes + scopes = CommonPlatform.Scopes + actual_tool_chain_tag = shell_environment.GetBuildVars().GetValue( + "TOOL_CHAIN_TAG", "" + ) + if actual_tool_chain_tag.upper().startswith("GCC"): + scopes += ("gcc_aarch64_linux",) + return scopes def GetName(self): ''' Get the name of the repo, platform, or product being build ''' @@ -278,8 +333,39 @@ def PlatformPostBuild(self): op_fv = os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"), "FV") logging.info("Building TF-A") + + shell_environment.CheckpointBuildVars() # checkpoint our config before we mess with it + if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB": + if os.name == 'nt': + # If this is a Windows build, we need to demolish the path and inject the VC variables of interest + # otherwise the build could pick up wrong tools + shell_environment.GetEnvironment().set_path('') + self.InjectVcVarsOfInterests(["LIB", "Path"]) + + clang_exe = "clang.exe" + choco_path = shell_environment.GetEnvironment().get_shell_var("CHOCOLATEYINSTALL") + shell_environment.GetEnvironment().insert_path(os.path.join(choco_path, "bin")) + shell_environment.GetEnvironment().insert_path(shell_environment.GetEnvironment().get_shell_var("CLANG_BIN")) + + # Need to build fiptool separately because the build system will override LIB with LIBC for firmware builds + cmd = "make" + args = " fiptool MAKEFLAGS= LIB=\"" + shell_environment.GetEnvironment().get_shell_var("LIB") + "\"" + ret = RunCmd(cmd, args, workingdir=self.env.GetValue("ARM_TFA_PATH")) + if ret != 0: + return ret + # Then we can make the firmware images with the fiptool built above + else: + clang_exe = "clang" + cmd = "make" - args = "CROSS_COMPILE=" + shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") + if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB": + args = "CC="+clang_exe + elif self.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": + args = "CROSS_COMPILE=" + shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX") + args += " -j $(nproc)" + else: + logging.error("Unsupported toolchain") + return -1 args += " PLAT=" + self.env.GetValue("QEMU_PLATFORM").lower() args += " ARCH=" + self.env.GetValue("TARGET_ARCH").lower() args += " DEBUG=" + str(1 if self.env.GetValue("TARGET").lower() == 'debug' else 0) @@ -288,11 +374,13 @@ def PlatformPostBuild(self): # args += " FEATURE_DETECTION=1" # Enforces support for features enabled. args += " BL32=" + os.path.join(op_fv, "BL32_AP_MM.fd") args += " all fip" - args += " -j $(nproc)" ret = RunCmd(cmd, args, workingdir= self.env.GetValue("ARM_TFA_PATH")) if ret != 0: return ret + # Revert the build vars to the original state + shell_environment.RevertBuildVars() + # Now that BL31 is built with BL32 supplied, patch BL1 and BL31 built fip.bin into the SECURE_FLASH0.fd op_tfa = os.path.join ( self.env.GetValue("ARM_TFA_PATH"), "build", diff --git a/Platforms/QemuSbsaPkg/QemuSbsaPkg.dsc b/Platforms/QemuSbsaPkg/QemuSbsaPkg.dsc index 8b15f6dd4..d53b9cba5 100644 --- a/Platforms/QemuSbsaPkg/QemuSbsaPkg.dsc +++ b/Platforms/QemuSbsaPkg/QemuSbsaPkg.dsc @@ -464,9 +464,6 @@ [LibraryClasses.common.DXE_RUNTIME_DRIVER] MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf CapsuleLib|MdeModulePkg/Library/DxeCapsuleLibNull/DxeCapsuleLibNull.inf -!if $(TARGET) != RELEASE - DebugLib|MdePkg/Library/DxeRuntimeDebugLibSerialPort/DxeRuntimeDebugLibSerialPort.inf -!endif VariablePolicyLib|MdeModulePkg/Library/VariablePolicyLib/VariablePolicyLibRuntimeDxe.inf ResetSystemLib|MdeModulePkg/Library/RuntimeResetSystemLib/RuntimeResetSystemLib.inf @@ -626,9 +623,19 @@ # gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiACPIReclaimMemory|0x40 gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiACPIMemoryNVS|0x0 +!if $(TOOL_CHAIN_TAG) == GCC5 # This is really odd on why CLANGPDB has runtime memory consumption differences gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiReservedMemoryType|0x505 gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesData|0x258 gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesCode|0x260 +!else +!if $(TARGET) == RELEASE + gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiReservedMemoryType|0x505 +!else + gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiReservedMemoryType|0x30 +!endif + gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesData|0x40 + gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesCode|0x300 +!endif gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiBootServicesCode|0x5DC gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiBootServicesData|0x2EE0 gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiLoaderCode|0x14 @@ -1365,7 +1372,7 @@ MdeModulePkg/Universal/FaultTolerantWriteDxe/FaultTolerantWriteStandaloneMm.inf MdeModulePkg/Universal/Variable/RuntimeDxe/VariableStandaloneMm.inf { - DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLib.inf + DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLibStandaloneMm.inf NULL|MdeModulePkg/Library/VarCheckUefiLib/VarCheckUefiLib.inf NULL|MdeModulePkg/Library/VarCheckPolicyLib/VarCheckPolicyLibStandaloneMm.inf BaseMemoryLib|MdePkg/Library/BaseMemoryLib/BaseMemoryLib.inf @@ -1388,18 +1395,26 @@ GCC:RELEASE_*_*_CC_FLAGS = -DMDEPKG_NDEBUG + # Exception tables are required for stack walks in the debugger. + MSFT:*_*_AARCH64_GENFW_FLAGS = --keepexceptiontable + GCC:*_*_AARCH64_GENFW_FLAGS = --keepexceptiontable + # # Disable deprecated APIs. # RVCT:*_*_*_CC_FLAGS = -DDISABLE_NEW_DEPRECATED_INTERFACES GCC:*_*_*_CC_FLAGS = -DDISABLE_NEW_DEPRECATED_INTERFACES +[BuildOptions.common.EDKII.SEC,BuildOptions.common.EDKII.MM_CORE_STANDALONE] + GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:0x1000 /FILEALIGN:0x1000 + [BuildOptions.common.EDKII.DXE_CORE,BuildOptions.common.EDKII.DXE_DRIVER,BuildOptions.common.EDKII.UEFI_DRIVER,BuildOptions.common.EDKII.UEFI_APPLICATION,BuildOptions.common.EDKII.MM_CORE_STANDALONE,BuildOptions.common.EDKII.MM_STANDALONE] - GCC:*_*_*_DLINK_FLAGS = -z common-page-size=0x1000 + GCC:*_GCC5_*_DLINK_FLAGS = -z common-page-size=0x1000 + GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:0x1000 [BuildOptions.common.EDKII.DXE_RUNTIME_DRIVER] - GCC:*_*_ARM_DLINK_FLAGS = -z common-page-size=0x1000 - GCC:*_*_AARCH64_DLINK_FLAGS = -z common-page-size=0x10000 + GCC:*_GCC5_AARCH64_DLINK_FLAGS = -z common-page-size=0x10000 + GCC:*_CLANGPDB_AARCH64_DLINK_FLAGS = /ALIGN:0x10000 RVCT:*_*_ARM_DLINK_FLAGS = --scatter $(EDK_TOOLS_PATH)/Scripts/Rvct-Align4K.sct [BuildOptions.AARCH64.EDKII.MM_CORE_STANDALONE,BuildOptions.AARCH64.EDKII.MM_STANDALONE]