Adding base tool build for Linux ARM (#362)

Please ensure you have read the [contribution docs](https://github.com/microsoft/mu/blob/master/CONTRIBUTING.md) prior to submitting the pull request. In particular, [pull request guidelines](https://github.com/microsoft/mu/blob/master/CONTRIBUTING.md#pull-request-best-practices). This change added the build script to cross compile the base tool binaries for Linux ARM system. The needed libuuid system library is pulled from source file and rebuilt to support the corresponding library dependencies. Individual tools' makefiles are also updated to link the cross compiled library as well. This is the first step of #369. For each item, place an "x" in between `[` and `]` if true. Example: `[x]`. _(you can also check items in the GitHub UI)_ - [x] Impacts functionality? - **Functionality** - Does the change ultimately impact how firmware functions? - Examples: Add a new library, publish a new PPI, update an algorithm, ... - [ ] Impacts security? - **Security** - Does the change have a direct security impact on an application, flow, or firmware? - Examples: Crypto algorithm change, buffer overflow fix, parameter validation improvement, ... - [ ] Breaking change? - **Breaking change** - Will anyone consuming this change experience a break in build or boot behavior? - Examples: Add a new library class, move a module to a different repo, call a function in a new library class in a pre-existing module, ... - [ ] Includes tests? - **Tests** - Does the change include any explicit test code? - Examples: Unit tests, integration tests, robot tests, ... - [ ] Includes documentation? - **Documentation** - Does the change contain explicit documentation additions outside direct code modifications (and comments)? - Examples: Update readme file, add feature readme file, link to documentation on an a separate Web page, ... This was tested functional on Linux ARM host system building QEMU virtual platforms (Q35 and SBSA). Update the basetool nuget version, once it is published.
microsoft · May 9, 2023 · 177c41f · 177c41f
1 parent 0e06769
commit 177c41f
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 16 deletions.
diff --git a/.azurepipelines/BaseTools-Build-For-Publication.yml b/.azurepipelines/BaseTools-Build-For-Publication.yml
@@ -30,6 +30,14 @@ variables:
 
 jobs:
   - job: linux_build
+    # Use matrix to speed up the build process
+    strategy:
+      matrix:
+        TARGET_x64:
+          Build.Targets: 'X64'
+        TARGET_AArch64:
+          Build.Targets: 'AARCH64'
+
     displayName: Linux Build
     pool:
       vmImage: ubuntu-20.04
@@ -60,15 +68,15 @@ jobs:
       displayName: Update Environment
       inputs:
         filename: stuart_update
-        arguments: -c .pytool/CISettings.py
+        arguments: -c .pytool/CISettings.py TOOL_CHAIN_TAG=GCC5 -a $(Build.Targets)
 
     - template: Steps/BuildBaseTools.yml@mu_devops
       parameters:
-        extra_parameters: '--skip_path_env'
+        extra_parameters: '--skip_path_env -a $(Build.Targets)'
         tool_chain_tag: 'GCC5'
 
     - publish: $(linux_build_output_path)
-      artifact: $(linux_artifact_name)
+      artifact: $(linux_artifact_name)_$(Build.Targets)
       displayName: Publish To Pipeline
       condition: SucceededOrFailed()
 
@@ -176,7 +184,8 @@ jobs:
 
     - powershell:
         mkdir $(Build.StagingDirectory)/$(temp_publication_directory);
-        mv $(Pipeline.Workspace)/$(linux_artifact_name) $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-x86;
+        mv $(Pipeline.Workspace)/$(linux_artifact_name)_X64 $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-x86;
+        mv $(Pipeline.Workspace)/$(linux_artifact_name)_AARCH64 $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-ARM-64;
         mv $(Pipeline.Workspace)/$(windows_artifact_name)_IA32 $(Build.StagingDirectory)/$(temp_publication_directory)/Windows-x86;
         mv $(Pipeline.Workspace)/$(windows_artifact_name)_ARM $(Build.StagingDirectory)/$(temp_publication_directory)/Windows-ARM;
       displayName: Stage Package Files

diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py
@@ -12,8 +12,9 @@
 import logging
 import argparse
 import multiprocessing
+import shutil
 from edk2toolext import edk2_logging
-from edk2toolext.environment import self_describing_environment
+from edk2toolext.environment import self_describing_environment, shell_environment
 from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
 from edk2toollib.utility_functions import RunCmd
 from edk2toollib.utility_functions import GetHostInfo # MU_CHANGE: Need to check if this is cross compilation
@@ -30,8 +31,9 @@ def ParseCommandLineOptions(self):
         # MU_CHANGE
         ParserObj.add_argument("-s", "--skip_path_env", dest="skip_env", default=False, action='store_true',
                                help="Skip the creation of the path_env descriptor file")
-        ParserObj.add_argument("-a", "--target_arch", dest="arch", default='IA32', choices=['IA32', 'ARM'],
-                               help="Specify the architecture of the built base tools")
+        ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
+                               help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
+                               "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
         args = ParserObj.parse_args()
         self.tool_chain_tag = args.tct
         self.target_arch = args.arch
@@ -47,8 +49,17 @@ def GetWorkspaceRoot(self):
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
 
-        # for now don't use scopes
-        return ('global',)
+        # MU_CHANGE STARTs: Adding scope for cross compilers when building for ARM/AARCH64
+        scopes = ('global',)
+        if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
+            if self.target_arch is None:
+                return scopes
+            if "AARCH64" in self.target_arch:
+                scopes += ("gcc_aarch64_linux",)
+            if "ARM" in self.target_arch:
+                scopes += ("gcc_arm_linux",)
+        return scopes
+        # MU_CHANGE ENDs
 
     def GetLoggingLevel(self, loggerType):
         ''' Get the logging level for a given type (return Logging.Level)
@@ -120,6 +131,10 @@ def Go(self):
 
         if self.tool_chain_tag.lower().startswith("vs"):
             # MU_CHANGE: Specify target architecture
+            if self.target_arch is None:
+                # Put a default as IA32
+                self.target_arch = "IA32"
+
             if self.target_arch == "IA32":
                 VcToolChainArch = "x86"
                 TargetInfoArch = "x86"
@@ -175,6 +190,81 @@ def Go(self):
             return ret
 
         elif self.tool_chain_tag.lower().startswith("gcc"):
+            # MU_CHANGE STARTs: Specify target architecture
+            # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
+            # this script is BUILDING the base tools.
+            prefix = None
+            TargetInfoArch = None
+            if self.target_arch is not None:
+                shell_env.set_shell_var('HOST_ARCH', self.target_arch)
+
+                if "AARCH64" in self.target_arch:
+                    # now check for install dir.  If set then set the Prefix
+                    install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")
+
+                    # make GCC5_AARCH64_PREFIX to align with tools_def.txt
+                    prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
+                    shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
+                    TargetInfoArch = "ARM"
+
+                elif "ARM" in self.target_arch:
+                    # now check for install dir.  If set then set the Prefix
+                    install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
+
+                    # make GCC5_ARM_PREFIX to align with tools_def.txt
+                    prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
+                    shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
+                    TargetInfoArch = "ARM"
+
+                else:
+                    TargetInfoArch = "x86"
+
+            # Otherwise, the built binary arch will be consistent with the host system
+
+            # Added logic to support cross compilation scenarios
+            HostInfo = GetHostInfo()
+            if TargetInfoArch != HostInfo.arch:
+                # this is defaulting to the version that comes with Ubuntu 20.04
+                ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
+                work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
+                pack_name = f"util-linux-{ver}"
+                unzip_dir = os.path.join(work_dir, pack_name)
+
+                if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
+                    os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
+                if os.path.isdir(unzip_dir):
+                    shutil.rmtree(unzip_dir)
+
+                # cross compiling, need to rebuild libuuid for the target
+                ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
+                if ret != 0:
+                    raise Exception(f"Failed to download libuuid version {ver} - {ret}")
+
+                ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
+                if ret != 0:
+                    raise Exception(f"Failed to untar the downloaded file {ret}")
+
+                # configure the source to use the cross compiler
+                pack_name = f"util-linux-{ver}"
+                if "AARCH64" in self.target_arch:
+                    ret = RunCmd("sh", f"./configure --host=aarch64-linux  -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
+                elif "ARM" in self.target_arch:
+                    ret = RunCmd("sh", f"./configure --host=arm-linux  -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
+                if ret != 0:
+                    raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")
+
+                ret = RunCmd("make", "", workingdir=unzip_dir)
+                if ret != 0:
+                    raise Exception(f"Failed to build the libuuid with our gcc {ret}")
+
+                shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
+                shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))
+
+            ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
+            if ret != 0:
+                raise Exception("Failed to build.")
+            # MU_CHANGE ENDs
+
             cpu_count = self.GetCpuThreads()
             ret = RunCmd("make", f"-C .  -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
             if ret != 0:

diff --git a/BaseTools/Source/C/DevicePath/GNUmakefile b/BaseTools/Source/C/DevicePath/GNUmakefile
@@ -29,6 +29,12 @@ ifeq ($(CYGWIN), CYGWIN)
 endif
 
 ifeq ($(LINUX), Linux)
+# MU_CHANGE STARTs: Support cross compiling
+ifndef CROSS_LIB_UUID
   LIBS += -luuid
+else
+  LIBS += -L$(CROSS_LIB_UUID)
+endif
+# MU_CHANGE ENDs
 endif
 
diff --git a/BaseTools/Source/C/GenFv/GNUmakefile b/BaseTools/Source/C/GenFv/GNUmakefile
@@ -14,6 +14,17 @@ include $(MAKEROOT)/Makefiles/app.makefile
 
 LIBS = -lCommon
 ifeq ($(CYGWIN), CYGWIN)
-  LIBS += -L/lib/e2fsprogs
+  LIBS += -L/lib/e2fsprogs -luuid
+endif
+
+ifeq ($(LINUX), Linux)
+# MU_CHANGE STARTs: Support cross compiling
+ifndef CROSS_LIB_UUID
+  LIBS += -luuid
+else
+  LIBS += -L$(CROSS_LIB_UUID)
+  BUILD_CFLAGS += -D__CROSS_LIB_UUID__ -I $(CROSS_LIB_UUID_INC)
+endif
+# MU_CHANGE ENDs
 endif
 
diff --git a/BaseTools/Source/C/GenFv/GenFvInternalLib.c b/BaseTools/Source/C/GenFv/GenFvInternalLib.c
@@ -14,6 +14,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent
 // Include files
 //
 
+#if defined(__FreeBSD__)
+#include <uuid.h>
+#elif defined(__GNUC__)
+// MU_CHANGE STARTs: Support cross compiling
+#if !defined(__CROSS_LIB_UUID__)
+#include <uuid/uuid.h>
+#else
+#include <uuid.h>
+#endif
+// MU_CHANGE ENDs
+#endif
 #ifdef __GNUC__
 #include <sys/stat.h>
 #endif
@@ -2409,7 +2420,6 @@ Routine Description:
   bSecCore |= 0x6F; //JAL opcode
 
   memcpy(FvImage->FileImage, &bSecCore, sizeof(bSecCore));
-
   return EFI_SUCCESS;
 }
 

diff --git a/BaseTools/Source/C/GenFw/GNUmakefile b/BaseTools/Source/C/GenFw/GNUmakefile
@@ -18,6 +18,12 @@ ifeq ($(CYGWIN), CYGWIN)
 endif
 
 ifeq ($(LINUX), Linux)
+# MU_CHANGE STARTs: Support cross compiling
+ifndef CROSS_LIB_UUID
   LIBS += -luuid
+else
+  LIBS += -L$(CROSS_LIB_UUID)
+endif
+# MU_CHANGE ENDs
 endif
 
diff --git a/BaseTools/Source/C/GenSec/GNUmakefile b/BaseTools/Source/C/GenSec/GNUmakefile
@@ -18,6 +18,12 @@ ifeq ($(CYGWIN), CYGWIN)
 endif
 
 ifeq ($(LINUX), Linux)
+# MU_CHANGE STARTs: Support cross compiling
+ifndef CROSS_LIB_UUID
   LIBS += -luuid
+else
+  LIBS += -L$(CROSS_LIB_UUID)
+endif
+# MU_CHANGE ENDs
 endif
 
diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile
@@ -51,11 +51,13 @@ BUILD_AS ?= $(CLANG_BIN)clang
 BUILD_AR ?= $(CLANG_BIN)llvm-ar
 BUILD_LD ?= $(CLANG_BIN)llvm-ld
 else
-BUILD_CC ?= gcc
-BUILD_CXX ?= g++
-BUILD_AS ?= gcc
-BUILD_AR ?= ar
-BUILD_LD ?= ld
+# MU_CHANGE STARTs: Support GCC prefix
+BUILD_CC ?= $(GCC_PREFIX)gcc
+BUILD_CXX ?= $(GCC_PREFIX)g++
+BUILD_AS ?= $(GCC_PREFIX)gcc
+BUILD_AR ?= $(GCC_PREFIX)ar
+BUILD_LD ?= $(GCC_PREFIX)ld
+# MU_CHANGE ENDs
 endif
 LINKER ?= $(BUILD_CC)
 ifeq ($(HOST_ARCH), IA32)