From 43a77d2bd180ef91184d327bb56cb4c53bbbf2c5 Mon Sep 17 00:00:00 2001 From: Ashraf Ali S Date: Wed, 31 Jan 2024 23:08:58 +0530 Subject: [PATCH 1/3] Optimize CollectPlatformGuids GenerateByteArrayValue API to reduce Incremental build time Signed-off-by: Ashraf Ali S --- edk2basetools/AutoGen/WorkspaceAutoGen.py | 16 ++--- edk2basetools/Workspace/DscBuildData.py | 83 +++++++++++++++++++---- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/edk2basetools/AutoGen/WorkspaceAutoGen.py b/edk2basetools/AutoGen/WorkspaceAutoGen.py index ba51a461..e9defebd 100644 --- a/edk2basetools/AutoGen/WorkspaceAutoGen.py +++ b/edk2basetools/AutoGen/WorkspaceAutoGen.py @@ -160,22 +160,18 @@ def ValidateBuildTarget(self): def CollectPlatformGuids(self): oriInfList = [] - oriPkgSet = set() - PlatformPkg = set() + pkgSet = set() for Arch in self.ArchList: Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain] oriInfList = Platform.Modules for ModuleFile in oriInfList: ModuleData = self.BuildDatabase[ModuleFile, Platform._Arch, Platform._Target, Platform._Toolchain] - oriPkgSet.update(ModuleData.Packages) - for Pkg in oriPkgSet: - Guids = Pkg.Guids - GlobalData.gGuidDict.update(Guids) + pkgSet.update(ModuleData.Packages) if Platform.Packages: - PlatformPkg.update(Platform.Packages) - for Pkg in PlatformPkg: - Guids = Pkg.Guids - GlobalData.gGuidDict.update(Guids) + pkgSet.update(Platform.Packages) + for Pkg in pkgSet: + Guids = Pkg.Guids + GlobalData.gGuidDict.update(Guids) @cached_property def FdfProfile(self): diff --git a/edk2basetools/Workspace/DscBuildData.py b/edk2basetools/Workspace/DscBuildData.py index 38a52833..b19cc051 100644 --- a/edk2basetools/Workspace/DscBuildData.py +++ b/edk2basetools/Workspace/DscBuildData.py @@ -37,6 +37,8 @@ from edk2basetools.Common.Misc import SaveFileOnChange from edk2basetools.Workspace.BuildClassObject import PlatformBuildClassObject, StructurePcd, PcdClassObject, ModuleBuildClassObject from collections import OrderedDict, defaultdict +import json +import shutil def _IsFieldValueAnArray (Value): Value = Value.strip() @@ -56,6 +58,7 @@ def _IsFieldValueAnArray (Value): PcdValueInitName = 'PcdValueInit' PcdValueCommonName = 'PcdValueCommon' +StructuredPcdsDataName = 'StructuredPcdsData.json' PcdMainCHeader = ''' /** @@ -2750,6 +2753,23 @@ def ParseCCFlags(self, ccflag): ccflags.add(item) i +=1 return ccflags + + def GetStructurePcdSet (self, OutputValueFile): + if not os.path.isfile(OutputValueFile): + EdkLogger.error("GetStructurePcdSet", FILE_NOT_FOUND, ExtraData=OutputValueFile) + return [] + File = open (OutputValueFile, 'r') + FileBuffer = File.readlines() + File.close() + + #start update structure pcd final value + StructurePcdSet = [] + for Pcd in FileBuffer: + PcdValue = Pcd.split ('|') + PcdInfo = PcdValue[0].split ('.') + StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip())) + return StructurePcdSet + def GenerateByteArrayValue (self, StructuredPcds): # # Generate/Compile/Run C application to determine if there are any flexible array members @@ -2757,6 +2777,54 @@ def GenerateByteArrayValue (self, StructuredPcds): if not StructuredPcds: return + StructuredPcdsData = {} + + for PcdName in StructuredPcds: + Pcd = StructuredPcds[PcdName] + TokenSpaceGuidCName = Pcd.TokenSpaceGuidCName + TokenCName = Pcd.TokenCName + + # Create a key using TokenSpaceGuidCName and TokenCName + StructuredPcdsData[f"{TokenSpaceGuidCName}_{TokenCName}"] = { + "DefaultValueFromDec": Pcd.DefaultValueFromDec, + "DefaultValues": Pcd.DefaultValues, + "PcdFieldValueFromComm": Pcd.PcdFieldValueFromComm, + "PcdFieldValueFromFdf": Pcd.PcdFieldValueFromFdf, + "DefaultFromDSC": Pcd.DefaultFromDSC + } + + # + # If the output path doesn't exists then create it + # + if not os.path.exists(self.OutputPath): + os.makedirs(self.OutputPath) + + StructuredPcdsDataPath = os.path.join(self.OutputPath, self._Arch, StructuredPcdsDataName) + PcdRecordOutputValueFile = os.path.join(self.OutputPath, self._Arch, 'Output.txt') + + if not os.path.exists(os.path.dirname(StructuredPcdsDataPath)): + os.makedirs(os.path.dirname(StructuredPcdsDataPath)) + # + # Check if the StructuredPcdsData.json exists or not + # if exits then it might be a incremental build then check if the StructuredPcdsData has been changed or not. + # if changed then proceed further, if not changed then return the stored data from earlier build + # + if os.path.isfile(StructuredPcdsDataPath): + with open(StructuredPcdsDataPath, 'r') as file: + StoredStructuredPcdsData = json.load(file) + if StructuredPcdsData == StoredStructuredPcdsData: + return self.GetStructurePcdSet(PcdRecordOutputValueFile) + else: + # update the record as PCD Input has been changed in incremental build + with open(StructuredPcdsDataPath, 'w') as file: + json.dump(StructuredPcdsData, file, indent=2) + else: + # + # 1st build, create the StructuredPcdsData.json + # + with open(StructuredPcdsDataPath, 'w') as file: + json.dump(StructuredPcdsData, file, indent=2) + InitByteValue = "" CApp = PcdMainCHeader @@ -2832,8 +2900,6 @@ def GenerateByteArrayValue (self, StructuredPcds): CApp = CApp + PcdMainCEntry + '\n' - if not os.path.exists(self.OutputPath): - os.makedirs(self.OutputPath) CAppBaseFileName = os.path.join(self.OutputPath, PcdValueInitName) SaveFileOnChange(CAppBaseFileName + '.c', CApp, False) @@ -3042,17 +3108,10 @@ def GenerateByteArrayValue (self, StructuredPcds): if returncode != 0: EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s\n%s\n%s\n' % (Command, StdOut, StdErr)) + # Copy update output file for each Arch + shutil.copyfile(OutputValueFile, PcdRecordOutputValueFile) #start update structure pcd final value - File = open (OutputValueFile, 'r') - FileBuffer = File.readlines() - File.close() - - StructurePcdSet = [] - for Pcd in FileBuffer: - PcdValue = Pcd.split ('|') - PcdInfo = PcdValue[0].split ('.') - StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip())) - return StructurePcdSet + return self.GetStructurePcdSet(OutputValueFile) @staticmethod def NeedUpdateOutput(OutputFile, ValueCFile, StructureInput): From 7f57cbd05ada6ad690846ba16bddd6e7d3cf270f Mon Sep 17 00:00:00 2001 From: Ashraf Ali S Date: Fri, 2 Feb 2024 20:49:29 +0530 Subject: [PATCH 2/3] add PcdFiledValueFromDscComponent Signed-off-by: Ashraf Ali S --- edk2basetools/Workspace/DscBuildData.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/edk2basetools/Workspace/DscBuildData.py b/edk2basetools/Workspace/DscBuildData.py index b19cc051..036298ba 100644 --- a/edk2basetools/Workspace/DscBuildData.py +++ b/edk2basetools/Workspace/DscBuildData.py @@ -2790,7 +2790,8 @@ def GenerateByteArrayValue (self, StructuredPcds): "DefaultValues": Pcd.DefaultValues, "PcdFieldValueFromComm": Pcd.PcdFieldValueFromComm, "PcdFieldValueFromFdf": Pcd.PcdFieldValueFromFdf, - "DefaultFromDSC": Pcd.DefaultFromDSC + "DefaultFromDSC": Pcd.DefaultFromDSC, + "PcdFiledValueFromDscComponent": Pcd.PcdFiledValueFromDscComponent } # From b10e5fda2b702f163a4fd54634cf49d6d7fca3b4 Mon Sep 17 00:00:00 2001 From: Ashraf Ali S Date: Wed, 7 Feb 2024 22:53:25 +0530 Subject: [PATCH 3/3] Updated for comments for the Patch V3 Below are the comments from V3 patch. and this PR will address those. Besides the PCD default value, I think there are still 2 more things need to check. These 2 things could impact the output of the PcdValueInit.exe. 1. Macros definition in BuildOption There would be a case that the Macros are used in Structure Pcd's definition in header file. User can change the structure pcd's struct def via changing the Macro value from command line or dsc file. 2. The header file list defined in DEC The [Includes] section in dec list folder that contains the structure pcd definition header file. There may be case that user changes the folder under [Includes] to change the header file to change the structure pcd struct definition. Signed-off-by: Ashraf Ali S --- edk2basetools/Workspace/DscBuildData.py | 121 ++++++++++++++---------- 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/edk2basetools/Workspace/DscBuildData.py b/edk2basetools/Workspace/DscBuildData.py index 036298ba..3a4fe99b 100644 --- a/edk2basetools/Workspace/DscBuildData.py +++ b/edk2basetools/Workspace/DscBuildData.py @@ -2756,7 +2756,7 @@ def ParseCCFlags(self, ccflag): def GetStructurePcdSet (self, OutputValueFile): if not os.path.isfile(OutputValueFile): - EdkLogger.error("GetStructurePcdSet", FILE_NOT_FOUND, ExtraData=OutputValueFile) + EdkLogger.error("GetStructurePcdSet", FILE_NOT_FOUND, "Output.txt doesn't exist", ExtraData=OutputValueFile) return [] File = open (OutputValueFile, 'r') FileBuffer = File.readlines() @@ -2770,6 +2770,46 @@ def GetStructurePcdSet (self, OutputValueFile): StructurePcdSet.append((PcdInfo[0], PcdInfo[1], PcdInfo[2], PcdInfo[3], PcdValue[2].strip())) return StructurePcdSet + def GetBuildOptionsValueList(self): + CC_FLAGS = LinuxCFLAGS + if sys.platform == "win32": + CC_FLAGS = WindowsCFLAGS + BuildOptions = OrderedDict() + for Options in self.BuildOptions: + if Options[2] != EDKII_NAME: + continue + Family = Options[0] + if Family and Family != self.ToolChainFamily: + continue + Target, Tag, Arch, Tool, Attr = Options[1].split("_") + if Tool != 'CC': + continue + if Attr != "FLAGS": + continue + if Target == TAB_STAR or Target == self._Target: + if Tag == TAB_STAR or Tag == self._Toolchain: + if 'COMMON' not in BuildOptions: + BuildOptions['COMMON'] = set() + if Arch == TAB_STAR: + BuildOptions['COMMON']|= self.ParseCCFlags(self.BuildOptions[Options]) + if Arch in self.SupArchList: + if Arch not in BuildOptions: + BuildOptions[Arch] = set() + BuildOptions[Arch] |= self.ParseCCFlags(self.BuildOptions[Options]) + + if BuildOptions: + ArchBuildOptions = {arch:flags for arch,flags in BuildOptions.items() if arch != 'COMMON'} + if len(ArchBuildOptions.keys()) == 1: + BuildOptions['COMMON'] |= (list(ArchBuildOptions.values())[0]) + elif len(ArchBuildOptions.keys()) > 1: + CommonBuildOptions = reduce(lambda x,y: x&y, ArchBuildOptions.values()) + BuildOptions['COMMON'] |= CommonBuildOptions + ValueList = [item for item in BuildOptions['COMMON'] if item.startswith((r"/U","-U"))] + ValueList.extend([item for item in BuildOptions['COMMON'] if item.startswith((r"/D", "-D"))]) + CC_FLAGS += " ".join(ValueList) + return CC_FLAGS + + def GenerateByteArrayValue (self, StructuredPcds): # # Generate/Compile/Run C application to determine if there are any flexible array members @@ -2778,6 +2818,10 @@ def GenerateByteArrayValue (self, StructuredPcds): return StructuredPcdsData = {} + StoredStructuredPcdObjectPaths = {} + SkipPcdValueInit = False + + CC_FLAGS = self.GetBuildOptionsValueList() for PcdName in StructuredPcds: Pcd = StructuredPcds[PcdName] @@ -2794,6 +2838,8 @@ def GenerateByteArrayValue (self, StructuredPcds): "PcdFiledValueFromDscComponent": Pcd.PcdFiledValueFromDscComponent } + # Store the CC Flags + StructuredPcdsData["CC_FLAGS"] = CC_FLAGS # # If the output path doesn't exists then create it # @@ -2813,18 +2859,23 @@ def GenerateByteArrayValue (self, StructuredPcds): if os.path.isfile(StructuredPcdsDataPath): with open(StructuredPcdsDataPath, 'r') as file: StoredStructuredPcdsData = json.load(file) + # OBJECTS will have the modified time, which needs to be checked later + StoredStructuredPcdObjectPaths = StoredStructuredPcdsData.pop("OBJECTS", {}) + if StructuredPcdsData == StoredStructuredPcdsData: - return self.GetStructurePcdSet(PcdRecordOutputValueFile) - else: - # update the record as PCD Input has been changed in incremental build - with open(StructuredPcdsDataPath, 'w') as file: - json.dump(StructuredPcdsData, file, indent=2) - else: - # - # 1st build, create the StructuredPcdsData.json - # - with open(StructuredPcdsDataPath, 'w') as file: - json.dump(StructuredPcdsData, file, indent=2) + SkipPcdValueInit = True + for filename, file_mtime in StoredStructuredPcdObjectPaths.items(): + f_mtime = os.path.getmtime(filename) + # + # check if the include_file are modified or not, + # if modified then generate the PcdValueInit + # + if f_mtime != file_mtime: + SkipPcdValueInit = False + break + + if SkipPcdValueInit: + return self.GetStructurePcdSet(PcdRecordOutputValueFile) InitByteValue = "" CApp = PcdMainCHeader @@ -2957,42 +3008,6 @@ def GenerateByteArrayValue (self, StructuredPcds): IncSearchList.append(inc) MakeApp = MakeApp + '\n' - CC_FLAGS = LinuxCFLAGS - if sys.platform == "win32": - CC_FLAGS = WindowsCFLAGS - BuildOptions = OrderedDict() - for Options in self.BuildOptions: - if Options[2] != EDKII_NAME: - continue - Family = Options[0] - if Family and Family != self.ToolChainFamily: - continue - Target, Tag, Arch, Tool, Attr = Options[1].split("_") - if Tool != 'CC': - continue - if Attr != "FLAGS": - continue - if Target == TAB_STAR or Target == self._Target: - if Tag == TAB_STAR or Tag == self._Toolchain: - if 'COMMON' not in BuildOptions: - BuildOptions['COMMON'] = set() - if Arch == TAB_STAR: - BuildOptions['COMMON']|= self.ParseCCFlags(self.BuildOptions[Options]) - if Arch in self.SupArchList: - if Arch not in BuildOptions: - BuildOptions[Arch] = set() - BuildOptions[Arch] |= self.ParseCCFlags(self.BuildOptions[Options]) - - if BuildOptions: - ArchBuildOptions = {arch:flags for arch,flags in BuildOptions.items() if arch != 'COMMON'} - if len(ArchBuildOptions.keys()) == 1: - BuildOptions['COMMON'] |= (list(ArchBuildOptions.values())[0]) - elif len(ArchBuildOptions.keys()) > 1: - CommonBuildOptions = reduce(lambda x,y: x&y, ArchBuildOptions.values()) - BuildOptions['COMMON'] |= CommonBuildOptions - ValueList = [item for item in BuildOptions['COMMON'] if item.startswith((r"/U","-U"))] - ValueList.extend([item for item in BuildOptions['COMMON'] if item.startswith((r"/D", "-D"))]) - CC_FLAGS += " ".join(ValueList) MakeApp += CC_FLAGS if sys.platform == "win32": @@ -3013,7 +3028,9 @@ def GenerateByteArrayValue (self, StructuredPcds): SearchPathList.append(os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], "BaseTools/Source/C/Common"))) SearchPathList.extend(str(item) for item in IncSearchList) IncFileList = GetDependencyList(IncludeFileFullPaths, SearchPathList) + StructuredPcdsData["OBJECTS"] = {} for include_file in IncFileList: + StructuredPcdsData["OBJECTS"][include_file] = os.path.getmtime(include_file) MakeApp += "$(OBJECTS) : %s\n" % include_file if sys.platform == "win32": PcdValueCommonPath = os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], "Source\C\Common\PcdValueCommon.c")) @@ -3109,8 +3126,16 @@ def GenerateByteArrayValue (self, StructuredPcds): if returncode != 0: EdkLogger.warn('Build', COMMAND_FAILURE, 'Can not collect output from command: %s\n%s\n%s\n' % (Command, StdOut, StdErr)) + # + # In 1st build create the StructuredPcdsData.json + # update the record as PCD Input has been changed if its incremental build + # + with open(StructuredPcdsDataPath, 'w') as file: + json.dump(StructuredPcdsData, file, indent=2) + # Copy update output file for each Arch shutil.copyfile(OutputValueFile, PcdRecordOutputValueFile) + #start update structure pcd final value return self.GetStructurePcdSet(OutputValueFile)