Enable integration tests

Integration tests to run platform builds with architectures IA32, X64
for targets DEBUG, RELEASE for platforms OvmfPkg, ArmVirtPkg, and
EmulatorPkg.

Signed-off-by: Joey Vagedes <[email protected]>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,64 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["new", "bug"]
+
+body:
+  - type: markdown
+    attributes:
+      value: Thanks for taking the time to file this bug!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: "How can we get in touch with you if we need more info?"
+      placeholder: ex. [email protected]
+    validations:
+      required: false
+  - type: textarea
+    id: bug-description
+    attributes:
+      label: Describe the Bug
+      description: "A clear and concise description of what the bug is."
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: Reproduction steps
+      description: "How did you find this bug? Walk us through it step by step."
+      placeholder: |
+        1.
+        2.
+        3.
+        ...
+    validations:
+      required: true
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: "A clear and concise description of what you expected to happen."
+    validations:
+      required: true
+  - type: textarea
+    id: env
+    attributes:
+      label: Execution Environment
+      description: "List the OS, python version, system environment variables, etc."
+    validations:
+      required: false
+  - type: textarea
+    id: pip
+    attributes:
+      label: Pip packages
+      description: "If python related, run 'python -m pip list' and copy the output here."
+    validations:
+      required: false
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+      description: "Any other additional context about the problem not covered by the above."
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,38 @@
+name: Feature Request
+description: Submit a feature request
+title: "[Feature]: "
+labels: ["new", "enhancement"]
+
+body:
+  - type: markdown
+    attributes:
+      value: Thanks for suggesting an idea for this project!
+  - type: textarea
+    id: description
+    attributes:
+      label: What does the feature solve?
+      description: "Is your feature request related to a problem?"
+      placeholder: "Ex. I'm always frustrated when..."
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution
+      description: "A clear and concise description of what you want to happen"
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Have you considered any alternatives?
+      description: "A clear and concise description of any alternative solutions, features, or workarounds you've considered"
+    validations:
+      required: false
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+      description: "Any other additional context about the problem not covered by the above."
+    validations:
+      required: false

edk2basetools/Common/Expression.py

@@ -1026,7 +1026,7 @@ def __call__(self, RealValue=False, Depth=0):
                                     Size += ItemSize
 
                             if Size > 0:
-                                PcdValue = '{' + ','.join(AllPcdValueList) + '}'
+                                PcdValue = '{' + ', '.join(AllPcdValueList) + '}'
                         else:
                             raise  BadExpression("Type: %s, Value: %s, %s"%(self.PcdType, PcdValue, Value))
 

edk2basetools/Common/VpdInfoFile.py

@@ -248,8 +248,8 @@ def CallExtenalBPDGTool(ToolPath, VpdFileName):
         PopenObject.wait()
 
     if PopenObject.returncode != 0:
-        EdkLogger.debug(EdkLogger.DEBUG_1, "Fail to call BPDG tool", str(error))
+        EdkLogger.debug(EdkLogger.DEBUG_1, "Fail to call BPDG tool", str(error.decode()))
         EdkLogger.error("BPDG", BuildToolError.COMMAND_FAILURE, "Fail to execute BPDG tool with exit code: %d, the error message is: \n %s" % \
-                            (PopenObject.returncode, str(error)))
+                            (PopenObject.returncode, str(error.decode())))
 
     return PopenObject.returncode

edk2basetools/CommonDataClass/FdfClass.py

@@ -190,6 +190,18 @@ def __init__(self):
         self.FvParentAddr = None
         self.IncludeFvSection = False
 
+## SubType GUID section data in FDF
+#
+#
+class SubTypeGuidSectionClassObject (SectionClassObject) :
+    ## The constructor
+    #
+    #   @param  self        The object pointer
+    #
+    def __init__(self):
+        SectionClassObject.__init__(self)
+        self.SubTypeGuid = None
+
 ## UI section data in FDF
 #
 #

edk2basetools/Ecc/EccToolError.py

@@ -186,7 +186,7 @@
     ERROR_META_DATA_FILE_CHECK_BINARY_INF_IN_FDF : "An INF file is specified in the FDF file, but not in the DSC file, therefore the INF file must be for a Binary module only",
     ERROR_META_DATA_FILE_CHECK_PCD_DUPLICATE : "Duplicate PCDs found",
     ERROR_META_DATA_FILE_CHECK_PCD_FLASH : "PCD settings in the FDF file should only be related to flash",
-    ERROR_META_DATA_FILE_CHECK_PCD_NO_USE : "There should be no PCDs declared in INF files that are not specified in in either a DSC or FDF file",
+    ERROR_META_DATA_FILE_CHECK_PCD_NO_USE : "There should be no PCDs declared in INF files that are not specified in either a DSC or FDF file",
     ERROR_META_DATA_FILE_CHECK_DUPLICATE_GUID : "Duplicate GUID found",
     ERROR_META_DATA_FILE_CHECK_DUPLICATE_PROTOCOL : "Duplicate PROTOCOL found",
     ERROR_META_DATA_FILE_CHECK_DUPLICATE_PPI : "Duplicate PPI found",

edk2basetools/FMMT/FMMT.py

@@ -24,7 +24,8 @@
                         If not given TargetFvName, all the existed target Ffs will be deleted'")
 parser.add_argument("-e", "--Extract", dest="Extract", nargs='+',
                     help="Extract a Ffs Info: '-e inputfile TargetFvName(Optional) TargetFfsName outputfile\
-                        If not given TargetFvName, the first found target Ffs will be extracted'")
+                        If not given TargetFvName, the first found target Ffs will be extracted.\
+                        If only given TargetFvName, not given TargetFfsName, the TargetFv will be extracted to output file'")
 parser.add_argument("-a", "--Add", dest="Add", nargs='+',
                     help="Add a Ffs into a FV:'-a inputfile TargetFvName newffsfile outputfile'")
 parser.add_argument("-r", "--Replace", dest="Replace", nargs='+',
@@ -40,6 +41,8 @@
                         FmmtConf file saves the target guidtool used in compress/uncompress process.\
                         If do not provide, FMMT tool will search the inputfile folder for FmmtConf.ini firstly, if not found,\
                         the FmmtConf.ini saved in FMMT tool's folder will be used as default.")
+parser.add_argument("-s", "--ShrinkFv", dest="ShrinkFv", nargs='+',
+                    help="Shrink the Fv file: '-s InputFvfile OutputFvfile")
 
 def print_banner():
     print("")
@@ -110,6 +113,9 @@ def Replace(self,inputfile: str, Ffs_name: str, newffsfile: str, outputfile: str
         else:
             ReplaceFfs(inputfile, self.CheckFfsName(Ffs_name), newffsfile, outputfile)
 
+    def Shrink(self,inputfile: str, outputfile: str) -> None:
+        self.SetDestPath(inputfile)
+        ShrinkFv(inputfile, outputfile)
 
 def main():
     args=parser.parse_args()
@@ -141,6 +147,8 @@ def main():
                 fmmt.Replace(args.Replace[0],args.Replace[2],args.Replace[3],args.Replace[4],args.Replace[1])
             else:
                 fmmt.Replace(args.Replace[0],args.Replace[1],args.Replace[2],args.Replace[3])
+        elif args.ShrinkFv:
+            fmmt.Shrink(args.ShrinkFv[0], args.ShrinkFv[1])
         else:
             parser.print_help()
     except Exception as e:

edk2basetools/FMMT/core/FMMTOperation.py

@@ -63,9 +63,10 @@ def DeleteFfs(inputfile: str, TargetFfs_name: str, outputfile: str, Fv_name: str
     FmmtParser.WholeFvTree.FindNode(TargetFfs_name, FmmtParser.WholeFvTree.Findlist)
     # Choose the Specfic DeleteFfs with Fv info
     if Fv_name:
-        for item in FmmtParser.WholeFvTree.Findlist:
-            if item.Parent.key != Fv_name and item.Parent.Data.Name != Fv_name:
-                FmmtParser.WholeFvTree.Findlist.remove(item)
+        FindNum = len(FmmtParser.WholeFvTree.Findlist)
+        for index in range(FindNum-1, -1, -1):
+            if FmmtParser.WholeFvTree.Findlist[index].Parent.key != Fv_name and FmmtParser.WholeFvTree.Findlist[index].Parent.Data.Name != Fv_name:
+                FmmtParser.WholeFvTree.Findlist.remove(FmmtParser.WholeFvTree.Findlist[index])
     Status = False
     if FmmtParser.WholeFvTree.Findlist != []:
         for Delete_Ffs in FmmtParser.WholeFvTree.Findlist:
@@ -149,9 +150,10 @@ def ReplaceFfs(inputfile: str, Ffs_name: str, newffsfile: str, outputfile: str,
     new_ffs.Data.PadData = GetPadSize(new_ffs.Data.Size, FFS_COMMON_ALIGNMENT) * b'\xff'
     FmmtParser.WholeFvTree.FindNode(Ffs_name, FmmtParser.WholeFvTree.Findlist)
     if Fv_name:
-        for item in FmmtParser.WholeFvTree.Findlist:
-            if item.Parent.key != Fv_name and item.Parent.Data.Name != Fv_name:
-                FmmtParser.WholeFvTree.Findlist.remove(item)
+        FindNum = len(FmmtParser.WholeFvTree.Findlist)
+        for index in range(FindNum-1, -1, -1):
+            if FmmtParser.WholeFvTree.Findlist[index].Parent.key != Fv_name and FmmtParser.WholeFvTree.Findlist[index].Parent.Data.Name != Fv_name:
+                FmmtParser.WholeFvTree.Findlist.remove(FmmtParser.WholeFvTree.Findlist[index])
     if FmmtParser.WholeFvTree.Findlist != []:
         for TargetFfs in FmmtParser.WholeFvTree.Findlist:
             FfsMod = FvHandler(newFmmtParser.WholeFvTree.Child[0], TargetFfs)
@@ -180,18 +182,51 @@ def ExtractFfs(inputfile: str, Ffs_name: str, outputfile: str, Fv_name: str=None
     logger.debug('Done!')
     FmmtParser.WholeFvTree.FindNode(Ffs_name, FmmtParser.WholeFvTree.Findlist)
     if Fv_name:
-        for item in FmmtParser.WholeFvTree.Findlist:
-            if item.Parent.key != Fv_name and item.Parent.Data.Name != Fv_name:
-                FmmtParser.WholeFvTree.Findlist.remove(item)
+        FindNum = len(FmmtParser.WholeFvTree.Findlist)
+        for index in range(FindNum-1, -1, -1):
+            if FmmtParser.WholeFvTree.Findlist[index].Parent.key != Fv_name and FmmtParser.WholeFvTree.Findlist[index].Parent.Data.Name != Fv_name:
+                FmmtParser.WholeFvTree.Findlist.remove(FmmtParser.WholeFvTree.Findlist[index])
     if FmmtParser.WholeFvTree.Findlist != []:
         TargetNode = FmmtParser.WholeFvTree.Findlist[0]
-        TargetFv = TargetNode.Parent
-        if TargetFv.Data.Header.Attributes & EFI_FVB2_ERASE_POLARITY:
-            TargetNode.Data.Header.State = c_uint8(
-                ~TargetNode.Data.Header.State)
-        FinalData = struct2stream(TargetNode.Data.Header) + TargetNode.Data.Data
-        with open(outputfile, "wb") as f:
-            f.write(FinalData)
-        logger.debug('Extract ffs data is saved in {}.'.format(outputfile))
+        if TargetNode.type == FV_TREE or SEC_FV_TREE or DATA_FV_TREE:
+            FinalData = struct2stream(TargetNode.Data.Header) + TargetNode.Data.Data
+            with open(outputfile, "wb") as f:
+                f.write(FinalData)
+            logger.debug('Extract fv data is saved in {}.'.format(outputfile))
+        else:
+            TargetFv = TargetNode.Parent
+            if TargetFv.Data.Header.Attributes & EFI_FVB2_ERASE_POLARITY:
+                TargetNode.Data.Header.State = c_uint8(
+                    ~TargetNode.Data.Header.State)
+            FinalData = struct2stream(TargetNode.Data.Header) + TargetNode.Data.Data
+            with open(outputfile, "wb") as f:
+                f.write(FinalData)
+            logger.debug('Extract ffs data is saved in {}.'.format(outputfile))
     else:
-        logger.error('Target Ffs not found!!!')
+        logger.error('Target Ffs/Fv not found!!!')
+
+def ShrinkFv(inputfile: str, outputfile: str) -> None:
+    if not os.path.exists(inputfile):
+        logger.error("Invalid inputfile, can not open {}.".format(inputfile))
+        raise Exception("Process Failed: Invalid inputfile!")
+    # 1. Data Prepare
+    with open(inputfile, "rb") as f:
+        whole_data = f.read()
+    FmmtParser = FMMTParser(inputfile, ROOT_TREE)
+    # 2. DataTree Create
+    logger.debug('Parsing inputfile data......')
+    FmmtParser.ParserFromRoot(FmmtParser.WholeFvTree, whole_data)
+    logger.debug('Done!')
+    TargetFv = FmmtParser.WholeFvTree.Child[0]
+    if TargetFv:
+        FvMod = FvHandler(TargetFv)
+        Status = FvMod.ShrinkFv()
+    else:
+        logger.error('Target Fv not found!!!')
+    # 4. Data Encapsulation
+    if Status:
+        logger.debug('Start encapsulating data......')
+        FmmtParser.Encapsulation(FmmtParser.WholeFvTree, False)
+        with open(outputfile, "wb") as f:
+            f.write(FmmtParser.FinalData)
+        logger.debug('Encapsulated data is saved in {}.'.format(outputfile))

edk2basetools/FMMT/core/FvHandler.py

@@ -145,7 +145,7 @@ def ModifyFvSystemGuid(TargetFv) -> None:
             TargetFv.Data.Data += struct2stream(item.Data.Header)+ item.Data.Data + item.Data.PadData
 
 class FvHandler:
-    def __init__(self, NewFfs, TargetFfs) -> None:
+    def __init__(self, NewFfs, TargetFfs=None) -> None:
         self.NewFfs = NewFfs
         self.TargetFfs = TargetFfs
         self.Status = False
@@ -155,7 +155,6 @@ def __init__(self, NewFfs, TargetFfs) -> None:
     def CompressData(self, TargetTree) -> None:
         TreePath = TargetTree.GetTreePath()
         pos = len(TreePath)
-        self.Status = False
         while pos:
             if not self.Status:
                 if TreePath[pos-1].type == SECTION_TREE and TreePath[pos-1].Data.Type == 0x02:
@@ -487,7 +486,6 @@ def AddFfs(self) -> bool:
                     ~self.NewFfs.Data.Header.State)
             # If TargetFv have enough free space, just move part of the free space to NewFfs, split free space to NewFfs and new free space.
             if TargetLen < 0:
-                self.Status = True
                 self.TargetFfs.Data.Data = b'\xff' * (-TargetLen)
                 TargetFv.Data.Free_Space = (-TargetLen)
                 TargetFv.Data.ModFvExt()
@@ -498,13 +496,14 @@ def AddFfs(self) -> bool:
                 ModifyFfsType(self.NewFfs)
                 # Recompress from the Fv node to update all the related node data.
                 self.CompressData(TargetFv)
-            elif TargetLen == 0:
                 self.Status = True
+            elif TargetLen == 0:
                 TargetFv.Child.remove(self.TargetFfs)
                 TargetFv.insertChild(self.NewFfs)
                 ModifyFfsType(self.NewFfs)
                 # Recompress from the Fv node to update all the related node data.
                 self.CompressData(TargetFv)
+                self.Status = True
             # If TargetFv do not have enough free space, need move part of the free space of TargetFv's parent Fv to TargetFv/NewFfs.
             else:
                 if TargetFv.type == FV_TREE:
@@ -639,3 +638,29 @@ def DeleteFfs(self) -> bool:
         self.Status = True
         logger.debug('Done!')
         return self.Status
+
+    def ShrinkFv(self) -> bool:
+        TargetFv = self.NewFfs
+        TargetFv.Data.Data = b''
+        if not TargetFv.Data.Free_Space:
+            self.Status = True
+        else:
+            BlockSize = TargetFv.Data.Header.BlockMap[0].Length
+            New_Free_Space = TargetFv.Data.Free_Space%BlockSize
+            Removed_Space = TargetFv.Data.Free_Space - New_Free_Space
+            TargetFv.Child[-1].Data.Data = b'\xff' * New_Free_Space
+            TargetFv.Data.Size -= Removed_Space
+            TargetFv.Data.Header.Fvlength = TargetFv.Data.Size
+            ModifyFvSystemGuid(TargetFv)
+            for item in TargetFv.Child:
+                if item.type == FFS_FREE_SPACE:
+                    TargetFv.Data.Data += item.Data.Data + item.Data.PadData
+                else:
+                    TargetFv.Data.Data += struct2stream(item.Data.Header)+ item.Data.Data + item.Data.PadData
+            TargetFv.Data.ModFvExt()
+            TargetFv.Data.ModFvSize()
+            TargetFv.Data.ModExtHeaderData()
+            ModifyFvExtData(TargetFv)
+            TargetFv.Data.ModCheckSum()
+            self.Status = True
+        return self.Status

edk2basetools/GenFds/FdfParser.py

@@ -42,6 +42,7 @@
 from .DepexSection import DepexSection
 from .CompressSection import CompressSection
 from .GuidSection import GuidSection
+from .SubTypeGuidSection import SubTypeGuidSection
 from .Capsule import EFI_CERT_TYPE_PKCS7_GUID, EFI_CERT_TYPE_RSA2048_SHA256_GUID, Capsule
 from .CapsuleData import CapsuleFfs, CapsulePayload, CapsuleFv, CapsuleFd, CapsuleAnyFile, CapsuleAfile
 from .RuleComplexFile import RuleComplexFile
@@ -2892,6 +2893,27 @@ def _GetLeafSection(self, Obj):
             DepexSectionObj.Expression = self._SkippedChars.rstrip(T_CHAR_BRACE_R)
             Obj.SectionList.append(DepexSectionObj)
 
+        elif self._IsKeyword("SUBTYPE_GUID"):
+            if AlignValue == 'Auto':
+                raise Warning("Auto alignment can only be used in PE32 or TE section ", self.FileName, self.CurrentLineNumber)
+            SubTypeGuidValue = None
+            if not self._GetNextGuid():
+                raise Warning.Expected("GUID", self.FileName, self.CurrentLineNumber)
+            else:
+                SubTypeGuidValue = self._Token
+
+            if not self._IsToken(TAB_EQUAL_SPLIT):
+                raise Warning.ExpectedEquals(self.FileName, self.CurrentLineNumber)
+            if not self._GetNextToken():
+                raise Warning.Expected("section file path", self.FileName, self.CurrentLineNumber)
+            FileName = self._Token
+
+            SubTypeGuidSectionObj = SubTypeGuidSection()
+            SubTypeGuidSectionObj.Alignment = AlignValue
+            SubTypeGuidSectionObj.SubTypeGuid = SubTypeGuidValue
+            SubTypeGuidSectionObj.SectFileName = FileName
+            Obj.SectionList.append(SubTypeGuidSectionObj)
+
         else:
             if not self._GetNextWord():
                 raise Warning.Expected("section type", self.FileName, self.CurrentLineNumber)
@@ -3996,7 +4018,7 @@ def _CheckRuleSectionFileType(self, SectionType, FileType):
             if FileType not in {BINARY_FILE_TYPE_PE32, "SEC_PE32"}:
                 raise Warning(WarningString % FileType, self.FileName, self.CurrentLineNumber)
         elif SectionType == BINARY_FILE_TYPE_PIC:
-            if FileType not in {BINARY_FILE_TYPE_PIC, BINARY_FILE_TYPE_PIC}:
+            if FileType not in {BINARY_FILE_TYPE_PIC, "SEC_PIC"}:
                 raise Warning(WarningString % FileType, self.FileName, self.CurrentLineNumber)
         elif SectionType == BINARY_FILE_TYPE_TE:
             if FileType not in {BINARY_FILE_TYPE_TE, "SEC_TE"}: