Add script for release Nuget validation

onnxruntime/test/onnx/main.cc

@@ -444,6 +444,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
       {"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"},
       {"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"},
       {"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398" },
+	  {"zfnet512", "disable temporarily"}
 };
 
 #ifdef USE_NGRAPH

tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml

@@ -166,6 +166,12 @@ jobs:
       FolderPath: '$(Build.ArtifactStagingDirectory)'
       DoEsrp: ${{ parameters.DoEsrp }}
 
+  - template: ../../templates/validate-nuget.yml
+    parameters:
+      NugetPath: '$(Build.ArtifactStagingDirectory)'
+      PlatformsSupported: 'win-x64,linux-x64,osx-x64'
+      VerifyNugetSigning: ${{ parameters.DoEsrp }}
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline NuGet Artifact'
     inputs:

tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml

@@ -222,6 +222,12 @@ parameters:
       FolderPath: '$(Build.ArtifactStagingDirectory)'
       DoEsrp: ${{ parameters.DoEsrp }}
 
+  - template: ../../templates/validate-nuget.yml
+    parameters:
+      NugetPath: '$(Build.ArtifactStagingDirectory)'
+      PlatformsSupported: 'win-x64,win-x86,win10-arm,linux-x64,osx-x64'
+      VerifyNugetSigning: ${{ parameters.DoEsrp }}
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline NuGet Artifact'
     inputs:

tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml

@@ -155,6 +155,12 @@ jobs:
       FolderPath: '$(Build.ArtifactStagingDirectory)'
       DoEsrp: ${{ parameters.DoEsrp }}
 
+  - template: ../../templates/validate-nuget.yml
+    parameters:
+      NugetPath: '$(Build.ArtifactStagingDirectory)'
+      PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64'
+      VerifyNugetSigning: ${{ parameters.DoEsrp }}
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline NuGet Artifact'
     inputs:

tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml

@@ -112,6 +112,12 @@ jobs:
       FolderPath: '$(Build.ArtifactStagingDirectory)'
       DoEsrp: ${{ parameters.DoEsrp }}
 
+  - template: ../../templates/validate-nuget.yml
+    parameters:
+      NugetPath: '$(Build.ArtifactStagingDirectory)'
+      PlatformsSupported: 'win-x64,linux-x64'
+      VerifyNugetSigning: ${{ parameters.DoEsrp }}
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline NuGet Artifact'
     inputs:

tools/ci_build/github/azure-pipelines/templates/validate-nuget.yml

@@ -0,0 +1,34 @@
+parameters:
+  NugetPath: ''
+  PlatformsSupported: ''
+  VerifyNugetSigning: ''
+
+steps:
+    - task: PowerShell@2
+      displayName: 'Download AzCopy (used for download test data script)'
+      inputs:
+        targetType: 'inline'
+        script: |
+          Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\azcopy.exe https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe
+
+    - task: CmdLine@1
+      displayName: 'Download Python'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\azcopy.exe'
+        arguments: 'copy https://onnxruntimetestdata.blob.core.windows.net/models/Miniconda3-4.7.10-Windows-x86_64.exe $(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe'
+      timeoutInMinutes: 10
+
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+
+    - task: CmdLine@1
+      displayName: 'Validate Nuget'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\nuget\validate_nuget.py --nuget_path ${{parameters.NugetPath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}}'
+        workingDirectory: "$(Build.BinariesDirectory)"
+      continueOnError: false

tools/nuget/validate_nuget.py

@@ -0,0 +1,141 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import sys
+import os
+import zipfile # Available Python 3.2 or higher
+import glob
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="ONNX Runtime create nuget spec script",
+                                     usage='')
+    # Main arguments
+    parser.add_argument("--nuget_path", required=True, help="Path containing the Nuget to be validated. Must only contain only one Nuget within this.")
+    parser.add_argument("--platforms_supported", required=True, help="Comma separated list (no space). Ex: linux-x64,win-x86,osx-x64")
+    parser.add_argument("--verify_nuget_signing", required=True, help="Flag inidicating if Nuget package signing is to be verified. Only accepets 'true' or 'false'")
+
+    return parser.parse_args()
+
+def check_exists(path):
+    return os.path.exists(path)
+
+def is_windows():
+    return sys.platform.startswith("win")
+
+def check_if_dlls_are_present(platforms_supported, zip_file):
+    platforms = platforms_supported.strip().split(",")
+    for platform in platforms:
+        if platform.startswith("win"):
+            path = "runtimes/" + platform + "/native/onnxruntime.dll"
+            print('Checking path: ' + path)
+            if (not path in zip_file.namelist()):
+               print("onnxruntime.dll not found for " + platform)
+               print(zip_file.namelist())
+               raise Exception("onnxruntime.dll not found for " + platform)
+
+        elif platform.startswith("linux"):
+            path = "runtimes/" + platform + "/native/libonnxruntime.so"
+            print('Checking path: ' + path)
+            if (not path in zip_file.namelist()):
+               print("libonnxruntime.so not found for " + platform)
+               raise Exception("libonnxruntime.so not found for " + platform)
+
+        elif platform.startswith("osx"):
+            path = "runtimes/" + platform + "/native/libonnxruntime.dylib"
+            print('Checking path: ' + path)        
+            if (not path in zip_file.namelist()):
+               print("libonnxruntime.dylib not found for " + platform)
+               raise Exception("libonnxruntime.dylib not found for " + platform)
+
+        else:
+            raise Exception("Unsupported platform: " + platform)
+
+def check_if_nuget_is_signed(nuget_path):
+    code_sign_summary_file = glob.glob(os.path.join(nuget_path, '*.md'))
+    print(os.path.join(nuget_path)  + '*.md')
+    if (len(code_sign_summary_file) != 1):
+        print('CodeSignSummary files found in path: ')
+        print(code_sign_summary_file)
+        raise Exception('No CodeSignSummary files / more than one CodeSignSummary files found in the given path.')
+
+    print('CodeSignSummary file: ' + code_sign_summary_file[0])
+
+    with open(code_sign_summary_file[0]) as f:
+        contents = f.read()
+        return "Pass" in contents
+
+    return False    
+
+def main():   
+    args = parse_arguments()
+
+    files = os.listdir(args.nuget_path)
+    nuget_packages_found_in_path = [i for i in files if i.endswith('.nupkg')]
+    if (len(nuget_packages_found_in_path) != 1):
+        print('Nuget packages found in path: ')
+        print(nuget_packages_found_in_path)
+        raise Exception('No Nuget packages / more than one Nuget packages found in the given path.')
+
+    nuget_file_name = nuget_packages_found_in_path[0]
+    full_nuget_path = os.path.join(args.nuget_path, nuget_file_name)
+
+    exit_code = 0
+
+    nupkg_copy_name = "NugetCopy" + ".nupkg" 
+    zip_copy_name = "NugetCopy" + ".zip"
+    zip_file = None
+
+    # Remove any residual files
+    if check_exists(nupkg_copy_name):
+        os.remove(nupkg_copy_name)
+
+    if check_exists(zip_copy_name):
+        os.remove(zip_copy_name)
+
+    # Do all validations here         
+    try:
+        if not is_windows():
+            raise Exception('Nuget validation is currently supported only on Windows')
+
+        # Make a copy of the Nuget package
+        print('Making a copy of the Nuget and extracting its contents') 
+        os.system("copy "  + full_nuget_path  + " " + nupkg_copy_name)
+
+        # Convert nupkg to zip
+        os.rename(nupkg_copy_name, zip_copy_name)
+        zip_file = zipfile.ZipFile(zip_copy_name)
+
+        # Check if the relevant dlls are present in the Nuget/Zip
+        print('Checking if the Nuget contains relevant dlls')
+        check_if_dlls_are_present(args.platforms_supported, zip_file)
+
+        # Check if the Nuget has been signed
+        if (args.verify_nuget_signing != 'true' and args.verify_nuget_signing != 'false'):
+            raise Exception('Parameter verify_nuget_signing accepts only true or false as an argument');
+
+        if (args.verify_nuget_signing == 'true'):
+            print('Verifying if Nuget has been signed')
+            if(not check_if_nuget_is_signed(args.nuget_path)):
+                print('Nuget signing verification failed')
+                raise Exception('Nuget signing verification failed')
+
+    except:
+        exit_code = 1
+
+    finally:
+        print('Cleaning up after Nuget validation')
+
+        if zip_file is not None:
+            zip_file.close()
+
+        if check_exists(zip_copy_name):
+            os.remove(zip_copy_name)
+
+        if exit_code == 0:
+            print('Nuget validation was successful')
+        else:
+            raise Exception('Nuget validation was unsuccessful')
+
+if __name__ == "__main__":
+    sys.exit(main())