test: Add CRI benchmarks for container operations (#1569)

* test: Add CRI benchmarks for container operations Add CRI API benchmarks to `cri-containerd.test.exe`, and update helper functions in `cri-containerd` to accept `tb testing.TB` instead of `t *testing.T`. Switched `cri-containerd` to `test\internal\flag` so that `test\internal\require` can be used to check if features are present since that implementation already mirrors what `cri-containerd` does, but accepts a `testing.TB`, and deleted old `test/internal/stringsetflag.go` file. Renamed `scripts/Test-Functionl.ps1` to `scripts/Run-Tests.ps1`, and updated it to run both functional and CRI tests and benchmarks. Signed-off-by: Hamza El-Saawy <[email protected]> * PR: clarifying comment Signed-off-by: Hamza El-Saawy <[email protected]> --------- Signed-off-by: Hamza El-Saawy <[email protected]>
microsoft · Jan 31, 2023 · db7a679 · db7a679
1 parent d310213
commit db7a679
Show file tree

Hide file tree

Showing 17 changed files with 544 additions and 327 deletions.
diff --git a/scripts/Test-Functional.ps1 → scripts/Run-Tests.ps1 b/scripts/Test-Functional.ps1 → scripts/Run-Tests.ps1
@@ -1,4 +1,4 @@
-# ex: .\scripts\Test-Functional.ps1 -Action Bench -Count 2 -BenchTime "2x"
+# ex: .\scripts\Run-Tests.ps1 -vb -TestExe Functional -Count 2 -BenchTime "2x"
 
 [CmdletBinding()]
 param (
@@ -8,12 +8,25 @@ param (
     $Action = 'Bench',
 
     [string]
-    $Note = '',
+    $Note,
 
     [string]
     $OutDirectory = '.\test\results',
 
+    [string]
+    $TestDirectory = '.\bin\test',
+
+    [ValidateSet('Functional', 'CRI')]
+    [string]
+    $TestExe = 'CRI',
+
+    [string]
+    $BenchstatPath = 'benchstat.exe',
+
     # test parameters
+    [switch]
+    $Shuffle,
+
     [int]
     $Count = 1,
 
@@ -28,34 +41,41 @@ param (
     $TestVerbose,
 
     [string]
-    $Run = '',
+    $Run,
 
-    [string]
-    $Feature = ''
+    [string[]]
+    $Features
 )
-
+$ErrorActionPreference = 'Stop'
 Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force
 
+$exe = Switch ($TestExe) {
+    'Functional' { 'functional.test.exe'; break }
+    'CRI' { 'cri-containerd.test.exe'; break }
+}
+$test = Join-Path $TestDirectory $exe
+
 $date = Get-Date
 $testcmd, $out = New-TestCommand `
     -Action $Action `
-    -Path .\bin\test\functional.exe `
-    -Name functional `
+    -Path $test `
+    -Name ($exe -replace '.test.exe$', '') `
     -OutDirectory $OutDirectory `
     -Date $date `
     -Note $Note `
+    -Shuffle:$Shuffle `
     -TestVerbose:$TestVerbose `
     -Count $Count `
     -BenchTime $BenchTime `
     -Timeout $Timeout `
     -Run $Run `
-    -Feature $Feature `
+    -Features $Features `
     -Verbose:$Verbose
 
 Invoke-TestCommand `
     -TestCmd $testcmd `
     -OutputFile $out `
-    -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
+    -OutputCmd (& { if ( $Action -eq 'Bench' ) { $BenchstatPath } }) `
     -Preamble `
     -Date $Date `
     -Note $Note `

diff --git a/scripts/Test-LCOW-UVM.ps1 b/scripts/Test-LCOW-UVM.ps1
@@ -1,4 +1,4 @@
-#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Action Bench -BootFilesPath C:\ContainerPlat\LinuxBootFiles\ -MountGCSTest -Count 2 -Benchtime '3s'
+#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Count 2 -Benchtime '3s'
 # benchstat via  `go install golang.org/x/perf/cmd/benchstat@latest`
 
 [CmdletBinding()]
@@ -12,6 +12,9 @@ param (
     $Note = '',
 
     # test parameters
+    [switch]
+    $Shuffle,
+
     [int]
     $Count = 1,
 
@@ -26,16 +29,16 @@ param (
     $TestVerbose,
 
     [string]
-    $Run = '',
-
-    [string]
-    $CodePath = '.',
+    $Run,
 
     [string]
     $OutDirectory = '.\test\results',
 
     # uvm parameters
 
+    [string]
+    $UVMBootPath = '.\bin\cmd\uvmboot.exe',
+
     [string]
     $BootFilesPath = 'C:\ContainerPlat\LinuxBootFiles',
 
@@ -61,19 +64,18 @@ param (
     $GCSTestPath = '.\bin\test\gcs.test',
 
     [switch]
-    $MountGCSTest,
+    $SkipGCSTestMount,
 
-    [string]
-    $Feature = ''
+    [string[]]
+    $Features
 )
-
+$ErrorActionPreference = 'Stop'
 Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force
 
-$CodePath = Resolve-Path $CodePath
-$OutDirectory = Resolve-Path $OutDirectory
 $BootFilesPath = Resolve-Path $BootFilesPath
 $ContainerRootFSPath = Resolve-Path $ContainerRootFSPath
 $GCSTestPath = Resolve-Path $GCSTestPath
+$UVMBootPath = Resolve-Path $UVMBootPath
 
 $shell = ( $Action -eq 'Shell' )
 
@@ -83,7 +85,7 @@ if ( $shell ) {
     $date = Get-Date
     $waitfiles = "$ContainerRootFSMount"
     $gcspath = 'gcs.test'
-    if ( $MountGCSTest ) {
+    if ( -not $SkipGCSTestMount ) {
         $waitfiles += ",$GCSTestMount"
         $gcspath = "$GCSTestMount/gcs.test"
     }
@@ -104,26 +106,27 @@ if ( $shell ) {
         -OutDirectory $OutDirectory `
         -Date $date `
         -Note $Note `
+        -Shuffle:$Shuffle `
         -TestVerbose:$TestVerbose `
         -Count $Count `
         -BenchTime $BenchTime `
         -Timeout $Timeout `
         -Run $Run `
-        -Feature $Feature `
+        -Features $Features `
         -Verbose:$Verbose
 
     $testcmd += " `'-rootfs-path=$ContainerRootFSMount`' "
     $cmd = $pre + $testcmd
 }
 
-$boot = '.\bin\tool\uvmboot.exe -gcs lcow ' + `
+$boot = "$UVMBootPath -gcs lcow " + `
     '-fwd-stdout -fwd-stderr -output-handling stdout ' + `
     "-boot-files-path $BootFilesPath " + `
     "-root-fs-type $BootFSType " + `
     '-kernel-file vmlinux ' + `
     "-mount-scsi `"$ContainerRootFSPath,$ContainerRootFSMount`" "
 
-if ( $MountGCSTest ) {
+if ( -not $SkipGCSTestMount ) {
     $boot += "-share `"$GCSTestPath,$GCSTestMount`" "
 }
 
@@ -140,8 +143,8 @@ $boot += " -exec `"$cmd`" "
 Invoke-TestCommand `
     -TestCmd $boot `
     -TestCmdPreamble $testcmd `
-    -OutputFile (&{ if ( $Action -ne 'Shell' ) { $out } }) `
-    -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
+    -OutputFile (& { if ( $Action -ne 'Shell' ) { $out } }) `
+    -OutputCmd (& { if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
     -Preamble `
     -Date $Date `
     -Note $Note `

diff --git a/scripts/Testing.psm1 b/scripts/Testing.psm1
@@ -1,10 +1,10 @@
 function New-TestCommand {
     [CmdletBinding()]
     param (
+        [Parameter(Mandatory)]
         [ValidateSet('Test', 'Bench', 'List')]
-        [alias('a')]
         [string]
-        $Action = 'Bench',
+        $Action,
 
         [Parameter(Mandatory)]
         [string]
@@ -16,73 +16,82 @@ function New-TestCommand {
 
         [Parameter(Mandatory)]
         [string]
-        $OutDirectory ,
+        $OutDirectory,
+
+        [string]
+        $OutFile,
 
         [DateTime]
         $Date = (Get-Date),
 
         [string]
-        $Note = '',
+        $Note,
 
         # test parameters
-        [alias('tv')]
         [switch]
-        $TestVerbose = $false,
+        $Shuffle,
+
+        [switch]
+        $TestVerbose,
 
         [int]
-        $Count = 1,
+        $Count,
 
         [string]
-        $BenchTime = '5s',
+        $BenchTime,
 
         [string]
-        $Timeout = '10m',
+        $Timeout,
 
         [string]
-        $Run = '',
+        $Run,
 
-        [string]
-        $Feature = ''
+        [string[]]
+        $Features
     )
-
-    $OutDirectory = Resolve-Path $OutDirectory
     Write-Verbose "creating $OutDirectory"
-
     New-Item -ItemType 'directory' -Path $OutDirectory -Force > $null
 
-    $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.shuffle=on`' `'-test.count=$Count`' "
+    $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.count=$Count`' "
+
+    if ( $Shuffle ) {
+        $testcmd += '''-test.shuffle=on'' '
+    }
 
     if ( $TestVerbose ) {
-        $testcmd += ' ''-test.v'' '
+        $testcmd += '''-test.v'' '
     }
 
     switch ( $Action ) {
         'List' {
-            if ( $Run -eq '' ) {
+            if ( -not $Run ) {
                 $Run = '.'
             }
-            $testcmd += " `'-test.list=$Run`' "
+            $testcmd += "`'-test.list=$Run`' "
         }
         'Test' {
-            if ( $Run -ne '' ) {
-                $testcmd += " `'-test.run=$Run`' "
+            if ( $Run ) {
+                $testcmd += "`'-test.run=$Run`' "
             }
         }
         'Bench' {
-            if ( $Run -eq '' ) {
+            if ( -not $Run ) {
                 $Run = '.'
             }
-            $testcmd += ' ''-test.run=^#'' ''-test.benchmem'' ' + `
-                " `'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' "
+            $testcmd += '''-test.run=^#'' ''-test.benchmem'' ' + `
+                "`'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' "
         }
     }
 
-    if ( $Feature -ne '' ) {
-        $testcmd += " `'-feature=$Feature`' "
+    foreach ( $Feature in $Features ) {
+        $Feature = $Feature -replace ' ', ''
+        if ( $Feature ) {
+            $testcmd += "`'-feature=$Feature`' "
+        }
     }
 
-    $f = $Name + '-' + $Action
-    if ($Note -ne '' ) {
+    $f = $Name + '-' + ($Action.ToLower())
+    if ( $Note ) {
         $f += '-' + $Note
     }
     $out = Join-Path $OutDirectory "$f-$(Get-Date -Date $date -Format FileDateTime).txt"
@@ -101,7 +110,7 @@ function Invoke-TestCommand {
         $TestCmdPreamble = $TestCmd,
 
         [string]
-        $OutputFile = 'nul',
+        $OutputFile = '',
 
         [string]
         $OutputCmd,
@@ -115,30 +124,33 @@ function Invoke-TestCommand {
         [string]
         $Note
     )
+    Write-Verbose "Running command: $TestCmd"
 
-    if ($OutputFile -eq '' ) {
+    if ( -not $OutputFile ) {
         $OutputFile = 'nul'
+    } else {
+        Write-Verbose "Saving output to: $OutputFile"
     }
 
-    Write-Verbose "Saving output to: $OutputFile"
+
     if ( $Preamble ) {
         & {
             Write-Output "test.date: $(Get-Date -Date $Date -UFormat '%FT%R%Z' -AsUTC)"
-            if ( $Note -ne '' ) {
+            if ( $Note ) {
                 Write-Output "note: $Note"
             }
             Write-Output "test.command: $TestCmdPreamble"
-            Write-Output "pkg.commit: $(git rev-parse HEAD)"
-        } | Tee-Object -Append -FilePath $OutputFile
+            if ( Get-Command -ErrorAction Ignore 'git' ) {
+                Write-Output "pkg.commit: $(git rev-parse HEAD 2>$null)"
+            }
+        } | Tee-Object -Encoding utf8 -FilePath $OutputFile
     }
+    Invoke-Expression $TestCmd |
+        Tee-Object -Encoding utf8 -Append -FilePath $OutputFile
 
-    Write-Verbose "Running command: $TestCmd"
-    Invoke-Expression $TestCmd | Tee-Object -Append -FilePath $OutputFile
-
-    if ( $OutputCmd -ne '' -and $OutputFile -ne 'nul' ) {
+    if ( $OutputCmd -and $OutputFile -ne 'nul' ) {
         $oc = "$OutputCmd $OutputFile"
         Write-Verbose "Running command: $oc"
         Invoke-Expression $oc
     }
-
-}
+}