diff --git a/scripts/Test-Functional.ps1 b/scripts/Run-Tests.ps1 similarity index 55% rename from scripts/Test-Functional.ps1 rename to scripts/Run-Tests.ps1 index 5884cd6d82..1120afd139 100644 --- a/scripts/Test-Functional.ps1 +++ b/scripts/Run-Tests.ps1 @@ -1,4 +1,4 @@ -# ex: .\scripts\Test-Functional.ps1 -Action Bench -Count 2 -BenchTime "2x" +# ex: .\scripts\Run-Tests.ps1 -vb -TestExe Functional -Count 2 -BenchTime "2x" [CmdletBinding()] param ( @@ -8,12 +8,25 @@ param ( $Action = 'Bench', [string] - $Note = '', + $Note, [string] $OutDirectory = '.\test\results', + [string] + $TestDirectory = '.\bin\test', + + [ValidateSet('Functional', 'CRI')] + [string] + $TestExe = 'CRI', + + [string] + $BenchstatPath = 'benchstat.exe', + # test parameters + [switch] + $Shuffle, + [int] $Count = 1, @@ -28,34 +41,41 @@ param ( $TestVerbose, [string] - $Run = '', + $Run, - [string] - $Feature = '' + [string[]] + $Features ) - +$ErrorActionPreference = 'Stop' Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force +$exe = Switch ($TestExe) { + 'Functional' { 'functional.test.exe'; break } + 'CRI' { 'cri-containerd.test.exe'; break } +} +$test = Join-Path $TestDirectory $exe + $date = Get-Date $testcmd, $out = New-TestCommand ` -Action $Action ` - -Path .\bin\test\functional.exe ` - -Name functional ` + -Path $test ` + -Name ($exe -replace '.test.exe$', '') ` -OutDirectory $OutDirectory ` -Date $date ` -Note $Note ` + -Shuffle:$Shuffle ` -TestVerbose:$TestVerbose ` -Count $Count ` -BenchTime $BenchTime ` -Timeout $Timeout ` -Run $Run ` - -Feature $Feature ` + -Features $Features ` -Verbose:$Verbose Invoke-TestCommand ` -TestCmd $testcmd ` -OutputFile $out ` - -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` + -OutputCmd (& { if ( $Action -eq 'Bench' ) { $BenchstatPath } }) ` -Preamble ` -Date $Date ` -Note $Note ` diff --git a/scripts/Test-LCOW-UVM.ps1 b/scripts/Test-LCOW-UVM.ps1 index fbea64e2f5..0570776bee 100644 --- a/scripts/Test-LCOW-UVM.ps1 +++ b/scripts/Test-LCOW-UVM.ps1 @@ -1,4 +1,4 @@ -#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Action Bench -BootFilesPath C:\ContainerPlat\LinuxBootFiles\ -MountGCSTest -Count 2 -Benchtime '3s' +#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Count 2 -Benchtime '3s' # benchstat via `go install golang.org/x/perf/cmd/benchstat@latest` [CmdletBinding()] @@ -12,6 +12,9 @@ param ( $Note = '', # test parameters + [switch] + $Shuffle, + [int] $Count = 1, @@ -26,16 +29,16 @@ param ( $TestVerbose, [string] - $Run = '', - - [string] - $CodePath = '.', + $Run, [string] $OutDirectory = '.\test\results', # uvm parameters + [string] + $UVMBootPath = '.\bin\cmd\uvmboot.exe', + [string] $BootFilesPath = 'C:\ContainerPlat\LinuxBootFiles', @@ -61,19 +64,18 @@ param ( $GCSTestPath = '.\bin\test\gcs.test', [switch] - $MountGCSTest, + $SkipGCSTestMount, - [string] - $Feature = '' + [string[]] + $Features ) - +$ErrorActionPreference = 'Stop' Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force -$CodePath = Resolve-Path $CodePath -$OutDirectory = Resolve-Path $OutDirectory $BootFilesPath = Resolve-Path $BootFilesPath $ContainerRootFSPath = Resolve-Path $ContainerRootFSPath $GCSTestPath = Resolve-Path $GCSTestPath +$UVMBootPath = Resolve-Path $UVMBootPath $shell = ( $Action -eq 'Shell' ) @@ -83,7 +85,7 @@ if ( $shell ) { $date = Get-Date $waitfiles = "$ContainerRootFSMount" $gcspath = 'gcs.test' - if ( $MountGCSTest ) { + if ( -not $SkipGCSTestMount ) { $waitfiles += ",$GCSTestMount" $gcspath = "$GCSTestMount/gcs.test" } @@ -104,26 +106,27 @@ if ( $shell ) { -OutDirectory $OutDirectory ` -Date $date ` -Note $Note ` + -Shuffle:$Shuffle ` -TestVerbose:$TestVerbose ` -Count $Count ` -BenchTime $BenchTime ` -Timeout $Timeout ` -Run $Run ` - -Feature $Feature ` + -Features $Features ` -Verbose:$Verbose $testcmd += " `'-rootfs-path=$ContainerRootFSMount`' " $cmd = $pre + $testcmd } -$boot = '.\bin\tool\uvmboot.exe -gcs lcow ' + ` +$boot = "$UVMBootPath -gcs lcow " + ` '-fwd-stdout -fwd-stderr -output-handling stdout ' + ` "-boot-files-path $BootFilesPath " + ` "-root-fs-type $BootFSType " + ` '-kernel-file vmlinux ' + ` "-mount-scsi `"$ContainerRootFSPath,$ContainerRootFSMount`" " -if ( $MountGCSTest ) { +if ( -not $SkipGCSTestMount ) { $boot += "-share `"$GCSTestPath,$GCSTestMount`" " } @@ -140,8 +143,8 @@ $boot += " -exec `"$cmd`" " Invoke-TestCommand ` -TestCmd $boot ` -TestCmdPreamble $testcmd ` - -OutputFile (&{ if ( $Action -ne 'Shell' ) { $out } }) ` - -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` + -OutputFile (& { if ( $Action -ne 'Shell' ) { $out } }) ` + -OutputCmd (& { if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` -Preamble ` -Date $Date ` -Note $Note ` diff --git a/scripts/Testing.psm1 b/scripts/Testing.psm1 index 3b79edc710..f993817fec 100644 --- a/scripts/Testing.psm1 +++ b/scripts/Testing.psm1 @@ -1,10 +1,10 @@ function New-TestCommand { [CmdletBinding()] param ( + [Parameter(Mandatory)] [ValidateSet('Test', 'Bench', 'List')] - [alias('a')] [string] - $Action = 'Bench', + $Action, [Parameter(Mandatory)] [string] @@ -16,73 +16,82 @@ function New-TestCommand { [Parameter(Mandatory)] [string] - $OutDirectory , + $OutDirectory, + + [string] + $OutFile, [DateTime] $Date = (Get-Date), [string] - $Note = '', + $Note, # test parameters - [alias('tv')] [switch] - $TestVerbose = $false, + $Shuffle, + + [switch] + $TestVerbose, [int] - $Count = 1, + $Count, [string] - $BenchTime = '5s', + $BenchTime, [string] - $Timeout = '10m', + $Timeout, [string] - $Run = '', + $Run, - [string] - $Feature = '' + [string[]] + $Features ) - - $OutDirectory = Resolve-Path $OutDirectory Write-Verbose "creating $OutDirectory" - New-Item -ItemType 'directory' -Path $OutDirectory -Force > $null - $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.shuffle=on`' `'-test.count=$Count`' " + $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.count=$Count`' " + + if ( $Shuffle ) { + $testcmd += '''-test.shuffle=on'' ' + } if ( $TestVerbose ) { - $testcmd += ' ''-test.v'' ' + $testcmd += '''-test.v'' ' } switch ( $Action ) { 'List' { - if ( $Run -eq '' ) { + if ( -not $Run ) { $Run = '.' } - $testcmd += " `'-test.list=$Run`' " + $testcmd += "`'-test.list=$Run`' " } 'Test' { - if ( $Run -ne '' ) { - $testcmd += " `'-test.run=$Run`' " + if ( $Run ) { + $testcmd += "`'-test.run=$Run`' " } } 'Bench' { - if ( $Run -eq '' ) { + if ( -not $Run ) { $Run = '.' } - $testcmd += ' ''-test.run=^#'' ''-test.benchmem'' ' + ` - " `'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' " + $testcmd += '''-test.run=^#'' ''-test.benchmem'' ' + ` + "`'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' " } } - if ( $Feature -ne '' ) { - $testcmd += " `'-feature=$Feature`' " + foreach ( $Feature in $Features ) { + $Feature = $Feature -replace ' ', '' + if ( $Feature ) { + $testcmd += "`'-feature=$Feature`' " + } } - $f = $Name + '-' + $Action - if ($Note -ne '' ) { + $f = $Name + '-' + ($Action.ToLower()) + if ( $Note ) { $f += '-' + $Note } $out = Join-Path $OutDirectory "$f-$(Get-Date -Date $date -Format FileDateTime).txt" @@ -101,7 +110,7 @@ function Invoke-TestCommand { $TestCmdPreamble = $TestCmd, [string] - $OutputFile = 'nul', + $OutputFile = '', [string] $OutputCmd, @@ -115,30 +124,33 @@ function Invoke-TestCommand { [string] $Note ) + Write-Verbose "Running command: $TestCmd" - if ($OutputFile -eq '' ) { + if ( -not $OutputFile ) { $OutputFile = 'nul' + } else { + Write-Verbose "Saving output to: $OutputFile" } - Write-Verbose "Saving output to: $OutputFile" + if ( $Preamble ) { & { Write-Output "test.date: $(Get-Date -Date $Date -UFormat '%FT%R%Z' -AsUTC)" - if ( $Note -ne '' ) { + if ( $Note ) { Write-Output "note: $Note" } Write-Output "test.command: $TestCmdPreamble" - Write-Output "pkg.commit: $(git rev-parse HEAD)" - } | Tee-Object -Append -FilePath $OutputFile + if ( Get-Command -ErrorAction Ignore 'git' ) { + Write-Output "pkg.commit: $(git rev-parse HEAD 2>$null)" + } + } | Tee-Object -Encoding utf8 -FilePath $OutputFile } + Invoke-Expression $TestCmd | + Tee-Object -Encoding utf8 -Append -FilePath $OutputFile - Write-Verbose "Running command: $TestCmd" - Invoke-Expression $TestCmd | Tee-Object -Append -FilePath $OutputFile - - if ( $OutputCmd -ne '' -and $OutputFile -ne 'nul' ) { + if ( $OutputCmd -and $OutputFile -ne 'nul' ) { $oc = "$OutputCmd $OutputFile" Write-Verbose "Running command: $oc" Invoke-Expression $oc } - -} \ No newline at end of file +} diff --git a/test/cri-containerd/container_bench_test.go b/test/cri-containerd/container_bench_test.go new file mode 100644 index 0000000000..5fa1e69303 --- /dev/null +++ b/test/cri-containerd/container_bench_test.go @@ -0,0 +1,234 @@ +//go:build windows && functional +// +build windows,functional + +package cri_containerd + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/test/internal/require" +) + +var _containerBenchmarkTests = []struct { + Name string + Feature string + Runtime string + SandboxOpts []SandboxConfigOpt + Image string + Command []string +}{ + { + Name: "LCOW", + Feature: featureLCOW, + Runtime: lcowRuntimeHandler, + Image: imageLcowAlpine, + Command: []string{"ash", "-c", "tail -f /dev/null"}, + }, + { + Name: "WCOW_Hypervisor", + Feature: featureWCOWHypervisor, + Runtime: wcowHypervisorRuntimeHandler, + Image: imageWindowsNanoserver, + Command: []string{"cmd", "/c", "ping -t 127.0.0.1"}, + }, + { + Name: "WCOW_Process", + Feature: featureWCOWProcess, + Runtime: wcowProcessRuntimeHandler, + Image: imageWindowsNanoserver, + Command: []string{"cmd", "/c", "ping -t 127.0.0.1"}, + }, +} + +func BenchmarkPodCreate(b *testing.B) { + require.Build(b, osversion.RS5) + client := newTestRuntimeClient(b) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, tt := range _containerBenchmarkTests { + b.StopTimer() + b.ResetTimer() + b.Run(tt.Name, func(b *testing.B) { + requireFeatures(b, tt.Feature) + + switch tt.Feature { + case featureLCOW: + pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) + case featureWCOWHypervisor, featureWCOWProcess: + pullRequiredImages(b, []string{tt.Image}) + } + + sandboxRequest := getRunPodSandboxRequest(b, tt.Runtime, tt.SandboxOpts...) + + for i := 0; i < b.N; i++ { + b.StartTimer() + podID := runPodSandbox(b, client, ctx, sandboxRequest) + b.StopTimer() + + stopPodSandbox(b, client, ctx, podID) + removePodSandbox(b, client, ctx, podID) + } + }) + } +} + +func BenchmarkContainerCreate(b *testing.B) { + require.Build(b, osversion.RS5) + client := newTestRuntimeClient(b) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // + // new pod per container + // + b.Run("NewPod", func(b *testing.B) { + for _, tt := range _containerBenchmarkTests { + b.StopTimer() + b.ResetTimer() + b.Run(tt.Name, func(b *testing.B) { + requireFeatures(b, tt.Feature) + + switch tt.Feature { + case featureLCOW: + pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) + case featureWCOWHypervisor, featureWCOWProcess: + pullRequiredImages(b, []string{tt.Image}) + } + + sandboxRequest := getRunPodSandboxRequest(b, tt.Runtime, tt.SandboxOpts...) + for i := 0; i < b.N; i++ { + podID := runPodSandbox(b, client, ctx, sandboxRequest) + + request := getCreateContainerRequest(podID, b.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) + + b.StartTimer() + containerID := createContainer(b, client, ctx, request) + b.StopTimer() + + removeContainer(b, client, ctx, containerID) + stopPodSandbox(b, client, ctx, podID) + removePodSandbox(b, client, ctx, podID) + } + }) + } + }) + + // + // same pod for containers + // + b.Run("SamePod", func(b *testing.B) { + for _, tt := range _containerBenchmarkTests { + b.StopTimer() + b.ResetTimer() + b.Run(tt.Name, func(b *testing.B) { + requireFeatures(b, tt.Feature) + + switch tt.Feature { + case featureLCOW: + pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) + case featureWCOWHypervisor, featureWCOWProcess: + pullRequiredImages(b, []string{tt.Image}) + } + sandboxRequest := getRunPodSandboxRequest(b, tt.Runtime, tt.SandboxOpts...) + podID := runPodSandbox(b, client, ctx, sandboxRequest) + + for i := 0; i < b.N; i++ { + request := getCreateContainerRequest(podID, b.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) + + b.StartTimer() + containerID := createContainer(b, client, ctx, request) + b.StopTimer() + + removeContainer(b, client, ctx, containerID) + } + + stopPodSandbox(b, client, ctx, podID) + removePodSandbox(b, client, ctx, podID) + }) + } + }) +} + +func BenchmarkContainerStart(b *testing.B) { + require.Build(b, osversion.RS5) + client := newTestRuntimeClient(b) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // + // new pod per container + // + b.Run("NewPod", func(b *testing.B) { + for _, tt := range _containerBenchmarkTests { + b.StopTimer() + b.ResetTimer() + b.Run(tt.Name, func(b *testing.B) { + requireFeatures(b, tt.Feature) + + switch tt.Feature { + case featureLCOW: + pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) + case featureWCOWHypervisor, featureWCOWProcess: + pullRequiredImages(b, []string{tt.Image}) + } + + sandboxRequest := getRunPodSandboxRequest(b, tt.Runtime, tt.SandboxOpts...) + for i := 0; i < b.N; i++ { + podID := runPodSandbox(b, client, ctx, sandboxRequest) + + request := getCreateContainerRequest(podID, b.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) + containerID := createContainer(b, client, ctx, request) + + b.StartTimer() + startContainer(b, client, ctx, containerID) + b.StopTimer() + + stopContainer(b, client, ctx, containerID) + removeContainer(b, client, ctx, containerID) + stopPodSandbox(b, client, ctx, podID) + removePodSandbox(b, client, ctx, podID) + } + }) + } + }) + + // + // same pod for containers + // + b.Run("SamePod", func(b *testing.B) { + for _, tt := range _containerBenchmarkTests { + b.StopTimer() + b.ResetTimer() + b.Run(tt.Name, func(b *testing.B) { + requireFeatures(b, tt.Feature) + + switch tt.Feature { + case featureLCOW: + pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) + case featureWCOWHypervisor, featureWCOWProcess: + pullRequiredImages(b, []string{tt.Image}) + } + sandboxRequest := getRunPodSandboxRequest(b, tt.Runtime, tt.SandboxOpts...) + podID := runPodSandbox(b, client, ctx, sandboxRequest) + + for i := 0; i < b.N; i++ { + request := getCreateContainerRequest(podID, b.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) + containerID := createContainer(b, client, ctx, request) + + b.StartTimer() + startContainer(b, client, ctx, containerID) + b.StopTimer() + + stopContainer(b, client, ctx, containerID) + removeContainer(b, client, ctx, containerID) + } + + stopPodSandbox(b, client, ctx, podID) + removePodSandbox(b, client, ctx, podID) + }) + } + }) +} diff --git a/test/cri-containerd/helper_container_test.go b/test/cri-containerd/helper_container_test.go index c129400a7c..6bae896eb9 100644 --- a/test/cri-containerd/helper_container_test.go +++ b/test/cri-containerd/helper_container_test.go @@ -13,48 +13,48 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) -func createContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.CreateContainerRequest) string { - t.Helper() +func createContainer(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.CreateContainerRequest) string { + tb.Helper() response, err := client.CreateContainer(ctx, request) if err != nil { - t.Fatalf("failed CreateContainer in sandbox: %s, with: %v", request.PodSandboxId, err) + tb.Fatalf("failed CreateContainer in sandbox: %s, with: %v", request.PodSandboxId, err) } return response.ContainerId } -func startContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() +func startContainer(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { + tb.Helper() _, err := client.StartContainer(ctx, &runtime.StartContainerRequest{ ContainerId: containerID, }) if err != nil { - t.Fatalf("failed StartContainer request for container: %s, with: %v", containerID, err) + tb.Fatalf("failed StartContainer request for container: %s, with: %v", containerID, err) } } -func stopContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() - stopContainerWithTimeout(t, client, ctx, containerID, 0) +func stopContainer(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { + tb.Helper() + stopContainerWithTimeout(tb, client, ctx, containerID, 0) } -func stopContainerWithTimeout(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string, timeout int64) { - t.Helper() +func stopContainerWithTimeout(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string, timeout int64) { + tb.Helper() _, err := client.StopContainer(ctx, &runtime.StopContainerRequest{ ContainerId: containerID, Timeout: timeout, }) if err != nil { - t.Fatalf("failed StopContainer request for container: %s, with: %v", containerID, err) + tb.Fatalf("failed StopContainer request for container: %s, with: %v", containerID, err) } } -func removeContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() +func removeContainer(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { + tb.Helper() _, err := client.RemoveContainer(ctx, &runtime.RemoveContainerRequest{ ContainerId: containerID, }) if err != nil { - t.Fatalf("failed RemoveContainer request for container: %s, with: %v", containerID, err) + tb.Fatalf("failed RemoveContainer request for container: %s, with: %v", containerID, err) } } @@ -74,25 +74,25 @@ func getCreateContainerRequest(podID string, name string, image string, command } } -func getContainerStatus(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) runtime.ContainerState { - t.Helper() - return getContainerStatusFull(t, client, ctx, containerID).State +func getContainerStatus(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) runtime.ContainerState { + tb.Helper() + return getContainerStatusFull(tb, client, ctx, containerID).State } -func assertContainerState(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string, state runtime.ContainerState) { - t.Helper() - if st := getContainerStatus(t, client, ctx, containerID); st != state { - t.Fatalf("got container %q state %q; wanted %v", containerID, st.String(), state.String()) +func assertContainerState(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string, state runtime.ContainerState) { + tb.Helper() + if st := getContainerStatus(tb, client, ctx, containerID); st != state { + tb.Fatalf("got container %q state %q; wanted %v", containerID, st.String(), state.String()) } } -func getContainerStatusFull(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) *runtime.ContainerStatus { - t.Helper() +func getContainerStatusFull(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) *runtime.ContainerStatus { + tb.Helper() response, err := client.ContainerStatus(ctx, &runtime.ContainerStatusRequest{ ContainerId: containerID, }) if err != nil { - t.Fatalf("failed ContainerStatus request for container: %s, with: %v", containerID, err) + tb.Fatalf("failed ContainerStatus request for container: %s, with: %v", containerID, err) } return response.Status } @@ -101,17 +101,17 @@ func getContainerStatusFull(t *testing.T, client runtime.RuntimeServiceClient, c // an error if the expected container state isn't reached within 30 seconds. func requireContainerState( ctx context.Context, - t *testing.T, + tb testing.TB, client runtime.RuntimeServiceClient, containerID string, expectedState runtime.ContainerState, ) { - t.Helper() - require.NoError(t, func() error { + tb.Helper() + require.NoError(tb, func() error { start := time.Now() var lastState runtime.ContainerState for { - lastState = getContainerStatus(t, client, ctx, containerID) + lastState = getContainerStatus(tb, client, ctx, containerID) if lastState == expectedState { return nil } diff --git a/test/cri-containerd/helper_cri_plugins_test.go b/test/cri-containerd/helper_cri_plugins_test.go index 0b64caeb1e..bf018dcf49 100644 --- a/test/cri-containerd/helper_cri_plugins_test.go +++ b/test/cri-containerd/helper_cri_plugins_test.go @@ -10,33 +10,33 @@ import ( cri "github.com/kevpar/cri/pkg/api/v1" ) -func newTestPluginClient(t *testing.T) cri.CRIPluginServiceClient { - t.Helper() +func newTestPluginClient(tb testing.TB) cri.CRIPluginServiceClient { + tb.Helper() ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) defer cancel() conn, err := createGRPCConn(ctx) if err != nil { - t.Fatalf("failed to dial runtime client: %v", err) + tb.Fatalf("failed to dial runtime client: %v", err) } return cri.NewCRIPluginServiceClient(conn) } -func resetContainer(t *testing.T, client cri.CRIPluginServiceClient, ctx context.Context, containerID string) { - t.Helper() +func resetContainer(tb testing.TB, client cri.CRIPluginServiceClient, ctx context.Context, containerID string) { + tb.Helper() _, err := client.ResetContainer(ctx, &cri.ResetContainerRequest{ ContainerId: containerID, }) if err != nil { - t.Fatalf("failed ResetContainer request for container: %s, with: %v", containerID, err) + tb.Fatalf("failed ResetContainer request for container: %s, with: %v", containerID, err) } } -func resetPodSandbox(t *testing.T, client cri.CRIPluginServiceClient, ctx context.Context, podID string) { - t.Helper() +func resetPodSandbox(tb testing.TB, client cri.CRIPluginServiceClient, ctx context.Context, podID string) { + tb.Helper() _, err := client.ResetPodSandbox(ctx, &cri.ResetPodSandboxRequest{ PodSandboxId: podID, }) if err != nil { - t.Fatalf("failed ResetPodSandbox request for container: %s, with: %v", podID, err) + tb.Fatalf("failed ResetPodSandbox request for container: %s, with: %v", podID, err) } } diff --git a/test/cri-containerd/helper_exec_test.go b/test/cri-containerd/helper_exec_test.go index a82c174169..f6bcb17f86 100644 --- a/test/cri-containerd/helper_exec_test.go +++ b/test/cri-containerd/helper_exec_test.go @@ -17,20 +17,20 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) -func execSync(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecSyncRequest) *runtime.ExecSyncResponse { - t.Helper() +func execSync(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecSyncRequest) *runtime.ExecSyncResponse { + tb.Helper() response, err := client.ExecSync(ctx, request) if err != nil { - t.Fatalf("failed ExecSync request with: %v", err) + tb.Fatalf("failed ExecSync request with: %v", err) } return response } -func execRequest(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecRequest) string { - t.Helper() +func execRequest(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecRequest) string { + tb.Helper() response, err := client.Exec(ctx, request) if err != nil { - t.Fatalf("failed Exec request with: %v", err) + tb.Fatalf("failed Exec request with: %v", err) } return response.Url } @@ -78,12 +78,12 @@ func execInHost(ctx context.Context, client shimdiag.ShimDiagService, args []str } // shimDiagExecOutput is a small wrapper on top of execInHost, that returns the exec output -func shimDiagExecOutput(ctx context.Context, t *testing.T, podID string, cmd []string) string { - t.Helper() +func shimDiagExecOutput(ctx context.Context, tb testing.TB, podID string, cmd []string) string { + tb.Helper() shimName := fmt.Sprintf("k8s.io-%s", podID) shim, err := shimdiag.GetShim(shimName) if err != nil { - t.Fatalf("failed to find shim %v: %v", shimName, err) + tb.Fatalf("failed to find shim %v: %v", shimName, err) } shimClient := shimdiag.NewShimDiagClient(shim) @@ -94,10 +94,10 @@ func shimDiagExecOutput(ctx context.Context, t *testing.T, podID string, cmd []s exitCode, err := execInHost(ctx, shimClient, cmd, nil, bw, bwErr) if err != nil { - t.Fatalf("failed to exec request in the host with: %v and %v", err, bufErr.String()) + tb.Fatalf("failed to exec request in the host with: %v and %v", err, bufErr.String()) } if exitCode != 0 { - t.Fatalf("exec request in host failed with exit code %v: %v", exitCode, bufErr.String()) + tb.Fatalf("exec request in host failed with exit code %v: %v", exitCode, bufErr.String()) } return strings.TrimSpace(bufOut.String()) diff --git a/test/cri-containerd/helper_gmsa_test.go b/test/cri-containerd/helper_gmsa_test.go index a07cac953c..ad49428042 100644 --- a/test/cri-containerd/helper_gmsa_test.go +++ b/test/cri-containerd/helper_gmsa_test.go @@ -30,15 +30,15 @@ func generateCredSpec(path string) error { // Tries to generate a cred spec to use for gmsa test cases. Returns the cred // spec and an error if any. -func gmsaSetup(t *testing.T) string { - t.Helper() - csPath := filepath.Join(t.TempDir(), "credspec.json") +func gmsaSetup(tb testing.TB) string { + tb.Helper() + csPath := filepath.Join(tb.TempDir(), "credspec.json") if err := generateCredSpec(csPath); err != nil { - t.Fatal(err) + tb.Fatal(err) } credSpec, err := os.ReadFile(csPath) if err != nil { - t.Fatalf("failed to read credential spec: %s", err) + tb.Fatalf("failed to read credential spec: %s", err) } return string(credSpec) } diff --git a/test/cri-containerd/helper_sandbox_test.go b/test/cri-containerd/helper_sandbox_test.go index 47f7a45dea..73b1d5ebf0 100644 --- a/test/cri-containerd/helper_sandbox_test.go +++ b/test/cri-containerd/helper_sandbox_test.go @@ -37,58 +37,58 @@ func WithSandboxLabels(labels map[string]string) SandboxConfigOpt { } } -func runPodSandbox(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.RunPodSandboxRequest) string { - t.Helper() +func runPodSandbox(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.RunPodSandboxRequest) string { + tb.Helper() response, err := client.RunPodSandbox(ctx, request) if err != nil { - t.Fatalf("failed RunPodSandbox request with: %v", err) + tb.Fatalf("failed RunPodSandbox request with: %v", err) } return response.PodSandboxId } -func stopPodSandbox(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podID string) { - t.Helper() +func stopPodSandbox(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string) { + tb.Helper() _, err := client.StopPodSandbox(ctx, &runtime.StopPodSandboxRequest{ PodSandboxId: podID, }) if err != nil { - t.Fatalf("failed StopPodSandbox for sandbox: %s, request with: %v", podID, err) + tb.Fatalf("failed StopPodSandbox for sandbox: %s, request with: %v", podID, err) } } -func removePodSandbox(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podID string) { - t.Helper() +func removePodSandbox(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string) { + tb.Helper() _, err := client.RemovePodSandbox(ctx, &runtime.RemovePodSandboxRequest{ PodSandboxId: podID, }) if err != nil { - t.Fatalf("failed RemovePodSandbox for sandbox: %s, request with: %v", podID, err) + tb.Fatalf("failed RemovePodSandbox for sandbox: %s, request with: %v", podID, err) } } -func getPodSandboxStatus(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podID string) *runtime.PodSandboxStatus { - t.Helper() +func getPodSandboxStatus(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string) *runtime.PodSandboxStatus { + tb.Helper() status, err := client.PodSandboxStatus(ctx, &runtime.PodSandboxStatusRequest{ PodSandboxId: podID, }) if err != nil { - t.Fatalf("failed PodSandboxStatus for sandbox: %s, request with: %v", podID, err) + tb.Fatalf("failed PodSandboxStatus for sandbox: %s, request with: %v", podID, err) } return status.Status } -func assertPodSandboxState(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podID string, state runtime.PodSandboxState) { - t.Helper() - if st := getPodSandboxStatus(t, client, ctx, podID).State; st != state { - t.Fatalf("got pod sandbox %q state %q; wanted %v", podID, st.String(), state.String()) +func assertPodSandboxState(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string, state runtime.PodSandboxState) { + tb.Helper() + if st := getPodSandboxStatus(tb, client, ctx, podID).State; st != state { + tb.Fatalf("got pod sandbox %q state %q; wanted %v", podID, st.String(), state.String()) } } -func getTestSandboxConfig(t *testing.T, opts ...SandboxConfigOpt) *runtime.PodSandboxConfig { - t.Helper() +func getTestSandboxConfig(tb testing.TB, opts ...SandboxConfigOpt) *runtime.PodSandboxConfig { + tb.Helper() c := &runtime.PodSandboxConfig{ Metadata: &runtime.PodSandboxMetadata{ - Name: t.Name(), + Name: tb.Name(), Namespace: testNamespace, }, } @@ -107,17 +107,17 @@ func getTestSandboxConfig(t *testing.T, opts ...SandboxConfigOpt) *runtime.PodSa for _, o := range opts { if err := o(c); err != nil { - t.Helper() - t.Fatalf("failed to apply PodSandboxConfig option: %s", err) + tb.Helper() + tb.Fatalf("failed to apply PodSandboxConfig option: %s", err) } } return c } -func getRunPodSandboxRequest(t *testing.T, runtimeHandler string, sandboxOpts ...SandboxConfigOpt) *runtime.RunPodSandboxRequest { - t.Helper() +func getRunPodSandboxRequest(tb testing.TB, runtimeHandler string, sandboxOpts ...SandboxConfigOpt) *runtime.RunPodSandboxRequest { + tb.Helper() return &runtime.RunPodSandboxRequest{ - Config: getTestSandboxConfig(t, sandboxOpts...), + Config: getTestSandboxConfig(tb, sandboxOpts...), RuntimeHandler: runtimeHandler, } } diff --git a/test/cri-containerd/helper_service_test.go b/test/cri-containerd/helper_service_test.go index 6c30c94a53..5ac7a34acd 100644 --- a/test/cri-containerd/helper_service_test.go +++ b/test/cri-containerd/helper_service_test.go @@ -79,16 +79,16 @@ func stopService(serviceName string) error { return nil } -func startContainerd(t *testing.T) { - t.Helper() +func startContainerd(tb testing.TB) { + tb.Helper() if err := startService(*flagContainerdServiceName); err != nil { - t.Fatal(err) + tb.Fatal(err) } } -func stopContainerd(t *testing.T) { - t.Helper() +func stopContainerd(tb testing.TB) { + tb.Helper() if err := stopService(*flagContainerdServiceName); err != nil { - t.Fatal(err) + tb.Fatal(err) } } diff --git a/test/cri-containerd/helper_update_utilities_test.go b/test/cri-containerd/helper_update_utilities_test.go index 60edf69df7..de4756fea6 100644 --- a/test/cri-containerd/helper_update_utilities_test.go +++ b/test/cri-containerd/helper_update_utilities_test.go @@ -28,33 +28,33 @@ func createJobObjectsGetUtilArgs(ctx context.Context, cid, toolPath string, opti return args } -func checkLCOWResourceLimit(t *testing.T, ctx context.Context, client runtime.RuntimeServiceClient, cid, path string, expected uint64) { - t.Helper() +func checkLCOWResourceLimit(tb testing.TB, ctx context.Context, client runtime.RuntimeServiceClient, cid, path string, expected uint64) { + tb.Helper() cmd := []string{"cat", path} containerExecReq := &runtime.ExecSyncRequest{ ContainerId: cid, Cmd: cmd, Timeout: 20, } - r := execSync(t, client, ctx, containerExecReq) + r := execSync(tb, client, ctx, containerExecReq) if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d to cat path: %s", r.ExitCode, r.Stderr) + tb.Fatalf("failed with exit code %d to cat path: %s", r.ExitCode, r.Stderr) } output := strings.TrimSpace(string(r.Stdout)) bytesActual, err := strconv.ParseUint(output, 10, 0) if err != nil { - t.Fatalf("could not parse output %s: %s", output, err) + tb.Fatalf("could not parse output %s: %s", output, err) } if bytesActual != expected { - t.Fatalf("expected to have a memory limit of %v, instead got %v", expected, bytesActual) + tb.Fatalf("expected to have a memory limit of %v, instead got %v", expected, bytesActual) } } -func checkWCOWResourceLimit(t *testing.T, ctx context.Context, runtimeHandler, shimName, cid, query string, expected uint64) { - t.Helper() +func checkWCOWResourceLimit(tb testing.TB, ctx context.Context, runtimeHandler, shimName, cid, query string, expected uint64) { + tb.Helper() shim, err := shimdiag.GetShim(shimName) if err != nil { - t.Fatalf("failed to find shim %v: %v", shimName, err) + tb.Fatalf("failed to find shim %v: %v", shimName, err) } shimClient := shimdiag.NewShimDiagClient(shim) @@ -65,7 +65,7 @@ func checkWCOWResourceLimit(t *testing.T, ctx context.Context, runtimeHandler, s } else { guestPath = podJobObjectUtilPath if err := shareInUVM(ctx, shimClient, testJobObjectUtilFilePath, guestPath, false); err != nil { - t.Fatalf("failed to share test utility into pod: %v", err) + tb.Fatalf("failed to share test utility into pod: %v", err) } } @@ -80,19 +80,19 @@ func checkWCOWResourceLimit(t *testing.T, ctx context.Context, runtimeHandler, s exitCode, err := execInHost(ctx, shimClient, args, nil, bw, bwErr) if err != nil { - t.Fatalf("failed to exec request in the host with: %v and %v", err, bufErr.String()) + tb.Fatalf("failed to exec request in the host with: %v and %v", err, bufErr.String()) } if exitCode != 0 { - t.Fatalf("exec request in host failed with exit code %v: %v", exitCode, bufErr.String()) + tb.Fatalf("exec request in host failed with exit code %v: %v", exitCode, bufErr.String()) } // validate the results value := strings.TrimSpace(strings.TrimPrefix(buf.String(), query+": ")) limitActual, err := strconv.ParseUint(value, 10, 0) if err != nil { - t.Fatalf("could not parse output %s: %s", buf.String(), err) + tb.Fatalf("could not parse output %s: %s", buf.String(), err) } if limitActual != expected { - t.Fatalf("expected to have a limit of %v, instead got %v", expected, limitActual) + tb.Fatalf("expected to have a limit of %v, instead got %v", expected, limitActual) } } diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index 333a719dd1..228ffb4e91 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -14,7 +14,6 @@ import ( "time" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/internal" "github.com/containerd/containerd" eventtypes "github.com/containerd/containerd/api/events" eventsapi "github.com/containerd/containerd/api/services/events/v1" @@ -26,6 +25,9 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "github.com/Microsoft/hcsshim/test/internal/constants" + testflag "github.com/Microsoft/hcsshim/test/internal/flag" + "github.com/Microsoft/hcsshim/test/internal/require" + _ "github.com/Microsoft/hcsshim/test/internal/manifest" ) @@ -33,6 +35,7 @@ const ( connectTimeout = time.Second * 10 testNamespace = "cri-containerd-test" + lcowRuntimeHandler = "runhcs-lcow" wcowProcessRuntimeHandler = "runhcs-wcow-process" wcowHypervisorRuntimeHandler = "runhcs-wcow-hypervisor" wcowHypervisor17763RuntimeHandler = "runhcs-wcow-hypervisor-17763" @@ -48,24 +51,27 @@ const ( testVMServiceAddress = "C:\\ContainerPlat\\vmservice.sock" testVMServiceBinary = "C:\\Containerplat\\vmservice.exe" - lcowRuntimeHandler = "runhcs-lcow" - imageLcowK8sPause = "mcr.microsoft.com/oss/kubernetes/pause:3.1" - imageLcowAlpine = "mcr.microsoft.com/mirror/docker/library/alpine:3.16" - imageLcowAlpineCoreDump = "cplatpublic.azurecr.io/stackoverflow-alpine:latest" - imageLcowCosmos = "cosmosarno/spark-master:2.4.1_2019-04-18_8e864ce" - imageLcowCustomUser = "cplatpublic.azurecr.io/linux_custom_user:latest" - imageWindowsProcessDump = "cplatpublic.azurecr.io/crashdump:latest" - imageWindowsArgsEscaped = "cplatpublic.azurecr.io/argsescaped:latest" - imageWindowsTimezone = "cplatpublic.azurecr.io/timezone:latest" - imageJobContainerHNS = "cplatpublic.azurecr.io/jobcontainer_hns:latest" - imageJobContainerETW = "cplatpublic.azurecr.io/jobcontainer_etw:latest" - imageJobContainerVHD = "cplatpublic.azurecr.io/jobcontainer_vhd:latest" - imageJobContainerCmdline = "cplatpublic.azurecr.io/jobcontainer_cmdline:latest" - imageJobContainerWorkDir = "cplatpublic.azurecr.io/jobcontainer_workdir:latest" - alpineAspNet = "mcr.microsoft.com/dotnet/core/aspnet:3.1-alpine3.11" - alpineAspnetUpgrade = "mcr.microsoft.com/dotnet/core/aspnet:3.1.2-alpine3.11" + imageLcowK8sPause = "mcr.microsoft.com/oss/kubernetes/pause:3.1" + imageLcowAlpine = "mcr.microsoft.com/mirror/docker/library/alpine:3.16" + imageLcowAlpineCoreDump = "cplatpublic.azurecr.io/stackoverflow-alpine:latest" + imageLcowCosmos = "cosmosarno/spark-master:2.4.1_2019-04-18_8e864ce" + imageLcowCustomUser = "cplatpublic.azurecr.io/linux_custom_user:latest" + alpineAspNet = "mcr.microsoft.com/dotnet/core/aspnet:3.1-alpine3.11" + alpineAspnetUpgrade = "mcr.microsoft.com/dotnet/core/aspnet:3.1.2-alpine3.11" + + imageWindowsProcessDump = "cplatpublic.azurecr.io/crashdump:latest" + imageWindowsArgsEscaped = "cplatpublic.azurecr.io/argsescaped:latest" + imageWindowsTimezone = "cplatpublic.azurecr.io/timezone:latest" + + imageJobContainerHNS = "cplatpublic.azurecr.io/jobcontainer_hns:latest" + imageJobContainerETW = "cplatpublic.azurecr.io/jobcontainer_etw:latest" + imageJobContainerVHD = "cplatpublic.azurecr.io/jobcontainer_vhd:latest" + imageJobContainerCmdline = "cplatpublic.azurecr.io/jobcontainer_cmdline:latest" + imageJobContainerWorkDir = "cplatpublic.azurecr.io/jobcontainer_workdir:latest" + gracefulTerminationServercore = "cplatpublic.azurecr.io/servercore-gracefultermination-repro:latest" gracefulTerminationNanoserver = "cplatpublic.azurecr.io/nanoserver-gracefultermination-repro:latest" + // Default account name for use with GMSA related tests. This will not be // present/you will not have access to the account on your machine unless // your environment is configured properly. @@ -88,7 +94,7 @@ var ( // Flags var ( - flagFeatures = testutilities.NewStringSetFlag() + flagFeatures = testflag.NewFeatureFlag(allFeatures) flagCRIEndpoint = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.") flagVirtstack = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers") flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service") @@ -125,14 +131,6 @@ var allFeatures = []string{ featureCRIPlugin, } -func init() { - // Flag definitions must be in init rather than TestMain, as TestMain isn't - // called if -help is passed, but we want the feature usage to show up. - flag.Var(flagFeatures, "feature", fmt.Sprintf( - "specifies which sets of functionality to test. can be set multiple times\n"+ - "supported features: %v", allFeatures)) -} - func TestMain(m *testing.M) { flag.Parse() os.Exit(m.Run()) @@ -141,33 +139,25 @@ func TestMain(m *testing.M) { // requireFeatures checks in flagFeatures to validate that each required feature // was enabled, and skips the test if any are missing. If the flagFeatures set // is empty, the function returns (by default all features are enabled). -func requireFeatures(t *testing.T, features ...string) { - t.Helper() - set := flagFeatures.ValueSet() - if len(set) == 0 { - return - } - for _, feature := range features { - if _, ok := set[feature]; !ok { - t.Skipf("skipping test due to feature not set: %s", feature) - } - } +func requireFeatures(tb testing.TB, features ...string) { + tb.Helper() + require.Features(tb, flagFeatures, features...) } // requireBinary checks if `binary` exists in the same directory as the test // binary. // Returns full binary path if it exists, otherwise, skips the test. -func requireBinary(t *testing.T, binary string) string { - t.Helper() +func requireBinary(tb testing.TB, binary string) string { + tb.Helper() executable, err := os.Executable() if err != nil { - t.Skipf("error locating executable: %s", err) + tb.Skipf("error locating executable: %s", err) return "" } baseDir := filepath.Dir(executable) binaryPath := filepath.Join(baseDir, binary) if _, err := os.Stat(binaryPath); os.IsNotExist(err) { - t.Skipf("binary not found: %s", binaryPath) + tb.Skipf("binary not found: %s", binaryPath) return "" } return binaryPath @@ -198,35 +188,35 @@ func createGRPCConn(ctx context.Context) (*grpc.ClientConn, error) { return grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithContextDialer(dialer)) } -func newTestRuntimeClient(t *testing.T) runtime.RuntimeServiceClient { - t.Helper() +func newTestRuntimeClient(tb testing.TB) runtime.RuntimeServiceClient { + tb.Helper() ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) defer cancel() conn, err := createGRPCConn(ctx) if err != nil { - t.Fatalf("failed to dial runtime client: %v", err) + tb.Fatalf("failed to dial runtime client: %v", err) } return runtime.NewRuntimeServiceClient(conn) } -func newTestEventService(t *testing.T) containerd.EventService { - t.Helper() +func newTestEventService(tb testing.TB) containerd.EventService { + tb.Helper() ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) defer cancel() conn, err := createGRPCConn(ctx) if err != nil { - t.Fatalf("Failed to create a client connection %v", err) + tb.Fatalf("Failed to create a client connection %v", err) } return containerd.NewEventServiceFromClient(eventsapi.NewEventsClient(conn)) } -func newTestImageClient(t *testing.T) runtime.ImageServiceClient { - t.Helper() +func newTestImageClient(tb testing.TB) runtime.ImageServiceClient { + tb.Helper() ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) defer cancel() conn, err := createGRPCConn(ctx) if err != nil { - t.Fatalf("failed to dial runtime client: %v", err) + tb.Fatalf("failed to dial runtime client: %v", err) } return runtime.NewImageServiceClient(conn) } @@ -269,36 +259,36 @@ func convertEvent(e *types.Any) (string, interface{}, error) { return id, evt, nil } -func pullRequiredImages(t *testing.T, images []string, opts ...SandboxConfigOpt) { - t.Helper() +func pullRequiredImages(tb testing.TB, images []string, opts ...SandboxConfigOpt) { + tb.Helper() opts = append(opts, WithSandboxLabels(map[string]string{ "sandbox-platform": "windows/amd64", // Not required for Windows but makes the test safer depending on defaults in the config. })) - pullRequiredImagesWithOptions(t, images, opts...) + pullRequiredImagesWithOptions(tb, images, opts...) } -func pullRequiredLCOWImages(t *testing.T, images []string, opts ...SandboxConfigOpt) { - t.Helper() +func pullRequiredLCOWImages(tb testing.TB, images []string, opts ...SandboxConfigOpt) { + tb.Helper() opts = append(opts, WithSandboxLabels(map[string]string{ "sandbox-platform": "linux/amd64", })) - pullRequiredImagesWithOptions(t, images, opts...) + pullRequiredImagesWithOptions(tb, images, opts...) } -func pullRequiredImagesWithOptions(t *testing.T, images []string, opts ...SandboxConfigOpt) { - t.Helper() +func pullRequiredImagesWithOptions(tb testing.TB, images []string, opts ...SandboxConfigOpt) { + tb.Helper() if len(images) < 1 { return } - client := newTestImageClient(t) + client := newTestImageClient(tb) ctx, cancel := context.WithCancel(context.Background()) defer cancel() sb := &runtime.PodSandboxConfig{} for _, o := range opts { if err := o(sb); err != nil { - t.Fatalf("failed to apply PodSandboxConfig option: %s", err) + tb.Fatalf("failed to apply PodSandboxConfig option: %s", err) } } @@ -310,18 +300,18 @@ func pullRequiredImagesWithOptions(t *testing.T, images []string, opts ...Sandbo SandboxConfig: sb, }) if err != nil { - t.Fatalf("failed PullImage for image: %s, with error: %v", image, err) + tb.Fatalf("failed PullImage for image: %s, with error: %v", image, err) } } } -func removeImages(t *testing.T, images []string) { - t.Helper() +func removeImages(tb testing.TB, images []string) { + tb.Helper() if len(images) < 1 { return } - client := newTestImageClient(t) + client := newTestImageClient(tb) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -332,7 +322,7 @@ func removeImages(t *testing.T, images []string) { }, }) if err != nil { - t.Fatalf("failed removeImage for image: %s, with error: %v", image, err) + tb.Fatalf("failed removeImage for image: %s, with error: %v", image, err) } } } diff --git a/test/functional/main_test.go b/test/functional/main_test.go index 877b9c6cdd..b05a2dafd2 100644 --- a/test/functional/main_test.go +++ b/test/functional/main_test.go @@ -90,13 +90,13 @@ var ( flagFeatures = testflag.NewFeatureFlag(allFeatures) flagContainerdNamespace = flag.String("ctr-namespace", hcsOwner, "containerd `namespace` to use when creating OCI specs") - flagLCOWLayerPaths = testflag.NewStringSlice("lcow-layer-paths", + flagLCOWLayerPaths = testflag.NewStringSet("lcow-layer-paths", "comma separated list of image layer `paths` to use as LCOW container rootfs. "+ - "If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.") + "If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.", true) //nolint:unused // will be used when WCOW tests are updated - flagWCOWLayerPaths = testflag.NewStringSlice("wcow-layer-paths", + flagWCOWLayerPaths = testflag.NewStringSet("wcow-layer-paths", "comma separated list of image layer `paths` to use as WCOW uVM and container rootfs. "+ - "If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.") + "If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.", true) flagLayerTempDir = flag.String("layer-temp-dir", "", "`directory` to unpack image layers to, if not provided. Leave empty to use os.TempDir.") flagLinuxBootFilesPath = flag.String("linux-bootfiles", "", @@ -133,7 +133,7 @@ func TestMain(m *testing.M) { logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.SetLevel(flagLogLevel.Level) - logrus.Debugf("using features %q", flagFeatures.S.Strings()) + logrus.Debugf("using features: %s", flagFeatures.Strings()) images := []*layers.LazyImageLayers{alpineImagePaths, nanoserverImagePaths, servercoreImagePaths} for _, l := range images { @@ -179,7 +179,7 @@ func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptio func requireFeatures(tb testing.TB, features ...string) { tb.Helper() - require.Features(tb, flagFeatures.S, features...) + require.Features(tb, flagFeatures, features...) } func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW { @@ -202,7 +202,7 @@ func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW { // Otherwise, it pulls an appropriate image. func linuxImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() - if ss := flagLCOWLayerPaths.S.Strings(); len(ss) > 0 { + if ss := flagLCOWLayerPaths.Strings(); len(ss) > 0 { return ss } return alpineImagePaths.Layers(ctx, tb) @@ -215,7 +215,7 @@ func linuxImageLayers(ctx context.Context, tb testing.TB) []string { //nolint:deadcode,unused // will be used when WCOW tests are updated func windowsImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() - if ss := flagWCOWLayerPaths.S.Strings(); len(ss) > 0 { + if ss := flagWCOWLayerPaths.Strings(); len(ss) > 0 { return ss } return nanoserverImagePaths.Layers(ctx, tb) diff --git a/test/gcs/main_test.go b/test/gcs/main_test.go index 593612e3d9..bc466a64a3 100644 --- a/test/gcs/main_test.go +++ b/test/gcs/main_test.go @@ -86,7 +86,7 @@ func setup() (err error) { // test2json does not consume stderr logrus.SetOutput(os.Stdout) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) - logrus.Infof("using features %q", flagFeatures.S.Strings()) + logrus.Debugf("using features: %s", flagFeatures.Strings()) // should already start in gcs cgroup if !*flagJoinGCSCgroup { @@ -171,5 +171,5 @@ func getTransport() transport.Transport { func requireFeatures(tb testing.TB, features ...string) { tb.Helper() - require.Features(tb, flagFeatures.S, features...) + require.Features(tb, flagFeatures, features...) } diff --git a/test/internal/flag/flag.go b/test/internal/flag/flag.go index 37e8423202..b76c6a1be8 100644 --- a/test/internal/flag/flag.go +++ b/test/internal/flag/flag.go @@ -12,69 +12,73 @@ import ( const FeatureFlagName = "feature" -func NewFeatureFlag(all []string) *StringSlice { - return NewStringSlice(FeatureFlagName, - "the sets of functionality to test; can be set multiple times, or separated with commas. "+ - "Supported features: "+strings.Join(all, ", "), +func NewFeatureFlag(all []string) *StringSet { + return NewStringSet(FeatureFlagName, + "set of `features` to test; can be set multiple times, with a comma-separated list, or both "+ + "(supported features: "+strings.Join(all, ", ")+")", + false, ) } -// StringSlice is a type to be used with the standard library's flag.Var -// function as a custom flag value, similar to "github.com/urfave/cli".StringSlice. +// StringSet is a type to be used with the standard library's flag.Var +// function as a custom flag value, similar to "github.com/urfave/cli".StringSet, +// but it only tracks unique instances. // It takes either a comma-separated list of strings, or repeated invocations. -type StringSlice struct { - S StringSet +type StringSet struct { + s map[string]struct{} + // cs indicates if the set is case sensitive or not + cs bool } -var _ flag.Value = &StringSlice{} +var _ flag.Value = &StringSet{} -// NewStringSetFlag returns a new StringSetFlag with an empty set. -func NewStringSlice(name, usage string) *StringSlice { - ss := &StringSlice{ - S: make(StringSet), +// NewStringSet returns a new StringSetFlag with an empty set. +func NewStringSet(name, usage string, caseSensitive bool) *StringSet { + ss := &StringSet{ + s: make(map[string]struct{}), + cs: caseSensitive, } flag.Var(ss, name, usage) return ss } // Strings returns a string slice of the flags provided to the flag -func (ss *StringSlice) Strings() []string { - return ss.S.Strings() +func (ss *StringSet) Strings() []string { + a := make([]string, 0, len(ss.s)) + for k := range ss.s { + a = append(a, k) + } + + return a +} + +func (ss *StringSet) String() string { + return "[" + strings.Join(ss.Strings(), ", ") + "]" } -func (ss *StringSlice) String() string { - return ss.S.String() +func (ss *StringSet) Len() int { return len(ss.s) } + +func (ss *StringSet) IsSet(s string) bool { + _, ok := ss.s[ss.standardize(s)] + return ok } // Set is called by `flag` each time the flag is seen when parsing the // command line. -func (ss *StringSlice) Set(s string) error { +func (ss *StringSet) Set(s string) error { for _, f := range strings.Split(s, ",") { - f = Standardize(f) - ss.S[f] = struct{}{} + ss.s[ss.standardize(f)] = struct{}{} } - return nil } -type StringSet map[string]struct{} - -func (ss StringSet) Strings() []string { - a := make([]string, 0, len(ss)) - for k := range ss { - a = append(a, k) - } - - return a -} - -func (ss StringSet) String() string { - return "[" + strings.Join(ss.Strings(), ", ") + "]" -} - // Standardize formats the feature flag s to be consistent (ie, trim and to lowercase) -func Standardize(s string) string { - return strings.ToLower(strings.TrimSpace(s)) +func (ss *StringSet) standardize(s string) string { + s = strings.TrimSpace(s) + if !ss.cs { + s = strings.ToLower(s) + } + return s } // LogrusLevel is a flag that accepts logrus logging levels, as strings. diff --git a/test/internal/require/requires.go b/test/internal/require/requires.go index bfb14f0315..3bf77b54cf 100644 --- a/test/internal/require/requires.go +++ b/test/internal/require/requires.go @@ -11,14 +11,13 @@ import ( // Features checks the wanted features are present in given, // and skips the test if any are missing. If the given set is empty, // the function returns (by default all features are enabled). -func Features(tb testing.TB, given flag.StringSet, want ...string) { +func Features(tb testing.TB, given *flag.StringSet, want ...string) { tb.Helper() - if len(given) == 0 { + if given.Len() == 0 { return } for _, f := range want { - ff := flag.Standardize(f) - if _, ok := given[ff]; !ok { + if !given.IsSet(f) { tb.Skipf("skipping test due to feature not set: %s", f) } } diff --git a/test/internal/stringsetflag.go b/test/internal/stringsetflag.go deleted file mode 100644 index 561f4ed275..0000000000 --- a/test/internal/stringsetflag.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build windows - -package internal - -import "github.com/Microsoft/hcsshim/test/internal/flag" - -// StringSetFlag is a type to be used with the standard library's flag.Var -// function as a custom flag value. It accumulates the arguments passed each -// time the flag is used into a set. -type StringSetFlag struct { - set flag.StringSet -} - -// NewStringSetFlag returns a new StringSetFlag with an empty set. -func NewStringSetFlag() StringSetFlag { - return StringSetFlag{ - set: make(map[string]struct{}), - } -} - -func (ssf StringSetFlag) String() string { - b := "[" - i := 0 - for k := range ssf.set { - if i > 0 { - b = b + ", " - } - b = b + k - i++ - } - b = b + "]" - return b -} - -// Set is called by `flag` each time the flag is seen when parsing the -// command line. -func (ssf StringSetFlag) Set(s string) error { - ssf.set[s] = struct{}{} - return nil -} - -// ValueSet returns the internal set of what values have been seen. -func (ssf StringSetFlag) ValueSet() flag.StringSet { - return ssf.set -}