Add new flags to just setup, teardown or run tests only

.buildkite/scripts/integration_tests.sh

@@ -38,6 +38,7 @@ usage() {
 PARALLEL_TARGET="test-check-packages-parallel"
 FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
+SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
 TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
@@ -115,7 +116,7 @@ with_docker
 echo "--- install docker-compose plugin"
 with_docker_compose_plugin
 
-if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
+if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
     echo "--- install kubectl & kind"
     with_kubernetes
 fi

docs/howto/system_testing.md

@@ -601,7 +601,6 @@ A sample workflow would look like:
 
 ### Running system tests without cleanup (technical preview)
 
-**Disclaimer**: just tested with service deployer `docker`.
 
 By default, `elastic-package test system` command always performs these steps to run tests for a given package:
 1. Setup:
@@ -634,6 +633,8 @@ This process is repeated for each combination of:
 It's possible also to run these steps independently. For that it is required to set which configuration file (`--config-file`)
 and which variant (`--variant`), if any, is going to be used to start and configure the service for these tests.
 
+**NOTE**: Currently, there is just support for packages using the following service deployers: `docker`, `agent` (custom agents) and `k8s`.
+
 Then, each step can be run using one of these flags:
 - Run the setup (`--setup`), after this command is executed:
     - Service container will be kept running.

internal/servicedeployer/factory.go

@@ -53,7 +53,15 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 	switch serviceDeployerName {
 	case "k8s":
 		if _, err := os.Stat(serviceDeployerPath); err == nil {
-			return NewKubernetesServiceDeployer(options.Profile, serviceDeployerPath, options.StackVersion)
+			opts := KubernetesServiceDeployerOptions{
+				Profile:        options.Profile,
+				DefinitionsDir: serviceDeployerPath,
+				StackVersion:   options.StackVersion,
+				RunSetup:       options.RunSetup,
+				RunTestsOnly:   options.RunTestsOnly,
+				RunTearDown:    options.RunTearDown,
+			}
+			return NewKubernetesServiceDeployer(opts)
 		}
 	case "docker":
 		dockerComposeYMLPath := filepath.Join(serviceDeployerPath, "docker-compose.yml")
@@ -88,6 +96,9 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 		}
 		return NewCustomAgentDeployer(opts)
 	case "tf":
+		if options.RunSetup || options.RunTearDown || options.RunTestsOnly {
+			return nil, errors.New("terraform service deployer not supported to run by steps")
+		}
 		if _, err := os.Stat(serviceDeployerPath); err == nil {
 			return NewTerraformServiceDeployer(serviceDeployerPath)
 		}

internal/servicedeployer/kubernetes.go

@@ -27,6 +27,20 @@ type KubernetesServiceDeployer struct {
 	profile        *profile.Profile
 	definitionsDir string
 	stackVersion   string
+
+	runSetup     bool
+	runTestsOnly bool
+	runTearDown  bool
+}
+
+type KubernetesServiceDeployerOptions struct {
+	Profile        *profile.Profile
+	DefinitionsDir string
+	StackVersion   string
+
+	RunSetup     bool
+	RunTestsOnly bool
+	RunTearDown  bool
 }
 
 type kubernetesDeployedService struct {
@@ -75,11 +89,14 @@ func (s *kubernetesDeployedService) SetContext(sc ServiceContext) error {
 var _ DeployedService = new(kubernetesDeployedService)
 
 // NewKubernetesServiceDeployer function creates a new instance of KubernetesServiceDeployer.
-func NewKubernetesServiceDeployer(profile *profile.Profile, definitionsPath string, stackVersion string) (*KubernetesServiceDeployer, error) {
+func NewKubernetesServiceDeployer(opts KubernetesServiceDeployerOptions) (*KubernetesServiceDeployer, error) {
 	return &KubernetesServiceDeployer{
-		profile:        profile,
-		definitionsDir: definitionsPath,
-		stackVersion:   stackVersion,
+		profile:        opts.Profile,
+		definitionsDir: opts.DefinitionsDir,
+		stackVersion:   opts.StackVersion,
+		runSetup:       opts.RunSetup,
+		runTestsOnly:   opts.RunTestsOnly,
+		runTearDown:    opts.RunTearDown,
 	}, nil
 }
 
@@ -91,19 +108,29 @@ func (ksd KubernetesServiceDeployer) SetUp(ctxt ServiceContext) (DeployedService
 		return nil, fmt.Errorf("kind context verification failed: %w", err)
 	}
 
-	err = kind.ConnectToElasticStackNetwork(ksd.profile)
-	if err != nil {
-		return nil, fmt.Errorf("can't connect control plane to Elastic stack network: %w", err)
+	if ksd.runTearDown || ksd.runTestsOnly {
+		logger.Debug("Skip connect kind to Elastic stack network")
+	} else {
+		err = kind.ConnectToElasticStackNetwork(ksd.profile)
+		if err != nil {
+			return nil, fmt.Errorf("can't connect control plane to Elastic stack network: %w", err)
+		}
 	}
 
-	err = installElasticAgentInCluster(ksd.profile, ksd.stackVersion)
-	if err != nil {
-		return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err)
+	if ksd.runTearDown || ksd.runTestsOnly {
+		logger.Debug("Skip install Elastic Agent in cluster")
+	} else {
+		err = installElasticAgentInCluster(ksd.profile, ksd.stackVersion)
+		if err != nil {
+			return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err)
+		}
 	}
 
-	err = ksd.installCustomDefinitions()
-	if err != nil {
-		return nil, fmt.Errorf("can't install custom definitions in the Kubernetes cluster: %w", err)
+	if !ksd.runTearDown {
+		err = ksd.installCustomDefinitions()
+		if err != nil {
+			return nil, fmt.Errorf("can't install custom definitions in the Kubernetes cluster: %w", err)
+		}
 	}
 
 	ctxt.Name = kind.ControlPlaneContainerName

internal/testrunner/runners/system/runner.go

@@ -106,7 +106,6 @@ type runner struct {
 	stackVersion    kibana.VersionInfo
 	locationManager *locations.LocationManager
 
-	serviceStateFolder   string
 	serviceStateFilePath string
 
 	// Execution order of following handlers is defined in runner.TearDown() method.
@@ -365,8 +364,7 @@ func (r *runner) initRun() error {
 	if err != nil {
 		return fmt.Errorf("reading service logs directory failed: %w", err)
 	}
-	r.serviceStateFolder = testrunner.StateFolderPath(r.options.Profile.ProfilePath)
-	r.serviceStateFilePath = filepath.Join(r.serviceStateFolder, testrunner.ServiceStateFileName)
+	r.serviceStateFilePath = filepath.Join(testrunner.StateFolderPath(r.options.Profile.ProfilePath), testrunner.ServiceStateFileName)
 
 	r.dataStreamPath, found, err = packages.FindDataStreamRootForPath(r.options.TestFolder.Path)
 	if err != nil {
@@ -995,7 +993,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic
 }
 
 func (r *runner) removeServiceStateDir() error {
-	dirPath := r.serviceStateFolder
+	dirPath := filepath.Dir(r.serviceStateFilePath)
 	err := os.RemoveAll(dirPath)
 	if err != nil {
 		return fmt.Errorf("failed to remove directory %q: %w", dirPath, err)
@@ -1004,7 +1002,7 @@ func (r *runner) removeServiceStateDir() error {
 }
 
 func (r *runner) createServiceStateDir() error {
-	dirPath := r.serviceStateFolder
+	dirPath := filepath.Dir(r.serviceStateFilePath)
 	err := os.MkdirAll(dirPath, 0755)
 	if err != nil {
 		return fmt.Errorf("mkdir failed (path: %s): %w", dirPath, err)