Clean elastic-package output

.buildkite/pipeline.trigger.integration.tests.sh

@@ -9,6 +9,7 @@ echo "  - group: \":terminal: Integration test suite\""
 echo "    key: \"integration-tests\""
 echo "    steps:"
 
+
 # Integration tests related stack command
 STACK_COMMAND_TESTS=(
     test-stack-command-default
@@ -20,13 +21,16 @@ STACK_COMMAND_TESTS=(
 
 for test in ${STACK_COMMAND_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t ${test}"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
     echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
     echo "          - build/elastic-stack-status/*/*"
+    if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+        echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+    fi
 done
 
 CHECK_PACKAGES_TESTS=(
@@ -37,13 +41,16 @@ CHECK_PACKAGES_TESTS=(
 )
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t ${test}"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/check-*/logs/*.log"
     echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
+    if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+        echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+    fi
     if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
     fi
@@ -54,40 +61,56 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
     echo "      - label: \":go: Running integration test: ${package_name}\""
     echo "        key: \"integration-parallel-${package_name}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
     echo "        env:"
     echo "          UPLOAD_SAFE_LOGS: 1"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
+    if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+        echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+    fi
 done
 
 popd > /dev/null
 
 echo "      - label: \":go: Running integration test: test-build-zip\""
-echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
+echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/build-zip/logs/*.log"
 echo "          - build/packages/*.sig"
+if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+    echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+fi
 
 echo "      - label: \":go: Running integration test: test-install-zip\""
-echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip"
+echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t test-install-zip"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip/logs/*.log"
+if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+    echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+fi
 
 echo "      - label: \":go: Running integration test: test-install-zip-shellinit\""
-echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip-shellinit"
+echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t test-install-zip-shellinit"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"
+if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+    echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+fi
 
 echo "      - label: \":go: Running integration test: test-profiles-command\""
-echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
+echo "        command: bash ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
 echo "        agents:"
 echo "          provider: \"gcp\""
+if [ "x${CI_DEBUG_LOG_FOLDER_PATH}" != "x" ]; then
+    echo "        artifact_paths:"
+    echo "          - ${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/**/*.log"
+fi

.buildkite/pipeline.yml

@@ -2,8 +2,11 @@ env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'
+  CI_DEBUG_LOG_FOLDER_PATH: './build'
+  CI_DEBUG_LOG_PREFIX: 'elastic-package-debug-output'
 
 steps:
   - label: ":go: Run check-static"
@@ -55,6 +58,11 @@ steps:
     agents:
       provider: "gcp"  # junit plugin requires docker
 
+  - label: ":mag_right: Check for errors"
+    command: "bash .buildkite/scripts/check_for_errors.sh"
+    agents:
+      provider: "gcp"
+
   - label: ":github: Release"
     key: "release"
     if: |

.buildkite/scripts/check_for_errors.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -euo pipefail
+
+
+mkdir -p build
+buildkite-agent artifact download "build/output-logs/*" build/
+
+for package_type in $(ls build/output-logs/); do
+    for output_file in $(ls build/output-logs/${package_type}); do
+        output="build/output-logs/${package_type}/${output_file}"
+        echo "Any error on ${output}?"
+
+        if grep -E "Error:" ${output}; then
+            echo "Found errors":
+
+            errors=$(grep -E "Error:" ${output})
+            echo "${errors}"
+
+            cat <<EOF >> markdown.md
+- Error found in ${package_type}
+  > ${errors}
+EOF
+        fi
+    done
+done
+
+if [ -f markdown.md ]; then
+    cat markdown.md | buildkite-agent annotate --style error
+fi

internal/compose/compose.go

@@ -31,15 +31,19 @@ const (
 	waitForHealthyInterval = 1 * time.Second
 )
 
-var DisableANSIComposeEnv = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_ANSI")
+var (
+	DisableANSIComposeEnv             = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_ANSI")
+	DisablePullProgressInformationEnv = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION")
+)
 
 // Project represents a Docker Compose project.
 type Project struct {
 	name             string
 	composeFilePaths []string
 
-	dockerComposeV1 bool
-	disableANSI     bool
+	dockerComposeV1                bool
+	disableANSI                    bool
+	disablePullProgressInformation bool
 }
 
 // Config represents a Docker Compose configuration file.
@@ -192,14 +196,19 @@ func NewProject(name string, paths ...string) (*Project, error) {
 		c.disableANSI = true
 	}
 
+	v, ok = os.LookupEnv(DisablePullProgressInformationEnv)
+	if ok && strings.ToLower(v) != "false" {
+		c.disablePullProgressInformation = true
+	}
+
 	return &c, nil
 }
 
 // Up brings up a Docker Compose project.
 func (p *Project) Up(opts CommandOptions) error {
 	args := p.baseArgs()
 	args = append(args, "up")
-	if p.disableANSI {
+	if p.disablePullProgressInformation {
 		args = append(args, "--quiet-pull")
 	}
 	args = append(args, opts.ExtraArgs...)
@@ -277,6 +286,9 @@ func (p *Project) Config(opts CommandOptions) (*Config, error) {
 func (p *Project) Pull(opts CommandOptions) error {
 	args := p.baseArgs()
 	args = append(args, "pull")
+	if p.disablePullProgressInformation {
+		args = append(args, "--quiet")
+	}
 	args = append(args, opts.ExtraArgs...)
 	args = append(args, opts.Services...)
 

internal/stack/compose.go

@@ -13,7 +13,6 @@ import (
 	"github.com/elastic/elastic-package/internal/compose"
 	"github.com/elastic/elastic-package/internal/docker"
 	"github.com/elastic/elastic-package/internal/install"
-	"github.com/elastic/elastic-package/internal/logger"
 )
 
 type ServiceStatus struct {
@@ -207,7 +206,6 @@ func dockerComposeStatus() ([]ServiceStatus, error) {
 		if err != nil {
 			return nil, err
 		}
-		logger.Debugf("Adding Service: \"%v\"", service.Name)
 		services = append(services, *service)
 	}
 

scripts/test-check-packages.sh

@@ -2,11 +2,31 @@
 
 set -euxo pipefail
 
+DEFAULT_DEBUG_LOG_FILE=elastic-package-debug-output.log
+
+run_elastic_package_command() {
+    local command="elastic-package $@"
+    if [ "x${CI_DEBUG_LOG_FOLDER_PATH:-}" != "x" ]; then
+        local full_path="${OLDPWD}/${CI_DEBUG_LOG_FOLDER_PATH}/output-logs/${PACKAGE_TEST_TYPE:-other}"
+        if [ -n "${PACKAGE_UNDER_TEST:-}" ]; then
+            full_path="${full_path}-${PACKAGE_UNDER_TEST}"
+        fi
+        full_path="${full_path}/${CI_DEBUG_LOG_FILE_PATH:-$DEFAULT_DEBUG_LOG_FILE}"
+
+        local folder=$(dirname ${full_path})
+        mkdir -p ${folder}
+
+        ${command} 2>&1 | tee -a ${full_path}
+    else
+        ${command}
+    fi
+}
+
 cleanup() {
   r=$?
 
   # Dump stack logs
-  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}"
+  run_elastic_package_command stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}"
 
   if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then
     # Dump kubectl details
@@ -20,13 +40,13 @@ cleanup() {
   fi
 
   # Take down the stack
-  elastic-package stack down -v
+  run_elastic_package_command stack down -v
 
   # Clean used resources
   for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
     (
       cd $d
-      elastic-package clean -v
+      run_elastic_package_command clean -v
     )
   done
 
@@ -42,18 +62,18 @@ OLDPWD=$PWD
 for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
-    elastic-package check -v
+    run_elastic_package_command check -v
   )
 done
 cd -
 
 # Update the stack
-elastic-package stack update -v
+run_elastic_package_command stack update -v
 
 # Boot up the stack
-elastic-package stack up -d -v
+run_elastic_package_command stack up -d -v
 
-elastic-package stack status
+run_elastic_package_command stack status
 
 if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then
   # Boot up the kind cluster
@@ -66,30 +86,33 @@ eval "$(elastic-package stack shellinit)"
 for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
-    elastic-package install -v
+
+    run_elastic_package_command install -v
 
     if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then
+      # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop
       if [ "${PACKAGE_UNDER_TEST:-*}" == "pipeline_benchmark" ]; then
         rm -rf "${OLDPWD}/build/benchmark-results"
-        elastic-package benchmark pipeline -v --report-format xUnit --report-output file --fail-on-missing
-        
+        run_elastic_package_command benchmark pipeline -v --report-format xUnit --report-output file --fail-on-missing
+
         rm -rf "${OLDPWD}/build/benchmark-results-old"
         mv "${OLDPWD}/build/benchmark-results" "${OLDPWD}/build/benchmark-results-old"
-        
-        elastic-package benchmark pipeline -v --report-format json --report-output file --fail-on-missing
-        
-        elastic-package report --fail-on-missing benchmark \
+
+        run_elastic_package_command benchmark pipeline -v --report-format json --report-output file --fail-on-missing
+
+        run_elastic_package_command report --fail-on-missing benchmark \
           --new ${OLDPWD}/build/benchmark-results \
           --old ${OLDPWD}/build/benchmark-results-old \
           --threshold 1 --report-output-path="${OLDPWD}/build/benchreport"
       fi
       if [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then
-        elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
+        run_elastic_package_command benchmark system --benchmark logs-benchmark -v
       fi
     else
       # defer-cleanup is set to a short period to verify that the option is available
-      elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage
+      run_elastic_package_command test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage
     fi
   )
 cd -
 done
+