From 614e2955cba1c040676c58a9fb3852d48bbbdefb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 17:32:55 +0100 Subject: [PATCH 01/19] Use pytest instead of nosetests Nosetests has been in maintenance mode for years, its last release was released in 2015. When evolving the testing framework for Python tests it would be better to do it based on a maintained test tool. --- auditbeat/magefile.go | 2 +- dev-tools/mage/check.go | 15 ++++++----- dev-tools/mage/pytest.go | 26 ++++++++++---------- dev-tools/mage/target/integtest/integtest.go | 2 +- dev-tools/mage/target/unittest/unittest.go | 2 +- docs/devguide/metricset-details.asciidoc | 2 +- docs/devguide/modules-dev-guide.asciidoc | 2 +- docs/devguide/python.asciidoc | 4 +-- docs/devguide/testing.asciidoc | 5 ++-- filebeat/magefile.go | 4 +-- libbeat/scripts/Makefile | 8 +++--- libbeat/tests/system/requirements.txt | 3 +-- metricbeat/Makefile | 2 +- packetbeat/SUPPORT_PROTOCOL.md | 2 +- packetbeat/magefile.go | 2 +- packetbeat/tests/system/README.md | 4 +-- x-pack/auditbeat/magefile.go | 2 +- x-pack/filebeat/magefile.go | 4 +-- x-pack/metricbeat/magefile.go | 4 +-- 19 files changed, 47 insertions(+), 48 deletions(-) diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index e9d80728ac4..31f14eb517a 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -167,5 +167,5 @@ func GoUnitTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go index 11767ae7d3c..be9c70cd0f1 100644 --- a/dev-tools/mage/check.go +++ b/dev-tools/mage/check.go @@ -43,12 +43,12 @@ import ( // if it finds any modifications. If executed in in verbose mode it will write // the results of 'git diff' to stdout to indicate what changes have been made. // -// It checks the file permissions of nosetests test cases and YAML files. +// It checks the file permissions of python test cases and YAML files. // It checks .go source files using 'go vet'. func Check() error { fmt.Println(">> check: Checking source code for common problems") - mg.Deps(GoVet, CheckNosetestsNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat) + mg.Deps(GoVet, CheckPythonTestNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat) changes, err := GitDiffIndex() if err != nil { @@ -124,16 +124,15 @@ func GitDiff() error { return err } -// CheckNosetestsNotExecutable checks that none of the nosetests files are -// executable. Nosetests silently skips executable .py files and we don't want -// this to happen. -func CheckNosetestsNotExecutable() error { +// CheckPythonTestNotExecutable checks that none of the python test files are +// executable. They are silently skipped and we don't want this to happen. +func CheckPythonTestNotExecutable() error { if runtime.GOOS == "windows" { // Skip windows because it doesn't have POSIX permissions. return nil } - tests, err := FindFiles(nosetestsTestFiles...) + tests, err := FindFiles(pythonTestFiles...) if err != nil { return err } @@ -151,7 +150,7 @@ func CheckNosetestsNotExecutable() error { } if len(executableTestFiles) > 0 { - return errors.Errorf("nosetests files cannot be executable because "+ + return errors.Errorf("python test files cannot be executable because "+ "they will be skipped. Fix permissions of %v", executableTestFiles) } return nil diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 539fc8b3f1b..e1c4c6439cd 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -56,7 +56,7 @@ var ( pythonVirtualenvLock sync.Mutex // More globs may be needed in the future if tests are added in more places. - nosetestsTestFiles = []string{ + pythonTestFiles = []string{ "tests/system/test_*.py", "module/*/test_*.py", "module/*/*/test_*.py", @@ -77,7 +77,7 @@ func init() { } // PythonTestArgs are the arguments used for the "python*Test" targets and they -// define how "nosetests" is invoked. +// define how python tests are invoked. type PythonTestArgs struct { TestName string // Test name used in logging. Env map[string]string // Env vars to add to the current env. @@ -108,8 +108,8 @@ func DefaultPythonTestUnitArgs() PythonTestArgs { return makePythonTestArgs("Uni // checking for INTEGRATION_TEST=1 in the test code. func DefaultPythonTestIntegrationArgs() PythonTestArgs { return makePythonTestArgs("Integration") } -// PythonNoseTest invokes "nosetests" via a Python virtualenv. -func PythonNoseTest(params PythonTestArgs) error { +// PythonTest executes python tests via a Python virtualenv. +func PythonTest(params PythonTestArgs) error { fmt.Println(">> python test:", params.TestName, "Testing") ve, err := PythonVirtualenv() @@ -117,32 +117,32 @@ func PythonNoseTest(params PythonTestArgs) error { return err } - nosetestsEnv := map[string]string{ + pytestEnv := map[string]string{ // activate sets this. Not sure if it's ever needed. "VIRTUAL_ENV": ve, } if IsInIntegTestEnv() { - nosetestsEnv["INTEGRATION_TESTS"] = "1" + pytestEnv["INTEGRATION_TESTS"] = "1" } for k, v := range params.Env { - nosetestsEnv[k] = v + pytestEnv[k] = v } - nosetestsOptions := []string{ + pytestOptions := []string{ "--process-timeout=90", "--with-timer", } if mg.Verbose() { - nosetestsOptions = append(nosetestsOptions, "-v") + pytestOptions = append(pytestOptions, "-v") } if params.XUnitReportFile != "" { - nosetestsOptions = append(nosetestsOptions, + pytestOptions = append(pytestOptions, "--with-xunit", "--xunit-file="+createDir(params.XUnitReportFile), ) } - testFiles, err := FindFiles(nosetestsTestFiles...) + testFiles, err := FindFiles(pythonTestFiles...) if err != nil { return err } @@ -154,13 +154,13 @@ func PythonNoseTest(params PythonTestArgs) error { // We check both the VE and the normal PATH because on Windows if the // requirements are met by the globally installed package they are not // installed to the VE. - nosetestsPath, err := LookVirtualenvPath(ve, "nosetests") + pytestPath, err := LookVirtualenvPath(ve, "pytest") if err != nil { return err } defer fmt.Println(">> python test:", params.TestName, "Testing Complete") - _, err = sh.Exec(nosetestsEnv, os.Stdout, os.Stderr, nosetestsPath, append(nosetestsOptions, testFiles...)...) + _, err = sh.Exec(pytestEnv, os.Stdout, os.Stderr, pytestPath, append(pytestOptions, testFiles...)...) return err // TODO: Aggregate all the individual code coverage reports and generate diff --git a/dev-tools/mage/target/integtest/integtest.go b/dev-tools/mage/target/integtest/integtest.go index 142a713effc..3d56eb9b78b 100644 --- a/dev-tools/mage/target/integtest/integtest.go +++ b/dev-tools/mage/target/integtest/integtest.go @@ -72,6 +72,6 @@ func PythonIntegTest(ctx context.Context) error { } return devtools.RunIntegTest("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs()) }, whitelistedEnvVars...) } diff --git a/dev-tools/mage/target/unittest/unittest.go b/dev-tools/mage/target/unittest/unittest.go index 859849bac1e..891b5c23309 100644 --- a/dev-tools/mage/target/unittest/unittest.go +++ b/dev-tools/mage/target/unittest/unittest.go @@ -61,5 +61,5 @@ func GoUnitTest(ctx context.Context) error { func PythonUnitTest() error { mg.SerialDeps(pythonTestDeps...) mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } diff --git a/docs/devguide/metricset-details.asciidoc b/docs/devguide/metricset-details.asciidoc index cdb5a57a157..531337447ee 100644 --- a/docs/devguide/metricset-details.asciidoc +++ b/docs/devguide/metricset-details.asciidoc @@ -298,7 +298,7 @@ module such as the `apache` module. To do this, you can: . Run `cd tests/system` to change to the folder that contains the integration tests. -. Run `INTEGRATION_TESTS=true nosetests test_apache.py`, +. Run `INTEGRATION_TESTS=true pytest test_apache.py`, remembering to replace `test_apache.py` with your own test file. [float] diff --git a/docs/devguide/modules-dev-guide.asciidoc b/docs/devguide/modules-dev-guide.asciidoc index d25161a079c..b8aa133f1f8 100644 --- a/docs/devguide/modules-dev-guide.asciidoc +++ b/docs/devguide/modules-dev-guide.asciidoc @@ -488,4 +488,4 @@ locally for a specific module, using the following procedure under Filebeat dire . Create python env: `make python-env` . Source python env: `./build/python-env/bin/activate` . Create the testing binary: `make filebeat.test` -. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx nosetests tests/system/test_modules.py` +. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx pytest tests/system/test_modules.py` diff --git a/docs/devguide/python.asciidoc b/docs/devguide/python.asciidoc index 4f9902af205..923166fe62b 100644 --- a/docs/devguide/python.asciidoc +++ b/docs/devguide/python.asciidoc @@ -52,14 +52,14 @@ of these virtual environments: is created by `make` or `mage` targets when needed. Virtual environments can also be used without `make` or `mage`, this is usual -for example when running individual system tests with `nosetests`. There are two +for example when running individual system tests with `pytest`. There are two ways to run commands from the virtual environment: * "Activating" the virtual environment in your current terminal running `source ./build/python-env/bin/activate`. Virtual environment can be deactivated by running `deactivate`. * Directly running commands from the virtual environment path. For example - `nosetests` can be executed as `./build/python-env/bin/nosetests`. + `pytest` can be executed as `./build/python-env/bin/pytest`. To recreate a virtual environment, remove its directory. All virtual environments are also removed with `make clean`. diff --git a/docs/devguide/testing.asciidoc b/docs/devguide/testing.asciidoc index a21a8d65ce0..75a476d5c5a 100644 --- a/docs/devguide/testing.asciidoc +++ b/docs/devguide/testing.asciidoc @@ -8,7 +8,8 @@ In general there are two major test suites: * Tests written in Go * Tests written in Python -The tests written in Go use the https://golang.org/pkg/testing/[Go Testing package]. The tests written in Python depend on http://nose.readthedocs.io/en/latest/[nosetests] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs. +The tests written in Go use the https://golang.org/pkg/testing/[Go Testing +package]. The tests written in Python depend on https://docs.pytest.org/en/latest/[pytest] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs. For both of the above test suites so called integration tests exists. Integration tests in Beats are tests which require an external system like Elasticsearch to test if the integration with this service works as expected. Beats provides in its testsuite docker containers and docker-compose files to start these environments but a developer can run the required services also locally. @@ -23,7 +24,7 @@ All Go tests are in the same package as the tested code itself and have the post ==== Running Python Tests -The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `nosetests` has to be run inside `tests/system`. +The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `pytest` has to be run inside `tests/system`. To automate all these steps into one `make system-tests` can be run. This creates the binary, the environment and runs all tests which do not require and external service. diff --git a/filebeat/magefile.go b/filebeat/magefile.go index b18450a0f25..c7afac2aff0 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -204,7 +204,7 @@ func GoIntegTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } // PythonIntegTest executes the python system tests in the integration environment (Docker). @@ -219,6 +219,6 @@ func PythonIntegTest(ctx context.Context) error { mg.Deps(devtools.BuildSystemTestBinary) args := devtools.DefaultPythonTestIntegrationArgs() args.Env["MODULES_PATH"] = devtools.CWD("module") - return devtools.PythonNoseTest(args) + return devtools.PythonTest(args) }, "GENERATE", "TESTING_FILEBEAT_MODULES", "TESTING_FILEBEAT_FILESETS") } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 9d1733018ab..b5e39a94180 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -82,7 +82,7 @@ REVIEWDOG_REPO?=github.com/reviewdog/reviewdog/cmd/reviewdog PROCESSES?= 4 TIMEOUT?= 90 PYTHON_TEST_FILES?=$(shell find . -type f -name 'test_*.py' -not -path "*/build/*" -not -path "*/vendor/*" 2>/dev/null) -NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling nosetests +PYTEST_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests @@ -238,7 +238,7 @@ integration-tests-environment: prepare-tests build-image .PHONY: system-tests system-tests: ## @testing Runs the system tests system-tests: prepare-tests ${BEAT_NAME}.test python-env - . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} nosetests ${PYTHON_TEST_FILES} ${NOSETESTS_OPTIONS} + . ${PYTHON_ENV}/bin/activate; INTEGRATION_TESTS=${INTEGRATION_TESTS} TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} pytest ${PYTHON_TEST_FILES} ${PYTEST_OPTIONS} ${PYTHON_ENV_EXE} ${ES_BEATS}/dev-tools/aggregate_coverage.py -o ${COVERAGE_DIR}/system.cov ${BUILD_DIR}/system-tests/run # Runs the system tests @@ -256,7 +256,7 @@ system-tests-environment: prepare-tests build-image .PHONY: fast-system-tests fast-system-tests: ## @testing Runs system tests without coverage reports and in parallel fast-system-tests: ${BEAT_NAME}.test python-env - . ${PYTHON_ENV}/bin/activate; nosetests ${PYTHON_TEST_FILES} ${NOSETESTS_OPTIONS} + . ${PYTHON_ENV}/bin/activate; pytest ${PYTHON_TEST_FILES} ${PYTEST_OPTIONS} # Runs the go based stress tests .PHONY: stress-tests @@ -275,7 +275,7 @@ benchmark-tests: ## @testing Runs benchmarks (NOT YET IMPLEMENTED) # Run load tests .PHONY: load-tests load-tests: ## @testing Runs load tests - . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 nosetests ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a 'load' + . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 pytest ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a 'load' # Sets up the virtual python environment .PHONY: python-env diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 52c36033406..231b428fb6a 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -15,8 +15,6 @@ ipaddress==1.0.19 Jinja2==2.10.1 jsonschema==2.6.0 MarkupSafe==1.0 -nose==1.3.7 -nose-timer==0.7.1 pycodestyle==2.4.0 PyYAML==4.2b1 Pillow==7.0.0 @@ -32,3 +30,4 @@ jsondiff==1.1.2 semver==2.8.1 stomp.py==4.1.22 deepdiff==4.2.0 +pytest==5.3.5 diff --git a/metricbeat/Makefile b/metricbeat/Makefile index fd2c60935e7..6023d19a5e1 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -70,7 +70,7 @@ run-module: test-module: ## @testing Tests the given module. Needs $MODULE as param an run-module must be started first. test-module: python-env update metricbeat.test go test -tags=integration ${BEAT_PATH}/module/${MODULE}/... -v - . ${PYTHON_ENV}/bin/activate && INTEGRATION_TESTS=1 nosetests module/${MODULE} + . ${PYTHON_ENV}/bin/activate && INTEGRATION_TESTS=1 pytest module/${MODULE} .PHONY: assets assets: diff --git a/packetbeat/SUPPORT_PROTOCOL.md b/packetbeat/SUPPORT_PROTOCOL.md index 035eb793b85..5bc304f6745 100644 --- a/packetbeat/SUPPORT_PROTOCOL.md +++ b/packetbeat/SUPPORT_PROTOCOL.md @@ -16,7 +16,7 @@ Test suites are based on [pcap files](./tests/system/pcaps), that are dumps from tcpdump -s 0 port 27017 -i docker0 -w tests/system/pcaps/mongodb_find.pcap -# Nosetests +# Python tests The 'tests' directory contains tests written in python that run the full packetbeat program. You can add some of yours based on the pcaps files of you test dataset. diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 7793ed32ec0..1e2d588695f 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -207,7 +207,7 @@ func GoUnitTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.SerialDeps(Fields, devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } // ----------------------------------------------------------------------------- diff --git a/packetbeat/tests/system/README.md b/packetbeat/tests/system/README.md index 370ba15b4e5..05f8e479bbb 100644 --- a/packetbeat/tests/system/README.md +++ b/packetbeat/tests/system/README.md @@ -1,7 +1,7 @@ # System tests for Packetbeat This folder contains the system tests for Packetbeat. The system tests -are written in Python and they make use of the nose framework. +are written in Python and they make use of the pytest framework. ## Running @@ -13,4 +13,4 @@ the setup and run all the tests with: Running a single test, e.g.: . env/bin/activate - nosetests test_0002_thrift_basics.py:Test.test_thrift_integration + pytest test_0002_thrift_basics.py:Test.test_thrift_integration diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 39b1316ffd3..1e04c877662 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -142,7 +142,7 @@ func GoUnitTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.SerialDeps(Fields, devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } // ----------------------------------------------------------------------------- diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index ccdb4de5c6c..b01e85d11d4 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -172,7 +172,7 @@ func GoIntegTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } // PythonIntegTest executes the python system tests in the integration environment (Docker). @@ -187,6 +187,6 @@ func PythonIntegTest(ctx context.Context) error { mg.Deps(devtools.BuildSystemTestBinary) args := devtools.DefaultPythonTestIntegrationArgs() args.Env["MODULES_PATH"] = devtools.CWD("module") - return devtools.PythonNoseTest(args) + return devtools.PythonTest(args) }, "GENERATE", "TESTING_FILEBEAT_MODULES", "TESTING_FILEBEAT_FILESETS") } diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index 5e30c711cc4..c4af7a43195 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -139,7 +139,7 @@ func GoUnitTest(ctx context.Context) error { // PythonUnitTest executes the python system tests. func PythonUnitTest() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs()) } // IntegTest executes integration tests (it uses Docker to run the tests). @@ -163,6 +163,6 @@ func PythonIntegTest(ctx context.Context) error { } return devtools.RunIntegTest("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs()) }) } From c6e2fd975fc9cee987786aa64f5fc9368cbc899c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 18:12:02 +0100 Subject: [PATCH 02/19] Fix options --- dev-tools/mage/pytest.go | 6 ++---- libbeat/scripts/Makefile | 4 ++-- libbeat/tests/system/requirements.txt | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index e1c4c6439cd..c19063b58ab 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -129,16 +129,14 @@ func PythonTest(params PythonTestArgs) error { } pytestOptions := []string{ - "--process-timeout=90", - "--with-timer", + "--timeout=90", } if mg.Verbose() { pytestOptions = append(pytestOptions, "-v") } if params.XUnitReportFile != "" { pytestOptions = append(pytestOptions, - "--with-xunit", - "--xunit-file="+createDir(params.XUnitReportFile), + "--junit-xml="+createDir(params.XUnitReportFile), ) } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index b5e39a94180..93ffa5f05a6 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -82,7 +82,7 @@ REVIEWDOG_REPO?=github.com/reviewdog/reviewdog/cmd/reviewdog PROCESSES?= 4 TIMEOUT?= 90 PYTHON_TEST_FILES?=$(shell find . -type f -name 'test_*.py' -not -path "*/build/*" -not -path "*/vendor/*" 2>/dev/null) -PYTEST_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest +PYTEST_OPTIONS?=--timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests @@ -275,7 +275,7 @@ benchmark-tests: ## @testing Runs benchmarks (NOT YET IMPLEMENTED) # Run load tests .PHONY: load-tests load-tests: ## @testing Runs load tests - . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 pytest ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a 'load' + . ${PYTHON_ENV}/bin/activate; LOAD_TESTS=1 pytest ${PYTHON_TEST_FILES} --processes=$(PROCESSES) --timeout=$(TIMEOUT) -a 'load' # Sets up the virtual python environment .PHONY: python-env diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 231b428fb6a..638de4039cd 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -31,3 +31,4 @@ semver==2.8.1 stomp.py==4.1.22 deepdiff==4.2.0 pytest==5.3.5 +pytest-timeout==1.3.4 From d406a6578cb4af4319a0f97c7f06a993d9a80d2d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 18:29:12 +0100 Subject: [PATCH 03/19] Fix output --- dev-tools/mage/pytest.go | 1 + pytest.ini | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index c19063b58ab..0568dd685c0 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -130,6 +130,7 @@ func PythonTest(params PythonTestArgs) error { pytestOptions := []string{ "--timeout=90", + "--durations=20", } if mg.Verbose() { pytestOptions = append(pytestOptions, "-v") diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..56711bbf764 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +junit_family=xunit1 From decc979c68af1415977dd68edec42bbdd2e2956b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 19:35:32 +0100 Subject: [PATCH 04/19] Remove nose dependencies in tests --- filebeat/tests/system/test_crawler.py | 11 +++--- filebeat/tests/system/test_load.py | 10 +++--- filebeat/tests/system/test_publisher.py | 4 +-- filebeat/tests/system/test_registrar.py | 14 ++++---- .../tests/system/test_registrar_upgrade.py | 5 +-- filebeat/tests/system/test_tcp_tls.py | 10 +++--- heartbeat/tests/system/heartbeat.py | 3 +- heartbeat/tests/system/test_base.py | 3 +- heartbeat/tests/system/test_icmp.py | 13 ++++--- heartbeat/tests/system/test_monitor.py | 9 +++-- heartbeat/tests/system/test_telemetry.py | 23 ++++++------ libbeat/tests/system/idxmgmt.py | 8 ++--- libbeat/tests/system/test_ca_pinning.py | 13 ++++--- .../system/test_cmd_setup_index_management.py | 35 +++++++++---------- libbeat/tests/system/test_cmd_test.py | 6 ++-- libbeat/tests/system/test_dashboard.py | 29 ++++++++------- libbeat/tests/system/test_ilm.py | 27 +++++++------- libbeat/tests/system/test_migration.py | 6 ++-- libbeat/tests/system/test_monitoring.py | 13 ++++--- libbeat/tests/system/test_template.py | 23 ++++++------ metricbeat/module/apache/test_apache.py | 4 +-- metricbeat/module/consul/test_consul.py | 4 +-- metricbeat/module/docker/test_docker.py | 1 - .../elasticsearch/test_elasticsearch.py | 9 +++-- metricbeat/module/haproxy/test_haproxy.py | 1 - metricbeat/module/http/test_http.py | 1 - metricbeat/module/jolokia/test_jolokia.py | 1 - metricbeat/module/kibana/test_kibana.py | 5 ++- metricbeat/module/logstash/test_logstash.py | 3 +- metricbeat/module/mongodb/test_mongodb.py | 4 +-- metricbeat/module/mysql/test_mysql.py | 4 +-- .../module/postgresql/test_postgresql.py | 8 ++--- metricbeat/module/redis/test_redis.py | 10 +++--- metricbeat/module/uwsgi/test_uwsgi.py | 4 +-- metricbeat/module/zookeeper/test_zookeeper.py | 8 ++--- .../tests/system/test_autodiscover_jolokia.py | 1 - metricbeat/tests/system/test_config.py | 1 - metricbeat/tests/system/test_template.py | 4 +-- packetbeat/tests/system/test_0015_udpjson.py | 8 ++--- pytest.ini | 5 +++ x-pack/metricbeat/module/mssql/test_mssql.py | 5 ++- 41 files changed, 161 insertions(+), 195 deletions(-) diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 2b22e7e830b..f3b5d0877a6 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- - -from filebeat import BaseTest - import codecs import os +import shutil import time import unittest -from nose.plugins.skip import Skip, SkipTest -import shutil +from filebeat import BaseTest # Additional tests to be added: # * Check what happens when file renamed -> no recrawling should happen @@ -767,7 +764,7 @@ def test_file_no_permission(self): """ if os.name != "nt" and os.geteuid() == 0: # root ignores permission flags, so we have to skip the test - raise SkipTest + raise unittest.SkipTest self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", @@ -789,7 +786,7 @@ def test_file_no_permission(self): if os.name == "nt": - raise SkipTest + raise unittest.SkipTest # TODO: Currently skipping this test on windows as it requires `pip install win32api` # which seems to have windows only dependencies. # To solve this problem a requirements_windows.txt could be introduced which would diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index b53a8d18032..de3a66dd3e3 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -1,12 +1,10 @@ -from filebeat import BaseTest import os import logging import logging.handlers import json import time import unittest -from nose.plugins.skip import Skip, SkipTest -from nose.plugins.attrib import attr +import pytest """ Test filebeat under different load scenarios @@ -25,7 +23,7 @@ def test_no_missing_events(self): if os.name == "nt": # This test is currently skipped on windows because very fast file # rotation cannot happen when harvester has file handler still open. - raise SkipTest + raise unittest.SkipTest log_file = self.working_dir + "/log/test.log" os.mkdir(self.working_dir + "/log/") @@ -106,7 +104,7 @@ def test_no_missing_events(self): assert len(entry_list) == total_lines @unittest.skipUnless(LOAD_TESTS, "load test") - @attr('load') + @pytest.mark.load def test_large_number_of_files(self): """ Tests the number of files filebeat can open on startup @@ -148,7 +146,7 @@ def test_large_number_of_files(self): assert len(data) == number_of_files @unittest.skipUnless(LOAD_TESTS, "load test") - @attr('load') + @pytest.mark.load def test_concurrent_harvesters(self): """ Test large number of files on startup if harvester overlap happens and would create too many events diff --git a/filebeat/tests/system/test_publisher.py b/filebeat/tests/system/test_publisher.py index 227f09d1050..3f018cab6f3 100644 --- a/filebeat/tests/system/test_publisher.py +++ b/filebeat/tests/system/test_publisher.py @@ -1,11 +1,9 @@ -from filebeat import BaseTest - import os import platform import time import shutil import json -from nose.plugins.skip import Skip, SkipTest +from filebeat import BaseTest # Additional tests: to be implemented diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index d5e9027fc9c..9401e474a97 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -7,9 +7,7 @@ import stat import time import unittest - from filebeat import BaseTest -from nose.plugins.skip import SkipTest # Additional tests: to be implemented @@ -163,7 +161,7 @@ def test_registry_file_default_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry" registry_file = os.path.join(registry_home, "filebeat/data.json") @@ -197,7 +195,7 @@ def test_registry_file_custom_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry" registry_file = os.path.join(registry_home, "filebeat/data.json") @@ -232,7 +230,7 @@ def test_registry_file_update_permissions(self): if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest registry_home = "a/b/c/registry_x" registry_file = os.path.join(registry_home, "filebeat/data.json") @@ -363,7 +361,7 @@ def test_rotating_file_inode(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" @@ -453,7 +451,7 @@ def test_restart_continue(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" @@ -527,7 +525,7 @@ def test_rotating_file_with_restart(self): ) if os.name == "nt": - raise SkipTest + raise unittest.SkipTest os.mkdir(self.working_dir + "/log/") testfile_path = self.working_dir + "/log/input" diff --git a/filebeat/tests/system/test_registrar_upgrade.py b/filebeat/tests/system/test_registrar_upgrade.py index 84d5ac8b4ff..fbc6311612e 100644 --- a/filebeat/tests/system/test_registrar_upgrade.py +++ b/filebeat/tests/system/test_registrar_upgrade.py @@ -3,9 +3,6 @@ import os import json - -from nose.plugins.skip import Skip, SkipTest - from filebeat import BaseTest @@ -55,7 +52,7 @@ def prepare_log(self): # test is current skipped on windows, due to FileStateOS must match the # current OS format. if os.name == "nt": - raise SkipTest + raise unittest.SkipTest self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*" diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index 83813a5ccaa..f022376f101 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -1,9 +1,9 @@ -from filebeat import BaseTest import socket import ssl import unittest +import pytest from os import path -from nose.tools import raises, assert_raises +from filebeat import BaseTest NUMBER_OF_EVENTS = 2 @@ -86,7 +86,7 @@ def test_tcp_over_tls_and_verify_valid_server_without_mutual_auth(self): sock.close() - @raises(ssl.SSLError) + @pytest.raises(ssl.SSLError) def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): """ Test filebeat TCP with TLS with an invalid cacert and not requiring mutual auth. @@ -125,7 +125,7 @@ def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): ca_certs=CERTIFICATE2, do_handshake_on_connect=True) tls.connect((config.get('host'), config.get('port'))) - @raises(ssl.SSLError) + @pytest.raises(ssl.SSLError) def test_tcp_over_tls_mutual_auth_fails(self): """ Test filebeat TCP with TLS with default setting to enforce client auth, with bad client certificates @@ -265,7 +265,7 @@ def test_tcp_tls_with_a_plain_text_socket(self): # The TLS handshake will close the connection, resulting in a broken pipe. # no events should be written on disk. - with assert_raises(IOError): + with pytest.raises(IOError): for n in range(0, 100000): sock.send(bytes("Hello World: " + str(n) + "\n", "utf-8")) diff --git a/heartbeat/tests/system/heartbeat.py b/heartbeat/tests/system/heartbeat.py index 3c93396c7ed..70b7faf1ac3 100644 --- a/heartbeat/tests/system/heartbeat.py +++ b/heartbeat/tests/system/heartbeat.py @@ -2,7 +2,6 @@ import sys import http.server import threading -import nose.tools sys.path.append(os.path.join(os.path.dirname( __file__), '../../../libbeat/tests/system')) @@ -69,7 +68,7 @@ def monitors_dir(self): return self.working_dir + "/monitors.d/" def assert_last_status(self, status): - nose.tools.eq_(self.last_output_line()["monitor.status"], status) + self.assertEqual(self.last_output_line()["monitor.status"], status) def setup_dynamic(self, extra_beat_args=[]): os.mkdir(self.monitors_dir()) diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 683ffde212a..876c9ee43dd 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -4,7 +4,6 @@ from heartbeat import BaseTest from elasticsearch import Elasticsearch from beat.beat import INTEGRATION_TESTS -import nose.tools class Test(BaseTest): @@ -180,7 +179,7 @@ def test_dataset(self): heartbeat_proc.check_kill_and_wait() for output in self.read_output(): - nose.tools.assert_equal( + self.assertEqual( output["event.dataset"], "uptime", "Check for event.dataset in {} failed".format(output) diff --git a/heartbeat/tests/system/test_icmp.py b/heartbeat/tests/system/test_icmp.py index 5e7fc551552..7f61a7430f8 100644 --- a/heartbeat/tests/system/test_icmp.py +++ b/heartbeat/tests/system/test_icmp.py @@ -1,15 +1,14 @@ +import logging import os -import unittest import platform import socket -import sys -from heartbeat import BaseTest -from elasticsearch import Elasticsearch -from beat.beat import INTEGRATION_TESTS -import nose.tools -import logging import subprocess +import sys import time +import unittest +from beat.beat import INTEGRATION_TESTS +from elasticsearch import Elasticsearch +from heartbeat import BaseTest class Test(BaseTest): diff --git a/heartbeat/tests/system/test_monitor.py b/heartbeat/tests/system/test_monitor.py index 9f5aa8d43c1..371ad775dd3 100644 --- a/heartbeat/tests/system/test_monitor.py +++ b/heartbeat/tests/system/test_monitor.py @@ -1,8 +1,7 @@ +import os +import unittest from heartbeat import BaseTest from parameterized import parameterized -import os -from nose.plugins.skip import SkipTest -import nose.tools class Test(BaseTest): @@ -83,7 +82,7 @@ def test_http_delayed(self): try: proc = self.start_beat() self.wait_until(lambda: self.output_has(lines=1)) - nose.tools.assert_greater_equal( + self.assertGreaterEqual( self.last_output_line()['http.rtt.total.us'], delay) finally: proc.check_kill_and_wait() @@ -125,7 +124,7 @@ def test_http_json(self, expected_status, body): self.assert_last_status(expected_status) if expected_status == "down": - nose.tools.eq_(self.last_output_line()["http.response.body.content"], body) + self.assertEqual(self.last_output_line()["http.response.body.content"], body) else: assert "http.response.body.content" not in self.last_output_line() finally: diff --git a/heartbeat/tests/system/test_telemetry.py b/heartbeat/tests/system/test_telemetry.py index d6420c2c56b..1575a5eed57 100644 --- a/heartbeat/tests/system/test_telemetry.py +++ b/heartbeat/tests/system/test_telemetry.py @@ -1,11 +1,10 @@ -from heartbeat import BaseTest +import unittest import urllib.request import urllib.error import urllib.parse import json -import nose.tools import os -from nose.plugins.skip import SkipTest +from heartbeat import BaseTest class Test(BaseTest): @@ -18,12 +17,12 @@ def test_telemetry(self): Test that telemetry metrics are correctly registered and increment / decrement """ # This test is flaky https://github.com/elastic/beats/issues/8966 - raise SkipTest + raise unittest.SkipTest if os.name == "nt": # This test is currently skipped on windows because file permission # configuration isn't implemented on Windows yet - raise SkipTest + raise unittest.SkipTest server = self.start_server("hello world", 200) try: @@ -90,8 +89,7 @@ def test_telemetry(self): self.proc.check_kill_and_wait() server.shutdown() - @staticmethod - def assert_state(expected={}): + def assert_state(self, expected={}): stats = json.loads(urllib.request.urlopen( "http://localhost:5066/state").read()) @@ -104,23 +102,22 @@ def assert_state(expected={}): endpoints = proto_expected.get("endpoints", 0) total_monitors += monitors total_endpoints += endpoints - nose.tools.assert_dict_equal(stats['heartbeat'][proto], { + self.assertDictEqual(stats['heartbeat'][proto], { 'monitors': monitors, 'endpoints': endpoints, }) - nose.tools.assert_equal(stats['heartbeat']['monitors'], total_monitors) - nose.tools.assert_equal( + self.assertEqual(stats['heartbeat']['monitors'], total_monitors) + self.assertEqual( stats['heartbeat']['endpoints'], total_endpoints) - @staticmethod - def assert_stats(expected={}): + def assert_stats(self, expected={}): stats = json.loads(urllib.request.urlopen( "http://localhost:5066/stats").read()) for proto in ("http", "tcp", "icmp"): proto_expected = expected.get(proto, {}) - nose.tools.assert_dict_equal(stats['heartbeat'][proto], { + self.assertDictEqual(stats['heartbeat'][proto], { 'monitor_starts': proto_expected.get("monitor_starts", 0), 'monitor_stops': proto_expected.get("monitor_stops", 0), 'endpoint_starts': proto_expected.get("endpoint_starts", 0), diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 095533f7f89..83dbd89368d 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -1,7 +1,7 @@ -from elasticsearch import NotFoundError -from nose.tools import raises import datetime import unittest +import pytest +from elasticsearch import NotFoundError class IdxMgmt(unittest.TestCase): @@ -52,7 +52,7 @@ def delete_policy(self, policy): except NotFoundError: pass - @raises(NotFoundError) + @pytest.raises(NotFoundError) def assert_index_template_not_loaded(self, template): self._client.transport.perform_request('GET', '/_template/' + template) @@ -86,7 +86,7 @@ def assert_alias_created(self, alias, pattern=None): assert name in resp assert resp[name]["aliases"][alias]["is_write_index"] == True - @raises(NotFoundError) + @pytest.raises(NotFoundError) def assert_policy_not_created(self, policy): self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index 1e0dd6f6a43..a5192ff73b2 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -1,11 +1,10 @@ -from base import BaseTest -from idxmgmt import IdxMgmt +import logging import os -from nose.plugins.attrib import attr +import pytest import unittest -import logging -from nose.tools import raises +from base import BaseTest from elasticsearch import RequestError +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -16,7 +15,7 @@ class TestCAPinning(BaseTest): """ @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_sending_events_with_a_good_sha256(self): """ Test Sending events while using ca pinning with a good sha256 @@ -48,7 +47,7 @@ def test_sending_events_with_a_good_sha256(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_sending_events_with_a_bad_sha256(self): """ Test Sending events while using ca pinning with a bad sha256 diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 1ce764d640e..9cbf0ec346d 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -1,11 +1,10 @@ -from base import BaseTest -from idxmgmt import IdxMgmt +import logging import os -from nose.plugins.attrib import attr +import pytest import unittest -import logging -from nose.tools import raises +from base import BaseTest from elasticsearch import RequestError +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -47,7 +46,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_default(self): """ Test setup --index-management with default config @@ -64,8 +63,8 @@ def test_setup_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') - @raises(RequestError) + @pytest.mark.integration + @pytest.raises(RequestError) def test_setup_default(self): """ Test setup --index-management with default config @@ -83,7 +82,7 @@ def test_setup_default(self): self.idxmgmt.delete_policy(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_disabled(self): """ Test setup --index-management when ilm disabled @@ -99,7 +98,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_ilm_disabled(self): """ Test setup --index-management when ilm disabled @@ -115,7 +114,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_policy_name(self): """ Test setup --index-management when policy_name is configured @@ -130,7 +129,7 @@ def test_setup_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_ilm_policy_no_overwrite(self): """ Test setup --index-management respects overwrite configuration @@ -177,7 +176,7 @@ def test_setup_ilm_policy_no_overwrite(self): assert "hot" in resp[policy_name]["policy"]["phases"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_rollover_alias(self): """ Test setup --index-management when ilm.rollover_alias is configured @@ -193,7 +192,7 @@ def test_setup_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_rollover_alias_with_fieldref(self): """ Test setup --index-management when ilm.rollover_alias is configured and using field reference. @@ -212,7 +211,7 @@ def test_setup_rollover_alias_with_fieldref(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_name_and_pattern(self): """ Test setup --index-management ignores template.name and template.pattern when ilm is enabled @@ -229,7 +228,7 @@ def test_setup_template_name_and_pattern(self): self.idxmgmt.assert_alias_created(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_name_and_pattern_on_ilm_disabled(self): """ Test setup --index-management respects template.name and template.pattern when ilm is disabled @@ -248,7 +247,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_with_opts(self): """ Test setup --index-management with config options @@ -270,7 +269,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_overwrite_template_on_ilm_policy_created(self): """ Test setup --index-management overwrites template when new ilm policy is created diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index 77b2d8f4b64..acb29697d94 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -1,8 +1,8 @@ -from base import BaseTest import os import logging import unittest -from nose.plugins.attrib import attr +import pytest +from base import BaseTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -41,7 +41,7 @@ def test_bad_config(self): assert self.log_contains("Config OK") is False @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_output(self): """ Test test output works diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 47564455c0d..d856625f593 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -2,11 +2,10 @@ import os import os.path import subprocess -from nose.plugins.attrib import attr import unittest -from unittest import SkipTest import requests import semver +from unittest import SkipTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -14,7 +13,7 @@ class Test(BaseTest): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_load_without_dashboard(self): """ Test loading without dashboards @@ -39,7 +38,7 @@ def test_load_without_dashboard(self): assert self.log_contains("Skipping loading dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_load_dashboard(self): """ Test loading dashboards @@ -64,7 +63,7 @@ def test_load_dashboard(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_load_dashboard_into_space(self, create_space=True): """ Test loading dashboards into Kibana space @@ -98,7 +97,7 @@ def test_load_dashboard_into_space(self, create_space=True): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_load_only_index_patterns(self): """ Test loading dashboards @@ -124,7 +123,7 @@ def test_load_only_index_patterns(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): """ Test testbeat export dashboard can export dashboards @@ -148,7 +147,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_export_dashboard_cmd_export_dashboard_by_id(self): """ Test testbeat export dashboard can export dashboards @@ -170,7 +169,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): """ Test testbeat export dashboard fails gracefully when dashboard with unknown ID is requested @@ -191,7 +190,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): assert self.log_contains("error exporting dashboard: Not found") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_export_dashboard_cmd_export_dashboard_from_yml(self): """ Test testbeat export dashboard can export dashboards from dashboards YAML file @@ -224,7 +223,7 @@ def test_export_dashboard_cmd_export_dashboard_from_yml(self): os.remove(exported_dashboard_path) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): """ Test testbeat export dashboard fails gracefully when cannot find YAML file @@ -246,7 +245,7 @@ def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): assert self.log_contains("error opening the list of dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_dev_tool_export_dashboard_by_id(self): """ Test dev-tools/cmd/dashboards exports dashboard and removes unsupported characters @@ -272,7 +271,7 @@ def test_dev_tool_export_dashboard_by_id(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_dev_tool_export_dashboard_by_id_unknown_id(self): """ Test dev-tools/cmd/dashboards fails gracefully when dashboard with unknown ID is requested @@ -288,7 +287,7 @@ def test_dev_tool_export_dashboard_by_id_unknown_id(self): assert p.returncode != 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_dev_tool_export_dashboard_by_id_from_space(self): """ Test dev-tools/cmd/dashboards exports dashboard from Kibana space @@ -319,7 +318,7 @@ def test_dev_tool_export_dashboard_by_id_from_space(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_dev_tool_export_dashboard_from_yml(self): """ Test dev-tools/cmd/dashboards exports dashboard from dashboards YAML file diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 293544f8dd6..85461587299 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -1,12 +1,11 @@ -from base import BaseTest -from idxmgmt import IdxMgmt import os -from nose.plugins.attrib import attr import unittest import shutil import datetime import logging import json +from base import BaseTest +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -37,7 +36,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_ilm_default(self): """ Test ilm default settings to load ilm policy, write alias and ilm template @@ -55,7 +54,7 @@ def test_ilm_default(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_ilm_disabled(self): """ Test ilm disabled to not load ilm related components @@ -72,7 +71,7 @@ def test_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_policy_name(self): """ Test setting ilm policy name @@ -92,7 +91,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_rollover_alias(self): """ Test settings ilm rollover alias @@ -111,7 +110,7 @@ def test_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_pattern(self): """ Test setting ilm pattern @@ -131,7 +130,7 @@ def test_pattern(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_pattern_date(self): """ Test setting ilm pattern with date @@ -187,7 +186,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_ilm_policy_and_template(self): """ Test combination of ilm policy and template setup @@ -204,7 +203,7 @@ def test_setup_ilm_policy_and_template(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_ilm_default(self): """ Test ilm policy setup with default config @@ -221,7 +220,7 @@ def test_setup_ilm_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_ilm_disabled(self): """ Test ilm policy setup when ilm disabled @@ -238,7 +237,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_policy_name(self): """ Test ilm policy setup when policy_name is configured @@ -254,7 +253,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_rollover_alias(self): """ Test ilm policy setup when rollover_alias is configured diff --git a/libbeat/tests/system/test_migration.py b/libbeat/tests/system/test_migration.py index fec98ef39f7..81ab3b679dd 100644 --- a/libbeat/tests/system/test_migration.py +++ b/libbeat/tests/system/test_migration.py @@ -1,11 +1,9 @@ -from base import BaseTest -from nose.plugins.attrib import attr -from elasticsearch import Elasticsearch, TransportError - import logging import os import shutil import unittest +from base import BaseTest +from elasticsearch import Elasticsearch, TransportError INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 250747757db..da8d6cfab21 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -1,12 +1,11 @@ -from base import BaseTest import os -from elasticsearch import Elasticsearch import re -from nose.plugins.attrib import attr import unittest import requests import random import string +from base import BaseTest +from elasticsearch import Elasticsearch INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -21,7 +20,7 @@ def setUp(self): self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_via_output_cluster(self): """ Test shipping monitoring data via the elasticsearch output cluster. @@ -58,7 +57,7 @@ def test_via_output_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_direct_to_monitoring_cluster(self): """ Test shipping monitoring data directly to the monitoring cluster. @@ -92,7 +91,7 @@ def test_direct_to_monitoring_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_compare(self): """ Test that monitoring docs are the same, regardless of how they are shipped. @@ -155,7 +154,7 @@ def test_compare(self): self.assert_same_structure(indirect_beats_stats_doc['beats_stats'], direct_beats_stats_doc['beats_stats']) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_cluster_uuid_setting(self): """ Test that monitoring.cluster_uuid setting may be set without any other monitoring.* settings diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index c1c4f5ea32c..26e8f88221f 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -1,11 +1,10 @@ -from base import BaseTest -from idxmgmt import IdxMgmt import os -from nose.plugins.attrib import attr import unittest import shutil import logging import json +from base import BaseTest +from idxmgmt import IdxMgmt INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -83,7 +82,7 @@ def test_index_with_pattern_name(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_json_template(self): """ Test loading of json based template @@ -140,7 +139,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_template_default(self): """ Test run cmd with default settings for template @@ -157,7 +156,7 @@ def test_template_default(self): self.idxmgmt.assert_docs_written_to_alias(self.index_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_template_disabled(self): """ Test run cmd does not load template when disabled in config @@ -201,7 +200,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup(self): """ Test setup cmd with template and ilm-policy subcommands @@ -216,7 +215,7 @@ def test_setup(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_default(self): """ Test template setup with default config @@ -235,7 +234,7 @@ def test_setup_template_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_disabled(self): """ Test template setup when ilm disabled @@ -254,7 +253,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_with_opts(self): """ Test template setup with config options @@ -275,7 +274,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_setup_template_with_ilm_changed_pattern(self): """ Test template setup with changed ilm.rollover_alias config @@ -290,7 +289,7 @@ def test_setup_template_with_ilm_changed_pattern(self): self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_template_created_on_ilm_policy_created(self): """ Test template setup overwrites template when new ilm policy is created diff --git a/metricbeat/module/apache/test_apache.py b/metricbeat/module/apache/test_apache.py index f47d046bdae..b4aab34bb57 100644 --- a/metricbeat/module/apache/test_apache.py +++ b/metricbeat/module/apache/test_apache.py @@ -1,9 +1,9 @@ import os import unittest -from nose.plugins.attrib import attr import urllib.request import urllib.error import urllib.parse +import pytest import time import semver import sys @@ -39,7 +39,7 @@ class ApacheStatusTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['apache'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_output(self): """ Apache module outputs an event. diff --git a/metricbeat/module/consul/test_consul.py b/metricbeat/module/consul/test_consul.py index 5ee0f5ac2a8..ed56ca5702f 100644 --- a/metricbeat/module/consul/test_consul.py +++ b/metricbeat/module/consul/test_consul.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -27,7 +27,7 @@ class ConsulAgentTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['consul'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_output(self): """ Consul agent module outputs an event. diff --git a/metricbeat/module/docker/test_docker.py b/metricbeat/module/docker/test_docker.py index 22a86784e02..8eacc289358 100644 --- a/metricbeat/module/docker/test_docker.py +++ b/metricbeat/module/docker/test_docker.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index f8e2762b8cf..08813a49342 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -2,14 +2,13 @@ import sys import os import unittest -from elasticsearch import Elasticsearch, TransportError, client -from parameterized import parameterized -from nose.plugins.skip import SkipTest import urllib.request import urllib.error import urllib.parse import json import semver +from elasticsearch import Elasticsearch, TransportError, client +from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) @@ -317,10 +316,10 @@ def start_basic(self): def check_skip(self, metricset): if metricset == 'ccr' and not self.is_ccr_available(): - raise SkipTest("elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") + raise unittest.SkipTest("elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") if metricset == 'enrich' and not self.is_enrich_available(): - raise SkipTest("elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") + raise unittest.SkipTest("elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") def is_ccr_available(self): es_version = self.get_version() diff --git a/metricbeat/module/haproxy/test_haproxy.py b/metricbeat/module/haproxy/test_haproxy.py index 40ab58f847f..39ec3cfe6d9 100644 --- a/metricbeat/module/haproxy/test_haproxy.py +++ b/metricbeat/module/haproxy/test_haproxy.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/http/test_http.py b/metricbeat/module/http/test_http.py index ce9e1c81b27..92aa693552b 100644 --- a/metricbeat/module/http/test_http.py +++ b/metricbeat/module/http/test_http.py @@ -3,7 +3,6 @@ import sys import time import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat diff --git a/metricbeat/module/jolokia/test_jolokia.py b/metricbeat/module/jolokia/test_jolokia.py index 509b9f5681e..db6055fe91e 100644 --- a/metricbeat/module/jolokia/test_jolokia.py +++ b/metricbeat/module/jolokia/test_jolokia.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) diff --git a/metricbeat/module/kibana/test_kibana.py b/metricbeat/module/kibana/test_kibana.py index 08e7657a1e8..cbd4f365979 100644 --- a/metricbeat/module/kibana/test_kibana.py +++ b/metricbeat/module/kibana/test_kibana.py @@ -6,7 +6,6 @@ import urllib.error import urllib.parse import urllib.request -from nose.plugins.skip import SkipTest sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -27,12 +26,12 @@ def test_status(self): if env == "2x" or env == "5x": # Skip for 5.x and 2.x tests as Kibana endpoint not available - raise SkipTest + raise unittest.SkipTest version = self.get_version() if semver.compare(version, "6.4.0") == -1: # Skip for Kibana versions < 6.4.0 as Kibana endpoint not available - raise SkipTest + raise unittest.SkipTest self.render_config_template(modules=[{ "name": "kibana", diff --git a/metricbeat/module/logstash/test_logstash.py b/metricbeat/module/logstash/test_logstash.py index 4d5f7c233fc..b526515fdf0 100644 --- a/metricbeat/module/logstash/test_logstash.py +++ b/metricbeat/module/logstash/test_logstash.py @@ -7,7 +7,6 @@ import urllib.error import urllib.parse import urllib.request -from nose.plugins.skip import SkipTest sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -41,7 +40,7 @@ def test_xpack(self): version = self.get_version() if semver.compare(version, "7.3.0") == -1: # Skip for Logstash versions < 7.3.0 as necessary APIs not available - raise SkipTest + raise unittest.SkipTest self.render_config_template(modules=[{ "name": "logstash", diff --git a/metricbeat/module/mongodb/test_mongodb.py b/metricbeat/module/mongodb/test_mongodb.py index c62801b9853..7b01efc40f7 100644 --- a/metricbeat/module/mongodb/test_mongodb.py +++ b/metricbeat/module/mongodb/test_mongodb.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -15,7 +15,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mongodb'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_status(self): """ MongoDB module outputs an event. diff --git a/metricbeat/module/mysql/test_mysql.py b/metricbeat/module/mysql/test_mysql.py index 2422422a685..c1b8d6cc736 100644 --- a/metricbeat/module/mysql/test_mysql.py +++ b/metricbeat/module/mysql/test_mysql.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -18,7 +18,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mysql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_status(self): """ MySQL module outputs an event. diff --git a/metricbeat/module/postgresql/test_postgresql.py b/metricbeat/module/postgresql/test_postgresql.py index 8dded22cd32..e8643a436f4 100644 --- a/metricbeat/module/postgresql/test_postgresql.py +++ b/metricbeat/module/postgresql/test_postgresql.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -32,7 +32,7 @@ def get_hosts(self): ) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_activity(self): """ PostgreSQL module outputs an event. @@ -59,7 +59,7 @@ def test_activity(self): assert "state" in evt["postgresql"]["activity"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_database(self): """ PostgreSQL module outputs an event. @@ -89,7 +89,7 @@ def test_database(self): assert "deadlocks" in evt["postgresql"]["database"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_bgwriter(self): """ PostgreSQL module outputs an event. diff --git a/metricbeat/module/redis/test_redis.py b/metricbeat/module/redis/test_redis.py index 98ffb982ca3..f223de4bba9 100644 --- a/metricbeat/module/redis/test_redis.py +++ b/metricbeat/module/redis/test_redis.py @@ -1,8 +1,8 @@ import os +import pytest import redis import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -29,7 +29,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['redis'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_info(self): """ Test redis info metricset @@ -59,7 +59,7 @@ def test_info(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_keyspace(self): """ Test redis keyspace metricset @@ -95,7 +95,7 @@ def test_keyspace(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_key(self): """ Test redis key metricset @@ -133,7 +133,7 @@ def test_key(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_module_processors(self): """ Test local processors for Redis info event. diff --git a/metricbeat/module/uwsgi/test_uwsgi.py b/metricbeat/module/uwsgi/test_uwsgi.py index 6f58c0d6911..389a608426b 100644 --- a/metricbeat/module/uwsgi/test_uwsgi.py +++ b/metricbeat/module/uwsgi/test_uwsgi.py @@ -1,8 +1,8 @@ import logging import os +import pytest import sys import unittest -from nose.plugins.attrib import attr from parameterized import parameterized sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) @@ -56,7 +56,7 @@ def common_checks(self, output): @parameterized.expand(["http", "tcp"]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_status(self, proto): """ uWSGI module outputs an event. diff --git a/metricbeat/module/zookeeper/test_zookeeper.py b/metricbeat/module/zookeeper/test_zookeeper.py index 65db80bf48b..039fa550d15 100644 --- a/metricbeat/module/zookeeper/test_zookeeper.py +++ b/metricbeat/module/zookeeper/test_zookeeper.py @@ -1,7 +1,7 @@ import os +import pytest import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) import metricbeat @@ -21,7 +21,7 @@ class ZooKeeperMntrTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['zookeeper'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_output(self): """ ZooKeeper mntr module outputs an event. @@ -55,7 +55,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_output(self): """ ZooKeeper server module outputs an event. @@ -83,7 +83,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_connection(self): """ ZooKeeper server module outputs an event. diff --git a/metricbeat/tests/system/test_autodiscover_jolokia.py b/metricbeat/tests/system/test_autodiscover_jolokia.py index 727e4e62807..51d5dddd5a8 100644 --- a/metricbeat/tests/system/test_autodiscover_jolokia.py +++ b/metricbeat/tests/system/test_autodiscover_jolokia.py @@ -1,7 +1,6 @@ import os import metricbeat import unittest -from nose.plugins.attrib import attr class Test(metricbeat.BaseTest): diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py index d1cd2702265..a09a23f5ede 100644 --- a/metricbeat/tests/system/test_config.py +++ b/metricbeat/tests/system/test_config.py @@ -1,7 +1,6 @@ import os from metricbeat import BaseTest import unittest -from nose.plugins.attrib import attr import urllib.request import urllib.error import urllib.parse diff --git a/metricbeat/tests/system/test_template.py b/metricbeat/tests/system/test_template.py index 5899796524d..75b366ab15d 100644 --- a/metricbeat/tests/system/test_template.py +++ b/metricbeat/tests/system/test_template.py @@ -1,7 +1,7 @@ import os import metricbeat import json -from nose.plugins.skip import SkipTest +import unittest class Test(metricbeat.BaseTest): @@ -12,7 +12,7 @@ def test_export_template(self): """ if os.name == "nt": - raise SkipTest + raise unittest.SkipTest self.render_config_template("metricbeat", os.path.join(self.working_dir, diff --git a/packetbeat/tests/system/test_0015_udpjson.py b/packetbeat/tests/system/test_0015_udpjson.py index 454dd3a3523..06e38f49990 100644 --- a/packetbeat/tests/system/test_0015_udpjson.py +++ b/packetbeat/tests/system/test_0015_udpjson.py @@ -1,11 +1,11 @@ -from packetbeat import BaseTest -from nose.tools import nottest import socket +import unittest +from packetbeat import BaseTest +@unittest.skip("udpjson not supported anymore") class Test(BaseTest): - @nottest def test_udpjson_config(self): """ Should start with sniffer and udpjson inputs configured. @@ -21,7 +21,6 @@ def test_udpjson_config(self): assert all([o["type"] == "mysql" for o in objs]) assert len(objs) == 7 - @nottest def test_only_udpjson_config(self): """ It should be possible to start without the sniffer configured. @@ -39,7 +38,6 @@ def test_only_udpjson_config(self): packetbeat.kill_and_wait() - @nottest def test_send_udpjson_msg(self): """ It should be possible to send a UDP message and read it from diff --git a/pytest.ini b/pytest.ini index 56711bbf764..3bb37ea8469 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,7 @@ [pytest] junit_family=xunit1 + +addopts = --strict-markers +markers = + load: load tests + integration: integration tests diff --git a/x-pack/metricbeat/module/mssql/test_mssql.py b/x-pack/metricbeat/module/mssql/test_mssql.py index fe839cb8969..94de71cfeac 100644 --- a/x-pack/metricbeat/module/mssql/test_mssql.py +++ b/x-pack/metricbeat/module/mssql/test_mssql.py @@ -1,7 +1,6 @@ import os import sys import unittest -from nose.plugins.attrib import attr sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) from xpack_metricbeat import XPackTest, metricbeat @@ -18,7 +17,7 @@ class Test(XPackTest): COMPOSE_SERVICES = ['mssql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_status(self): """ MSSQL module outputs an event. @@ -47,7 +46,7 @@ def test_status(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') + @pytest.mark.integration def test_performance(self): """ MSSQL module outputs an event. From 608a9c655b352bec83ae5fdba314a4b6b0bdd20c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 21:48:47 +0100 Subject: [PATCH 05/19] Tag integration tests --- libbeat/tests/system/test_ca_pinning.py | 4 +-- .../system/test_cmd_setup_index_management.py | 24 ++++++++--------- libbeat/tests/system/test_cmd_test.py | 2 +- libbeat/tests/system/test_dashboard.py | 26 +++++++++---------- libbeat/tests/system/test_ilm.py | 22 ++++++++-------- libbeat/tests/system/test_monitoring.py | 8 +++--- libbeat/tests/system/test_template.py | 18 ++++++------- metricbeat/module/apache/test_apache.py | 2 +- metricbeat/module/consul/test_consul.py | 2 +- metricbeat/module/mongodb/test_mongodb.py | 2 +- metricbeat/module/mysql/test_mysql.py | 2 +- .../module/postgresql/test_postgresql.py | 6 ++--- metricbeat/module/redis/test_redis.py | 8 +++--- metricbeat/module/uwsgi/test_uwsgi.py | 2 +- metricbeat/module/zookeeper/test_zookeeper.py | 6 ++--- pytest.ini | 4 +-- x-pack/metricbeat/module/mssql/test_mssql.py | 4 +-- 17 files changed, 71 insertions(+), 71 deletions(-) diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py index a5192ff73b2..4c1480b82a0 100644 --- a/libbeat/tests/system/test_ca_pinning.py +++ b/libbeat/tests/system/test_ca_pinning.py @@ -15,7 +15,7 @@ class TestCAPinning(BaseTest): """ @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_sending_events_with_a_good_sha256(self): """ Test Sending events while using ca pinning with a good sha256 @@ -47,7 +47,7 @@ def test_sending_events_with_a_good_sha256(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_sending_events_with_a_bad_sha256(self): """ Test Sending events while using ca pinning with a bad sha256 diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 9cbf0ec346d..85c51ede7f9 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -46,7 +46,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_default(self): """ Test setup --index-management with default config @@ -63,7 +63,7 @@ def test_setup_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') @pytest.raises(RequestError) def test_setup_default(self): """ @@ -82,7 +82,7 @@ def test_setup_default(self): self.idxmgmt.delete_policy(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_disabled(self): """ Test setup --index-management when ilm disabled @@ -98,7 +98,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_ilm_disabled(self): """ Test setup --index-management when ilm disabled @@ -114,7 +114,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_policy_name(self): """ Test setup --index-management when policy_name is configured @@ -129,7 +129,7 @@ def test_setup_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_ilm_policy_no_overwrite(self): """ Test setup --index-management respects overwrite configuration @@ -176,7 +176,7 @@ def test_setup_ilm_policy_no_overwrite(self): assert "hot" in resp[policy_name]["policy"]["phases"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_rollover_alias(self): """ Test setup --index-management when ilm.rollover_alias is configured @@ -192,7 +192,7 @@ def test_setup_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_rollover_alias_with_fieldref(self): """ Test setup --index-management when ilm.rollover_alias is configured and using field reference. @@ -211,7 +211,7 @@ def test_setup_rollover_alias_with_fieldref(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_name_and_pattern(self): """ Test setup --index-management ignores template.name and template.pattern when ilm is enabled @@ -228,7 +228,7 @@ def test_setup_template_name_and_pattern(self): self.idxmgmt.assert_alias_created(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_name_and_pattern_on_ilm_disabled(self): """ Test setup --index-management respects template.name and template.pattern when ilm is disabled @@ -247,7 +247,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_with_opts(self): """ Test setup --index-management with config options @@ -269,7 +269,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_overwrite_template_on_ilm_policy_created(self): """ Test setup --index-management overwrites template when new ilm policy is created diff --git a/libbeat/tests/system/test_cmd_test.py b/libbeat/tests/system/test_cmd_test.py index acb29697d94..38f15ef095f 100644 --- a/libbeat/tests/system/test_cmd_test.py +++ b/libbeat/tests/system/test_cmd_test.py @@ -41,7 +41,7 @@ def test_bad_config(self): assert self.log_contains("Config OK") is False @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_output(self): """ Test test output works diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index d856625f593..86b4355c7f7 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -13,7 +13,7 @@ class Test(BaseTest): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_load_without_dashboard(self): """ Test loading without dashboards @@ -38,7 +38,7 @@ def test_load_without_dashboard(self): assert self.log_contains("Skipping loading dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_load_dashboard(self): """ Test loading dashboards @@ -63,7 +63,7 @@ def test_load_dashboard(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_load_dashboard_into_space(self, create_space=True): """ Test loading dashboards into Kibana space @@ -97,7 +97,7 @@ def test_load_dashboard_into_space(self, create_space=True): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_load_only_index_patterns(self): """ Test loading dashboards @@ -123,7 +123,7 @@ def test_load_only_index_patterns(self): assert self.log_contains("Kibana dashboards successfully loaded") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): """ Test testbeat export dashboard can export dashboards @@ -147,7 +147,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id(self): """ Test testbeat export dashboard can export dashboards @@ -169,7 +169,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id(self): assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): """ Test testbeat export dashboard fails gracefully when dashboard with unknown ID is requested @@ -190,7 +190,7 @@ def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): assert self.log_contains("error exporting dashboard: Not found") is True @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_from_yml(self): """ Test testbeat export dashboard can export dashboards from dashboards YAML file @@ -223,7 +223,7 @@ def test_export_dashboard_cmd_export_dashboard_from_yml(self): os.remove(exported_dashboard_path) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): """ Test testbeat export dashboard fails gracefully when cannot find YAML file @@ -245,7 +245,7 @@ def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): assert self.log_contains("error opening the list of dashboards") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id(self): """ Test dev-tools/cmd/dashboards exports dashboard and removes unsupported characters @@ -271,7 +271,7 @@ def test_dev_tool_export_dashboard_by_id(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id_unknown_id(self): """ Test dev-tools/cmd/dashboards fails gracefully when dashboard with unknown ID is requested @@ -287,7 +287,7 @@ def test_dev_tool_export_dashboard_by_id_unknown_id(self): assert p.returncode != 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_by_id_from_space(self): """ Test dev-tools/cmd/dashboards exports dashboard from Kibana space @@ -318,7 +318,7 @@ def test_dev_tool_export_dashboard_by_id_from_space(self): os.remove("output.json") @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_dev_tool_export_dashboard_from_yml(self): """ Test dev-tools/cmd/dashboards exports dashboard from dashboards YAML file diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 85461587299..8b637d6afb8 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -36,7 +36,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_ilm_default(self): """ Test ilm default settings to load ilm policy, write alias and ilm template @@ -54,7 +54,7 @@ def test_ilm_default(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_ilm_disabled(self): """ Test ilm disabled to not load ilm related components @@ -71,7 +71,7 @@ def test_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_policy_name(self): """ Test setting ilm policy name @@ -91,7 +91,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_rollover_alias(self): """ Test settings ilm rollover alias @@ -110,7 +110,7 @@ def test_rollover_alias(self): self.idxmgmt.assert_alias_created(self.custom_alias) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_pattern(self): """ Test setting ilm pattern @@ -130,7 +130,7 @@ def test_pattern(self): self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_pattern_date(self): """ Test setting ilm pattern with date @@ -186,7 +186,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_ilm_policy_and_template(self): """ Test combination of ilm policy and template setup @@ -203,7 +203,7 @@ def test_setup_ilm_policy_and_template(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_ilm_default(self): """ Test ilm policy setup with default config @@ -220,7 +220,7 @@ def test_setup_ilm_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_ilm_disabled(self): """ Test ilm policy setup when ilm disabled @@ -237,7 +237,7 @@ def test_setup_ilm_disabled(self): self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_policy_name(self): """ Test ilm policy setup when policy_name is configured @@ -253,7 +253,7 @@ def test_policy_name(self): self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_rollover_alias(self): """ Test ilm policy setup when rollover_alias is configured diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index da8d6cfab21..d8d461b1970 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -20,7 +20,7 @@ def setUp(self): self.es_monitoring = Elasticsearch([self.get_elasticsearch_monitoring_url()]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_via_output_cluster(self): """ Test shipping monitoring data via the elasticsearch output cluster. @@ -57,7 +57,7 @@ def test_via_output_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_direct_to_monitoring_cluster(self): """ Test shipping monitoring data directly to the monitoring cluster. @@ -91,7 +91,7 @@ def test_direct_to_monitoring_cluster(self): self.assert_monitoring_doc_contains_fields(monitoring_doc_type, field_names) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_compare(self): """ Test that monitoring docs are the same, regardless of how they are shipped. @@ -154,7 +154,7 @@ def test_compare(self): self.assert_same_structure(indirect_beats_stats_doc['beats_stats'], direct_beats_stats_doc['beats_stats']) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_cluster_uuid_setting(self): """ Test that monitoring.cluster_uuid setting may be set without any other monitoring.* settings diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 26e8f88221f..88d0bbcb9ac 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -82,7 +82,7 @@ def test_index_with_pattern_name(self): proc.check_kill_and_wait() @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_json_template(self): """ Test loading of json based template @@ -139,7 +139,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_template_default(self): """ Test run cmd with default settings for template @@ -156,7 +156,7 @@ def test_template_default(self): self.idxmgmt.assert_docs_written_to_alias(self.index_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_template_disabled(self): """ Test run cmd does not load template when disabled in config @@ -200,7 +200,7 @@ def render_config(self, **kwargs): ) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup(self): """ Test setup cmd with template and ilm-policy subcommands @@ -215,7 +215,7 @@ def test_setup(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_default(self): """ Test template setup with default config @@ -234,7 +234,7 @@ def test_setup_template_default(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_disabled(self): """ Test template setup when ilm disabled @@ -253,7 +253,7 @@ def test_setup_template_disabled(self): self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_with_opts(self): """ Test template setup with config options @@ -274,7 +274,7 @@ def test_setup_template_with_opts(self): assert index["number_of_shards"] == "2", index["number_of_shards"] @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_setup_template_with_ilm_changed_pattern(self): """ Test template setup with changed ilm.rollover_alias config @@ -289,7 +289,7 @@ def test_setup_template_with_ilm_changed_pattern(self): self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_template_created_on_ilm_policy_created(self): """ Test template setup overwrites template when new ilm policy is created diff --git a/metricbeat/module/apache/test_apache.py b/metricbeat/module/apache/test_apache.py index b4aab34bb57..396c60e3847 100644 --- a/metricbeat/module/apache/test_apache.py +++ b/metricbeat/module/apache/test_apache.py @@ -39,7 +39,7 @@ class ApacheStatusTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['apache'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_output(self): """ Apache module outputs an event. diff --git a/metricbeat/module/consul/test_consul.py b/metricbeat/module/consul/test_consul.py index ed56ca5702f..2702c144c3a 100644 --- a/metricbeat/module/consul/test_consul.py +++ b/metricbeat/module/consul/test_consul.py @@ -27,7 +27,7 @@ class ConsulAgentTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['consul'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_output(self): """ Consul agent module outputs an event. diff --git a/metricbeat/module/mongodb/test_mongodb.py b/metricbeat/module/mongodb/test_mongodb.py index 7b01efc40f7..bd53edd1060 100644 --- a/metricbeat/module/mongodb/test_mongodb.py +++ b/metricbeat/module/mongodb/test_mongodb.py @@ -15,7 +15,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mongodb'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_status(self): """ MongoDB module outputs an event. diff --git a/metricbeat/module/mysql/test_mysql.py b/metricbeat/module/mysql/test_mysql.py index c1b8d6cc736..4a8b34728ed 100644 --- a/metricbeat/module/mysql/test_mysql.py +++ b/metricbeat/module/mysql/test_mysql.py @@ -18,7 +18,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['mysql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_status(self): """ MySQL module outputs an event. diff --git a/metricbeat/module/postgresql/test_postgresql.py b/metricbeat/module/postgresql/test_postgresql.py index e8643a436f4..a8d286d5685 100644 --- a/metricbeat/module/postgresql/test_postgresql.py +++ b/metricbeat/module/postgresql/test_postgresql.py @@ -32,7 +32,7 @@ def get_hosts(self): ) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_activity(self): """ PostgreSQL module outputs an event. @@ -59,7 +59,7 @@ def test_activity(self): assert "state" in evt["postgresql"]["activity"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_database(self): """ PostgreSQL module outputs an event. @@ -89,7 +89,7 @@ def test_database(self): assert "deadlocks" in evt["postgresql"]["database"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_bgwriter(self): """ PostgreSQL module outputs an event. diff --git a/metricbeat/module/redis/test_redis.py b/metricbeat/module/redis/test_redis.py index f223de4bba9..1f6082b5626 100644 --- a/metricbeat/module/redis/test_redis.py +++ b/metricbeat/module/redis/test_redis.py @@ -29,7 +29,7 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['redis'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_info(self): """ Test redis info metricset @@ -59,7 +59,7 @@ def test_info(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_keyspace(self): """ Test redis keyspace metricset @@ -95,7 +95,7 @@ def test_keyspace(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_key(self): """ Test redis key metricset @@ -133,7 +133,7 @@ def test_key(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_module_processors(self): """ Test local processors for Redis info event. diff --git a/metricbeat/module/uwsgi/test_uwsgi.py b/metricbeat/module/uwsgi/test_uwsgi.py index 389a608426b..1659a7e3e4f 100644 --- a/metricbeat/module/uwsgi/test_uwsgi.py +++ b/metricbeat/module/uwsgi/test_uwsgi.py @@ -56,7 +56,7 @@ def common_checks(self, output): @parameterized.expand(["http", "tcp"]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_status(self, proto): """ uWSGI module outputs an event. diff --git a/metricbeat/module/zookeeper/test_zookeeper.py b/metricbeat/module/zookeeper/test_zookeeper.py index 039fa550d15..99184119cea 100644 --- a/metricbeat/module/zookeeper/test_zookeeper.py +++ b/metricbeat/module/zookeeper/test_zookeeper.py @@ -21,7 +21,7 @@ class ZooKeeperMntrTest(metricbeat.BaseTest): COMPOSE_SERVICES = ['zookeeper'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_output(self): """ ZooKeeper mntr module outputs an event. @@ -55,7 +55,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_output(self): """ ZooKeeper server module outputs an event. @@ -83,7 +83,7 @@ def test_output(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_connection(self): """ ZooKeeper server module outputs an event. diff --git a/pytest.ini b/pytest.ini index 3bb37ea8469..1daee08918d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ junit_family=xunit1 addopts = --strict-markers markers = - load: load tests - integration: integration tests + load: Load tests + tag(name): Tag tests with Go-like semantics diff --git a/x-pack/metricbeat/module/mssql/test_mssql.py b/x-pack/metricbeat/module/mssql/test_mssql.py index 94de71cfeac..b3f9dbacaba 100644 --- a/x-pack/metricbeat/module/mssql/test_mssql.py +++ b/x-pack/metricbeat/module/mssql/test_mssql.py @@ -17,7 +17,7 @@ class Test(XPackTest): COMPOSE_SERVICES = ['mssql'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_status(self): """ MSSQL module outputs an event. @@ -46,7 +46,7 @@ def test_status(self): self.assert_fields_are_documented(evt) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @pytest.mark.integration + @pytest.mark.tag('integration') def test_performance(self): """ MSSQL module outputs an event. From 27b82b599bb1b5e30080876721afa33de3161947 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 22:07:11 +0100 Subject: [PATCH 06/19] Fix format --- metricbeat/module/elasticsearch/test_elasticsearch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index 08813a49342..3bbff028d9d 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -316,10 +316,12 @@ def start_basic(self): def check_skip(self, metricset): if metricset == 'ccr' and not self.is_ccr_available(): - raise unittest.SkipTest("elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") + raise unittest.SkipTest( + "elasticsearch/ccr metricset system test only valid with Elasticsearch versions >= 6.5.0") if metricset == 'enrich' and not self.is_enrich_available(): - raise unittest.SkipTest("elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") + raise unittest.SkipTest( + "elasticsearch/enrich metricset system test only valid with Elasticsearch versions >= 7.5.0") def is_ccr_available(self): es_version = self.get_version() From 9c5a2de352aa3846fad46c0642ea136592e3208c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Mar 2020 22:52:30 +0100 Subject: [PATCH 07/19] Fix options also in makefile --- libbeat/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 93ffa5f05a6..6b0c27008cd 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -82,7 +82,7 @@ REVIEWDOG_REPO?=github.com/reviewdog/reviewdog/cmd/reviewdog PROCESSES?= 4 TIMEOUT?= 90 PYTHON_TEST_FILES?=$(shell find . -type f -name 'test_*.py' -not -path "*/build/*" -not -path "*/vendor/*" 2>/dev/null) -PYTEST_OPTIONS?=--timeout=$(TIMEOUT) --with-timer -v --with-xunit --xunit-file=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest +PYTEST_OPTIONS?=--timeout=$(TIMEOUT) --durations=20 --junit-xml=${BUILD_DIR}/TEST-system.xml ## @testing the options to pass when calling pytest TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests From 80043b0592061bc1901ef206b4a44165ef35be24 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jul 2020 19:08:29 +0200 Subject: [PATCH 08/19] Replace uses of NOSE env variables --- .ci/windows.groovy | 2 +- Jenkinsfile | 2 +- dev-tools/mage/target/integtest/integtest.go | 7 +++---- filebeat/magefile.go | 2 +- metricbeat/magefile.go | 7 +++---- x-pack/filebeat/magefile.go | 2 +- x-pack/metricbeat/magefile.go | 7 +++---- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.ci/windows.groovy b/.ci/windows.groovy index 8c83bde0a6a..ae4c19f836a 100644 --- a/.ci/windows.groovy +++ b/.ci/windows.groovy @@ -527,7 +527,7 @@ def dumpFilteredEnvironment(){ echo "PROCESSES: ${env.PROCESSES}" echo "TIMEOUT: ${env.TIMEOUT}" echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" diff --git a/Jenkinsfile b/Jenkinsfile index c415e7ef605..05471f74735 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1088,7 +1088,7 @@ def dumpFilteredEnvironment(){ echo "PROCESSES: ${env.PROCESSES}" echo "TIMEOUT: ${env.TIMEOUT}" echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" diff --git a/dev-tools/mage/target/integtest/integtest.go b/dev-tools/mage/target/integtest/integtest.go index 42e9633469d..7b7cae52819 100644 --- a/dev-tools/mage/target/integtest/integtest.go +++ b/dev-tools/mage/target/integtest/integtest.go @@ -74,14 +74,13 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(pythonTestDeps...) } - runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("NOSE_")...)...) + runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("PYTEST_")...)...) if err != nil { return err } diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 5661be14b74..0d68e5a86c4 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -218,7 +218,7 @@ func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.Deps(Fields) } - runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "NOSE_"), "GENERATE")...) + runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "PYTEST_"), "GENERATE")...) if err != nil { return err } diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 84474e39fe5..ca0372baa0b 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -191,14 +191,13 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). // Use MODULE=module to run only tests for `module`. -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } - runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("NOSE_")...) + runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("PYTEST_")...) if err != nil { return err } diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index d32a5e6139c..9c7f436e2e4 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -174,7 +174,7 @@ func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.Deps(Fields) } - runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "NOSE_"), "GENERATE")...) + runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "PYTEST_"), "GENERATE")...) if err != nil { return err } diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index f7f98a7c38e..4f15998df17 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -152,14 +152,13 @@ func GoIntegTest(ctx context.Context) error { // PythonIntegTest executes the python system tests in the integration // environment (Docker). // Use MODULE=module to run only tests for `module`. -// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern. -// Use any other NOSE_* environment variable to influence the behavior of -// nosetests. +// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. +// Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } - runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("NOSE_")...) + runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("PYTEST_")...) if err != nil { return err } From 1ac49e9f5545a73217aee1e3ec6b88ef3778bbed Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jul 2020 19:11:00 +0200 Subject: [PATCH 09/19] Replace module-specific python nose targets --- dev-tools/mage/pytest.go | 6 +++--- metricbeat/magefile.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 658efb11970..e562fdef95b 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -171,10 +171,10 @@ func PythonTest(params PythonTestArgs) error { // and HTML report. } -// PythonNoseTestForModule executes python system tests for modules. +// PythonTestForModule executes python system tests for modules. // // Use `MODULE=module` to run only tests for `module`. -func PythonNoseTestForModule(params PythonTestArgs) error { +func PythonTestForModule(params PythonTestArgs) error { if module := EnvOr("MODULE", ""); module != "" { params.Files = []string{ fmt.Sprintf("module/%s/test_*.py", module), @@ -182,7 +182,7 @@ func PythonNoseTestForModule(params PythonTestArgs) error { } params.TestName += "-" + module } - return PythonNoseTest(params) + return PythonTest(params) } // PythonVirtualenv constructs a virtualenv that contains the given modules as diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index ca0372baa0b..661239410b2 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -203,6 +203,6 @@ func PythonIntegTest(ctx context.Context) error { } return runner.Test("pythonIntegTest", func() error { mg.Deps(devtools.BuildSystemTestBinary) - return devtools.PythonNoseTestForModule(devtools.DefaultPythonTestIntegrationArgs()) + return devtools.PythonTestForModule(devtools.DefaultPythonTestIntegrationArgs()) }) } From 9eae7a8e94207751526f744cc232cf1297315a88 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jul 2020 19:32:52 +0200 Subject: [PATCH 10/19] Upgrade pytest --- libbeat/tests/system/requirements.txt | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index e19cdc82b8b..c073b49544b 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,36 +1,47 @@ +attrs==19.3.0 autopep8==1.3.5 backports.ssl-match-hostname==3.5.0.1 cached-property==1.4.2 certifi==2018.1.18 chardet==3.0.4 +deepdiff==4.2.0 docker==4.1.0 docker-compose==1.25.3 -docker-pycreds==0.4.0 dockerpty==0.4.1 +docker-pycreds==0.4.0 docopt==0.6.2 elasticsearch==7.1.0 enum34==1.1.6 idna==2.6 +importlib-metadata==1.7.0 +iniconfig==1.0.1 ipaddress==1.0.19 Jinja2==2.10.1 +jsondiff==1.1.2 jsonschema==2.6.0 +kafka-python==1.4.3 MarkupSafe==1.1.1 +more-itertools==8.4.0 +ordered-set==3.1.1 +packaging==20.4 +parameterized==0.7.0 +Pillow==7.2.0 +pluggy==0.13.1 +py==1.9.0 pycodestyle==2.4.0 +pyparsing==2.4.7 +pytest==6.0.1 +pytest-timeout==1.3.4 PyYAML==4.2b1 -Pillow>=7.1.0 redis==2.10.6 requests==2.20.0 -six==1.11.0 +semver==2.8.1 +six==1.15.0 +stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 +toml==0.10.1 urllib3==1.24.2 +wcwidth==0.2.5 websocket-client==0.47.0 -parameterized==0.7.0 -jsondiff==1.1.2 -semver==2.8.1 -stomp.py==4.1.22 -ordered-set==3.1.1 -deepdiff==4.2.0 -pytest==5.3.5 -pytest-timeout==1.3.4 -kafka-python==1.4.3 +zipp==3.1.0 From 8ebbe3390b56708a5a7a5618132486a55dfe8ec3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jul 2020 19:35:19 +0200 Subject: [PATCH 11/19] Upgrade deprecated dependencies --- heartbeat/docs/fields.asciidoc | 4 ++-- libbeat/tests/system/requirements.txt | 9 +++++---- packetbeat/docs/fields.asciidoc | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 1e2881bdbff..7b52078d294 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -7952,7 +7952,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- @@ -7963,7 +7963,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index c073b49544b..cae946fc916 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -7,8 +7,8 @@ chardet==3.0.4 deepdiff==4.2.0 docker==4.1.0 docker-compose==1.25.3 -dockerpty==0.4.1 docker-pycreds==0.4.0 +dockerpty==0.4.1 docopt==0.6.2 elasticsearch==7.1.0 enum34==1.1.6 @@ -16,9 +16,9 @@ idna==2.6 importlib-metadata==1.7.0 iniconfig==1.0.1 ipaddress==1.0.19 -Jinja2==2.10.1 +Jinja2==2.11.2 jsondiff==1.1.2 -jsonschema==2.6.0 +jsonschema==3.2.0 kafka-python==1.4.3 MarkupSafe==1.1.1 more-itertools==8.4.0 @@ -30,9 +30,10 @@ pluggy==0.13.1 py==1.9.0 pycodestyle==2.4.0 pyparsing==2.4.7 +pyrsistent==0.16.0 pytest==6.0.1 pytest-timeout==1.3.4 -PyYAML==4.2b1 +PyYAML==5.3.1 redis==2.10.6 requests==2.20.0 semver==2.8.1 diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index cb66e939a76..5100c738baf 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -10484,7 +10484,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- @@ -10495,7 +10495,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- @@ -10756,7 +10756,7 @@ Time at which the certificate is first considered valid. type: date -example: 2019-08-16 01:40:25 +example: 2019-08-16 01:40:25+00:00 -- @@ -10767,7 +10767,7 @@ Time at which the certificate is no longer considered valid. type: date -example: 2020-07-16 03:15:39 +example: 2020-07-16 03:15:39+00:00 -- From 36b2e58b6dcfcebc1571afb08fdaf464e80f58fa Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jul 2020 19:42:29 +0200 Subject: [PATCH 12/19] Print also PYTEST_OPTIONS in CI --- .ci/windows.groovy | 1 + Jenkinsfile | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/windows.groovy b/.ci/windows.groovy index ae4c19f836a..92826e80926 100644 --- a/.ci/windows.groovy +++ b/.ci/windows.groovy @@ -528,6 +528,7 @@ def dumpFilteredEnvironment(){ echo "TIMEOUT: ${env.TIMEOUT}" echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" + echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" diff --git a/Jenkinsfile b/Jenkinsfile index 05471f74735..06449db85b5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1089,6 +1089,7 @@ def dumpFilteredEnvironment(){ echo "TIMEOUT: ${env.TIMEOUT}" echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" + echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" From c81b1bbbbb62c4da3d603c9cd594c7ebdaee6bef Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 11:48:58 +0200 Subject: [PATCH 13/19] Fix more tests --- filebeat/tests/system/test_load.py | 4 +++- filebeat/tests/system/test_tcp_tls.py | 22 ++++++++++--------- libbeat/tests/system/idxmgmt.py | 8 +++---- .../system/test_cmd_setup_index_management.py | 5 +++-- libbeat/tests/system/test_dashboard.py | 3 +-- libbeat/tests/system/test_ilm.py | 10 +++++---- libbeat/tests/system/test_monitoring.py | 5 +++-- libbeat/tests/system/test_template.py | 8 ++++--- x-pack/metricbeat/module/mssql/test_mssql.py | 1 + 9 files changed, 38 insertions(+), 28 deletions(-) diff --git a/filebeat/tests/system/test_load.py b/filebeat/tests/system/test_load.py index de3a66dd3e3..6a35a6ce250 100644 --- a/filebeat/tests/system/test_load.py +++ b/filebeat/tests/system/test_load.py @@ -6,6 +6,8 @@ import unittest import pytest +from filebeat import BaseTest + """ Test filebeat under different load scenarios """ @@ -61,7 +63,7 @@ def test_no_missing_events(self): for i in range(total_lines): # Make sure each line has the same length line = format(i, str(line_length - 1)) - logger.debug("%d", i) + logger.warning("%d", i) # wait until all lines are read self.wait_until( diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index f022376f101..4001fd863c9 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -2,8 +2,9 @@ import ssl import unittest import pytest -from os import path + from filebeat import BaseTest +from os import path NUMBER_OF_EVENTS = 2 @@ -86,7 +87,6 @@ def test_tcp_over_tls_and_verify_valid_server_without_mutual_auth(self): sock.close() - @pytest.raises(ssl.SSLError) def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): """ Test filebeat TCP with TLS with an invalid cacert and not requiring mutual auth. @@ -123,9 +123,10 @@ def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP tls = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=CERTIFICATE2, do_handshake_on_connect=True) - tls.connect((config.get('host'), config.get('port'))) - @pytest.raises(ssl.SSLError) + with pytest.raises(ssl.SSLError): + tls.connect((config.get('host'), config.get('port'))) + def test_tcp_over_tls_mutual_auth_fails(self): """ Test filebeat TCP with TLS with default setting to enforce client auth, with bad client certificates @@ -162,12 +163,13 @@ def test_tcp_over_tls_mutual_auth_fails(self): tls = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=CERTIFICATE1, do_handshake_on_connect=True) - tls.connect((config.get('host'), config.get('port'))) - # In TLS 1.3 authentication failures are not detected by the initial - # connection and handshake. For the client to detect that authentication - # has failed (at least in python) it must wait for a server response - # so that the failure can be reported as an exception when it arrives. - tls.recv(1) + with pytest.raises(ssl.SSLError): + tls.connect((config.get('host'), config.get('port'))) + # In TLS 1.3 authentication failures are not detected by the initial + # connection and handshake. For the client to detect that authentication + # has failed (at least in python) it must wait for a server response + # so that the failure can be reported as an exception when it arrives. + tls.recv(1) def test_tcp_over_tls_mutual_auth_succeed(self): """ diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index 83dbd89368d..864d134a9c9 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -52,9 +52,9 @@ def delete_policy(self, policy): except NotFoundError: pass - @pytest.raises(NotFoundError) def assert_index_template_not_loaded(self, template): - self._client.transport.perform_request('GET', '/_template/' + template) + with pytest.raises(NotFoundError): + self._client.transport.perform_request('GET', '/_template/' + template) def assert_index_template_loaded(self, template): resp = self._client.transport.perform_request('GET', '/_template/' + template) @@ -86,9 +86,9 @@ def assert_alias_created(self, alias, pattern=None): assert name in resp assert resp[name]["aliases"][alias]["is_write_index"] == True - @pytest.raises(NotFoundError) def assert_policy_not_created(self, policy): - self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) + with pytest.raises(NotFoundError): + self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) def assert_policy_created(self, policy): resp = self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 85c51ede7f9..8a5b6100fad 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -2,6 +2,7 @@ import os import pytest import unittest + from base import BaseTest from elasticsearch import RequestError from idxmgmt import IdxMgmt @@ -64,7 +65,6 @@ def test_setup_default(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') - @pytest.raises(RequestError) def test_setup_default(self): """ Test setup --index-management with default config @@ -79,7 +79,8 @@ def test_setup_default(self): self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) # try deleting policy needs to raise an error as it is in use - self.idxmgmt.delete_policy(self.policy_name) + with pytest.raises(RequestError): + self.idxmgmt.delete_policy(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 8624645fc4c..879ebd1515f 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -1,12 +1,11 @@ import os import os.path +import pytest import re import requests import semver import subprocess import unittest -import requests -import semver from base import BaseTest from unittest import SkipTest diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 8b637d6afb8..3d37125f6e4 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -1,9 +1,11 @@ -import os -import unittest -import shutil import datetime -import logging import json +import logging +import os +import pytest +import shutil +import unittest + from base import BaseTest from idxmgmt import IdxMgmt diff --git a/libbeat/tests/system/test_monitoring.py b/libbeat/tests/system/test_monitoring.py index 1c8af50e890..a1fe0f3fad1 100644 --- a/libbeat/tests/system/test_monitoring.py +++ b/libbeat/tests/system/test_monitoring.py @@ -1,9 +1,10 @@ import os +import pytest +import random import re -import unittest import requests -import random import string +import unittest from base import BaseTest from elasticsearch import Elasticsearch diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 88d0bbcb9ac..67a34457484 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -1,8 +1,10 @@ +import json +import logging import os -import unittest +import pytest import shutil -import logging -import json +import unittest + from base import BaseTest from idxmgmt import IdxMgmt diff --git a/x-pack/metricbeat/module/mssql/test_mssql.py b/x-pack/metricbeat/module/mssql/test_mssql.py index b3f9dbacaba..225dde74fb2 100644 --- a/x-pack/metricbeat/module/mssql/test_mssql.py +++ b/x-pack/metricbeat/module/mssql/test_mssql.py @@ -1,4 +1,5 @@ import os +import pytest import sys import unittest From f49b774142e223708dd1fd2811fc68b22a0f6ca7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 13:57:11 +0200 Subject: [PATCH 14/19] Fix x-pack metricbeat template paths --- libbeat/tests/system/beat/beat.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 1c6a99ba73f..41c7cc1f319 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -134,14 +134,11 @@ def setUpClass(self): if not hasattr(self, 'test_binary'): self.test_binary = os.path.abspath(self.beat_path + "/" + self.beat_name + ".test") - template_paths = [ - self.beat_path, - os.path.abspath(os.path.join(self.beat_path, "../libbeat")) - ] if not hasattr(self, 'template_paths'): - self.template_paths = template_paths - else: - self.template_paths.append(template_paths) + self.template_paths = [ + self.beat_path, + os.path.abspath(os.path.join(self.beat_path, "../libbeat")) + ] # Create build path build_dir = self.beat_path + "/build" From 3826048666e5f5c80fa21459a2dc2986becf47a1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 15:11:32 +0200 Subject: [PATCH 15/19] Use SkipTest from unittest --- heartbeat/tests/system/test_monitor.py | 7 ++++--- libbeat/tests/system/test_dashboard.py | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/heartbeat/tests/system/test_monitor.py b/heartbeat/tests/system/test_monitor.py index 371ad775dd3..98ace615bd3 100644 --- a/heartbeat/tests/system/test_monitor.py +++ b/heartbeat/tests/system/test_monitor.py @@ -1,5 +1,6 @@ import os import unittest + from heartbeat import BaseTest from parameterized import parameterized @@ -33,7 +34,7 @@ def test_http(self, status_code): if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) @parameterized.expand([ @@ -63,7 +64,7 @@ def test_http_with_hosts_config(self, status_code): if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) def test_http_delayed(self): @@ -200,7 +201,7 @@ def test_tcp(self, url, status): self.assert_last_status(status) if os.name == "nt": # Currently skipped on Windows as fields.yml not generated - raise SkipTest + raise unittest.SkipTest self.assert_fields_are_documented(output[0]) finally: server.shutdown() diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 879ebd1515f..a7b3145c1e7 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -8,7 +8,6 @@ import unittest from base import BaseTest -from unittest import SkipTest INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -74,7 +73,7 @@ def test_load_dashboard_into_space(self, create_space=True): version = self.get_version() if semver.compare(version, "6.5.0") == -1: # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available - raise SkipTest + raise unittest.SkipTest self.render_config_template() if create_space: @@ -300,7 +299,7 @@ def test_dev_tool_export_dashboard_by_id_from_space(self): version = self.get_version() if semver.compare(version, "6.5.0") == -1: # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available - raise SkipTest + raise unittest.SkipTest self.test_load_dashboard_into_space(False) From 1c1eee5fda282ea157577d79913e2beb73ae23bc Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 15:46:24 +0200 Subject: [PATCH 16/19] Missing import --- filebeat/tests/system/test_registrar_upgrade.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filebeat/tests/system/test_registrar_upgrade.py b/filebeat/tests/system/test_registrar_upgrade.py index a6054e37297..91e0664d214 100644 --- a/filebeat/tests/system/test_registrar_upgrade.py +++ b/filebeat/tests/system/test_registrar_upgrade.py @@ -3,6 +3,8 @@ import os import json +import unittest + from filebeat import BaseTest From 7e465960ea8977a2eba3ddf23b949a59430b3259 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 16:17:45 +0200 Subject: [PATCH 17/19] Apply timeout only to test function, not to setup --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index 1daee08918d..cc816886a6e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,6 @@ addopts = --strict-markers markers = load: Load tests tag(name): Tag tests with Go-like semantics + +# Ignore setup and teardown for the timeout +timeout_func_only = True From 6b99dab32c8ce871d355c16e6d73470ec18961d9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 17:44:24 +0200 Subject: [PATCH 18/19] Increase timeout of dashboards test --- metricbeat/tests/system/test_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index 7df5152ccd6..f43d738f050 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -1,12 +1,15 @@ +import os +import pytest import re +import shutil import sys import unittest -import os -import shutil + from metricbeat import BaseTest -from elasticsearch import Elasticsearch + from beat.beat import INTEGRATION_TESTS from beat import common_tests +from elasticsearch import Elasticsearch class Test(BaseTest, common_tests.TestExportsMixin): @@ -54,6 +57,7 @@ def test_template(self): assert len(es.cat.templates(name='metricbeat-*', h='name')) > 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @pytest.mark.timeout(180) def test_dashboards(self): """ Test that the dashboards can be loaded with `setup --dashboards` From 7a44d0e49e5563e9b81ed3b6e1a4def2c8b5f0dd Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 6 Aug 2020 12:18:17 +0200 Subject: [PATCH 19/19] Fix timeout in test_dashboards --- metricbeat/tests/system/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index f43d738f050..1c42b967ac3 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -57,7 +57,7 @@ def test_template(self): assert len(es.cat.templates(name='metricbeat-*', h='name')) > 0 @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.timeout(180) + @pytest.mark.timeout(180, func_only=True) def test_dashboards(self): """ Test that the dashboards can be loaded with `setup --dashboards`