Introduce rally-tracks compatibility testing (#1564)

This commit modifies and extends `pytest-rally` so that it can be invoked from
within the Rally repo. This enables Rally developers and CI jobs to test
Rally changes against arbitrary revisions of local track repositories,
using arbitrary versions of Elasticsearch.

The actual contents of the tests live in the track repository. The
plugin will run any tests found in the `it` subdirectory of the provided track
repository by default, including those that are auto-generated by
`pytest-rally`.

By default, tests will be run against the `master` branch of the track
repository checked out in `$RALLY_HOME/benchmarks/tracks/default` (typically
`rally-tracks`), using a build of the `main` branch of Elasticsearch.

To run tests with these defaults, run the following from the root of this
repository:

`pytest it/track_repo_compatibility`

Here is an example invocation that overrides these defaults:

```
pytest it/track_repo_compatibility \
     --track-repository=/path/to/repo \
     --track-revision=some-branch \
     --distribution-version=8.3.2
```

.ci/build.sh

@@ -127,6 +127,11 @@ function it310 {
   make it310
 }
 
+function rally-tracks-compat {
+  install
+  make rally-tracks-compat
+}
+
 function license-scan {
   # turn nounset off because some of the following commands fail if nounset is turned on
   set +u

.ci/jobs/pull-request-rally-tracks-compat.yml

@@ -0,0 +1,27 @@
+---
+
+- job:
+    name: "elastic+rally+pull-request+rally-tracks-compat"
+    display-name: "elastic / rally # pull-request+rally-tracks-compat"
+    description: "rally-tracks integration tests using rally built from PR branch"
+    scm:
+      - git:
+          refspec: "+refs/pull/*:refs/remotes/origin/pr/* +refs/heads/*:refs/remotes/origin/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+rally/rally-tracks-compat.*'
+          github-hooks: true
+          status-context: "rally/rally-tracks-compat"
+          cancel-builds-on-update: true
+          black-list-labels:
+            - '>test-mute'
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld
+          set -o errexit
+          bash .ci/build.sh rally-tracks-compat

Makefile

@@ -67,6 +67,8 @@ install-user: venv-create
 install: install-user
 	# Also install development dependencies
 	. $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install -e .[develop]
+	. $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install git+https://github.com/elastic/pytest-rally.git
+
 
 clean: nondocs-clean docs-clean
 
@@ -125,6 +127,9 @@ it39: check-venv python-caches-clean tox-env-clean
 it310: check-venv python-caches-clean tox-env-clean
 	. $(VENV_ACTIVATE_FILE); tox -e py310-it
 
+rally-tracks-compat: check-venv python-caches-clean tox-env-clean
+	. $(VENV_ACTIVATE_FILE); tox -e rally-tracks-compat
+
 check-all: lint test it
 
 benchmark: check-venv

it/track_repo_compatibility/conftest.py

@@ -0,0 +1,69 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shlex
+import subprocess
+
+RALLY_HOME = os.getenv("RALLY_HOME", os.path.expanduser("~"))
+RALLY_CONFIG_DIR = os.path.join(RALLY_HOME, ".rally")
+TRACK_REPO_PATH = os.path.join(RALLY_CONFIG_DIR, "benchmarks", "tracks", "rally-tracks-compat")
+REMOTE_TRACK_REPO = "https://github.com/elastic/rally-tracks"
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("rally")
+    group.addoption(
+        "--track-repository",
+        action="store",
+        default=TRACK_REPO_PATH,
+        help=f"Path to a local track repository\n(default: {TRACK_REPO_PATH})",
+    )
+    group.addoption(
+        "--track-revision",
+        action="store",
+        default="master",
+        help="Track repository revision to test\ndefault: `master`",
+    )
+    group.addoption(
+        "--track-repository-test-directory",
+        action="store",
+        dest="track_repo_test_dir",
+        default="it",
+        help="Name of the directory containing the track repo's integration tests\n(default: `it`)",
+    )
+
+
+def pytest_cmdline_main(config):
+    repo = config.option.track_repository
+    if not os.path.isdir(repo):
+        # we're using the defaults, so perform an initial clone of rally-tracks
+        if repo == TRACK_REPO_PATH:
+            try:
+                subprocess.run(shlex.split(f"git clone {REMOTE_TRACK_REPO} {repo}"), text=True, capture_output=True, check=True)
+            except subprocess.CalledProcessError as e:
+                raise AssertionError(f"Unable to clone {REMOTE_TRACK_REPO} into {repo}: {e.stderr}")
+        # user has provided a non-default track repository, but it's not a valid path, so we raise an error
+        else:
+            raise AssertionError(f"Track repository not found: [{repo}]")
+
+    test_dir = config.option.track_repo_test_dir
+    config.args.append(os.path.join(repo, test_dir))
+
+
+def pytest_report_header(config):
+    return f"rally: track-repository={config.option.track_repository}, track-revision={config.option.track_revision}"

tox.ini

@@ -10,6 +10,7 @@ isolated_build = True
 envlist =
     py{38,39,310}-unit
     py{38,39,310}-it
+    rally-tracks-compat
 platform =
     linux|darwin
 
@@ -47,3 +48,8 @@ commands =
 
 whitelist_externals =
     pytest
+
+[testenv:rally-tracks-compat]
+deps = pytest-rally @ git+https://github.com/elastic/pytest-rally.git
+commands =
+        rally-tracks-compat: pytest it/track_repo_compatibility --junitxml=junit-{envname}.xml --log-cli-level=INFO