diff --git a/esrally/config.py b/esrally/config.py index 7d6df3300..35e20db3f 100644 --- a/esrally/config.py +++ b/esrally/config.py @@ -108,7 +108,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl class Config: EARLIEST_SUPPORTED_VERSION = 12 - CURRENT_CONFIG_VERSION = 15 + CURRENT_CONFIG_VERSION = 16 """ Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on @@ -457,11 +457,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install) config["distributions"] = {} - config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \ - "{{VERSION}}.tar.gz" - config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \ - "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz" - config["distributions"]["release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" config["distributions"]["release.cache"] = "true" config_file.store(config) @@ -708,6 +703,15 @@ def warn_if_plugin_build_task_is_in_use(config): current_version = 15 config["meta"]["config.version"] = str(current_version) + if current_version == 15 and target_version > current_version: + if "distributions" in config: + # Remove obsolete settings + config["distributions"].pop("release.1.url", None) + config["distributions"].pop("release.2.url", None) + config["distributions"].pop("release.url", None) + current_version = 16 + config["meta"]["config.version"] = str(current_version) + # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version) diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 47d6f82c5..34d324f97 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -40,8 +40,17 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins= else: es_src_dir = None distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir")) + + dist_cfg = {} + # car / plugin defines defaults... + dist_cfg.update(car.variables) + for plugin in plugins: + for k, v in plugin.variables: + dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v + # ... but the user can override it in rally.ini + dist_cfg.update(cfg.all_opts("distributions")) repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"), - distribution_config=cfg.all_opts("distributions"), + distribution_config=dist_cfg, version=es_version) suppliers.append(ElasticsearchDistributionSupplier(repo, distributions_root)) @@ -333,7 +342,7 @@ def prepare(self): def add(self, binaries): # if we have multiple plugin configurations for a plugin we will override entries here but as this is always the same # key-value pair this is ok. - plugin_url = self.repo.plugin_download_url(self.plugin.name) + plugin_url = self.repo.plugin_download_url(self.plugin.name, self.plugin.variables) if plugin_url: binaries[self.plugin.name] = plugin_url @@ -480,34 +489,35 @@ def __init__(self, name, distribution_config, version): @property def download_url(self): - major_version = versions.major_version(self.version) - version_url_key = "%s.%s.url" % (self.name, str(major_version)) - default_url_key = "%s.url" % self.name - return self._url_for(version_url_key, default_url_key) + # team repo + default_key = "{}_url".format(self.name) + # rally.ini + override_key = "{}.url".format(self.name) + return self._url_for(override_key, default_key) def plugin_download_url(self, plugin_name): - major_version = versions.major_version(self.version) - version_url_key = "plugin.%s.%s.%s.url" % (plugin_name, self.name, str(major_version)) - default_url_key = "plugin.%s.%s.url" % (plugin_name, self.name) - return self._url_for(version_url_key, default_url_key, mandatory=False) + # team repo + default_key = "plugin_{}_{}_url".format(plugin_name, self.name) + # rally.ini + override_key = "plugin.{}.{}.url".format(plugin_name, self.name) + return self._url_for(override_key, default_key, mandatory=False) - def _url_for(self, version_url_key, default_url_key, mandatory=True): + def _url_for(self, user_defined_key, default_key, mandatory=True): try: - if version_url_key in self.cfg: - url_template = self.cfg[version_url_key] + if user_defined_key in self.cfg: + url_template = self.cfg[user_defined_key] else: - url_template = self.cfg[default_url_key] + url_template = self.cfg[default_key] except KeyError: if mandatory: - raise exceptions.SystemSetupError("Neither version specific distribution config key [%s] nor a default distribution " - "config key [%s] is defined." % (version_url_key, default_url_key)) + raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key)) else: return None return url_template.replace("{{VERSION}}", self.version) @property def cache(self): - k = "%s.cache" % self.name + k = "{}.cache".format(self.name) try: raw_value = self.cfg[k] except KeyError: diff --git a/tests/config_test.py b/tests/config_test.py index 43ba46d27..fc9e35d6b 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -257,7 +257,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ print("%s::%s: %s" % (section, k, v)) self.assertTrue("meta" in config_store.config) - self.assertEqual("15", config_store.config["meta"]["config.version"]) + self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"]) self.assertTrue("system" in config_store.config) self.assertEqual("local", config_store.config["system"]["env.name"]) @@ -295,13 +295,6 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ self.assertEqual("False", config_store.config["defaults"]["preserve_benchmark_candidate"]) self.assertTrue("distributions" in config_store.config) - self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.1.url"]) - self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/" - "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.2.url"]) - self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.url"]) self.assertEqual("true", config_store.config["distributions"]["release.cache"]) @mock.patch("esrally.utils.jvm.is_early_access_release") @@ -379,7 +372,7 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i self.assertIsNotNone(config_store.config) self.assertTrue("meta" in config_store.config) - self.assertEqual("15", config_store.config["meta"]["config.version"]) + self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"]) self.assertTrue("system" in config_store.config) self.assertEqual("unittest-env", config_store.config["system"]["env.name"]) self.assertTrue("node" in config_store.config) @@ -408,13 +401,6 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i self.assertEqual("True", config_store.config["defaults"]["preserve_benchmark_candidate"]) self.assertTrue("distributions" in config_store.config) - self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.1.url"]) - self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/" - "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.2.url"]) - self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", - config_store.config["distributions"]["release.url"]) self.assertEqual("true", config_store.config["distributions"]["release.cache"]) @@ -475,6 +461,9 @@ def test_migrate_from_earliest_supported_to_latest(self, sleep, get_size): }, "runtime": { "java8.home": "/opt/jdk/8", + }, + "distributions": { + "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" } } @@ -818,3 +807,25 @@ def test_migrate_from_14_to_15_with_source_plugin_definition(self, major_version "plugin.x-pack.build.command", "https://esrally.readthedocs.io/en/latest/elasticsearch_plugins.html#running-a-benchmark-with-plugins""" ), string_buffer.read()) + + def test_migrate_from_15_to_16(self): + config_file = InMemoryConfigStore("test") + sample_config = { + "meta": { + "config.version": 15 + }, + "distributions": { + "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" + } + } + config_file.store(sample_config) + config.migrate(config_file, 15, 16, out=null_output) + + self.assertTrue(config_file.backup_created) + self.assertEqual("16", config_file.config["meta"]["config.version"]) + # does not remove section + self.assertIn("distributions", config_file.config) + self.assertNotIn("release.url", config_file.config["distributions"]) + + + diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py index 6cba55b96..65c6fd040 100644 --- a/tests/mechanic/supplier_test.py +++ b/tests/mechanic/supplier_test.py @@ -498,36 +498,31 @@ def test_create_suppliers_for_es_and_plugin_source_build(self): class DistributionRepositoryTests(TestCase): def test_release_repo_config_with_default_url(self): repo = supplier.DistributionRepository(name="release", distribution_config={ - "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/" - "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz", - "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", + "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", "release.cache": "true" }, version="5.5.0") self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.tar.gz", repo.download_url) self.assertTrue(repo.cache) - def test_release_repo_config_with_version_url(self): + def test_release_repo_config_with_user_url(self): repo = supplier.DistributionRepository(name="release", distribution_config={ - "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/" - "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz", - "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", + "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", + # user override + "release.url": "https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", "release.cache": "false" }, version="2.4.3") - self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.3/" - "elasticsearch-2.4.3.tar.gz", repo.download_url) + self.assertEqual("https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-2.4.3.tar.gz", repo.download_url) self.assertFalse(repo.cache) def test_missing_url(self): repo = supplier.DistributionRepository(name="miss", distribution_config={ - "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", + "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", "release.cache": "true" }, version="2.4.3") with self.assertRaises(exceptions.SystemSetupError) as ctx: # noinspection PyStatementEffect repo.download_url - self.assertEqual( - "Neither version specific distribution config key [miss.2.url] nor a default distribution config key [miss.url] is defined.", - ctx.exception.args[0]) + self.assertEqual("Neither config key [miss.url] nor [miss_url] is defined.", ctx.exception.args[0]) def test_missing_cache(self): repo = supplier.DistributionRepository(name="release", distribution_config={ @@ -547,3 +542,21 @@ def test_invalid_cache_value(self): # noinspection PyStatementEffect repo.cache self.assertEqual("Value [Invalid] for config key [release.cache] is not a valid boolean value.", ctx.exception.args[0]) + + def test_plugin_config_with_default_url(self): + repo = supplier.DistributionRepository(name="release", distribution_config={ + "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip" + }, version="5.5.0") + self.assertEqual("https://artifacts.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example")) + + def test_plugin_config_with_user_url(self): + repo = supplier.DistributionRepository(name="release", distribution_config={ + "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip", + # user override + "plugin.example.release.url": "https://mirror.example.org/downloads/plugins/example-{{VERSION}}.zip" + }, version="5.5.0") + self.assertEqual("https://mirror.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example")) + + def test_missing_plugin_config(self): + repo = supplier.DistributionRepository(name="release", distribution_config={}, version="5.5.0") + self.assertIsNone(repo.plugin_download_url("not existing"))