Get distribution download URL from car config

With this commit we read the download URL for an Elasticsearch
distribution from the car (and similarly for plugins). This is necessary
because with Elasticsearch 6.3 there will be two Elasticsearch
distributions: A pure OSS one and one that includes x-pack. Handling
this in rally.ini would have been possible but it is cleaner leveraging
our team repository infrastructure for this task.

esrally/config.py

@@ -108,7 +108,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl
 class Config:
     EARLIEST_SUPPORTED_VERSION = 12
 
-    CURRENT_CONFIG_VERSION = 15
+    CURRENT_CONFIG_VERSION = 16
 
     """
     Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on
@@ -457,11 +457,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals
         config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install)
 
         config["distributions"] = {}
-        config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
-                                                   "{{VERSION}}.tar.gz"
-        config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
-                                                   "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
-        config["distributions"]["release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
         config["distributions"]["release.cache"] = "true"
 
         config_file.store(config)
@@ -708,6 +703,15 @@ def warn_if_plugin_build_task_is_in_use(config):
         current_version = 15
         config["meta"]["config.version"] = str(current_version)
 
+    if current_version == 15 and target_version > current_version:
+        if "distributions" in config:
+            # Remove obsolete settings
+            config["distributions"].pop("release.1.url", None)
+            config["distributions"].pop("release.2.url", None)
+            config["distributions"].pop("release.url", None)
+        current_version = 16
+        config["meta"]["config.version"] = str(current_version)
+
     # all migrations done
     config_file.store(config)
     logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)

esrally/mechanic/supplier.py

@@ -40,8 +40,17 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins=
     else:
         es_src_dir = None
         distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir"))
+
+        dist_cfg = {}
+        # car / plugin defines defaults...
+        dist_cfg.update(car.variables)
+        for plugin in plugins:
+            for k, v in plugin.variables:
+                dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v
+        # ... but the user can override it in rally.ini
+        dist_cfg.update(cfg.all_opts("distributions"))
         repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"),
-                                      distribution_config=cfg.all_opts("distributions"),
+                                      distribution_config=dist_cfg,
                                       version=es_version)
         suppliers.append(ElasticsearchDistributionSupplier(repo, distributions_root))
 
@@ -333,7 +342,7 @@ def prepare(self):
     def add(self, binaries):
         # if we have multiple plugin configurations for a plugin we will override entries here but as this is always the same
         # key-value pair this is ok.
-        plugin_url = self.repo.plugin_download_url(self.plugin.name)
+        plugin_url = self.repo.plugin_download_url(self.plugin.name, self.plugin.variables)
         if plugin_url:
             binaries[self.plugin.name] = plugin_url
 
@@ -480,34 +489,35 @@ def __init__(self, name, distribution_config, version):
 
     @property
     def download_url(self):
-        major_version = versions.major_version(self.version)
-        version_url_key = "%s.%s.url" % (self.name, str(major_version))
-        default_url_key = "%s.url" % self.name
-        return self._url_for(version_url_key, default_url_key)
+        # team repo
+        default_key = "{}_url".format(self.name)
+        # rally.ini
+        override_key = "{}.url".format(self.name)
+        return self._url_for(override_key, default_key)
 
     def plugin_download_url(self, plugin_name):
-        major_version = versions.major_version(self.version)
-        version_url_key = "plugin.%s.%s.%s.url" % (plugin_name, self.name, str(major_version))
-        default_url_key = "plugin.%s.%s.url" % (plugin_name, self.name)
-        return self._url_for(version_url_key, default_url_key, mandatory=False)
+        # team repo
+        default_key = "plugin_{}_{}_url".format(plugin_name, self.name)
+        # rally.ini
+        override_key = "plugin.{}.{}.url".format(plugin_name, self.name)
+        return self._url_for(override_key, default_key, mandatory=False)
 
-    def _url_for(self, version_url_key, default_url_key, mandatory=True):
+    def _url_for(self, user_defined_key, default_key, mandatory=True):
         try:
-            if version_url_key in self.cfg:
-                url_template = self.cfg[version_url_key]
+            if user_defined_key in self.cfg:
+                url_template = self.cfg[user_defined_key]
             else:
-                url_template = self.cfg[default_url_key]
+                url_template = self.cfg[default_key]
         except KeyError:
             if mandatory:
-                raise exceptions.SystemSetupError("Neither version specific distribution config key [%s] nor a default distribution "
-                                                  "config key [%s] is defined." % (version_url_key, default_url_key))
+                raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key))
             else:
                 return None
         return url_template.replace("{{VERSION}}", self.version)
 
     @property
     def cache(self):
-        k = "%s.cache" % self.name
+        k = "{}.cache".format(self.name)
         try:
             raw_value = self.cfg[k]
         except KeyError:

tests/config_test.py

@@ -257,7 +257,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_
                 print("%s::%s: %s" % (section, k, v))
 
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("15", config_store.config["meta"]["config.version"])
+        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"])
 
         self.assertTrue("system" in config_store.config)
         self.assertEqual("local", config_store.config["system"]["env.name"])
@@ -295,13 +295,6 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_
         self.assertEqual("False", config_store.config["defaults"]["preserve_benchmark_candidate"])
 
         self.assertTrue("distributions" in config_store.config)
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.1.url"])
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                         "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.2.url"])
-        self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.url"])
         self.assertEqual("true", config_store.config["distributions"]["release.cache"])
 
     @mock.patch("esrally.utils.jvm.is_early_access_release")
@@ -379,7 +372,7 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i
 
         self.assertIsNotNone(config_store.config)
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("15", config_store.config["meta"]["config.version"])
+        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"])
         self.assertTrue("system" in config_store.config)
         self.assertEqual("unittest-env", config_store.config["system"]["env.name"])
         self.assertTrue("node" in config_store.config)
@@ -408,13 +401,6 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i
         self.assertEqual("True", config_store.config["defaults"]["preserve_benchmark_candidate"])
 
         self.assertTrue("distributions" in config_store.config)
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.1.url"])
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                         "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.2.url"])
-        self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.url"])
         self.assertEqual("true", config_store.config["distributions"]["release.cache"])
 
 
@@ -475,6 +461,9 @@ def test_migrate_from_earliest_supported_to_latest(self, sleep, get_size):
             },
             "runtime": {
                 "java8.home": "/opt/jdk/8",
+            },
+            "distributions": {
+                "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
             }
         }
 
@@ -818,3 +807,25 @@ def test_migrate_from_14_to_15_with_source_plugin_definition(self, major_version
                        "plugin.x-pack.build.command",
                        "https://esrally.readthedocs.io/en/latest/elasticsearch_plugins.html#running-a-benchmark-with-plugins"""
             ), string_buffer.read())
+
+    def test_migrate_from_15_to_16(self):
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 15
+            },
+            "distributions": {
+                "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 15, 16, out=null_output)
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("16", config_file.config["meta"]["config.version"])
+        # does not remove section
+        self.assertIn("distributions", config_file.config)
+        self.assertNotIn("release.url", config_file.config["distributions"])
+
+
+

tests/mechanic/supplier_test.py

@@ -498,36 +498,31 @@ def test_create_suppliers_for_es_and_plugin_source_build(self):
 class DistributionRepositoryTests(TestCase):
     def test_release_repo_config_with_default_url(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
-            "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                             "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "true"
         }, version="5.5.0")
         self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.tar.gz", repo.download_url)
         self.assertTrue(repo.cache)
 
-    def test_release_repo_config_with_version_url(self):
+    def test_release_repo_config_with_user_url(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
-            "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                             "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            # user override
+            "release.url": "https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "false"
         }, version="2.4.3")
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.3/"
-                         "elasticsearch-2.4.3.tar.gz", repo.download_url)
+        self.assertEqual("https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-2.4.3.tar.gz", repo.download_url)
         self.assertFalse(repo.cache)
 
     def test_missing_url(self):
         repo = supplier.DistributionRepository(name="miss", distribution_config={
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "true"
         }, version="2.4.3")
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
             # noinspection PyStatementEffect
             repo.download_url
-        self.assertEqual(
-            "Neither version specific distribution config key [miss.2.url] nor a default distribution config key [miss.url] is defined.",
-            ctx.exception.args[0])
+        self.assertEqual("Neither config key [miss.url] nor [miss_url] is defined.", ctx.exception.args[0])
 
     def test_missing_cache(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
@@ -547,3 +542,21 @@ def test_invalid_cache_value(self):
             # noinspection PyStatementEffect
             repo.cache
         self.assertEqual("Value [Invalid] for config key [release.cache] is not a valid boolean value.", ctx.exception.args[0])
+
+    def test_plugin_config_with_default_url(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={
+            "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip"
+        }, version="5.5.0")
+        self.assertEqual("https://artifacts.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example"))
+
+    def test_plugin_config_with_user_url(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={
+            "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip",
+            # user override
+            "plugin.example.release.url": "https://mirror.example.org/downloads/plugins/example-{{VERSION}}.zip"
+        }, version="5.5.0")
+        self.assertEqual("https://mirror.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example"))
+
+    def test_missing_plugin_config(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={}, version="5.5.0")
+        self.assertIsNone(repo.plugin_download_url("not existing"))