Fail Rally early if there are unused variables in track-params

esrally/exceptions.py

@@ -67,3 +67,11 @@ class InvalidSyntax(RallyError):
 
 class InvalidName(RallyError):
     pass
+
+
+class TrackConfigError(RallyError):
+    """
+    Thrown when something is wrong with the track config e.g. user supplied a track-param
+    that can't be set
+    """
+    pass

esrally/track/loader.py

@@ -15,9 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import copy
 import json
 import logging
 import os
+import re
 import glob
 import urllib.error
 import tempfile
@@ -26,9 +28,11 @@
 import jinja2.exceptions
 import jsonschema
 import tabulate
+
 from esrally import exceptions, time, PROGRAM_NAME
 from esrally.track import params, track
 from esrally.utils import io, convert, net, console, modules, repo
+from jinja2 import meta
 
 
 class TrackSyntaxError(exceptions.InvalidSyntax):
@@ -468,7 +472,61 @@ def prepare_bundled_document_set(self, document_set, data_root):
                 return False
 
 
-def render_template(loader, template_name, template_vars=None, glob_helper=lambda f: [], clock=time.Clock):
+class TemplateSource:
+    """
+    Prepares the fully assembled track file from file or string.
+    Doesn't render using jinja2, but embeds track fragments referenced with
+    rally.collect(parts=...
+    """
+
+    collect_parts_re = re.compile(r'''{{\ +?rally.collect\(parts="(.+?(?="))"\)\ +?}}''')
+
+    def __init__(self, base_path, template_file_name, source=io.FileSource, fileglobber=glob.glob):
+        self.base_path = base_path
+        self.template_file_name = template_file_name
+        self.source = source
+        self.fileglobber = fileglobber
+        self.assembled_source = None
+
+    def load_template_from_file(self):
+        loader = jinja2.FileSystemLoader(self.base_path)
+        try:
+            base_track = loader.get_source(jinja2.Environment(), self.template_file_name)
+        except jinja2.TemplateNotFound:
+            self.logger.exception("Could not track from [%s].", self.template_file_name)
+            raise TrackSyntaxError("Could not load track from '{}'".format(self.template_file_name))
+        self.assembled_source = self.replace_includes(self.base_path, base_track[0])
+
+    def load_template_from_string(self, template_source):
+        self.assembled_source = self.replace_includes(self.base_path, template_source)
+
+    def replace_includes(self, base_path, track_fragment):
+        match = TemplateSource.collect_parts_re.findall(track_fragment)
+        if match:
+            # Construct replacement dict for matched captures
+            repl = {}
+            for glob_pattern in match:
+                full_glob_path = os.path.join(base_path, glob_pattern)
+                sub_source = self.read_glob_files(full_glob_path)
+                repl[glob_pattern] = self.replace_includes(base_path=io.dirname(full_glob_path), track_fragment=sub_source)
+
+            def replstring(matchobj):
+                # matchobj.groups() is a tuple and first element contains the matched group id
+                return repl[matchobj.groups()[0]]
+
+            return TemplateSource.collect_parts_re.sub(replstring, track_fragment)
+        return track_fragment
+
+    def read_glob_files(self, pattern):
+        source = []
+        files = self.fileglobber(pattern)
+        for fname in files:
+            with self.source(fname, mode="rt") as fp:
+                source.append(fp.read())
+        return ",\n".join(source)
+
+
+def render_template(template_source, template_vars=None, glob_helper=lambda f: [], clock=time.Clock):
     macros = """
         {% macro collect(parts) -%}
             {% set comma = joiner() %}
@@ -479,23 +537,43 @@ def render_template(loader, template_name, template_vars=None, glob_helper=lambd
         {%- endmacro %}
     """
 
+    loader = jinja2.BaseLoader()
     # place helpers dict loader first to prevent users from overriding our macros.
     env = jinja2.Environment(
         loader=jinja2.ChoiceLoader([jinja2.DictLoader({"rally.helpers": macros}), loader])
     )
+
     if template_vars:
         for k, v in template_vars.items():
             env.globals[k] = v
     # ensure that user variables never override our internal variables
     env.globals["now"] = clock.now()
     env.globals["glob"] = glob_helper
     env.filters["days_ago"] = time.days_ago
-    template = env.get_template(template_name)
+    template = env.from_string(template_source)
 
     return template.render()
 
 
-def render_template_from_file(template_file_name, template_vars):
+def check_unused_track_params(assembled_source, unused_track_params):
+    if not unused_track_params or not unused_track_params.track_params:
+        return
+
+    j2env = jinja2.Environment()
+    # we don't need the following j2 filters/macros but we define them anyway to prevent parsing failures
+    j2env.globals["now"] = time.Clock()
+    # use dummy macro for glob, we don't require it to assemble the source
+    j2env.globals["glob"] = lambda c: ""
+    j2env.filters["days_ago"] = time.days_ago
+    ast = j2env.parse(assembled_source)
+    j2_variables = meta.find_undeclared_variables(ast)
+
+    for k in unused_track_params.track_params:
+        if k in j2_variables:
+            unused_track_params.remove_param(k)
+
+
+def render_template_from_file(template_file_name, template_vars, unused_track_params=None):
     def relative_glob(start, f):
         result = glob.glob(os.path.join(start, f))
         if result:
@@ -504,8 +582,11 @@ def relative_glob(start, f):
             return []
 
     base_path = io.dirname(template_file_name)
-    return render_template(loader=jinja2.FileSystemLoader(base_path),
-                           template_name=io.basename(template_file_name),
+    template_source = TemplateSource(base_path, io.basename(template_file_name))
+    template_source.load_template_from_file()
+    check_unused_track_params(template_source.assembled_source, unused_track_params)
+
+    return render_template(template_source=template_source.assembled_source,
                            template_vars=template_vars,
                            glob_helper=lambda f: relative_glob(base_path, f))
 
@@ -622,6 +703,24 @@ def post_process_for_test_mode(t):
     return t
 
 
+class UnusedTrackParams:
+    def __init__(self, track_params=None):
+        self.track_params = copy.deepcopy(track_params) if track_params else []
+
+    def remove_param(self, key):
+        self.track_params.remove(key)
+
+    def set_params(self, keys):
+        self.track_params = copy.deepcopy(keys)
+
+    def exit_if_not_empty(self):
+        if self.track_params:
+            raise exceptions.TrackConfigError(
+                "You've declared {} using track-param but they didn't override "
+                "any of the jinja2 variables defined in the track or index settings or "
+                "index templates.".format(self.track_params))
+
+
 class TrackFileReader:
     MINIMUM_SUPPORTED_TRACK_VERSION = 2
     MAXIMUM_SUPPORTED_TRACK_VERSION = 2
@@ -634,7 +733,10 @@ def __init__(self, cfg):
         with open(track_schema_file, mode="rt", encoding="utf-8") as f:
             self.track_schema = json.loads(f.read())
         self.track_params = cfg.opts("track", "params")
-        self.read_track = TrackSpecificationReader(self.track_params)
+        self.unused_track_params = UnusedTrackParams(list(self.track_params.keys()))
+        self.read_track = TrackSpecificationReader(
+            track_params=self.track_params,
+            unused_track_params=self.unused_track_params)
         self.logger = logging.getLogger(__name__)
 
     def read(self, track_name, track_spec_file, mapping_dir):
@@ -649,7 +751,7 @@ def read(self, track_name, track_spec_file, mapping_dir):
 
         self.logger.info("Reading track specification file [%s].", track_spec_file)
         try:
-            rendered = render_template_from_file(track_spec_file, self.track_params)
+            rendered = render_template_from_file(track_spec_file, self.track_params, unused_track_params=self.unused_track_params)
             # render the track to a temporary file instead of dumping it into the logs. It is easier to check for error messages
             # involving lines numbers and it also does not bloat Rally's log file so much.
             tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
@@ -677,7 +779,7 @@ def read(self, track_name, track_spec_file, mapping_dir):
         if TrackFileReader.MAXIMUM_SUPPORTED_TRACK_VERSION < track_version:
             raise exceptions.RallyError("Track {} requires a newer version of Rally. Please upgrade Rally (supported track version: {}, "
                                         "required track version: {}).".format(track_name, TrackFileReader.MAXIMUM_SUPPORTED_TRACK_VERSION,
-                                                                             track_version))
+                                                                              track_version))
         try:
             jsonschema.validate(track_spec, self.track_schema)
         except jsonschema.exceptions.ValidationError as ve:
@@ -733,9 +835,10 @@ class TrackSpecificationReader:
     Creates a track instances based on its parsed JSON description.
     """
 
-    def __init__(self, track_params=None, source=io.FileSource):
+    def __init__(self, track_params=None, unused_track_params=None, source=io.FileSource):
         self.name = None
         self.track_params = track_params if track_params else {}
+        self.unused_track_params = unused_track_params
         self.source = source
         self.logger = logging.getLogger(__name__)
 
@@ -750,7 +853,9 @@ def __call__(self, track_name, track_specification, mapping_dir):
                      for tpl in self._r(track_specification, "templates", mandatory=False, default_value=[])]
         corpora = self._create_corpora(self._r(track_specification, "corpora", mandatory=False, default_value=[]), indices)
         challenges = self._create_challenges(track_specification)
-
+        # at this point, *all* track params must have been referenced in the templates
+        if self.unused_track_params:
+            self.unused_track_params.exit_if_not_empty()
         return track.Track(name=self.name, meta_data=meta_data, description=description, challenges=challenges, indices=indices,
                            templates=templates, corpora=corpora)
 
@@ -779,27 +884,36 @@ def _create_index(self, index_spec, mapping_dir):
         index_name = self._r(index_spec, "name")
         body_file = self._r(index_spec, "body", mandatory=False)
         if body_file:
+            idx_body_tmpl_src = TemplateSource(mapping_dir, body_file, self.source)
             with self.source(os.path.join(mapping_dir, body_file), "rt") as f:
-                body = self._load_template(f.read(), "definition for index {} in {}".format(index_name, body_file))
+                idx_body_tmpl_src.load_template_from_string(f.read())
+                body = self._load_template(
+                    idx_body_tmpl_src.assembled_source,
+                    "definition for index {} in {}".format(index_name, body_file))
         else:
             body = None
 
         return track.Index(name=index_name, body=body, types=self._r(index_spec, "types", mandatory=False, default_value=[]))
 
     def _create_index_template(self, tpl_spec, mapping_dir):
         name = self._r(tpl_spec, "name")
+        template_file = self._r(tpl_spec, "template")
         index_pattern = self._r(tpl_spec, "index-pattern")
         delete_matching_indices = self._r(tpl_spec, "delete-matching-indices", mandatory=False, default_value=True)
-        template_file = os.path.join(mapping_dir, self._r(tpl_spec, "template"))
+        template_file = os.path.join(mapping_dir, template_file)
+        idx_tmpl_src = TemplateSource(mapping_dir, template_file, self.source)
         with self.source(template_file, "rt") as f:
-            template_content = self._load_template(f.read(), "definition for index template {} in {}".format(name, template_file))
+            idx_tmpl_src.load_template_from_string(f.read())
+            template_content = self._load_template(
+                idx_tmpl_src.assembled_source,
+                "definition for index template {} in {}".format(name, template_file))
         return track.IndexTemplate(name, index_pattern, template_content, delete_matching_indices)
 
     def _load_template(self, contents, description):
         self.logger.info("Loading template [%s].", description)
+        check_unused_track_params(contents, self.unused_track_params)
         try:
-            rendered = render_template(loader=jinja2.DictLoader({"default": contents}),
-                                       template_name="default",
+            rendered = render_template(contents,
                                        template_vars=self.track_params)
             return json.loads(rendered)
         except Exception as e: