Make possible to reference previous pillars from subsequent pillars, as they specified in the top file

doc/topics/pillar/index.rst

@@ -263,6 +263,51 @@ Since both pillar SLS files contained a ``bind`` key which contained a nested
 dictionary, the pillar dictionary's ``bind`` key contains the combined contents
 of both SLS files' ``bind`` keys.
 
+
+Referencing Other Pillars
+=========================
+
+.. versionadded:: Nitrogen
+
+It is possible to reference pillar values that are defined in other
+pillar files. To do this, place any pillar SLS files with referenced pillar
+values *before* referencing SLS files in the Top file.
+
+For example with such Top file:
+
+.. code-block:: yaml
+
+    base:
+      '*':
+        - settings
+        - postgres
+
+And such ``settings.sls`` file:
+
+.. code-block:: yaml
+
+    db:
+      name: qux
+      user: baz
+      password: supersecret
+
+Values from ``settings`` can be referenced from ``postgres`` like that:
+
+.. code-block:: yaml
+
+    postgres:
+      users:
+        {{ salt['pillar.get']('db:user', 'bar') }}:
+          ensure: present
+          password: {{ salt['pillar.get']('db:password', 'secret') }}
+      databases:
+        {{ salt['pillar.get']('db:name', 'foo') }}:
+          owner: '{{ salt['pillar.get']('db:user', 'bar') }}'
+          template: 'template0'
+          lc_ctype: 'en_US.UTF-8'
+          lc_collate: 'en_US.UTF-8'
+
+
 Including Other Pillars
 =======================
 

salt/pillar/__init__.py

@@ -396,6 +396,21 @@ def _get_envs(self):
             envs.update(list(self.opts['file_roots']))
         return envs
 
+    def __merge(self, *items):
+        '''
+        Merge 'items' according to Pillar's merge strategy and other options.
+        '''
+        return six.moves.reduce(
+            lambda a, b: merge(
+                a,
+                b,
+                self.merge_strategy,
+                self.opts.get('renderer', 'yaml'),
+                self.opts.get('pillar_merge_lists', False)
+            ),
+            items
+        )
+
     def get_tops(self):
         '''
         Gather the top files
@@ -598,10 +613,12 @@ def top_matches(self, top):
                             env_matches.append(item)
         return matches
 
-    def render_pstate(self, sls, saltenv, mods, defaults=None):
+    def render_pstate(self, sls, saltenv, mods, renderers=None, defaults=None):
         '''
         Collect a single pillar sls file and render it
         '''
+        if renderers is None:
+            renderers = self.rend
         if defaults is None:
             defaults = {}
         err = ''
@@ -641,7 +658,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None):
         state = None
         try:
             state = compile_template(fn_,
-                                     self.rend,
+                                     renderers,
                                      self.opts['renderer'],
                                      self.opts['renderer_blacklist'],
                                      self.opts['renderer_whitelist'],
@@ -698,12 +715,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None):
                                                 key_fragment: nstate
                                             }
 
-                                    state = merge(
-                                        state,
-                                        nstate,
-                                        self.merge_strategy,
-                                        self.opts.get('renderer', 'yaml'),
-                                        self.opts.get('pillar_merge_lists', False))
+                                    state = self.__merge(state, nstate)
 
                                 if err:
                                     errors += err
@@ -714,7 +726,10 @@ def render_pillar(self, matches, errors=None):
         Extract the sls pillar files from the matches and render them into the
         pillar
         '''
-        pillar = copy.copy(self.pillar_override)
+        pillar = self.__merge(
+            self.opts.get('pillar', {}),
+            self.pillar_override
+        )
         if errors is None:
             errors = []
         for saltenv, pstates in six.iteritems(matches):
@@ -735,7 +750,16 @@ def render_pillar(self, matches, errors=None):
                     pstatefiles.append(sls_match)
 
             for sls in pstatefiles:
-                pstate, mods, err = self.render_pstate(sls, saltenv, mods)
+                opts = dict(self.opts, pillar=pillar)
+                utils = salt.loader.utils(opts)
+                functions = salt.loader.minion_mods(opts, utils=utils)
+                renderers = salt.loader.render(opts, functions)
+                pstate, mods, err = self.render_pstate(
+                    sls,
+                    saltenv,
+                    mods,
+                    renderers=renderers
+                )
 
                 if err:
                     errors += err
@@ -754,12 +778,7 @@ def render_pillar(self, matches, errors=None):
                             )
                         )
                         continue
-                    pillar = merge(
-                        pillar,
-                        pstate,
-                        self.merge_strategy,
-                        self.opts.get('renderer', 'yaml'),
-                        self.opts.get('pillar_merge_lists', False))
+                    pillar = self.__merge(pillar, pstate)
 
         return pillar, errors
 
@@ -821,11 +840,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None):
         ext = None
         # Bring in CLI pillar data
         if self.pillar_override:
-            pillar = merge(pillar,
-                           self.pillar_override,
-                           self.merge_strategy,
-                           self.opts.get('renderer', 'yaml'),
-                           self.opts.get('pillar_merge_lists', False))
+            pillar = self.__merge(pillar, self.pillar_override)
 
         for run in self.opts['ext_pillar']:
             if not isinstance(run, dict):
@@ -858,12 +873,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None):
                         key, ''.join(traceback.format_tb(sys.exc_info()[2]))
                     )
             if ext:
-                pillar = merge(
-                    pillar,
-                    ext,
-                    self.merge_strategy,
-                    self.opts.get('renderer', 'yaml'),
-                    self.opts.get('pillar_merge_lists', False))
+                pillar = self.__merge(pillar, ext)
                 ext = None
         return pillar, errors
 
@@ -880,11 +890,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None):
                 self.rend = salt.loader.render(self.opts, self.functions)
                 matches = self.top_matches(top)
                 pillar, errors = self.render_pillar(matches, errors=errors)
-                pillar = merge(self.opts['pillar'],
-                               pillar,
-                               self.merge_strategy,
-                               self.opts.get('renderer', 'yaml'),
-                               self.opts.get('pillar_merge_lists', False))
+                pillar = self.__merge(self.opts['pillar'], pillar)
             else:
                 matches = self.top_matches(top)
                 pillar, errors = self.render_pillar(matches)
@@ -908,11 +914,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None):
             pillar['_errors'] = errors
 
         if self.pillar_override:
-            pillar = merge(pillar,
-                           self.pillar_override,
-                           self.merge_strategy,
-                           self.opts.get('renderer', 'yaml'),
-                           self.opts.get('pillar_merge_lists', False))
+            pillar = self.__merge(pillar, self.pillar_override)
 
         decrypt_errors = self.decrypt_pillar(pillar)
         if decrypt_errors:

tests/integration/files/pillar/base/sub.sls

@@ -1 +1,9 @@
 sub: sub_minion
+lowercase_knights:
+{% for knight in pillar.get('knights') %}
+  - {{ knight|lower }}
+{% endfor %}
+uppercase_knights:
+{% for knight in salt['pillar.get']('knights') %}
+  - {{ knight|upper }}
+{% endfor %}

tests/integration/modules/test_pillar.py

@@ -61,3 +61,17 @@ def test_pillar_items(self):
         self.assertDictContainsSubset(
             {'knights': ['Lancelot', 'Galahad', 'Bedevere', 'Robin']},
             get_items)
+
+    def test_referencing_preceding_pillars(self):
+        '''
+        Pillars, that come first in the top file can be referenced from
+        the subsequent pillars.
+        '''
+        items = self.run_function('pillar.items', minion_tgt='sub_minion')
+        self.assertDictContainsSubset(
+            {'lowercase_knights': ['lancelot', 'galahad',
+                                   'bedevere', 'robin'],
+             'uppercase_knights': ['LANCELOT', 'GALAHAD',
+                                   'BEDEVERE', 'ROBIN']},
+            items
+        )

tests/unit/test_pillar.py

@@ -22,6 +22,82 @@
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 class PillarTestCase(TestCase):
+    def test_referencing_preceding_pillars(self):
+        '''
+        Pillars, that come first in the top file can be referenced from
+        the subsequent pillars.
+        '''
+        opts = {
+            'renderer': 'py',
+            'renderer_blacklist': [],
+            'renderer_whitelist': [],
+            'state_top': '',
+            'pillar_roots': [],
+            'file_roots': {},
+            'extension_modules': ''
+        }
+        grains = {
+            'os': 'Ubuntu',
+            'os_family': 'Debian',
+            'oscodename': 'raring',
+            'osfullname': 'Ubuntu',
+            'osrelease': '13.04',
+            'kernel': 'Linux'
+        }
+        pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
+        top = tempfile.NamedTemporaryFile()
+        top.write(
+b'''#!yaml
+
+base:
+    '*':
+        - foo
+        - bar
+'''
+        )
+        top.flush()
+        foo = tempfile.NamedTemporaryFile()
+        foo.write(
+b'''#!py
+
+def run():
+    return {'foo': {'spam': 'eggs'}}
+'''
+        )
+        foo.flush()
+        bar = tempfile.NamedTemporaryFile()
+        bar.write(
+b'''#!py
+
+def run():
+    return {
+        'bar': {
+            'bacon': __pillar__.get('foo', {}).get('spam', 'cheddar'),
+            'sausage': __salt__['pillar.get']('foo:spam', 'cheddar')
+        }
+    }
+'''
+        )
+        bar.flush()
+
+        def get_state(sls, env):
+            return {
+                'foo': {'path': '', 'dest': foo.name},
+                'bar': {'path': '', 'dest': bar.name}
+            }[sls]
+
+        pillar.client = MagicMock(**{
+            'get_state.side_effect': get_state,
+            'cache_file.return_value': top.name
+        })
+        pillar.matcher = MagicMock()
+        pillar.matcher.confirm_top.return_value = True
+        actual_pillar_data = pillar.compile_pillar()
+        expected_pillar_data = {
+            'foo': {'spam': 'eggs'},
+            'bar': {'bacon': 'eggs', 'sausage': 'eggs'}
+        }
+        self.assertEqual(actual_pillar_data, expected_pillar_data)
 
     def tearDown(self):
         for attrname in ('generic_file', 'generic_minion_file', 'ssh_file', 'ssh_minion_file', 'top_file'):