Merge pull request #55749 from max-arnold/json-query-port

[master] Port json_query Jinja filter from Ansible

doc/topics/jinja/index.rst

@@ -1343,9 +1343,52 @@ Returns:
 
   'default'
 
+
+.. jinja_ref:: json_query
+
+``json_query``
+--------------
+
+.. versionadded:: Neon
+
+A port of Ansible ``json_query`` Jinja filter to make queries against JSON data using `JMESPath language`_.
+Could be used to filter ``pillar`` data, ``yaml`` maps, and together with :jinja_ref:`http_query`.
+Depends on the `jmespath`_ Python module.
+
+Examples:
+
+.. code-block:: jinja
+
+  Example 1: {{ [1, 2, 3, 4, [5, 6]] | json_query('[]') }}
+
+  Example 2: {{
+  {"machines": [
+    {"name": "a", "state": "running"},
+    {"name": "b", "state": "stopped"},
+    {"name": "c", "state": "running"}
+  ]} | json_query("machines[?state=='running'].name") }}
+
+  Example 3: {{
+  {"services": [
+    {"name": "http", "host": "1.2.3.4", "port": 80},
+    {"name": "smtp", "host": "1.2.3.5", "port": 25},
+    {"name": "ssh",  "host": "1.2.3.6", "port": 22},
+  ]} | json_query("services[].port") }}
+
+Returns:
+
+.. code-block:: text
+
+  Example 1: [1, 2, 3, 4, 5, 6]
+
+  Example 2: ['a', 'c']
+
+  Example 3: [80, 25, 22]
+
 .. _`builtin filters`: http://jinja.pocoo.org/docs/templates/#builtin-filters
 .. _`timelib`: https://github.com/pediapress/timelib/
-
+.. _`JMESPath language`: http://jmespath.org/
+.. _`jmespath`: https://github.com/jmespath/jmespath.py
 
 .. jinja_ref:: to_snake_case
 

doc/topics/releases/neon.rst

@@ -82,8 +82,11 @@ as well as managing keystore files.
             -----END CERTIFICATE-----
 
 
+Jinja enhancements
+==================
+
 Troubleshooting Jinja map files
-===============================
+-------------------------------
 
 A new :py:func:`execution module <salt.modules.jinja>` for ``map.jinja`` troubleshooting
 has been added.
@@ -109,6 +112,18 @@ The module can be also used to test ``json`` and ``yaml`` maps:
   salt myminion jinja.import_json myformula/defaults.json
 
 
+json_query filter
+-----------------
+
+A port of Ansible :jinja_ref:`json_query` Jinja filter has been added. It allows
+making queries against JSON data using `JMESPath language`_. Could be used to
+filter ``pillar`` data, ``yaml`` maps, and also useful with :jinja_ref:`http_query`.
+
+Depends on the `jmespath`_ Python module.
+
+.. _`JMESPath language`: http://jmespath.org/
+.. _`jmespath`: https://github.com/jmespath/jmespath.py
+
 Slot Syntax Updates
 ===================
 

requirements/static/darwin.in

@@ -6,6 +6,7 @@ croniter>=0.3.0,!=0.3.22
 dnspython
 docker
 futures>=2.0; python_version < '3.0'
+jmespath
 jsonschema
 junos-eznc
 jxmlease

requirements/static/linux.in

@@ -10,6 +10,7 @@ docker
 futures>=2.0; python_version < '3.0'
 GitPython
 hgtools
+jmespath
 jsonschema
 junos-eznc
 jxmlease

requirements/static/windows.in

@@ -7,6 +7,7 @@ dnspython
 # We require docker < 3.0.0 because after that they also start locking their pywin32 requirement, actually
 # pypiwin32, which after version 223 it just makes pywin32 a dep and installs nothing else
 docker<3.0.0
+jmespath
 jsonschema
 keyring==5.7.1
 kubernetes<4.0

salt/utils/data.py

@@ -31,6 +31,11 @@
 from salt.ext import six
 from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
 
+try:
+    import jmespath
+except ImportError:
+    jmespath = None
+
 log = logging.getLogger(__name__)
 
 
@@ -979,6 +984,38 @@ def stringify(data):
     return ret
 
 
+@jinja_filter('json_query')
+def json_query(data, expr):
+    '''
+    Query data using JMESPath language (http://jmespath.org).
+
+    Requires the https://github.com/jmespath/jmespath.py library.
+
+    :param data: A complex data structure to query
+    :param expr: A JMESPath expression (query)
+    :returns: The query result
+
+    .. code-block:: jinja
+
+        {"services": [
+            {"name": "http", "host": "1.2.3.4", "port": 80},
+            {"name": "smtp", "host": "1.2.3.5", "port": 25},
+            {"name": "ssh",  "host": "1.2.3.6", "port": 22},
+        ]} | json_query("services[].port") }}
+
+    will be rendered as:
+
+    .. code-block:: text
+
+        [80, 25, 22]
+    '''
+    if jmespath is None:
+        err = 'json_query requires jmespath module installed'
+        log.error(err)
+        raise RuntimeError(err)
+    return jmespath.search(expr, data)
+
+
 def _is_not_considered_falsey(value, ignore_types=()):
     '''
     Helper function for filter_falsey to determine if something is not to be

tests/unit/utils/test_data.py

@@ -628,6 +628,27 @@ def test_stringify(self):
             ['one', 'two', 'three', '4', '5']
         )
 
+    def test_json_query(self):
+        # Raises exception if jmespath module is not found
+        with patch('salt.utils.data.jmespath', None):
+            self.assertRaisesRegex(
+                RuntimeError, 'requires jmespath',
+                salt.utils.data.json_query, {}, '@'
+            )
+
+        # Test search
+        user_groups = {
+            'user1': {'groups': ['group1', 'group2', 'group3']},
+            'user2': {'groups': ['group1', 'group2']},
+            'user3': {'groups': ['group3']},
+        }
+        expression = '*.groups[0]'
+        primary_groups = ['group1', 'group1', 'group3']
+        self.assertEqual(
+            sorted(salt.utils.data.json_query(user_groups, expression)),
+            primary_groups
+        )
+
 
 class FilterFalseyTestCase(TestCase):
     '''

tests/unit/utils/test_jinja.py

@@ -1451,6 +1451,16 @@ def test_base64_decode(self):
                                      dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
         self.assertEqual(rendered, 'random')
 
+    def test_json_query(self):
+        '''
+        Test the `json_query` Jinja filter.
+        '''
+        rendered = render_jinja_tmpl(
+            "{{ [1, 2, 3] | json_query('[1]')}}",
+            dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)
+        )
+        self.assertEqual(rendered, '2')
+
     # def test_print(self):
     #     env = Environment(extensions=[SerializerExtension])
     #     source = '{% import_yaml "toto.foo" as docu %}'