[master] Porting #50428 to master

doc/topics/jinja/index.rst

@@ -1212,8 +1212,52 @@ Returns:
 
   'default'
 
+
+.. jinja_ref:: json_query
+
+``json_query``
+--------------
+
+.. versionadded:: Neon
+
+A port of Ansible ``json_query`` Jinja filter to make queries against JSON data using `JMESPath language`_.
+Could be used to filter ``pillar`` data, ``yaml`` maps, and together with :jinja_ref:`http_query`.
+Depends on the `jmespath`_ Python module.
+
+Examples:
+
+.. code-block:: jinja
+
+  Example 1: {{ [1, 2, 3, 4, [5, 6]] | json_query('[]') }}
+
+  Example 2: {{
+  {"machines": [
+    {"name": "a", "state": "running"},
+    {"name": "b", "state": "stopped"},
+    {"name": "b", "state": "running"}
+  ]} | json_query("machines[?state=='running'].name") }}
+
+  Example 3: {{
+  {"services": [
+    {"name": "http", "host": "1.2.3.4", "port": 80},
+    {"name": "smtp", "host": "1.2.3.5", "port": 25},
+    {"name": "ssh",  "host": "1.2.3.6", "port": 22},
+  ]} | json_query("services[].port") }}
+
+Returns:
+
+.. code-block:: text
+
+  Example 1: [1, 2, 3, 4, 5, 6]
+
+  Example 2: ['a', 'b']
+
+  Example 3: [80, 25, 22]
+
 .. _`builtin filters`: http://jinja.pocoo.org/docs/templates/#builtin-filters
 .. _`timelib`: https://github.com/pediapress/timelib/
+.. _`JMESPath language`: http://jmespath.org/
+.. _`jmespath`: https://github.com/jmespath/jmespath.py
 
 Networking Filters
 ------------------

requirements/static/linux.in

@@ -10,6 +10,7 @@ docker
 futures>=2.0; python_version < '3.0'
 GitPython
 hgtools
+jmespath
 jsonschema
 junos-eznc
 jxmlease

requirements/static/osx.in

@@ -6,6 +6,7 @@ croniter>=0.3.0,!=0.3.22
 dnspython
 docker
 futures>=2.0; python_version < '3.0'
+jmespath
 jsonschema
 junos-eznc
 jxmlease

requirements/static/py2.7/linux.txt

@@ -48,7 +48,7 @@ idna==2.8                 # via requests
 ipaddress==1.0.22         # via cryptography, docker, kubernetes
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py2.7/osx.txt

@@ -50,7 +50,7 @@ idna==2.8
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py2.7/windows.txt

@@ -48,7 +48,7 @@ ioloop==0.1a0
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.4/linux.txt

@@ -41,7 +41,7 @@ idna==2.8                 # via requests
 ipaddress==1.0.22         # via kubernetes
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.5/linux.txt

@@ -41,7 +41,7 @@ idna==2.8                 # via requests
 ipaddress==1.0.22         # via kubernetes
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.5/osx.txt

@@ -44,7 +44,7 @@ idna==2.8
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.5/windows.txt

@@ -42,7 +42,7 @@ ioloop==0.1a0
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.6/linux.txt

@@ -41,7 +41,7 @@ idna==2.8                 # via requests
 ipaddress==1.0.22         # via kubernetes
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.6/osx.txt

@@ -44,7 +44,7 @@ idna==2.8
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.6/windows.txt

@@ -42,7 +42,7 @@ ioloop==0.1a0
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.7/linux.txt

@@ -41,7 +41,7 @@ idna==2.8                 # via requests
 ipaddress==1.0.22         # via kubernetes
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.7/osx.txt

@@ -44,7 +44,7 @@ idna==2.8
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/py3.7/windows.txt

@@ -42,7 +42,7 @@ ioloop==0.1a0
 ipaddress==1.0.22
 jaraco.functools==2.0     # via tempora
 jinja2==2.10.1
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.4
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 jsonschema==2.6.0

requirements/static/windows.in

@@ -7,6 +7,7 @@ dnspython
 # We require docker < 3.0.0 because after that they also start locking their pywin32 requirement, actually
 # pypiwin32, which after version 223 it just makes pywin32 a dep and installs nothing else
 docker<3.0.0
+jmespath
 jsonschema
 keyring==5.7.1
 kubernetes<4.0

salt/utils/data.py

@@ -30,6 +30,11 @@
 from salt.ext import six
 from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
 
+try:
+    import jmespath
+except ImportError:
+    jmespath = None
+
 log = logging.getLogger(__name__)
 
 
@@ -976,3 +981,15 @@ def stringify(data):
             item = six.text_type(item)
         ret.append(item)
     return ret
+
+
+@jinja_filter('json_query')
+def json_query(data, expr):
+    '''
+    Query data using JMESPath language (http://jmespath.org).
+    '''
+    if jmespath is None:
+        err = 'json_query requires jmespath module installed'
+        log.error(err)
+        raise RuntimeError(err)
+    return jmespath.search(expr, data)

tests/unit/utils/test_data.py

@@ -598,3 +598,24 @@ def test_stringify(self):
             salt.utils.data.stringify(['one', 'two', str('three'), 4, 5]),  # future lint: disable=blacklisted-function
             ['one', 'two', 'three', '4', '5']
         )
+
+    def test_json_query(self):
+        # Raises exception if jmespath module is not found
+        with patch('salt.utils.data.jmespath', None):
+            self.assertRaisesRegex(
+                RuntimeError, 'requires jmespath',
+                salt.utils.data.json_query, {}, '@'
+            )
+
+        # Test search
+        user_groups = {
+            'user1': {'groups': ['group1', 'group2', 'group3']},
+            'user2': {'groups': ['group1', 'group2']},
+            'user3': {'groups': ['group3']},
+        }
+        expression = '*.groups[0]'
+        primary_groups = ['group1', 'group1', 'group3']
+        self.assertEqual(
+            sorted(salt.utils.data.json_query(user_groups, expression)),
+            primary_groups
+        )

tests/unit/utils/test_jinja.py

@@ -1322,6 +1322,16 @@ def test_base64_decode(self):
                                      dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
         self.assertEqual(rendered, 'random')
 
+    def test_json_query(self):
+        '''
+        Test the `json_query` Jinja filter.
+        '''
+        rendered = render_jinja_tmpl(
+            "{{ [1, 2, 3] | json_query('[1]')}}",
+            dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)
+        )
+        self.assertEqual(rendered, '2')
+
     # def test_print(self):
     #     env = Environment(extensions=[SerializerExtension])
     #     source = '{% import_yaml "toto.foo" as docu %}'