Replace yaml usage with a helper to ensure unicode is handled properly

Without allow_unicode=True, unicode characters are processed through the
str representer and on Python 2 are dumped as a Unicode code point (i.e.
a literal \u0414). This commit makes allow_unicode=True the default in
our salt.utils.yamlloader.safe_dump() helper. It also adds a new
salt.utils.yamlloader.dump() helper which wraps yaml.dump() and also
makes allow_unicode=True the default.

To make importing and using our custom yaml loader/dumper easier, a
convenience module called salt.utils.yaml has been added, which does a
wildcard import from both salt.utils.yamldumper and
salt.utils.yamlloader.

Refs to yaml.load/dump and yaml.safe_load/safe_dump have been updated to
salt.utils.yaml, to ensure that unicode is handled properly.

doc/ref/renderers/index.rst

@@ -144,17 +144,14 @@ Here is a simple YAML renderer example:
 
 .. code-block:: python
 
-    import yaml
+    import salt.utils.yaml
     from salt.utils.yamlloader import SaltYamlSafeLoader
     from salt.ext import six
 
     def render(yaml_data, saltenv='', sls='', **kws):
         if not isinstance(yaml_data, six.string_types):
             yaml_data = yaml_data.read()
-        data = yaml.load(
-            yaml_data,
-            Loader=SaltYamlSafeLoader
-        )
+        data = salt.utils.yaml.safe_load(yaml_data)
         return data if data else {}
 
 Full List of Renderers

salt/cloud/__init__.py

@@ -37,6 +37,7 @@
 import salt.utils.dictupdate
 import salt.utils.files
 import salt.utils.verify
+import salt.utils.yaml
 import salt.syspaths
 from salt.template import compile_template
 
@@ -48,7 +49,6 @@
         import Crypto.Random
     except ImportError:
         pass  # pycrypto < 2.1
-import yaml
 from salt.ext import six
 from salt.ext.six.moves import input  # pylint: disable=import-error,redefined-builtin
 
@@ -1393,7 +1393,7 @@ def run_profile(self, profile, names, vm_overrides=None):
 
         try:
             with salt.utils.files.fopen(self.opts['conf_file'], 'r') as mcc:
-                main_cloud_config = yaml.safe_load(mcc)
+                main_cloud_config = salt.utils.yaml.safe_load(mcc)
             if not main_cloud_config:
                 main_cloud_config = {}
         except KeyError:

salt/cloud/clouds/azurearm.py

@@ -59,13 +59,13 @@
 import logging
 import pprint
 import base64
-import yaml
 import collections
 import salt.cache
 import salt.config as config
 import salt.utils.cloud
 import salt.utils.data
 import salt.utils.files
+import salt.utils.yaml
 from salt.utils.versions import LooseVersion
 from salt.ext import six
 import salt.version
@@ -1070,7 +1070,7 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
     )
 
     if isinstance(kwargs.get('volumes'), six.string_types):
-        volumes = yaml.safe_load(kwargs['volumes'])
+        volumes = salt.utils.yaml.safe_load(kwargs['volumes'])
     else:
         volumes = kwargs.get('volumes')
 

salt/cloud/clouds/ec2.py

@@ -75,7 +75,6 @@
 import uuid
 import pprint
 import logging
-import yaml
 
 # Import libs for talking to the EC2 API
 import hmac
@@ -92,6 +91,7 @@
 import salt.utils.files
 import salt.utils.hashutils
 import salt.utils.json
+import salt.utils.yaml
 from salt._compat import ElementTree as ET
 import salt.utils.http as http
 import salt.utils.aws as aws
@@ -2840,7 +2840,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True):
         kwargs['instance_id'] = _get_node(name)['instanceId']
 
     if isinstance(kwargs['volumes'], six.string_types):
-        volumes = yaml.safe_load(kwargs['volumes'])
+        volumes = salt.utils.yaml.safe_load(kwargs['volumes'])
     else:
         volumes = kwargs['volumes']
 
@@ -4109,7 +4109,7 @@ def create_volume(kwargs=None, call=None, wait_to_finish=False):
     # Allow tags to be set upon creation
     if 'tags' in kwargs:
         if isinstance(kwargs['tags'], six.string_types):
-            tags = yaml.safe_load(kwargs['tags'])
+            tags = salt.utils.yaml.safe_load(kwargs['tags'])
         else:
             tags = kwargs['tags']
 

salt/cloud/clouds/joyent.py

@@ -57,7 +57,6 @@
 import base64
 import pprint
 import inspect
-import yaml
 import datetime
 from Crypto.Hash import SHA256
 from Crypto.PublicKey import RSA
@@ -70,6 +69,7 @@
 import salt.utils.files
 import salt.utils.http
 import salt.utils.json
+import salt.utils.yaml
 import salt.config as config
 from salt.cloud.libcloudfuncs import node_state
 from salt.exceptions import (
@@ -1121,6 +1121,6 @@ def query(action=None,
 
     if 'Content-Length' in result['headers']:
         content = result['text']
-        return_content = yaml.safe_load(content)
+        return_content = salt.utils.yaml.safe_load(content)
 
     return [result['status'], return_content]

salt/cloud/clouds/msazure.py

@@ -44,12 +44,12 @@
 import logging
 import pprint
 import time
-import yaml
 
 # Import salt libs
 import salt.config as config
 from salt.exceptions import SaltCloudSystemExit
 import salt.utils.cloud
+import salt.utils.yaml
 
 # Import 3rd-party libs
 from salt.ext import six
@@ -711,7 +711,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True):
         kwargs = {}
 
     if isinstance(kwargs['volumes'], six.string_types):
-        volumes = yaml.safe_load(kwargs['volumes'])
+        volumes = salt.utils.yaml.safe_load(kwargs['volumes'])
     else:
         volumes = kwargs['volumes']
 
@@ -805,7 +805,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True):
         kwargs = {}
 
     if isinstance(kwargs['volumes'], six.string_types):
-        volumes = yaml.safe_load(kwargs['volumes'])
+        volumes = salt.utils.yaml.safe_load(kwargs['volumes'])
     else:
         volumes = kwargs['volumes']
 
@@ -2731,7 +2731,7 @@ def set_storage_container_metadata(kwargs=None, storage_conn=None, call=None):
     if 'name' not in kwargs:
         raise SaltCloudSystemExit('An storage container name must be specified as "name"')
 
-    x_ms_meta_name_values = yaml.safe_load(
+    x_ms_meta_name_values = salt.utils.yaml.safe_load(
         kwargs.get('meta_name_values', '')
     )
 

salt/cloud/clouds/nova.py

@@ -206,13 +206,13 @@
 import logging
 import socket
 import pprint
-import yaml
 
 # Import Salt Libs
 from salt.ext import six
 import salt.utils.cloud
 import salt.utils.files
 import salt.utils.pycrypto
+import salt.utils.yaml
 import salt.client
 from salt.utils.openstack import nova
 try:
@@ -1276,7 +1276,7 @@ def volume_create_attach(name, call=None, **kwargs):
         )
 
     if type(kwargs['volumes']) is str:
-        volumes = yaml.safe_load(kwargs['volumes'])
+        volumes = salt.utils.yaml.safe_load(kwargs['volumes'])
     else:
         volumes = kwargs['volumes']
 

salt/config/__init__.py

@@ -15,7 +15,6 @@
 import codecs
 import logging
 import types
-import yaml
 from copy import deepcopy
 
 # pylint: disable=import-error,no-name-in-module
@@ -34,6 +33,7 @@
 import salt.utils.user
 import salt.utils.validate.path
 import salt.utils.xdg
+import salt.utils.yaml
 import salt.utils.yamlloader as yamlloader
 import salt.utils.zeromq
 import salt.syspaths
@@ -2098,11 +2098,8 @@ def _read_conf_file(path):
     log.debug('Reading configuration from {0}'.format(path))
     with salt.utils.files.fopen(path, 'r') as conf_file:
         try:
-            conf_opts = yamlloader.load(
-                conf_file.read(),
-                Loader=yamlloader.SaltYamlSafeLoader,
-            ) or {}
-        except yaml.YAMLError as err:
+            conf_opts = salt.utils.yaml.safe_load(conf_file) or {}
+        except salt.utils.yaml.YAMLError as err:
             message = 'Error parsing configuration file: {0} - {1}'.format(path, err)
             log.error(message)
             raise salt.exceptions.SaltConfigurationError(message)

salt/engines/slack.py

@@ -99,7 +99,6 @@
 import time
 import re
 import traceback
-import yaml
 
 log = logging.getLogger(__name__)
 
@@ -119,7 +118,7 @@
 import salt.utils.http
 import salt.utils.json
 import salt.utils.slack
-from salt.utils.yamldumper import OrderedDumper
+import salt.utils.yaml
 
 __virtualname__ = 'slack'
 
@@ -373,7 +372,7 @@ def message_text(self, m_data):
 
         # Convert UTF to string
         _text = salt.utils.json.dumps(_text)
-        _text = yaml.safe_load(_text)
+        _text = salt.utils.yaml.safe_load(_text)
 
         if not _text:
             raise ValueError('_text has no value')
@@ -561,7 +560,7 @@ def format_return_text(self, data, **kwargs):  # pylint: disable=unused-argument
         '''
         Print out YAML using the block mode
         '''
-        params = dict(Dumper=OrderedDumper)
+        params = {}
         if 'output_indent' not in __opts__:
             # default indentation
             params.update(default_flow_style=False)
@@ -573,7 +572,7 @@ def format_return_text(self, data, **kwargs):  # pylint: disable=unused-argument
             params.update(default_flow_style=True,
                           indent=0)
         try:
-            #return yaml.dump(data, **params).replace("\n\n", "\n")
+            #return salt.utils.yaml.safe_dump(data, **params).replace("\n\n", "\n")
             return salt.utils.json.dumps(data, sort_keys=True, indent=1)
         # pylint: disable=broad-except
         except Exception as exc:
@@ -627,7 +626,7 @@ def get_jobs_from_runner(self, outstanding_jids):
                 # pylint is tripping
                 # pylint: disable=missing-whitespace-after-comma
                 job_data = salt.utils.json.dumps({key:val['return'] for key, val in jid_result.items()})
-                results[jid] = yaml.load(job_data)
+                results[jid] = salt.utils.yaml.safe_load(job_data)
 
         return results
 
@@ -692,7 +691,7 @@ def run_commands_from_slack_async(self, message_generator, fire_all, tag, contro
                         content=return_text)
                     # Handle unicode return
                     log.debug('Got back {} via the slack client'.format(r))
-                    resp = yaml.safe_load(salt.utils.json.dumps(r))
+                    resp = salt.utils.yaml.safe_load(salt.utils.json.dumps(r))
                     if 'ok' in resp and resp['ok'] is False:
                         this_job['channel'].send_message('Error: {0}'.format(resp['error']))
                     del outstanding[jid]

salt/grains/extra.py

@@ -6,12 +6,12 @@
 import os
 
 # Import third party libs
-import yaml
 import logging
 
 # Import salt libs
 import salt.utils.files
 import salt.utils.platform
+import salt.utils.yaml
 
 __proxyenabled__ = ['*']
 log = logging.getLogger(__name__)
@@ -69,7 +69,7 @@ def config():
         log.debug('Loading static grains from %s', gfn)
         with salt.utils.files.fopen(gfn, 'rb') as fp_:
             try:
-                return yaml.safe_load(fp_.read())
+                return salt.utils.yaml.safe_load(fp_)
             except Exception:
                 log.warning("Bad syntax in grains file! Skipping.")
                 return {}

salt/minion.py

@@ -786,6 +786,7 @@ class SMinion(MinionBase):
     '''
     def __init__(self, opts):
         # Late setup of the opts grains, so we can log from the grains module
+        import salt.loader
         opts['grains'] = salt.loader.grains(opts)
         super(SMinion, self).__init__(opts)
 
@@ -803,8 +804,7 @@ def __init__(self, opts):
 
         # If configured, cache pillar data on the minion
         if self.opts['file_client'] == 'remote' and self.opts.get('minion_pillar_cache', False):
-            import yaml
-            from salt.utils.yamldumper import SafeOrderedDumper
+            import salt.utils.yaml
             pdir = os.path.join(self.opts['cachedir'], 'pillar')
             if not os.path.isdir(pdir):
                 os.makedirs(pdir, 0o700)
@@ -815,21 +815,11 @@ def __init__(self, opts):
                 penv = 'base'
             cache_top = {penv: {self.opts['id']: ['cache']}}
             with salt.utils.files.fopen(ptop, 'wb') as fp_:
-                fp_.write(
-                    yaml.dump(
-                        cache_top,
-                        Dumper=SafeOrderedDumper
-                    )
-                )
+                salt.utils.yaml.safe_dump(cache_top, fp_)
                 os.chmod(ptop, 0o600)
             cache_sls = os.path.join(pdir, 'cache.sls')
             with salt.utils.files.fopen(cache_sls, 'wb') as fp_:
-                fp_.write(
-                    yaml.dump(
-                        self.opts['pillar'],
-                        Dumper=SafeOrderedDumper
-                    )
-                )
+                salt.utils.yaml.safe_dump(self.opts['pillar'], fp_)
                 os.chmod(cache_sls, 0o600)
 
     def gen_modules(self, initial_load=False):

salt/modules/ansiblegate.py

@@ -33,13 +33,13 @@
 import sys
 import logging
 import importlib
-import yaml
 import fnmatch
 import subprocess
 
 import salt.utils.json
 from salt.exceptions import LoaderError, CommandExecutionError
-from salt.utils import timed_subprocess
+import salt.utils.timed_subprocess
+import salt.utils.yaml
 
 try:
     import ansible
@@ -153,11 +153,13 @@ def call(self, module, *args, **kwargs):
         js_args = str('{{"ANSIBLE_MODULE_ARGS": {args}}}')  # future lint: disable=blacklisted-function
         js_args = js_args.format(args=salt.utils.json.dumps(kwargs))
 
-        proc_out = timed_subprocess.TimedProc(["echo", "{0}".format(js_args)],
-                                              stdout=subprocess.PIPE, timeout=self.timeout)
+        proc_out = salt.utils.timed_subprocess.TimedProc(
+            ["echo", "{0}".format(js_args)],
+            stdout=subprocess.PIPE, timeout=self.timeout)
         proc_out.run()
-        proc_exc = timed_subprocess.TimedProc(['python', module.__file__],
-                                              stdin=proc_out.stdout, stdout=subprocess.PIPE, timeout=self.timeout)
+        proc_exc = salt.utils.timed_subprocess.TimedProc(
+            ['python', module.__file__],
+            stdin=proc_out.stdout, stdout=subprocess.PIPE, timeout=self.timeout)
         proc_exc.run()
 
         try:
@@ -244,7 +246,7 @@ def help(module=None, *args):
     ret = {}
     for docset in module.DOCUMENTATION.split('---'):
         try:
-            docset = yaml.load(docset)
+            docset = salt.utils.yaml.safe_load(docset)
             if docset:
                 doc.update(docset)
         except Exception as err: