ToUtf8IfNeeded is now ToCppStringCompatible

It's only meant to be used for talking to the C++ layer. For everything
else, ToBytes is a better pick.

ycmd/completers/all/identifier_completer.py

@@ -30,7 +30,7 @@
 from ycmd.completers.general_completer import GeneralCompleter
 from ycmd import identifier_utils
 from ycmd import utils
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToCppStringCompatible
 from ycmd import responses
 
 SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX'
@@ -54,8 +54,8 @@ def ComputeCandidates( self, request_data ):
       return []
 
     completions = self._completer.CandidatesForQueryAndType(
-      ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ),
-      ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) )
+      ToCppStringCompatible( utils.SanitizeQuery( request_data[ 'query' ] ) ),
+      ToCppStringCompatible( request_data[ 'filetypes' ][ 0 ] ) )
 
     completions = completions[ : self._max_candidates ]
     completions = _RemoveSmallCandidates(
@@ -80,11 +80,12 @@ def AddIdentifier( self, identifier, request_data ):
       return
 
     vector = ycm_core.StringVector()
-    vector.append( ToUtf8IfNeeded( identifier ) )
+    vector.append( ToCppStringCompatible( identifier ) )
     self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath )
-    self._completer.AddIdentifiersToDatabase( vector,
-                                              ToUtf8IfNeeded( filetype ),
-                                              ToUtf8IfNeeded( filepath ) )
+    self._completer.AddIdentifiersToDatabase(
+      vector,
+      ToCppStringCompatible( filetype ),
+      ToCppStringCompatible( filepath ) )
 
 
   def AddPreviousIdentifier( self, request_data ):
@@ -121,8 +122,8 @@ def AddBufferIdentifiers( self, request_data ):
         _IdentifiersFromBuffer( text,
                                 filetype,
                                 collect_from_comments_and_strings ),
-        ToUtf8IfNeeded( filetype ),
-        ToUtf8IfNeeded( filepath ) )
+        ToCppStringCompatible( filetype ),
+        ToCppStringCompatible( filepath ) )
 
 
   def AddIdentifiersFromTagFiles( self, tag_files ):
@@ -140,7 +141,7 @@ def AddIdentifiersFromTagFiles( self, tag_files ):
         continue
 
       self._tags_file_last_mtime[ tag_file ] = current_mtime
-      absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) )
+      absolute_paths_to_tag_files.append( ToCppStringCompatible( tag_file ) )
 
     if not absolute_paths_to_tag_files:
       return
@@ -152,12 +153,13 @@ def AddIdentifiersFromTagFiles( self, tag_files ):
   def AddIdentifiersFromSyntax( self, keyword_list, filetypes ):
     keyword_vector = ycm_core.StringVector()
     for keyword in keyword_list:
-      keyword_vector.append( ToUtf8IfNeeded( keyword ) )
+      keyword_vector.append( ToCppStringCompatible( keyword ) )
 
     filepath = SYNTAX_FILENAME + filetypes[ 0 ]
-    self._completer.AddIdentifiersToDatabase( keyword_vector,
-                                              ToUtf8IfNeeded( filetypes[ 0 ] ),
-                                              ToUtf8IfNeeded( filepath ) )
+    self._completer.AddIdentifiersToDatabase(
+      keyword_vector,
+      ToCppStringCompatible( filetypes[ 0 ] ),
+      ToCppStringCompatible( filepath ) )
 
 
   def OnFileReadyToParse( self, request_data ):
@@ -237,5 +239,5 @@ def _IdentifiersFromBuffer( text,
   idents = identifier_utils.ExtractIdentifiersFromText( text, filetype )
   vector = ycm_core.StringVector()
   for ident in idents:
-    vector.append( ToUtf8IfNeeded( ident ) )
+    vector.append( ToCppStringCompatible( ident ) )
   return vector

ycmd/completers/completer_utils.py

@@ -24,7 +24,7 @@
 from builtins import *  # noqa
 from future.utils import iteritems
 
-from ycmd.utils import ToUtf8IfNeeded, RunningInsideVim
+from ycmd.utils import ToCppStringCompatible, RunningInsideVim
 
 if RunningInsideVim():
   from ycm_client_support import FilterAndSortCandidates
@@ -158,8 +158,8 @@ def FiletypeCompleterExistsForFiletype( filetype ):
 
 def FilterAndSortCandidatesShim( candidates, sort_property, query ):
   return FilterAndSortCandidates( candidates,
-                                  ToUtf8IfNeeded( sort_property ),
-                                  ToUtf8IfNeeded( query ) )
+                                  ToCppStringCompatible( sort_property ),
+                                  ToCppStringCompatible( query ) )
 
 TRIGGER_REGEX_PREFIX = 're!'
 

ycmd/completers/cpp/clang_completer.py

@@ -31,7 +31,7 @@
 import textwrap
 from ycmd import responses
 from ycmd import extra_conf_store
-from ycmd.utils import ToUtf8IfNeeded
+from ycmd.utils import ToCppStringCompatible
 from ycmd.completers.completer import Completer
 from ycmd.completers.completer_utils import GetIncludeStatementValue
 from ycmd.completers.cpp.flags import Flags, PrepareFlagsForClang
@@ -75,10 +75,10 @@ def GetUnsavedFilesVector( self, request_data ):
         continue
 
       unsaved_file = ycm_core.UnsavedFile()
-      utf8_contents = ToUtf8IfNeeded( contents )
+      utf8_contents = ToCppStringCompatible( contents )
       unsaved_file.contents_ = utf8_contents
       unsaved_file.length_ = len( utf8_contents )
-      unsaved_file.filename_ = ToUtf8IfNeeded( filename )
+      unsaved_file.filename_ = ToCppStringCompatible( filename )
 
       files.append( unsaved_file )
     return files
@@ -89,7 +89,8 @@ def ComputeCandidatesInner( self, request_data ):
     if not filename:
       return
 
-    if self._completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
+    if self._completer.UpdatingTranslationUnit(
+        ToCppStringCompatible( filename ) ):
       raise RuntimeError( PARSING_FILE_MESSAGE )
 
     flags = self._FlagsForRequest( request_data )
@@ -101,7 +102,7 @@ def ComputeCandidatesInner( self, request_data ):
     column = request_data[ 'start_column' ]
     with self._files_being_compiled.GetExclusive( filename ):
       results = self._completer.CandidatesForLocationInFile(
-          ToUtf8IfNeeded( filename ),
+          ToCppStringCompatible( filename ),
           line,
           column,
           files,
@@ -160,7 +161,7 @@ def _LocationForGoTo( self, goto_function, request_data, reparse = True ):
     line = request_data[ 'line_num' ]
     column = request_data[ 'column_num' ]
     return getattr( self._completer, goto_function )(
-        ToUtf8IfNeeded( filename ),
+        ToCppStringCompatible( filename ),
         line,
         column,
         files,
@@ -249,11 +250,12 @@ def _GoToInclude( self, request_data ):
     return include_response
 
 
-  def _GetSemanticInfo( self,
-                        request_data,
-                        func,
-                        response_builder = responses.BuildDisplayMessageResponse,
-                        reparse = True ):
+  def _GetSemanticInfo(
+      self,
+      request_data,
+      func,
+      response_builder = responses.BuildDisplayMessageResponse,
+      reparse = True ):
     filename = request_data[ 'filepath' ]
     if not filename:
       raise ValueError( INVALID_FILE_MESSAGE )
@@ -267,7 +269,7 @@ def _GetSemanticInfo( self,
     column = request_data[ 'column_num' ]
 
     message = getattr( self._completer, func )(
-        ToUtf8IfNeeded( filename ),
+        ToCppStringCompatible( filename ),
         line,
         column,
         files,
@@ -296,7 +298,7 @@ def _FixIt( self, request_data ):
     column = request_data[ 'column_num' ]
 
     fixits = getattr( self._completer, "GetFixItsForLocationInFile" )(
-        ToUtf8IfNeeded( filename ),
+        ToCppStringCompatible( filename ),
         line,
         column,
         files,
@@ -319,7 +321,7 @@ def OnFileReadyToParse( self, request_data ):
 
     with self._files_being_compiled.GetExclusive( filename ):
       diagnostics = self._completer.UpdateTranslationUnit(
-        ToUtf8IfNeeded( filename ),
+        ToCppStringCompatible( filename ),
         self.GetUnsavedFilesVector( request_data ),
         flags )
 
@@ -331,7 +333,7 @@ def OnFileReadyToParse( self, request_data ):
 
   def OnBufferUnload( self, request_data ):
     self._completer.DeleteCachesForFile(
-        ToUtf8IfNeeded( request_data[ 'unloaded_buffer' ] ) )
+        ToCppStringCompatible( request_data[ 'unloaded_buffer' ] ) )
 
 
   def GetDetailedDiagnostic( self, request_data ):
@@ -386,7 +388,8 @@ def ConvertCompletionData( completion_data ):
     extra_menu_info = completion_data.ExtraMenuInfo(),
     kind = completion_data.kind_.name,
     detailed_info = completion_data.DetailedInfoForPreviewWindow(),
-    extra_data = { 'doc_string': completion_data.DocString() } if completion_data.DocString() else None )
+    extra_data = ( { 'doc_string': completion_data.DocString() }
+                   if completion_data.DocString() else None ) )
 
 
 def DiagnosticsToDiagStructure( diagnostics ):

ycmd/completers/cpp/flags.py

@@ -28,7 +28,7 @@
 import inspect
 import re
 from ycmd import extra_conf_store
-from ycmd.utils import ToUtf8IfNeeded, OnMac, OnWindows
+from ycmd.utils import ToCppStringCompatible, OnMac, OnWindows
 from ycmd.responses import NoExtraConfDetected
 
 INCLUDE_FLAGS = [ '-isystem', '-I', '-iquote', '-isysroot', '--sysroot',
@@ -206,7 +206,7 @@ def _SanitizeFlags( flags ):
 
   vector = ycm_core.StringVector()
   for flag in sanitized_flags:
-    vector.append( ToUtf8IfNeeded( flag ) )
+    vector.append( ToCppStringCompatible( flag ) )
   return vector
 
 

ycmd/completers/go/gocode_completer.py

@@ -87,7 +87,7 @@ def ComputeCandidatesInner( self, request_data ):
     if not filename:
       return
 
-    contents = utils.ToUtf8IfNeeded(
+    contents = utils.ToBytes(
         request_data[ 'file_data' ][ filename ][ 'contents' ] )
     offset = _ComputeOffset( contents, request_data[ 'line_num' ],
                              request_data[ 'column_num' ] )
@@ -200,7 +200,7 @@ def _GoToDefinition( self, request_data ):
       _logger.info( "godef GoTo request %s" % filename )
       if not filename:
         return
-      contents = utils.ToUtf8IfNeeded(
+      contents = utils.ToBytes(
           request_data[ 'file_data' ][ filename ][ 'contents' ] )
       offset = _ComputeOffset( contents, request_data[ 'line_num' ],
                                request_data[ 'column_num' ] )

ycmd/request_wrap.py

@@ -23,7 +23,7 @@
 standard_library.install_aliases()
 from builtins import *  # noqa
 
-from ycmd.utils import ToUnicode, ToUtf8IfNeeded
+from ycmd.utils import ToUnicode, ToBytes
 from ycmd.identifier_utils import StartOfLongestIdentifierEndingAtIndex
 from ycmd.request_validation import EnsureRequestValid
 
@@ -103,7 +103,7 @@ def CompletionStartColumn( line_value, column_num, filetype ):
   # NOTE: column_num and other numbers on the wire are byte indices, but we need
   # to walk codepoints for identifier checks.
 
-  utf8_line_value = ToUtf8IfNeeded( line_value )
+  utf8_line_value = ToBytes( line_value )
   unicode_line_value = ToUnicode( line_value )
   codepoint_column_num = len(
       str( utf8_line_value[ : column_num -1 ], 'utf8' ) ) + 1

ycmd/utils.py

@@ -52,12 +52,10 @@ def ReadFile( filepath ):
     return f.read()
 
 
-# Given an object, returns a str object that's utf-8 encoded.
-def ToUtf8IfNeeded( value ):
-  # NOTE: the C++ interop layer specifically wants py2 'str' objects and
-  # probably 'bytes' on py3. You likely want to wrap what this returns with
-  # future.utils's native() if talking to C++.
-
+# Given an object, returns a str object that's utf-8 encoded. This is meant to
+# be used exclusively when producing strings to be passed to the C++ Python
+# plugins. For other code, you likely want to use ToBytes below.
+def ToCppStringCompatible( value ):
   if isinstance( value, str ):
     return value.encode( 'utf8' )
   if isinstance( value, bytes ):
@@ -89,8 +87,7 @@ def ToBytes( value ):
   if type( value ) == bytes:
     return value
 
-  # This is meant to catch Python 2's str type and the str on Python 3, which is
-  # unicode.
+  # This is meant to catch Python 2's native str type.
   if isinstance( value, bytes ):
     return bytes( value, encoding = 'utf8' )