Auto merge of #919 - bstaletic:cache_includes, r=micbou

[READY] Cache include paths

Since @davits said he won't be able to finish #873, I've updated his PR according to the review comments.

All credits go to @davits for actually doing all the work here.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/ycmd/919)
<!-- Reviewable:end -->

ycmd/completers/cpp/clang_completer.py

@@ -39,8 +39,7 @@
                                         NoCompilationDatabase,
                                         UserIncludePaths )
 from ycmd.completers.cpp.ephemeral_values_set import EphemeralValuesSet
-from ycmd.completers.general.filename_completer import (
-    GenerateCandidatesForPaths )
+from ycmd.completers.cpp.include_cache import IncludeCache, IncludeList
 from ycmd.responses import NoExtraConfDetected, UnknownExtraConf
 
 CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] )
@@ -61,6 +60,7 @@ def __init__( self, user_options ):
       'max_diagnostics_to_display' ]
     self._completer = ycm_core.ClangCompleter()
     self._flags = Flags()
+    self._include_cache = IncludeCache()
     self._diagnostic_store = None
     self._files_being_compiled = EphemeralValuesSet()
     self._logger = logging.getLogger( __name__ )
@@ -119,30 +119,22 @@ def GetIncludePaths( self, request_data ):
     if quoted_include:
       include_paths.extend( quoted_include_paths )
 
-    paths = []
+    includes = IncludeList()
     for include_path in include_paths:
       unicode_path = ToUnicode( os.path.join( include_path, path_dir ) )
-      try:
-        # We need to pass a unicode string to get unicode strings out of
-        # listdir.
-        relative_paths = os.listdir( unicode_path )
-      except Exception:
-        self._logger.exception( 'Error while listing %s folder.', unicode_path )
-        relative_paths = []
+      includes.AddIncludes( self._include_cache.GetIncludes( unicode_path ) )
 
-      paths.extend( os.path.join( include_path, path_dir, relative_path ) for
-                    relative_path in relative_paths  )
-    return paths
+    return includes.GetIncludes()
 
 
   def ComputeCandidatesInner( self, request_data ):
     flags, filename = self._FlagsForRequest( request_data )
     if not flags:
       raise RuntimeError( NO_COMPILE_FLAGS_MESSAGE )
 
-    paths = self.GetIncludePaths( request_data )
-    if paths is not None:
-      return GenerateCandidatesForPaths( paths )
+    includes = self.GetIncludePaths( request_data )
+    if includes is not None:
+      return includes
 
     if self._completer.UpdatingTranslationUnit(
         ToCppStringCompatible( filename ) ):
@@ -325,9 +317,11 @@ def _GetSemanticInfo(
 
     return response_builder( message )
 
+
   def _ClearCompilationFlagCache( self ):
     self._flags.Clear()
 
+
   def _FixIt( self, request_data ):
     flags, filename = self._FlagsForRequest( request_data )
     if not flags:
@@ -351,6 +345,7 @@ def _FixIt( self, request_data ):
 
     return responses.BuildFixItResponse( fixits )
 
+
   def OnFileReadyToParse( self, request_data ):
     flags, filename = self._FlagsForRequest( request_data )
     if not flags:

ycmd/completers/cpp/flags.py

@@ -682,36 +682,38 @@ def _GetCompilationInfoForFile( database, file_name, file_extension ):
   return None
 
 
-def UserIncludePaths( flags, filename ):
-  quoted_include_paths = [ os.path.dirname( filename ) ]
+def UserIncludePaths( user_flags, filename ):
+  """
+  Returns a tuple ( quoted_include_paths, include_paths )
+
+  quoted_include_paths is a list of include paths that are only suitable for
+  quoted include statement.
+  include_paths is a list of include paths that can be used for angle bracketed
+  and quoted include statement.
+  """
+  quoted_include_paths = [ ToUnicode( os.path.dirname( filename ) ) ]
   include_paths = []
-  if flags:
-    quote_flag = '-iquote'
-    path_flags = ( [ '-isystem', '-I', '/I' ]
-                   if _ShouldAllowWinStyleFlags( flags )
-                   else [ '-isystem', '-I' ] )
+
+  if user_flags:
+    include_flags = { '-iquote':  quoted_include_paths,
+                      '-I':       include_paths,
+                      '-isystem': include_paths }
+    if _ShouldAllowWinStyleFlags( user_flags ):
+      include_flags[ '/I' ] = include_paths
 
     try:
-      it = iter( flags )
-      for flag in it:
-        flag_len = len( flag )
-        if flag.startswith( quote_flag ):
-          quote_flag_len = len( quote_flag )
-          # Add next flag to the include paths if current flag equals to
-          # '-iquote', or add remaining string otherwise.
-          quoted_include_path = ( next( it ) if flag_len == quote_flag_len else
-                                  flag[ quote_flag_len: ] )
-          if quoted_include_path:
-            quoted_include_paths.append( ToUnicode( quoted_include_path ) )
-        else:
-          for path_flag in path_flags:
-            if flag.startswith( path_flag ):
-              path_flag_len = len( path_flag )
-              include_path = ( next( it ) if flag_len == path_flag_len else
-                               flag[ path_flag_len: ] )
-              if include_path:
-                include_paths.append( ToUnicode( include_path ) )
-              break
+      it = iter( user_flags )
+      for user_flag in it:
+        user_flag_len = len( user_flag )
+        for flag in include_flags:
+          if user_flag.startswith( flag ):
+            flag_len = len( flag )
+            include_path = ( next( it ) if user_flag_len == flag_len else
+                             user_flag[ flag_len: ] )
+            if include_path:
+              container = include_flags[ flag ]
+              container.append( ToUnicode( include_path ) )
+            break
     except StopIteration:
       pass
 

ycmd/completers/cpp/include_cache.py

@@ -0,0 +1,142 @@
+# Copyright (C) 2017 Davit Samvelyan [email protected]
+#                    Synopsys.
+#               2018 ycmd contributors
+#
+# This file is part of ycmd.
+#
+# ycmd is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ycmd is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ycmd.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+# Not installing aliases from python-future; it's unreliable and slow.
+from builtins import *  # noqa
+
+from future.utils import iteritems
+
+import os
+import threading
+
+from collections import defaultdict, namedtuple
+from ycmd import responses
+from ycmd.completers.general.filename_completer import ( GetPathType,
+                                                         GetPathTypeName )
+
+import logging
+_logger = logging.getLogger( __name__ )
+
+""" Represents single include completion candidate.
+name is the name/string of the completion candidate,
+entry_type is an integer indicating whether the candidate is a
+'File', 'Dir' or both (See EXTRA_INFO_MAP in filename_completer). """
+IncludeEntry = namedtuple( 'IncludeEntry', [ 'name', 'entry_type' ] )
+
+
+class IncludeList( object ):
+  """
+  Helper class for combining include completion candidates from
+  several include paths.
+  self._includes is a dictionary whose keys are
+  IncludeEntry `name`s and values are IncludeEntry `entry_type`s.
+  """
+
+  def __init__( self ):
+    self._includes = defaultdict( int )
+
+
+  def AddIncludes( self, includes ):
+    for include in includes:
+      self._includes[ include.name ] |= include.entry_type
+
+
+  def GetIncludes( self ):
+    includes = []
+    for name, include_type in iteritems( self._includes ):
+      includes.append( responses.BuildCompletionData(
+        name, GetPathTypeName( include_type ) ) )
+    return includes
+
+
+class IncludeCache( object ):
+  """
+  Holds a dictionary representing the include path cache.
+  Dictionary keys are the include path directories.
+  Dictionary values are tuples whose first object
+  represents `mtime` of the dictionary key and the other
+  object is an IncludeList.
+  """
+
+  def __init__( self ):
+    self._cache = {}
+    self._cache_lock = threading.Lock()
+
+
+  def GetIncludes( self, path ):
+    includes = self._GetCached( path )
+
+    if includes is None:
+      includes = self._ListIncludes( path )
+      self._AddToCache( path, includes )
+
+    return includes
+
+
+  def _AddToCache( self, path, includes, mtime = None ):
+    if not mtime:
+      mtime = _GetModificationTime( path )
+    # mtime of 0 is "a magic value" to represent inaccessible directory mtime.
+    if mtime:
+      with self._cache_lock:
+        self._cache[ path ] = { 'mtime': mtime, 'includes': includes }
+
+
+  def _GetCached( self, path ):
+    includes = None
+    with self._cache_lock:
+      cache_entry = self._cache.get( path )
+    if cache_entry:
+      mtime = _GetModificationTime( path )
+      if mtime > cache_entry[ 'mtime' ]:
+        includes = self._ListIncludes( path )
+        self._AddToCache( path, includes, mtime )
+      else:
+        includes = cache_entry[ 'includes' ]
+
+    return includes
+
+
+  def _ListIncludes( self, path ):
+    try:
+      names = os.listdir( path )
+    except OSError:
+      _logger.exception( 'Can not list entries for include path %s.', path )
+      return []
+
+    includes = []
+    for name in names:
+      inc_path = os.path.join( path, name )
+      entry_type = GetPathType( inc_path )
+      includes.append( IncludeEntry( name, entry_type ) )
+
+    return includes
+
+
+def _GetModificationTime( path ):
+  try:
+    return os.path.getmtime( path )
+  except OSError:
+    _logger.exception( 'Can not get modification time for include path %s.',
+                       path )
+    return 0

ycmd/completers/general/filename_completer.py

@@ -25,14 +25,18 @@
 
 import logging
 import os
-from collections import defaultdict
 
 from ycmd.completers.completer import Completer
 from ycmd.utils import ( ExpandVariablesInPath, GetCurrentDirectory, OnWindows,
                          re, ToUnicode )
 from ycmd import responses
 
-EXTRA_INFO_MAP = { 1 : '[File]', 2 : '[Dir]', 3 : '[File&Dir]' }
+FILE = 1
+DIR = 2
+# This mapping is also used for the #include completion.
+# We have option 3, because some entries could simultaneously be
+# both a file and a directory.
+EXTRA_INFO_MAP = { FILE : '[File]', DIR : '[Dir]', 3 : '[File&Dir]' }
 
 _logger = logging.getLogger( __name__ )
 
@@ -104,8 +108,8 @@ def ComputeCandidatesInner( self, request_data ):
     # working directory of ycmd
     working_dir = request_data.get( 'working_dir' )
 
-    return GenerateCandidatesForPaths(
-      _GetAbsolutePaths(
+    return GeneratePathCompletionData(
+      _GetPathCompletionCandidates(
         path_dir,
         self.user_options[ 'filepath_completion_use_working_dir' ],
         filepath,
@@ -138,40 +142,43 @@ def _GetAbsolutePathForCompletions( path_dir,
                          path_dir )
 
 
-def _GetAbsolutePaths( path_dir, use_working_dir, filepath, working_dir ):
+def _GetPathCompletionCandidates( path_dir, use_working_dir,
+                                  filepath, working_dir ):
+
   absolute_path_dir = _GetAbsolutePathForCompletions( path_dir,
                                                       use_working_dir,
                                                       filepath,
                                                       working_dir )
-
+  entries = []
+  unicode_path = ToUnicode( absolute_path_dir )
   try:
     # We need to pass a unicode string to get unicode strings out of
     # listdir.
-    relative_paths = os.listdir( ToUnicode( absolute_path_dir ) )
+    relative_paths = os.listdir( unicode_path )
   except Exception:
     _logger.exception( 'Error while listing %s folder.', absolute_path_dir )
     relative_paths = []
 
-  return ( os.path.join( absolute_path_dir, relative_path )
-           for relative_path in relative_paths )
+  for rel_path in relative_paths:
+    absolute_path = os.path.join( unicode_path, rel_path )
+    entries.append( ( rel_path, GetPathType( absolute_path ) ) )
+
+  return entries
+
+
+def GetPathType( path ):
+  return DIR if os.path.isdir( path ) else FILE
+
 
+def GetPathTypeName( path_type ):
+  return EXTRA_INFO_MAP[ path_type ]
 
-def GenerateCandidatesForPaths( absolute_paths ):
-  extra_info = defaultdict( int )
-  basenames = []
-  for absolute_path in absolute_paths:
-    basename = os.path.basename( absolute_path )
-    if extra_info[ basename ] == 0:
-      basenames.append( basename )
-    is_dir = os.path.isdir( absolute_path )
-    extra_info[ basename ] |= ( 2 if is_dir else 1 )
 
+def GeneratePathCompletionData( entries ):
   completion_dicts = []
-  # Keep original ordering
-  for basename in basenames:
+  for entry in entries:
     completion_dicts.append(
-      responses.BuildCompletionData(
-        basename,
-        EXTRA_INFO_MAP[ extra_info[ basename ] ] ) )
+      responses.BuildCompletionData( entry[ 0 ],
+        GetPathTypeName( entry[ 1 ] ) ) )
 
   return completion_dicts