Reduce number of file reads in audit and scan runs

detect_secrets/core/audit.py

@@ -10,6 +10,11 @@
 from collections import defaultdict
 from copy import deepcopy
 
+try:
+    from functools import lru_cache
+except ImportError:  # pragma: no cover
+    from functools32 import lru_cache
+
 from ..plugins.common import initialize
 from ..plugins.common.filetype import determine_file_type
 from ..plugins.common.util import get_mapping_from_secret_type_to_class_name
@@ -28,8 +33,8 @@
 class SecretNotFoundOnSpecifiedLineError(Exception):
     def __init__(self, line):
         super(SecretNotFoundOnSpecifiedLineError, self).__init__(
-            'ERROR: Secret not found on line {}!\n'.format(line) +
-            'Try recreating your baseline to fix this issue.',
+            'ERROR: Secret not found on line {}!\n'.format(line)
+            + 'Try recreating your baseline to fix this issue.',
         )
 
 
@@ -529,17 +534,32 @@ def _handle_user_decision(decision, secret):
         del secret['is_secret']
 
 
-def _get_file_line(filename, line_number):
+@lru_cache(maxsize=1)
+def _open_file_with_cache(filename):
     """
-    Attempts to read a given line from the input file.
+    Reads the input file and returns the result as a string.
+
+    This caches opened files to ensure that the audit functionality
+    doesn't unnecessarily re-open the same file.
     """
     try:
         with codecs.open(filename, encoding='utf-8') as f:
-            return f.read().splitlines()[line_number - 1]  # line numbers are 1-indexed
-    except (OSError, IOError, IndexError):
+            return f.read()
+    except (OSError, IOError):
         return None
 
 
+def _get_file_line(filename, line_number):
+    """
+    Attempts to read a given line from the input file.
+    """
+    file_content = _open_file_with_cache(filename)
+    if not file_content:
+        return None
+
+    return file_content.splitlines()[line_number - 1]
+
+
 def _get_secret_with_context(
     filename,
     secret,
@@ -569,13 +589,20 @@ def _get_secret_with_context(
 
     :raises: SecretNotFoundOnSpecifiedLineError
     """
-    snippet = CodeSnippetHighlighter().get_code_snippet(
-        filename,
-        secret['line_number'],
-        lines_of_context=lines_of_context,
-    )
 
     try:
+        file_content = _open_file_with_cache(filename)
+        if not file_content:
+            raise SecretNotFoundOnSpecifiedLineError(secret['line_number'])
+
+        file_lines = file_content.splitlines()
+
+        snippet = CodeSnippetHighlighter().get_code_snippet(
+            file_lines,
+            secret['line_number'],
+            lines_of_context=lines_of_context,
+        )
+
         raw_secret_value = get_raw_secret_value(
             snippet.target_line,
             secret,

detect_secrets/core/bidirectional_iterator.py

@@ -21,7 +21,7 @@ def __next__(self):
 
         return result
 
-    def next(self):
+    def next(self):  # pragma: no cover
         return self.__next__()
 
     def step_back_on_next_iteration(self):
@@ -30,5 +30,5 @@ def step_back_on_next_iteration(self):
     def can_step_back(self):
         return self.index > 0
 
-    def __iter__(self):
+    def __iter__(self):  # pragma: no cover
         return self

detect_secrets/core/code_snippet.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import codecs
 import itertools
 
 from .color import AnsiColor
@@ -9,9 +8,10 @@
 
 class CodeSnippetHighlighter:
 
-    def get_code_snippet(self, filename, line_number, lines_of_context=5):
+    def get_code_snippet(self, file_lines, line_number, lines_of_context=5):
         """
-        :type filename: str
+        :type file_lines: iterable of str
+        :param file_lines: an iterator of lines in the file
 
         :type line_number: int
         :param line_number: line which you want to focus on
@@ -35,7 +35,7 @@ def get_code_snippet(self, filename, line_number, lines_of_context=5):
         return CodeSnippet(
             list(
                 itertools.islice(
-                    self._get_lines_in_file(filename),
+                    file_lines,
                     start_line,
                     end_line,
                 ),
@@ -44,19 +44,12 @@ def get_code_snippet(self, filename, line_number, lines_of_context=5):
             index_of_secret_in_output,
         )
 
-    def _get_lines_in_file(self, filename):
-        """
-        :rtype: list
-        """
-        with codecs.open(filename, encoding='utf-8') as file:
-            return file.read().splitlines()
-
 
 class CodeSnippet:
 
     def __init__(self, snippet, start_line, target_index):
         """
-        :type snippet: list
+        :type snippet: iterable and indexable of str
         :param snippet: lines of code extracted from file
 
         :type start_line: int

detect_secrets/plugins/base.py

@@ -53,7 +53,8 @@ def analyze(self, file, filename):
                                detect_secrets.core.potential_secret         }
         """
         potential_secrets = {}
-        for line_num, line in enumerate(file.readlines(), start=1):
+        file_lines = tuple(file.readlines())
+        for line_num, line in enumerate(file_lines, start=1):
             results = self.analyze_string(line, line_num, filename)
             if not self.should_verify:
                 potential_secrets.update(results)
@@ -62,7 +63,7 @@ def analyze(self, file, filename):
             filtered_results = {}
             for result in results:
                 snippet = CodeSnippetHighlighter().get_code_snippet(
-                    filename,
+                    file_lines,
                     result.lineno,
                     lines_of_context=LINES_OF_CONTEXT,
                 )

tests/core/audit_test.py

@@ -16,6 +16,11 @@
 from testing.util import uncolor
 
 
+@pytest.fixture(autouse=True)
+def reset_file_cache():
+    audit._open_file_with_cache.cache_clear()
+
+
 class TestAuditBaseline(object):
 
     def test_no_baseline(self, mock_printer):
@@ -740,7 +745,7 @@ def mock_open(
                 string.ascii_letters[:(end_line - secret_line)][::-1],
             ),
         )
-        return mock_open_base(data, 'detect_secrets.core.code_snippet.codecs.open')
+        return mock_open_base(data, 'detect_secrets.core.audit.codecs.open')
 
     @staticmethod
     def _make_string_into_individual_lines(string):