Use plugin analyze function in audit functionality

detect_secrets/core/audit.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import codecs
+import io
 import json
 import os
 import subprocess
@@ -10,10 +11,13 @@
 from collections import defaultdict
 from copy import deepcopy
 
+try:
+    from functools import lru_cache
+except ImportError:  # pragma: no cover
+    from functools32 import lru_cache
+
 from ..plugins.common import initialize
-from ..plugins.common.filetype import determine_file_type
 from ..plugins.common.util import get_mapping_from_secret_type_to_class_name
-from ..plugins.high_entropy_strings import HighEntropyStringsPlugin
 from ..util import get_git_remotes
 from ..util import get_git_sha
 from .baseline import merge_results
@@ -22,14 +26,13 @@
 from .color import AnsiColor
 from .color import colorize
 from .common import write_baseline_to_file
-from .potential_secret import PotentialSecret
 
 
 class SecretNotFoundOnSpecifiedLineError(Exception):
     def __init__(self, line):
         super(SecretNotFoundOnSpecifiedLineError, self).__init__(
-            'ERROR: Secret not found on line {}!\n'.format(line) +
-            'Try recreating your baseline to fix this issue.',
+            'ERROR: Secret not found on line {}!\n'.format(line)
+            + 'Try recreating your baseline to fix this issue.',
         )
 
 
@@ -228,16 +231,17 @@ def determine_audit_results(baseline, baseline_path):
     secret_type_to_plugin_name = get_mapping_from_secret_type_to_class_name()
 
     for filename, secret in all_secrets:
-        plaintext_line = _get_file_line(filename, secret['line_number'])
+        file_contents = _open_file_with_cache(filename)
+
         try:
             secret_plaintext = get_raw_secret_value(
-                secret_line=plaintext_line,
                 secret=secret,
                 plugin_settings=baseline['plugins_used'],
+                file_handle=io.StringIO(file_contents),
                 filename=filename,
             )
         except SecretNotFoundOnSpecifiedLineError:
-            secret_plaintext = plaintext_line
+            secret_plaintext = _get_file_line(filename, secret['line_number'])
 
         plugin_name = secret_type_to_plugin_name[secret['type']]
         audit_result = AUDIT_RESULT_TO_STRING[secret.get('is_secret')]
@@ -529,17 +533,32 @@ def _handle_user_decision(decision, secret):
         del secret['is_secret']
 
 
-def _get_file_line(filename, line_number):
+@lru_cache(maxsize=1)
+def _open_file_with_cache(filename):
     """
-    Attempts to read a given line from the input file.
+    Reads the input file and returns the result as a string.
+
+    This caches opened files to ensure that the audit functionality
+    doesn't unnecessarily re-open the same file.
     """
     try:
         with codecs.open(filename, encoding='utf-8') as f:
-            return f.read().splitlines()[line_number - 1]  # line numbers are 1-indexed
-    except (OSError, IOError, IndexError):
+            return f.read()
+    except (OSError, IOError):
         return None
 
 
+def _get_file_line(filename, line_number):
+    """
+    Attempts to read a given line from the input file.
+    """
+    file_content = _open_file_with_cache(filename)
+    if not file_content:
+        return None
+
+    return file_content.splitlines()[line_number - 1]
+
+
 def _get_secret_with_context(
     filename,
     secret,
@@ -569,17 +588,24 @@ def _get_secret_with_context(
 
     :raises: SecretNotFoundOnSpecifiedLineError
     """
-    snippet = CodeSnippetHighlighter().get_code_snippet(
-        filename,
-        secret['line_number'],
-        lines_of_context=lines_of_context,
-    )
 
     try:
+        file_content = _open_file_with_cache(filename)
+        if not file_content:
+            raise SecretNotFoundOnSpecifiedLineError(secret['line_number'])
+
+        file_lines = file_content.splitlines()
+
+        snippet = CodeSnippetHighlighter().get_code_snippet(
+            file_lines,
+            secret['line_number'],
+            lines_of_context=lines_of_context,
+        )
+
         raw_secret_value = get_raw_secret_value(
-            snippet.target_line,
             secret,
             plugin_settings,
+            io.StringIO(file_content),
             filename,
         )
 
@@ -597,21 +623,21 @@ def _get_secret_with_context(
 
 
 def get_raw_secret_value(
-    secret_line,
     secret,
     plugin_settings,
+    file_handle,
     filename,
 ):
     """
-    :type secret_line: str
-    :param secret_line: the line on which the secret is found
-
     :type secret: dict
     :param secret: see caller's docstring
 
     :type plugin_settings: list
     :param plugin_settings: see caller's docstring
 
+    :type file_handle: file object
+    :param file_handle: Open handle to file where the secret is
+
     :type filename: str
     :param filename: this is needed, because PotentialSecret uses this
         as a means of comparing whether two secrets are equal.
@@ -621,36 +647,15 @@ def get_raw_secret_value(
         plugin_settings,
     )
 
-    for raw_secret in raw_secret_generator(
-        plugin,
-        secret_line,
-        filetype=determine_file_type(filename),
-    ):
-        secret_obj = PotentialSecret(
-            plugin.secret_type,
-            filename,
-            secret=raw_secret,
-        )
-
-        # There could be more than two secrets on the same line.
-        # We only want to highlight the right one.
-        if secret_obj.secret_hash == secret['hashed_secret']:
-            return raw_secret
-    else:
-        raise SecretNotFoundOnSpecifiedLineError(secret['line_number'])
-
+    plugin_secrets = plugin.analyze(file_handle, filename)
 
-def raw_secret_generator(plugin, secret_line, filetype):
-    """Generates raw secrets by re-scanning the line, with the specified plugin
+    matching_secret = [
+        plugin_secret.secret_value
+        for plugin_secret in plugin_secrets
+        if plugin_secret.secret_hash == secret['hashed_secret']
+    ]
 
-    :type plugin: BasePlugin
-    :type secret_line: str
-    :type filetype: FileType
-    """
-    for raw_secret in plugin.secret_generator(secret_line, filetype=filetype):
-        yield raw_secret
+    if not matching_secret:
+        raise SecretNotFoundOnSpecifiedLineError(secret['line_number'])
 
-    if issubclass(plugin.__class__, HighEntropyStringsPlugin):
-        with plugin.non_quoted_string_regex(strict=False):
-            for raw_secret in plugin.secret_generator(secret_line):
-                yield raw_secret
+    return matching_secret[0]

detect_secrets/core/bidirectional_iterator.py

@@ -21,7 +21,7 @@ def __next__(self):
 
         return result
 
-    def next(self):
+    def next(self):  # pragma: no cover
         return self.__next__()
 
     def step_back_on_next_iteration(self):
@@ -30,5 +30,5 @@ def step_back_on_next_iteration(self):
     def can_step_back(self):
         return self.index > 0
 
-    def __iter__(self):
+    def __iter__(self):  # pragma: no cover
         return self

detect_secrets/core/code_snippet.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import codecs
 import itertools
 
 from .color import AnsiColor
@@ -9,9 +8,10 @@
 
 class CodeSnippetHighlighter:
 
-    def get_code_snippet(self, filename, line_number, lines_of_context=5):
+    def get_code_snippet(self, file_lines, line_number, lines_of_context=5):
         """
-        :type filename: str
+        :type file_lines: iterable of str
+        :param file_lines: an iterator of lines in the file
 
         :type line_number: int
         :param line_number: line which you want to focus on
@@ -35,7 +35,7 @@ def get_code_snippet(self, filename, line_number, lines_of_context=5):
         return CodeSnippet(
             list(
                 itertools.islice(
-                    self._get_lines_in_file(filename),
+                    file_lines,
                     start_line,
                     end_line,
                 ),
@@ -44,19 +44,12 @@ def get_code_snippet(self, filename, line_number, lines_of_context=5):
             index_of_secret_in_output,
         )
 
-    def _get_lines_in_file(self, filename):
-        """
-        :rtype: list
-        """
-        with codecs.open(filename, encoding='utf-8') as file:
-            return file.read().splitlines()
-
 
-class CodeSnippet:
+class CodeSnippet(object):
 
     def __init__(self, snippet, start_line, target_index):
         """
-        :type snippet: list
+        :type snippet: iterable and indexable of str
         :param snippet: lines of code extracted from file
 
         :type start_line: int

detect_secrets/plugins/base.py

@@ -53,7 +53,8 @@ def analyze(self, file, filename):
                                detect_secrets.core.potential_secret         }
         """
         potential_secrets = {}
-        for line_num, line in enumerate(file.readlines(), start=1):
+        file_lines = tuple(file.readlines())
+        for line_num, line in enumerate(file_lines, start=1):
             results = self.analyze_string(line, line_num, filename)
             if not self.should_verify:
                 potential_secrets.update(results)
@@ -62,7 +63,7 @@ def analyze(self, file, filename):
             filtered_results = {}
             for result in results:
                 snippet = CodeSnippetHighlighter().get_code_snippet(
-                    filename,
+                    file_lines,
                     result.lineno,
                     lines_of_context=LINES_OF_CONTEXT,
                 )

detect_secrets/plugins/high_entropy_strings.py

@@ -203,7 +203,7 @@ def _analyze_yaml_file(self, file, filename):
                 item = to_search.pop()
 
                 try:
-                    if '__line__' in item and not item['__line__'] in ignored_lines:
+                    if '__line__' in item and item['__line__'] not in ignored_lines:
                         potential_secrets.update(
                             self.analyze_string(
                                 item['__value__'],