Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use python instead of sed/wc during audit #120

Merged
merged 2 commits into from
Feb 1, 2019
Merged

Use python instead of sed/wc during audit #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c872ca
Use python instead of sed/wc during audit
Oct 19, 2018
7b312cd
fix: wrap highlight with try catch :tada:
Jan 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 14 additions & 31 deletions detect_secrets/core/audit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from __future__ import print_function

import codecs
import itertools
import json
import os
import subprocess
Expand Down Expand Up @@ -451,42 +453,23 @@ def _get_secret_with_context(

:raises: SecretNotFoundOnSpecifiedLineError
"""
secret_lineno = secret['line_number']
start_line = 1 if secret_lineno <= lines_of_context \
else secret_lineno - lines_of_context
end_line = secret_lineno + lines_of_context

output = subprocess.check_output([
'sed',
'-n', '{},{}p'.format(start_line, end_line),
filename,
]).decode('utf-8').splitlines()

trailing_lines_of_context = lines_of_context
if len(output) < end_line - start_line + 1:
# This handles the case of a short file.
num_lines_in_file = int(subprocess.check_output([
# https://stackoverflow.com/a/38870057
# - 'wc -l' cannot be used here because if the last char
# of the file isn't \n, then the last line isn't counted
'grep',
'-c',
'',
filename,
]).decode('utf-8').split()[0])
secret_line_idx = secret['line_number'] - 1
end_line = secret_line_idx + lines_of_context + 1

trailing_lines_of_context = lines_of_context - \
(end_line - num_lines_in_file)
if secret_line_idx <= lines_of_context:
start_line = 0
index_of_secret_in_output = secret_line_idx
else:
start_line = secret_line_idx - lines_of_context
index_of_secret_in_output = lines_of_context

# -1, because that's where the secret actually is (without it,
# it would just be the start of the context block).
# NOTE: index_of_secret_in_output should *always* be negative.
index_of_secret_in_output = -trailing_lines_of_context - 1
with codecs.open(filename, encoding='utf-8') as file:
output = list(itertools.islice(file.read().splitlines(), start_line, end_line))

try:
output[index_of_secret_in_output] = _highlight_secret(
output[index_of_secret_in_output],
secret_lineno,
secret['line_number'],
secret,
filename,
plugin_settings,
Expand All @@ -507,7 +490,7 @@ def _get_secret_with_context(
map(
lambda x: '{}:{}'.format(
colorize(
str(int(x[0]) + start_line),
str(int(x[0]) + start_line + 1),
AnsiColor.LIGHT_GREEN,
),
x[1],
Expand Down
47 changes: 20 additions & 27 deletions tests/core/audit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from detect_secrets.core import audit
from testing.factories import potential_secret_factory
from testing.mocks import mock_open as mock_open_base
from testing.mocks import mock_printer as mock_printer_base
from testing.util import uncolor

Expand Down Expand Up @@ -500,28 +501,24 @@ def run_logic(self, secret=None, secret_lineno=15, settings=None):
plugin_settings=settings,
)

@contextmanager
def _mock_sed_call(
def mock_open(
self,
start_line=10,
secret_line=15,
end_line=20,
line_containing_secret='BEGIN PRIVATE KEY',
):
with mock.patch(
'detect_secrets.core.audit.subprocess',
) as m:
m.check_output.return_value = '{}{}{}'.format(
self._make_string_into_individual_lines(
string.ascii_letters[:(secret_line - start_line)],
),
line_containing_secret + '\n',
self._make_string_into_individual_lines(
string.ascii_letters[:(end_line - secret_line)][::-1],
),
).encode()

yield m.check_output
data = '{}{}{}{}'.format(
'\n' * (start_line - 1),
self._make_string_into_individual_lines(
string.ascii_letters[:(secret_line - start_line)],
),
line_containing_secret + '\n',
self._make_string_into_individual_lines(
string.ascii_letters[:(end_line - secret_line)][::-1],
),
)
return mock_open_base(data, 'detect_secrets.core.audit.codecs.open')

@staticmethod
def _make_string_into_individual_lines(string):
Expand All @@ -533,16 +530,14 @@ def _make_string_into_individual_lines(string):
)

def test_basic(self, mock_printer):
with self._mock_sed_call(
with self.mock_open(
start_line=10,
secret_line=15,
end_line=20,
line_containing_secret='-----BEGIN PRIVATE KEY-----',
) as sed_call:
):
self.run_logic()

assert sed_call.call_args[0][0] == 'sed -n 10,20p filenameA'.split()

assert uncolor(mock_printer.message) == textwrap.dedent("""
Secret: 1 of 2
Filename: filenameA
Expand All @@ -564,18 +559,16 @@ def test_basic(self, mock_printer):
""")[1:-1]

def test_secret_at_top_of_file(self, mock_printer):
with self._mock_sed_call(
with self.mock_open(
start_line=1,
secret_line=1,
end_line=6,
line_containing_secret='-----BEGIN PRIVATE KEY-----',
) as sed_call:
):
self.run_logic(
secret_lineno=1,
)

assert sed_call.call_args[0][0] == 'sed -n 1,6p filenameA'.split()

assert uncolor(mock_printer.message) == textwrap.dedent("""
Secret: 1 of 2
Filename: filenameA
Expand All @@ -592,7 +585,7 @@ def test_secret_at_top_of_file(self, mock_printer):
""")[1:-1]

def test_secret_not_found(self, mock_printer):
with self._mock_sed_call(), pytest.raises(
with self.mock_open(), pytest.raises(
audit.SecretNotFoundOnSpecifiedLineError,
):
self.run_logic(
Expand All @@ -616,7 +609,7 @@ def test_secret_not_found(self, mock_printer):
""")[1:-1]

def test_hex_high_entropy_secret_in_yaml_file(self, mock_printer):
with self._mock_sed_call(
with self.mock_open(
line_containing_secret='api key: 123456789a',
):
self.run_logic(
Expand Down Expand Up @@ -655,7 +648,7 @@ def test_hex_high_entropy_secret_in_yaml_file(self, mock_printer):
""")[1:-1]

def test_keyword_secret_in_yaml_file(self, mock_printer):
with self._mock_sed_call(
with self.mock_open(
line_containing_secret='api_key: yerba',
):
self.run_logic(
Expand Down