Skip to content

Commit

Permalink
Remove deprecated code (#433)
Browse files Browse the repository at this point in the history
* Remove deprecated code

* Updating changelog

* Addressing review comments

* Removed support for old signatures which generated with +/- chars in git diff

* Fixing test names and removed comments around backwards_compatibility_prefix
emayuri-godaddy authored Jan 13, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent a3617f9 commit 24976f1
Showing 10 changed files with 85 additions and 307 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -2,6 +2,10 @@ vX.X.X - DD MMM YYYY
--------------------

Features:
* [#433](https://github.com/godaddy/tartufo/pull/433) - Dropped support for deprecated flags rules, b64, hex
and corresponding code around deprecated options. Removed support for old signatures which generated with +/-
chars in git diff.

* [#411](https://github.com/godaddy/tartufo/pull/411) - Drop support for python 3.6.
This version reached end of life several years ago, and end of security support at
the end of 2021. Users with a requirement to run tartufo on this python version
20 changes: 0 additions & 20 deletions README.md
Original file line number Diff line number Diff line change
@@ -45,9 +45,6 @@ Usage: tartufo [OPTIONS] COMMAND [ARGS]...
commit hook.

Options:
--rules FILENAME [DEPRECATED] Use the rule-patterns config
options instead. Path(s) to regex rules json
list file(s).
--default-regexes / --no-default-regexes
Whether to include the default regex list
when configuring search patterns. Only
@@ -108,23 +105,6 @@ Options:
likelihood that a given string will be
identified as suspicious. [default: 75;
0<=x<=100]
-b64, --b64-entropy-score TEXT [DEPRECATED] Use `--entropy-sensitivity`.
Modify the base64 entropy score. If a value
greater than the default (4.5 in a range of
0.0-6.0) is specified, tartufo lists higher
entropy base64 strings (longer or more
randomized strings. A lower value lists
lower entropy base64 strings (shorter or
less randomized strings).
-hex, --hex-entropy-score TEXT [DEPRECATED] Use `--entropy-sensitivity`.
Modify the hexadecimal entropy score. If a
value greater than the default (3.0 in a
range of 0.0-4.0) is specified, tartufo
lists higher entropy hexadecimal strings
(longer or more randomized strings). A lower
value lists lower entropy hexadecimal
strings (shorter or less randomized
strings).
-V, --version Show the version and exit.
-h, --help Show this message and exit.

25 changes: 0 additions & 25 deletions tartufo/cli.py
Original file line number Diff line number Diff line change
@@ -45,13 +45,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma
name="tartufo",
context_settings=dict(help_option_names=["-h", "--help"]),
)
@click.option(
"--rules",
multiple=True,
type=click.File("r"),
help="[DEPRECATED] Use the rule-patterns config options instead. Path(s) to regex "
"rules json list file(s).",
)
@click.option(
"--rule-patterns",
multiple=True,
@@ -250,24 +243,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma
Decreasing the scanner's sensitivity increases the likelihood that a given
string will be identified as suspicious.""",
)
@click.option(
"-b64",
"--b64-entropy-score",
help="""[DEPRECATED] Use `--entropy-sensitivity`. Modify the base64 entropy score. If
a value greater than the default (4.5 in a range of 0.0-6.0) is specified,
tartufo lists higher entropy base64 strings (longer or more randomized strings.
A lower value lists lower entropy base64 strings (shorter or less randomized
strings).""",
)
@click.option(
"-hex",
"--hex-entropy-score",
help="""[DEPRECATED] Use `--entropy-sensitivity`. Modify the hexadecimal entropy score.
If a value greater than the default (3.0 in a range of 0.0-4.0) is specified,
tartufo lists higher entropy hexadecimal strings (longer or more randomized
strings). A lower value lists lower entropy hexadecimal strings (shorter or less
randomized strings).""",
)
# The first positional argument here would be a hard-coded version, hence the `None`
@click.version_option(None, "-V", "--version")
@click.pass_context
13 changes: 1 addition & 12 deletions tartufo/config.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@
import pathlib
import re
import shutil
import warnings
from typing import (
Any,
Dict,
@@ -149,7 +148,6 @@ def read_pyproject_toml(

def configure_regexes(
include_default: bool = True,
rules_files: Optional[Iterable[TextIO]] = None,
rule_patterns: Optional[Iterable[Dict[str, str]]] = None,
rules_repo: Optional[str] = None,
rules_repo_files: Optional[Iterable[str]] = None,
@@ -186,16 +184,7 @@ def configure_regexes(
f"Invalid rule-pattern; both reason and pattern are required fields. Rule: {pattern}"
) from exc

if rules_files:
warnings.warn(
"Storing rules in a separate file is deprecated and will be removed "
"in tartufo 4.x. Please use the 'rule-patterns' config "
" option instead.",
DeprecationWarning,
)
all_files: List[TextIO] = list(rules_files)
else:
all_files = []
all_files = []
try:
cloned_repo = False
repo_path = None
99 changes: 3 additions & 96 deletions tartufo/scanner.py
Original file line number Diff line number Diff line change
@@ -25,7 +25,6 @@
Tuple,
IO,
)
import warnings

from cached_property import cached_property
import click
@@ -175,16 +174,6 @@ def compute_scaled_entropy_limit(self, maximum_bitrate: float) -> float:
def hex_entropy_limit(self) -> float:
"""Returns low entropy limit for suspicious hexadecimal encodings"""

# For backwards compatibility, allow the caller to manipulate this score
# # directly (but complain about it).
if self.global_options.hex_entropy_score:
warnings.warn(
"--hex-entropy-score is deprecated and will be removed in tartufo 4.x. "
"Please use --entropy-sensitivity instead.",
DeprecationWarning,
)
return self.global_options.hex_entropy_score

# Each hexadecimal digit represents a 4-bit number, so we want to scale
# the base score by this amount to account for the efficiency of the
# string representation we're examining.
@@ -194,16 +183,6 @@ def hex_entropy_limit(self) -> float:
def b64_entropy_limit(self) -> float:
"""Returns low entropy limit for suspicious base64 encodings"""

# For backwards compatibility, allow the caller to manipulate this score
# # directly (but complain about it).
if self.global_options.b64_entropy_score:
warnings.warn(
"--b64-entropy-score is deprecated and will be removed in tartufo 4.x. "
"Please use --entropy-sensitivity instead.",
DeprecationWarning,
)
return self.global_options.b64_entropy_score

# Each 4-character base64 group represents 3 8-bit bytes, i.e. an effective
# bit rate of 24/4 = 6 bits per character. We want to scale the base score
# by this amount to account for the efficiency of the string representation
@@ -249,7 +228,6 @@ def included_paths(self) -> List[Pattern]:
if self._included_paths is None:
self.logger.info("Initializing included paths")
patterns: Set[str] = set()
deprecated = False
for pattern in tuple(
self.global_options.include_path_patterns or []
) + tuple(self.config_data.get("include_path_patterns", [])):
@@ -260,21 +238,10 @@ def included_paths(self) -> List[Pattern]:
raise types.ConfigException(
"Required key path-pattern missing in include-path-patterns"
) from exc
elif isinstance(pattern, str):
deprecated = True
patterns.add(pattern)
else:
raise types.ConfigException(
f"{type(pattern).__name__} pattern is illegal in include-path-patterns"
)
if deprecated:
warnings.warn(
"Old format of --include-path-patterns option and config file setup include-path-patterns "
"= ['inclusion pattern'] has been deprecated and will be removed in tartufo 4.x. "
"Make sure all the inclusions are set up using new pattern i.e. include-path-patterns = "
"[{path-pattern='inclusion pattern',reason='reason for inclusion'}] in the config file",
DeprecationWarning,
)
self._included_paths = config.compile_path_rules(patterns)
return self._included_paths

@@ -298,7 +265,6 @@ def excluded_paths(self) -> List[Pattern]:
if self._excluded_paths is None:
self.logger.info("Initializing excluded paths")
patterns: Set[str] = set()
deprecated = False
for pattern in tuple(
self.global_options.exclude_path_patterns or []
) + tuple(self.config_data.get("exclude_path_patterns", [])):
@@ -309,21 +275,10 @@ def excluded_paths(self) -> List[Pattern]:
raise types.ConfigException(
"Required key path-pattern missing in exclude-path-patterns"
) from exc
elif isinstance(pattern, str):
deprecated = True
patterns.add(pattern)
else:
raise types.ConfigException(
f"{type(pattern).__name__} pattern is illegal in exclude-path-patterns"
)
if deprecated:
warnings.warn(
"Old format of --exclude-path-patterns option and config file setup exclude-path-patterns "
"= ['exclusion pattern'] has been deprecated and will be removed in tartufo 4.x. "
"Make sure all the exclusions are set up using new pattern i.e. exclude-path-patterns = "
"[{path-pattern='exclusion pattern',reason='reason for exclusion'}] in the config file",
DeprecationWarning,
)
self._excluded_paths = config.compile_path_rules(patterns)
return self._excluded_paths

@@ -338,7 +293,6 @@ def rules_regexes(self) -> Set[Rule]:
try:
self._rules_regexes = config.configure_regexes(
include_default=self.global_options.default_regexes,
rules_files=self.global_options.rules,
rule_patterns=self.global_options.rule_patterns,
rules_repo=self.global_options.git_rules_repo,
rules_repo_files=self.global_options.git_rules_files,
@@ -386,7 +340,6 @@ def excluded_signatures(self) -> Tuple[str, ...]:
"""
if self._excluded_signatures is None:
signatures: Set[str] = set()
deprecated = False
for signature in tuple(
self.global_options.exclude_signatures or []
) + tuple(self.config_data.get("exclude_signatures", [])):
@@ -397,21 +350,10 @@ def excluded_signatures(self) -> Tuple[str, ...]:
raise types.ConfigException(
"Required key signature missing in exclude-signatures"
) from exc
elif isinstance(signature, str):
deprecated = True
signatures.add(signature)
else:
raise types.ConfigException(
f"{type(signature).__name__} signature is illegal in exclude-signatures"
)
if deprecated:
warnings.warn(
"Configuring exclude-signatures as string has been deprecated and support for this format will "
"be removed in tartufo 4.x. Please update your exclude-signatures configuration to "
"an array of tables. For example: exclude-signatures = [{signature='signature', reason='The "
"reason of excluding the signature'}]",
DeprecationWarning,
)
self._excluded_signatures = tuple(signatures)
return self._excluded_signatures

@@ -599,73 +541,38 @@ def scan_entropy(
# If the chunk is diff output, the first character of each line is
# generated metadata ("+", "-", etc.) that is not part of actual
# repository content, and it should be ignored.
extra_char: Optional[str]
if chunk.is_diff:
extra_char = line[0]
analyze = line[1:]
else:
extra_char = None
analyze = line
analyze = line[1:] if chunk.is_diff else line
for word in analyze.split():
for string in util.find_strings_by_regex(word, BASE64_REGEX):
yield from self.evaluate_entropy_string(
chunk, analyze, string, self.b64_entropy_limit, extra_char
chunk, analyze, string, self.b64_entropy_limit
)
for string in util.find_strings_by_regex(word, HEX_REGEX):
yield from self.evaluate_entropy_string(
chunk, analyze, string, self.hex_entropy_limit, extra_char
chunk, analyze, string, self.hex_entropy_limit
)
extra_char = None

def evaluate_entropy_string(
self,
chunk: types.Chunk,
line: str,
string: str,
min_entropy_score: float,
backwards_compatibility_prefix: Optional[str],
) -> Generator[Issue, None, None]:
"""Check entropy string using entropy characters and score.
:param chunk: The chunk of data to check
:param line: Source line containing string of interest
:param string: String to check
:param min_entropy_score: Minimum entropy score to flag
:param backwards_compatibility_prefix: Possible prefix character
:return: Generator of issues flagged
If the string in "string" would result in an Issue (i.e. it has high
entropy and is not excluded), and backwards_compatibility_prefix is not
None, re-check for exclusions based on "prefix" + "string". This preserves
the utility of signatures generated by earlier tartufo versions which did
not handle "diff" chunks correctly.
"""

if not self.signature_is_excluded(string, chunk.file_path):
entropy_score = self.calculate_entropy(string)
if entropy_score > min_entropy_score:
if self.entropy_string_is_excluded(string, line, chunk.file_path):
self.logger.debug("line containing entropy was excluded: %s", line)
elif (
backwards_compatibility_prefix is not None
and self.signature_is_excluded(
backwards_compatibility_prefix + string, chunk.file_path
)
):
self.logger.debug(
"line containing entropy was excluded (old signature): %s", line
)
# We should tell the user to update their old signature
new_signature = util.generate_signature(string, chunk.file_path)
old_signature = util.generate_signature(
backwards_compatibility_prefix + string, chunk.file_path
)
warnings.warn(
f"Signature {old_signature} was generated by an old version of tartufo and is deprecated. "
"tartufo 4.x will not recognize this signature. "
f"Please update your configuration to use signature {new_signature} instead.",
DeprecationWarning,
)

else:
yield Issue(types.IssueType.Entropy, string, chunk)
20 changes: 4 additions & 16 deletions tartufo/types.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# pylint: disable=too-many-instance-attributes
import enum
from dataclasses import dataclass
from typing import Any, Dict, Optional, TextIO, Tuple, Pattern, Union
from typing import Any, Dict, Optional, TextIO, Tuple, Pattern


class IssueType(enum.Enum):
@@ -47,8 +47,6 @@ class OutputFormat(enum.Enum):
class GlobalOptions:
"""Configuration options for controlling scans and output
:param rules: External files containing lists of regex patterns to match
against
:param rule_patterns: Dictionaries containing regex patterns to match
against
:param default_regexes: Whether to include built-in regex patterns in the
@@ -78,17 +76,12 @@ class GlobalOptions:
:param quiet: Whether to suppress all output
:param log_timestamps: Whether to include timestamps in log output
:param output_format: What format should be output from the scan
:param b64_entropy_score: A number from 0.0 - 6.0 representing the
sensitivity of b64 entropy scans
:param hex_entropy_score: A number from 0.0 - 4.0 representing the
sensitivity of hex entropy scans
:param entropy_sensitivity: A number from 0 - 100 representing the
sensitivity of entropy scans. A value of 0 will detect totally non-random
values, while a value of 100 will detect only wholly random values.
"""

__slots__ = (
"rules",
"rule_patterns",
"default_regexes",
"entropy",
@@ -108,20 +101,17 @@ class GlobalOptions:
"quiet",
"log_timestamps",
"output_format",
"b64_entropy_score",
"hex_entropy_score",
"entropy_sensitivity",
)
rules: Tuple[TextIO, ...]
rule_patterns: Tuple[Dict[str, str], ...]
default_regexes: bool
entropy: bool
regex: bool
scan_filenames: bool
include_path_patterns: Union[Tuple[str, ...], Tuple[Dict[str, str], ...]]
exclude_path_patterns: Union[Tuple[str, ...], Tuple[Dict[str, str], ...]]
include_path_patterns: Tuple[Dict[str, str], ...]
exclude_path_patterns: Tuple[Dict[str, str], ...]
exclude_entropy_patterns: Tuple[Dict[str, str], ...]
exclude_signatures: Union[Tuple[Dict[str, str], ...], Tuple[str, ...]]
exclude_signatures: Tuple[Dict[str, str], ...]
output_dir: Optional[str]
temp_dir: Optional[str]
buffer_size: int
@@ -132,8 +122,6 @@ class GlobalOptions:
quiet: bool
log_timestamps: bool
output_format: Optional[OutputFormat]
b64_entropy_score: float
hex_entropy_score: float
entropy_sensitivity: int


32 changes: 15 additions & 17 deletions tests/test_base_scanner.py
Original file line number Diff line number Diff line change
@@ -61,8 +61,6 @@ def test_scan_aborts_due_to_invalid_regex(self, mock_config: mock.MagicMock):
def test_scan_iterates_through_all_chunks(self, mock_entropy: mock.MagicMock):
# Make sure we do at least one type of scan
self.options.entropy = True
self.options.b64_entropy_score = 4.5
self.options.hex_entropy_score = 3
test_scanner = TestScanner(self.options)
list(test_scanner.scan())
mock_entropy.assert_has_calls(
@@ -197,7 +195,9 @@ def test_included_paths_is_empty_if_not_specified(self):
def test_include_paths_are_calculated_if_specified(
self, mock_compile: mock.MagicMock
):
self.options.include_path_patterns = ("foo",)
self.options.include_path_patterns = (
{"path-pattern": "foo", "reason": "Testing exclude path pattern"},
)
test_scanner = TestScanner(self.options)
test_scanner.included_paths # pylint: disable=pointless-statement
mock_compile.assert_called_once_with({"foo"})
@@ -219,7 +219,9 @@ def test_excluded_paths_is_empty_if_not_specified(self):
def test_exclude_paths_are_calculated_if_specified(
self, mock_compile: mock.MagicMock
):
self.options.exclude_path_patterns = ("foo",)
self.options.exclude_path_patterns = (
{"path-pattern": "foo", "reason": "Testing exclude path pattern"},
)
test_scanner = TestScanner(self.options)
test_scanner.excluded_paths # pylint: disable=pointless-statement
mock_compile.assert_called_once_with({"foo"})
@@ -269,15 +271,13 @@ def test_populated_regex_list_does_not_recompute(self):

def test_regex_rules_are_computed_when_first_accessed(self):
self.options.default_regexes = True
self.options.rules = "foo" # type: ignore
self.options.rule_patterns = "oof" # type: ignore
self.options.git_rules_repo = "bar"
self.options.git_rules_files = "baz" # type: ignore
test_scanner = TestScanner(self.options)
test_scanner.rules_regexes # pylint: disable=pointless-statement
self.mock_configure.assert_called_once_with(
include_default=True,
rules_files="foo",
rule_patterns="oof",
rules_repo="bar",
rules_repo_files="baz",
@@ -289,7 +289,9 @@ class SignatureTests(ScannerTestCase):
def test_matched_signatures_are_excluded(self, mock_signature: mock.MagicMock):
mock_signature.return_value = "foo"
test_scanner = TestScanner(self.options)
self.options.exclude_signatures = ("foo",)
self.options.exclude_signatures = (
{"signature": "foo", "reason": "Testing exclude signature"},
)
self.assertTrue(test_scanner.signature_is_excluded("bar", "blah"))

@mock.patch("tartufo.util.generate_signature")
@@ -298,12 +300,16 @@ def test_unmatched_signatures_are_not_excluded(
):
mock_signature.return_value = "bar"
test_scanner = TestScanner(self.options)
self.options.exclude_signatures = ("foo",)
self.options.exclude_signatures = (
{"signature": "foo", "reason": "Testing exclude signature"},
)
self.assertFalse(test_scanner.signature_is_excluded("blah", "stuff"))

def test_signature_found_as_scan_match_is_excluded(self):
test_scanner = TestScanner(self.options)
self.options.exclude_signatures = ("ford_prefect",)
self.options.exclude_signatures = (
{"signature": "ford_prefect", "reason": "Testing exclude signature"},
)
self.assertTrue(test_scanner.signature_is_excluded("ford_prefect", "/earth"))


@@ -644,14 +650,6 @@ def test_sensitivity_high_end_calculation(self):
self.assertEqual(test_scanner.b64_entropy_limit, 6.0)
self.assertEqual(test_scanner.hex_entropy_limit, 4.0)

def test_sensitivity_deprecated_overrides(self):
self.options.b64_entropy_score = 11.1
self.options.hex_entropy_score = 22.2
test_scanner = TestScanner(self.options)

self.assertEqual(test_scanner.b64_entropy_limit, 11.1)
self.assertEqual(test_scanner.hex_entropy_limit, 22.2)

def test_calculate_entropy_minimum_calculation(self):

# We already know an empty string trivially has zero entropy.
70 changes: 3 additions & 67 deletions tests/test_config.py
Original file line number Diff line number Diff line change
@@ -16,67 +16,12 @@

class ConfigureRegexTests(unittest.TestCase):
def test_configure_regexes_rules_files_without_defaults(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(),)
expected_regexes = {
Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=None,
re_match_type=MatchType.Match,
re_match_scope=None,
),
Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
re_match_type=MatchType.Match,
re_match_scope=None,
),
}

actual_regexes = config.configure_regexes(
include_default=False, rules_files=rules_files
)
actual_regexes = config.configure_regexes(include_default=False)

self.assertEqual(
expected_regexes,
set(),
actual_regexes,
f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})",
)

def test_configure_regexes_rules_files_with_defaults(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(),)
with config.DEFAULT_PATTERN_FILE.open() as handle:
expected_regexes = config.load_rules_from_file(handle)
expected_regexes.add(
Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=None,
re_match_type=MatchType.Match,
re_match_scope=None,
)
)
expected_regexes.add(
Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
re_match_type=MatchType.Match,
re_match_scope=None,
)
)

actual_regexes = config.configure_regexes(
include_default=True, rules_files=rules_files
)

self.assertEqual(
expected_regexes,
actual_regexes,
f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})",
"The regexes dictionary should not have been been changed when no defaults or rules files are specified",
)

def test_configure_regexes_returns_just_default_regexes_by_default(self):
@@ -166,15 +111,6 @@ def test_configure_regexes_includes_rules_from_rules_repo(self):
f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})",
)

def test_loading_rules_from_file_raises_deprecation_warning(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(),)
with self.assertWarnsRegex(
DeprecationWarning,
"Storing rules in a separate file is deprecated and will be removed in tartufo 4.x. ",
):
config.configure_regexes(rules_files=rules_files)

def test_rule_patterns_without_defaults(self):
rule_patterns = [
{
10 changes: 3 additions & 7 deletions tests/test_folder_scanner.py
Original file line number Diff line number Diff line change
@@ -18,10 +18,10 @@ def test_scan_should_detect_entropy_and_not_binary(self):
folder_path = pathlib.Path(__file__).parent / "data" / "scan_folder"
recurse = True
self.global_options.entropy = True
self.global_options.b64_entropy_score = 4.5
self.global_options.hex_entropy_score = 3
self.global_options.exclude_signatures = ()
self.global_options.exclude_path_patterns = [r"donotscan\.txt"]
self.global_options.exclude_path_patterns = [
{"path-pattern": r"donotscan\.txt", "reason": "Reason to be excluded"}
]
self.global_options.buffer_size = 50000

test_scanner = scanner.FolderScanner(self.global_options, folder_path, recurse)
@@ -53,8 +53,6 @@ def test_scan_all_the_files_recursively(self):
recurse = True
self.global_options.entropy = True
self.global_options.exclude_signatures = ()
self.global_options.b64_entropy_score = 4.5
self.global_options.hex_entropy_score = 3
self.global_options.buffer_size = 50000

test_scanner = scanner.FolderScanner(self.global_options, folder_path, recurse)
@@ -73,8 +71,6 @@ def test_scan_only_root_level_files(self):
recurse = False
self.global_options.entropy = True
self.global_options.exclude_signatures = ()
self.global_options.b64_entropy_score = 4.5
self.global_options.hex_entropy_score = 3
self.global_options.buffer_size = 50000

test_scanner = scanner.FolderScanner(self.global_options, folder_path, recurse)
99 changes: 52 additions & 47 deletions tests/test_git_repo_scanner.py
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
import pygit2

from tartufo import scanner, types
from tartufo.types import GlobalOptions, GitOptions, TartufoException
from tartufo.types import GlobalOptions, GitOptions, TartufoException, ConfigException
from tests.helpers import generate_options


@@ -69,9 +69,16 @@ def test_load_repo_does_not_filter_submodules_when_requested(
def test_extra_inclusions_get_added(self, mock_load: mock.MagicMock):
mock_load.return_value = (
self.data_dir / "pyproject.toml",
{"include_path_patterns": ("tartufo/", "scripts/")},
{
"include_path_patterns": (
{"path-pattern": "tartufo/", "reason": "Inclusion reason"},
{"path-pattern": "scripts/", "reason": "Inclusion reason"},
)
},
)
self.global_options.include_path_patterns = (
{"path-pattern": "foo/", "reason": "Inclusion reason"},
)
self.global_options.include_path_patterns = ("foo/",)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, str(self.data_dir)
)
@@ -86,9 +93,17 @@ def test_extra_inclusions_get_added(self, mock_load: mock.MagicMock):
def test_extra_exclusions_get_added(self, mock_load: mock.MagicMock):
mock_load.return_value = (
self.data_dir / "pyproject.toml",
{"exclude_path_patterns": ("tests/", r"\.venv/", r".*\.egg-info/")},
{
"exclude_path_patterns": (
{"path-pattern": "tests/", "reason": "Exclusion reason"},
{"path-pattern": r"\.venv/", "reason": "Exclusion reason"},
{"path-pattern": r".*\.egg-info/", "reason": "Exclusion reason"},
)
},
)
self.global_options.exclude_path_patterns = (
{"path-pattern": "bar/", "reason": "Exclusion reason"},
)
self.global_options.exclude_path_patterns = ("bar/",)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, str(self.data_dir)
)
@@ -108,9 +123,15 @@ def test_extra_exclusions_get_added(self, mock_load: mock.MagicMock):
def test_extra_signatures_get_added(self, mock_load: mock.MagicMock):
mock_load.return_value = (
self.data_dir / "pyproject.toml",
{"exclude_signatures": ["foo"]},
{
"exclude_signatures": [
{"signature": "foo", "reason": "Reason to exclude signature"}
]
},
)
self.global_options.exclude_signatures = (
{"signature": "bar", "reason": "Reason to exclude signature"},
)
self.global_options.exclude_signatures = ("bar",)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, str(self.data_dir)
)
@@ -526,92 +547,76 @@ def test_scan_filename_enabled(self, mock_header_length):


class ExcludedSignaturesTests(ScannerTestCase):
def test_old_style_signatures_are_processed(self):
self.global_options.exclude_signatures = ["bar/"]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.excluded_signatures, ("bar/",))

def test_new_style_signatures_are_processed(self):
self.global_options.exclude_signatures = [
{"signature": "bar/", "reason": "path pattern"}
]
self.global_options.exclude_signatures = (
{"signature": "bar/", "reason": "path pattern"},
)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.excluded_signatures, ("bar/",))

def test_error_is_not_raised_when_two_styles_signatures_are_configured(self):
def test_error_is_raised_when_string_signature_is_used(self):
self.global_options.exclude_signatures = [
"foo/",
{"signature": "bar/", "reason": "path pattern"},
]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertCountEqual(test_scanner.excluded_signatures, ("foo/", "bar/"))
with self.assertRaisesRegex(
ConfigException, "str signature is illegal in exclude-signatures"
):
self.assertIsNone(test_scanner.excluded_signatures)


class IncludedPathsTests(ScannerTestCase):
def test_old_style_included_paths_are_processed(self):
self.global_options.include_path_patterns = ["bar/"]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.included_paths, [re.compile("bar/")])

def test_new_style_included_paths_are_processed(self):
self.global_options.include_path_patterns = [
{"path-pattern": "bar/", "reason": "path pattern"}
]
self.global_options.include_path_patterns = (
{"path-pattern": "bar/", "reason": "path pattern"},
)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.included_paths, [re.compile("bar/")])

def test_error_is_not_raised_when_two_styles_included_paths_are_configured(self):
def test_error_is_raised_when_string_include_path_is_used(self):
self.global_options.include_path_patterns = [
"foo/",
{"path-pattern": "bar/", "reason": "path pattern"},
]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertCountEqual(
test_scanner.included_paths, [re.compile("foo/"), re.compile("bar/")]
)
with self.assertRaisesRegex(
ConfigException, "str pattern is illegal in include-path-patterns"
):
self.assertIsNone(test_scanner.included_paths)


class ExcludedPathsTests(ScannerTestCase):
def test_old_style_excluded_paths_are_processed(self):
self.global_options.exclude_path_patterns = ["bar/"]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.excluded_paths, [re.compile("bar/")])

def test_new_style_excluded_paths_are_processed(self):
self.global_options.exclude_path_patterns = [
{"path-pattern": "bar/", "reason": "path pattern"}
]
self.global_options.exclude_path_patterns = (
{"path-pattern": "bar/", "reason": "path pattern"},
)
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertEqual(test_scanner.excluded_paths, [re.compile("bar/")])

@mock.patch("tartufo.scanner.GitScanner.filter_submodules", mock.MagicMock())
def test_error_is_not_raised_when_two_styles_excluded_paths_are_configured(self):
def test_error_is_raised_when_string_exclude_path_is_used(self):
self.global_options.exclude_path_patterns = [
"foo/",
{"path-pattern": "bar/", "reason": "path pattern"},
]
test_scanner = scanner.GitRepoScanner(
self.global_options, self.git_options, "."
)
self.assertCountEqual(
test_scanner.excluded_paths, [re.compile("foo/"), re.compile("bar/")]
)
with self.assertRaisesRegex(
ConfigException, "str pattern is illegal in exclude-path-patterns"
):
self.assertIsNone(test_scanner.excluded_paths)


if __name__ == "__main__":

0 comments on commit 24976f1

Please sign in to comment.