test: added experimental feature to main malware check, tests updated to use MACARON_PATH

Author: art1f1c3R

tests/malware_analyzer/pypi/test_pypi_sourcecode_analyzer.py

@@ -8,11 +8,13 @@
 
 import pytest
 
-import macaron
+from macaron import MACARON_PATH
 from macaron.errors import ConfigurationError, HeuristicAnalyzerValueError
 from macaron.malware_analyzer.pypi_heuristics.heuristics import HeuristicResult
 from macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer import PyPISourcecodeAnalyzer
 
+RESOURCES_PATH = os.path.join(MACARON_PATH, "resources")
+
 
 def test_no_resources() -> None:
     """Test for when the semgrep rules can't be found, so error."""
@@ -25,22 +27,22 @@ def test_no_defaults_section(mock_defaults: MagicMock) -> None:
     """Test for when the heuristics.pypi in defaults isn't defined at all, so error."""
     mock_defaults.has_section.side_effect = lambda _: False
     with pytest.raises(ConfigurationError):
-        _ = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+        _ = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
 
 
 @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults")
 def test_no_custom_path(mock_defaults: MagicMock) -> None:
     """Test for when a default path isn't provided, so the custom rule path should be None."""
     mock_defaults.has_section.side_effect = lambda section: section == "heuristic.pypi"
     mock_defaults.__getitem__.side_effect = lambda _: (MagicMock(get=MagicMock(return_value=None)))
-    analyzer = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+    analyzer = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
     assert analyzer.custom_rule_path is None
 
     mock_defaults.has_section.side_effect = lambda section: section == "heuristic.pypi"
     mock_defaults.__getitem__.side_effect = lambda section: (
         MagicMock(get=MagicMock(return_value="" if section == "heuristic.pypi" else None))
     )
-    analyzer = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+    analyzer = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
     assert analyzer.custom_rule_path is None
 
 
@@ -52,7 +54,7 @@ def test_nonexistent_rule_path(mock_defaults: MagicMock) -> None:
         MagicMock(get=MagicMock(return_value="some_random_path" if section == "heuristic.pypi" else None))
     )
     with pytest.raises(ConfigurationError):
-        _ = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+        _ = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
 
 
 @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults")
@@ -64,12 +66,12 @@ def test_invalid_custom_rules(mock_defaults: MagicMock) -> None:
         MagicMock(get=MagicMock(return_value=os.path.abspath(__file__) if section == "heuristic.pypi" else None))
     )
     with pytest.raises(ConfigurationError):
-        _ = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+        _ = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
 
 
 def test_no_sourcecode(pypi_package_json: MagicMock) -> None:
     """Test for when there is no source code available, so error."""
-    analyzer = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+    analyzer = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
 
     pypi_package_json.package_sourcecode_path = ""
 
@@ -103,7 +105,7 @@ def test_rules(
         MagicMock(get=MagicMock(return_value="" if section == "heuristic.pypi" else None))
     )
 
-    analyzer = PyPISourcecodeAnalyzer(resources_path=os.path.join(os.path.dirname(macaron.__file__), "resources"))
+    analyzer = PyPISourcecodeAnalyzer(resources_path=RESOURCES_PATH)
 
     pypi_package_json.package_sourcecode_path = sample_path
     analyzer.default_rule_path = os.path.join(analyzer.default_rule_path, rule_file)

tests/slsa_analyzer/checks/test_detect_malicious_metadata_check.py

@@ -7,10 +7,12 @@
 import os
 import urllib.parse
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import pytest
 from pytest_httpserver import HTTPServer
 
+from macaron import MACARON_PATH
 from macaron.config.defaults import load_defaults
 from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool
 from macaron.slsa_analyzer.checks.check_result import CheckResultType
@@ -22,20 +24,31 @@
 RESOURCE_PATH = Path(__file__).parent.joinpath("resources")
 
 
+@patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.global_config")
 @pytest.mark.parametrize(
-    ("purl", "expected"),
+    ("purl", "expected", "experimental"),
     [
         # TODO: This check is expected to FAIL for pkg:pypi/zlibxjson. However, after introducing the wheel presence heuristic,
         # a false negative has been introduced. Note that if the unit test were allowed to access the OSV
         # knowledge base, it would report the package as malware. However, we intentionally block unit tests
         # from reaching the network.
-        ("pkg:pypi/zlibxjson", CheckResultType.PASSED),
-        ("pkg:pypi/test", CheckResultType.UNKNOWN),
-        ("pkg:maven:test/test", CheckResultType.UNKNOWN),
+        pytest.param("pkg:pypi/zlibxjson", CheckResultType.PASSED, False, id="test_malicious_pypi_package"),
+        pytest.param("pkg:pypi/test", CheckResultType.UNKNOWN, False, id="test_unknown_pypi_package"),
+        pytest.param("pkg:maven:test/test", CheckResultType.UNKNOWN, False, id="test_non_pypi_package"),
+        # TODO: including source code analysis that detects flow from a remote point to a file write may assist in resolving
+        # the issue of this false negative.
+        pytest.param("pkg:pypi/zlibxjson", CheckResultType.PASSED, True, id="test_experimental_malicious_pypi_package"),
     ],
 )
 def test_detect_malicious_metadata(
-    httpserver: HTTPServer, tmp_path: Path, pip_tool: BaseBuildTool, macaron_path: Path, purl: str, expected: str
+    mock_global_config: MagicMock,
+    httpserver: HTTPServer,
+    tmp_path: Path,
+    pip_tool: BaseBuildTool,
+    macaron_path: Path,
+    purl: str,
+    expected: str,
+    experimental: bool,
 ) -> None:
     """Test that the check handles repositories correctly."""
     check = DetectMaliciousMetadataCheck()
@@ -44,6 +57,10 @@ def test_detect_malicious_metadata(
     ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl=purl)
     pypi_registry = PyPIRegistry()
     ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(pip_tool, pypi_registry)]
+    if experimental:
+        ctx.dynamic_data["analyze_source"] = True
+
+    mock_global_config.resources_path = os.path.join(MACARON_PATH, "resources")
 
     # Set up responses of PyPI endpoints using the httpserver plugin.
     with open(os.path.join(RESOURCE_PATH, "pypi_files", "zlibxjson.html"), encoding="utf8") as page: