fix(deps): pin bbot to v3.0.2 tag instead of dev branch

README.md

@@ -232,11 +232,11 @@ You can output a BBOT scan directly to BBOT server with the following preset:
 # bbot-server.yml
 
 output_modules:
-  - http
+  - webhook
 
 config:
   modules:
-    http:
+    webhook:
       # URL of BBOT Server
       url: http://localhost:8807/v1/events/
       # API Key header

bbot_server/modules/agents/agent.py

@@ -217,10 +217,10 @@ def make_agent_preset(self):
 
         # agent-specific overrides for output url etc.
         agent_preset = Preset(
-            output_modules=["http"],
+            output_modules=["webhook"],
             config={
                 "modules": {
-                    "http": {"url": self.scan_output_url, "headers": {bbcfg.auth_header: str(bbcfg.get_api_key())}}
+                    "webhook": {"url": self.scan_output_url, "headers": {bbcfg.auth_header: str(bbcfg.get_api_key())}}
                 }
             },
         )

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "pymongo>=4.15.3",
     "cloudcheck>=9.2.0",
     "textual>=7.5.0",
+    "omegaconf>=2.3.0,<3",
     "bbot",
 ]
 
@@ -51,7 +52,7 @@ default-groups = ["dev"]
 module-root = ""
 
 [tool.uv.sources]
-bbot = { git = "https://github.com/blacklanternsecurity/bbot", rev = "dev" }
+bbot = { git = "https://github.com/blacklanternsecurity/bbot", tag = "v3.0.2" }
 
 [tool.ruff]
 line-length = 119

tests/gen_scan_data.py

@@ -1,4 +1,5 @@
 import shutil
+import orjson
 import pytest_asyncio
 from pathlib import Path
 from datetime import datetime, timedelta, timezone
@@ -62,15 +63,18 @@ async def run(cls):
         speculate = scan.modules.get("speculate")
         if speculate is not None:
             speculate.open_port_consumers = True
-            speculate.emit_open_ports = not speculate.portscanner_enabled
-        events = []
-        async for e in scan.async_start():
-            event = Event(**e.json())
-            events.append(event)
-        events.sort(key=lambda x: x.timestamp)
+            speculate._always_emit_open_ports = speculate.portscanner is None
+        async for _ in scan.async_start():
+            pass
 
         out_file = scan.home / "output.json"
-        return events, out_file.read_text()
+        out_file_text = out_file.read_text()
+        # events are read back from the output file, not the live stream: bbot strips
+        # dns_children etc. from in-memory events once their module consumers finish
+        events = [Event(**orjson.loads(line)) for line in out_file_text.splitlines() if line.strip()]
+        events.sort(key=lambda x: x.timestamp)
+
+        return events, out_file_text
 
 
 class DummyScan1(DummyScan):

tests/test_applets/test_applet_events.py

@@ -57,13 +57,12 @@ async def setup(self):
 
     async def after_scan_1(self):
         events = [e async for e in self.bbot_server.list_events()]
-        # TODO: why does this change sometimes?
-        assert 30 <= len(events) <= 40
+        assert len(events) == len(self.scan1_events)
         assert len(self.event_messages) == len(events)
 
     async def after_scan_2(self):
         events = [e async for e in self.bbot_server.list_events()]
-        assert 60 <= len(events) <= 80
+        assert len(events) == len(self.scan1_events) + len(self.scan2_events)
         assert len(self.event_messages) == len(events)
 
         # filter events by type

tests/test_cli/test_cli_eventctl.py

@@ -8,6 +8,14 @@
 from tests.conftest import BBCTL_COMMAND, INGEST_PROCESSING_DELAY
 
 
+def assert_ingest_progress(stderr, num_events):
+    # the ingester logs progress every 10 events, counting from zero
+    last_milestone = ((num_events - 1) // 10) * 10
+    for milestone in range(10, last_milestone + 1, 10):
+        assert f"Ingested {milestone:,} events" in stderr
+    assert f"Ingested {last_milestone + 10:,} events" not in stderr
+
+
 def test_cli_events(bbot_server_http, bbot_worker, bbot_out_file, bbot_events):
     scan1_out_file, scan2_out_file = bbot_out_file
     scan1_events, scan2_events = bbot_events
@@ -26,10 +34,7 @@ def test_cli_events(bbot_server_http, bbot_worker, bbot_out_file, bbot_events):
     process = subprocess.run(BBCTL_COMMAND + ["event", "ingest", "-f", str(json_file)], capture_output=True, text=True)
     assert process.returncode == 0
     assert process.stdout == ""
-    assert "Ingested 10 events" in process.stderr
-    assert "Ingested 20 events" in process.stderr
-    assert "Ingested 30 events" in process.stderr
-    assert "Ingested 40 events" not in process.stderr
+    assert_ingest_progress(process.stderr, len(scan1_events))
 
     sleep(INGEST_PROCESSING_DELAY)
 
@@ -42,10 +47,7 @@ def test_cli_events(bbot_server_http, bbot_worker, bbot_out_file, bbot_events):
     process = subprocess.run(BBCTL_COMMAND + ["event", "ingest"], input=scan2_out_file, capture_output=True, text=True)
     assert process.returncode == 0
     assert process.stdout == ""
-    assert "Ingested 10 events" in process.stderr
-    assert "Ingested 20 events" in process.stderr
-    assert "Ingested 30 events" in process.stderr
-    assert "Ingested 40 events" not in process.stderr
+    assert_ingest_progress(process.stderr, len(scan2_events))
 
     sleep(INGEST_PROCESSING_DELAY)