diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..1aa3862 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,48 @@ +name: Bug report +description: Report a reproducible Ghost defect. +title: "bug: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for helping harden Ghost. Keep reports focused on authorized use cases and do not include private target data. + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What should Ghost have done? + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual behavior + description: What happened instead? + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Reproduction steps + description: Include commands, flags, and a safe public or synthetic target. + placeholder: | + 1. Run `ghost doctor` + 2. Run `ghost investigate demo_user --type username --modules username --no-ai --authorized` + 3. Observe ... + validations: + required: true + - type: input + id: version + attributes: + label: Ghost version or commit + placeholder: "0.1.0 / commit SHA" + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: OS, Python version, install method, and relevant optional tools. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..69fa0c9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,44 @@ +name: Feature request +description: Propose a focused, authorized-use improvement. +title: "feat: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Ghost prioritizes defensible investigation workflows: authorization, provenance, case files, and repeatable reports. + - type: textarea + id: problem + attributes: + label: Problem + description: What authorized workflow is blocked or painful today? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: Describe the smallest useful version. + validations: + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - CLI + - API + - Storage/case files + - Report provenance + - OSINT module + - Documentation/demo + - Other + validations: + required: true + - type: textarea + id: safety + attributes: + label: Safety and privacy notes + description: Explain how this avoids stalking, harassment, unauthorized surveillance, or private data leakage. + validations: + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..db3aea8 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,20 @@ +## What changed + +- + +## Why + +- + +## Proof + +- [ ] `python -m ruff check ghost tests` +- [ ] `python -m ruff format --check ghost tests` +- [ ] `python -m pytest -q` +- [ ] Screenshots or terminal output added/updated when user-facing behavior changed + +## Safety + +- [ ] This supports authorized security research, journalism, law enforcement, or self-audits +- [ ] No private target data, secrets, generated databases, reports, or investigation artifacts are committed +- [ ] New collection behavior is documented and scoped diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..72a909f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install package + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + + - name: Lint + run: | + python -m ruff check ghost tests + python -m ruff format --check ghost tests + + - name: Test + run: python -m pytest -q diff --git a/README.md b/README.md index ea103e8..6b2ab62 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,11 @@ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-3776AB.svg?style=for-the-badge&logo=python&logoColor=white)](https://python.org) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg?style=for-the-badge)](LICENSE) [![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey.svg?style=for-the-badge)](#installation) +[![CI](https://img.shields.io/github/actions/workflow/status/juliosuas/ghost/ci.yml?branch=main&style=for-the-badge&logo=githubactions&label=CI)](https://github.com/juliosuas/ghost/actions/workflows/ci.yml) [![GitHub Stars](https://img.shields.io/github/stars/juliosuas/ghost?style=for-the-badge&logo=github)](https://github.com/juliosuas/ghost/stargazers) [![GitHub Issues](https://img.shields.io/github/issues/juliosuas/ghost?style=for-the-badge)](https://github.com/juliosuas/ghost/issues) -**Multi-vector intelligence gathering with 500+ platform checks, AI-driven analysis, and professional reports.** +**Multi-vector intelligence gathering with durable case files, AI-assisted analysis, and professional reports.** [Quick Start](#-quick-start) · [Features](#-features) · [Installation](#-installation) · [Demo](docs/self-audit-demo.md) · [Roadmap](#-roadmap) · [Contributing](#-contributing) @@ -25,7 +26,8 @@ | Capability | Ghost | Maltego | SpiderFoot | Recon-ng | |---|:---:|:---:|:---:|:---:| | **AI-Powered Correlation** | ✅ | ❌ | ❌ | ❌ | -| **500+ Platform Checks** | ✅ | ✅¹ | ✅ | ~100 | +| **75+ Built-in Username Checks** | ✅ | ✅¹ | ✅ | ~100 | +| **SQLite Case Files & Provenance** | ✅ | ❌ | ✅ | ❌ | | **Professional HTML/PDF Reports** | ✅ | ✅ | ✅ | ❌ | | **Web Dashboard with Graphs** | ✅ | ✅ | ✅ | ❌ | | **Image/Face Analysis** | ✅ | ❌ | ❌ | ❌ | @@ -39,14 +41,12 @@ ## 📸 Screenshots -> **Coming soon** — Screenshots of the CLI, web dashboard, entity graph, and report output. - - ## ✨ Features @@ -54,7 +54,7 @@ | Module | Description | Status | |---|---|:---:| -| 🔤 **Username Enumeration** | Check 500+ platforms (social, forums, dating, adult) | ✅ | +| 🔤 **Username Enumeration** | Check 75+ built-in platforms; optional Sherlock expands coverage | ✅ | | 📧 **Email Intelligence** | Breach checks, account discovery, WHOIS, validation | ✅ | | 📱 **Phone OSINT** | Carrier lookup, location, social media association | ✅ | | 🌐 **Domain Recon** | WHOIS, DNS, subdomains, tech stack, SSL, Wayback | ✅ | @@ -111,7 +111,7 @@ python -m ghost.ui.cli # Investigate an email address python -m ghost.ui.cli --target "john.doe@example.com" --type email -# Username hunt across 500+ platforms +# Username hunt across built-in platforms python -m ghost.ui.cli --target "johndoe" --type username # Phone number lookup @@ -131,6 +131,13 @@ ghost list # Show one saved case by full ID or unique prefix ghost show 5f3a9c2e + +# Export/import portable case files for handoff or backup +ghost export 5f3a9c2e --output cases/johndoe.json +ghost import cases/johndoe.json --replace + +# Delete a local case file when retention is no longer needed +ghost delete 5f3a9c2e --yes ``` For a safe public walkthrough, use the [authorized self-audit demo](docs/self-audit-demo.md). @@ -172,8 +179,8 @@ write data to the wrong place. from ghost.core.investigator import GhostInvestigator investigator = GhostInvestigator() -report = investigator.investigate("johndoe", input_type="username") -report.export("report.html", format="html") +investigation = investigator.investigate("johndoe", input_type="username") +report_path = investigator.generate_report(investigation, format="html", output_path="report.html") ``` ### REST API @@ -223,6 +230,8 @@ Contributions are welcome! Here's how to get started: 4. **Push** to the branch: `git push origin feature/amazing-module` 5. **Open** a Pull Request +CI runs Ruff and pytest on Python 3.10, 3.11, and 3.12. PRs should include proof plus screenshots or terminal output when user-facing behavior changes. + ### Areas We Need Help - 🌍 **New OSINT modules** — More platforms, more data sources diff --git a/V2_PLAN.md b/V2_PLAN.md index 68afe79..0b14a97 100644 --- a/V2_PLAN.md +++ b/V2_PLAN.md @@ -119,3 +119,15 @@ Added the first case-file retrieval polish so Ghost's SQLite work is visible fro - `list_investigations()` now includes scope, authorized-use flag, risk, and summary so API/CLI users can audit case context without fetching every case one by one. Verified with `.venv/bin/python -m pytest -q` and `.venv/bin/python -m ruff check .`. + +## Batch 9 — public repo polish and portable cases + +Added the repo-level pieces that help Ghost look maintained and contribution-ready: + +- CI workflow for Python 3.10, 3.11, and 3.12 with Ruff and pytest. +- Issue templates for bugs and feature requests that force reproduction steps and safety/privacy context. +- Pull request template requiring proof, screenshots/terminal output for user-facing changes, and no committed private artifacts. +- README screenshots for `ghost doctor`, `ghost list`, and `ghost show`. +- CLI case-file portability commands: `ghost export`, `ghost import`, and `ghost delete`. + +This turns the SQLite response into a complete user story: run an authorized investigation, store it, retrieve it, export it for handoff/backup, import it elsewhere, and delete it when retention is no longer needed. diff --git a/docs/issue-1-response.md b/docs/issue-1-response.md index 91a8341..91c03de 100644 --- a/docs/issue-1-response.md +++ b/docs/issue-1-response.md @@ -13,6 +13,7 @@ What changed: - Investigations now store scope and authorized-use metadata so a case file can show why/under what authority it was created. - Reports include provenance: generated timestamp, target metadata, modules run, source URLs, source URL count, module errors, and global errors. - The CLI exposes saved case files with `ghost list` and `ghost show ` so persisted investigations are visible outside the web/API layer. +- Case files can now be exported, imported, and deleted from the CLI with `ghost export`, `ghost import`, and `ghost delete`. Current position: @@ -23,7 +24,7 @@ PostgreSQL is still the right roadmap direction for multi-user/team deployments, Near-term storage roadmap: 1. Keep hardening SQLite case-file workflows. -2. Add export/import for portable investigations. +2. Add signed report bundles and retention policies for teams. 3. Add a storage adapter interface. 4. Add Postgres support once the API/dashboard needs multi-user concurrency. diff --git a/docs/screenshots/case-list.svg b/docs/screenshots/case-list.svg new file mode 100644 index 0000000..f1dac66 --- /dev/null +++ b/docs/screenshots/case-list.svg @@ -0,0 +1,17 @@ + + Ghost saved case list screenshot + Terminal screenshot showing saved Ghost investigations in SQLite. + + + + + + ghost list + Ghost Investigations + ID Target Type Status Risk Authorized Scope + + 8f3a91c2 juliosuas username completed 10% yes authorized self-audit demo + c19e7b4a acme.com domain completed 20% yes client-owned asset audit + 4aa08d10 demo_user username completed 10% yes docs demo + SQLite case files make investigations retrievable after the report is generated. + diff --git a/docs/screenshots/case-show.svg b/docs/screenshots/case-show.svg new file mode 100644 index 0000000..5c43111 --- /dev/null +++ b/docs/screenshots/case-show.svg @@ -0,0 +1,20 @@ + + Ghost case detail screenshot + Terminal screenshot showing a Ghost case summary with authorization, scope, modules, and graph size. + + + + + + ghost show 8f3a91c2 + CASE FILE + ID: 8f3a91c2-53f8-4e82-96df-7cc4fd940f31 + Target: juliosuas + Type: username + Status: completed + Risk: 10% + Authorized: yes + Scope: authorized self-audit demo + Findings modules: username + Graph: 4 nodes, 3 links + diff --git a/docs/screenshots/doctor.svg b/docs/screenshots/doctor.svg new file mode 100644 index 0000000..c342abb --- /dev/null +++ b/docs/screenshots/doctor.svg @@ -0,0 +1,19 @@ + + Ghost doctor command screenshot + Terminal screenshot showing Ghost doctor readiness checks. + + + + + + ghost doctor + Ghost Doctor + Check Status Detail + + database OK sqlite:///ghost/data/ghost.db + enabled modules OK username, email, phone, social, domain + openai key WARN not configured; heuristic mode available + optional sherlock WARN not installed; built-in checks still work + storage permissions OK data directory writable + Use doctor before demos, deployments, and bug reports. + diff --git a/docs/self-audit-demo.md b/docs/self-audit-demo.md index c18564e..c89dcfe 100644 --- a/docs/self-audit-demo.md +++ b/docs/self-audit-demo.md @@ -37,7 +37,10 @@ ghost list # 4. Open the saved case by ID prefix ghost show -# 5. Inspect report provenance +# 5. Export a portable case file for handoff or backup +ghost export --output demo-case.json + +# 6. Inspect report provenance cat demo-report.json | jq '.provenance' ``` @@ -49,7 +52,8 @@ Capture these for README/demo material: 2. Investigation progress running with `--no-ai --authorized`. 3. `ghost list` showing saved cases with scope and authorization. 4. `ghost show ` case summary with modules and graph size. -5. JSON provenance block from `demo-report.json`. +5. `ghost export ` writing a portable JSON case file. +6. JSON provenance block from `demo-report.json`. ## Talk track diff --git a/ghost/ai/analyzer.py b/ghost/ai/analyzer.py index b7a8966..1fb0e10 100644 --- a/ghost/ai/analyzer.py +++ b/ghost/ai/analyzer.py @@ -25,12 +25,11 @@ async def analyze( try: import openai + client = openai.AsyncOpenAI(api_key=self.config.openai_api_key) # Prepare findings summary (truncated for token limits) - findings_text = json.dumps( - self._sanitize_findings(findings), indent=2, default=str - )[:10000] + findings_text = json.dumps(self._sanitize_findings(findings), indent=2, default=str)[:10000] correlations_text = json.dumps(correlations, indent=2, default=str)[:3000] @@ -91,9 +90,9 @@ def _sanitize_findings(self, findings: dict) -> dict: for module, data in findings.items(): if isinstance(data, dict): sanitized[module] = { - k: v for k, v in data.items() - if k not in ("raw", "encoding", "all_tags", "html") - and not isinstance(v, bytes) + k: v + for k, v in data.items() + if k not in ("raw", "encoding", "all_tags", "html") and not isinstance(v, bytes) } # Truncate long lists for k, v in sanitized[module].items(): @@ -129,13 +128,16 @@ def _fallback_analysis(self, findings: dict) -> dict: locations.add(val) # Simple risk scoring - risk_score = min(1.0, ( - (0.1 if profile_count > 5 else 0) + - (0.2 if profile_count > 15 else 0) + - (0.3 if breach_count > 0 else 0) + - (0.2 if breach_count > 5 else 0) + - 0.1 # Base risk for any digital presence - )) + risk_score = min( + 1.0, + ( + (0.1 if profile_count > 5 else 0) + + (0.2 if profile_count > 15 else 0) + + (0.3 if breach_count > 0 else 0) + + (0.2 if breach_count > 5 else 0) + + 0.1 # Base risk for any digital presence + ), + ) risk_level = "low" if risk_score < 0.4 else "medium" if risk_score < 0.7 else "high" diff --git a/ghost/ai/summarizer.py b/ghost/ai/summarizer.py index e39e420..1a0c292 100644 --- a/ghost/ai/summarizer.py +++ b/ghost/ai/summarizer.py @@ -23,6 +23,7 @@ async def _ai_summary(self, inv: dict) -> str: """Generate AI-powered executive summary.""" try: import openai + client = openai.AsyncOpenAI(api_key=self.config.openai_api_key) findings_brief = {} @@ -40,15 +41,15 @@ async def _ai_summary(self, inv: dict) -> str: prompt = f"""Write a concise executive summary (3-5 paragraphs) of this OSINT investigation. -Target: {inv.get('target')} -Type: {inv.get('input_type')} -Risk Score: {inv.get('risk_score', 'N/A')} +Target: {inv.get("target")} +Type: {inv.get("input_type")} +Risk Score: {inv.get("risk_score", "N/A")} Key Findings: {json.dumps(findings_brief, indent=2, default=str)[:5000]} AI Analysis: -{json.dumps(inv.get('ai_analysis', {}), indent=2, default=str)[:3000]} +{json.dumps(inv.get("ai_analysis", {}), indent=2, default=str)[:3000]} Write in a professional, objective tone suitable for a security report. Include: 1. Overview of the investigation scope @@ -60,7 +61,10 @@ async def _ai_summary(self, inv: dict) -> str: response = await client.chat.completions.create( model=self.config.openai_model, messages=[ - {"role": "system", "content": "You are a professional intelligence analyst writing executive summaries. Be concise, factual, and objective."}, + { + "role": "system", + "content": "You are a professional intelligence analyst writing executive summaries. Be concise, factual, and objective.", + }, {"role": "user", "content": prompt}, ], temperature=0.3, diff --git a/ghost/backend/db.py b/ghost/backend/db.py index 2b8ea16..1622dee 100644 --- a/ghost/backend/db.py +++ b/ghost/backend/db.py @@ -145,9 +145,7 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_findings_investigation ON findings(investigation_id); CREATE INDEX IF NOT EXISTS idx_relationships_investigation ON relationships(investigation_id); """) - existing_columns = { - row["name"] for row in conn.execute("PRAGMA table_info(investigations)").fetchall() - } + existing_columns = {row["name"] for row in conn.execute("PRAGMA table_info(investigations)").fetchall()} if "scope" not in existing_columns: conn.execute("ALTER TABLE investigations ADD COLUMN scope TEXT DEFAULT ''") if "authorized_use" not in existing_columns: @@ -160,26 +158,30 @@ def init_db(): # ── Investigation CRUD ────────────────────────────────────────────── + def save_investigation(inv_dict: dict): """Save or update a full investigation from Investigation.to_dict().""" with get_db() as conn: - conn.execute(""" + conn.execute( + """ INSERT OR REPLACE INTO investigations (id, target, input_type, scope, authorized_use, status, started_at, completed_at, summary, risk_score, errors) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - inv_dict["id"], - inv_dict["target"], - inv_dict["input_type"], - inv_dict.get("scope", ""), - 1 if inv_dict.get("authorized_use", False) else 0, - inv_dict["status"], - inv_dict["started_at"], - inv_dict["completed_at"], - inv_dict.get("summary", ""), - inv_dict.get("risk_score", 0.0), - json.dumps(inv_dict.get("errors", [])), - )) + """, + ( + inv_dict["id"], + inv_dict["target"], + inv_dict["input_type"], + inv_dict.get("scope", ""), + 1 if inv_dict.get("authorized_use", False) else 0, + inv_dict["status"], + inv_dict["started_at"], + inv_dict["completed_at"], + inv_dict.get("summary", ""), + inv_dict.get("risk_score", 0.0), + json.dumps(inv_dict.get("errors", [])), + ), + ) # Store each module's findings conn.execute("DELETE FROM findings WHERE investigation_id = ?", (inv_dict["id"],)) @@ -266,9 +268,9 @@ def _ensure_entity(etype, value, platform="", confidence=0.5, metadata=None): # Locations for loc in inv_dict.get("correlations", {}).get("locations", []): - _ensure_entity("location", loc.get("value", ""), metadata={ - k: loc[k] for k in ("lat", "lon", "source") if k in loc - }) + _ensure_entity( + "location", loc.get("value", ""), metadata={k: loc[k] for k in ("lat", "lon", "source") if k in loc} + ) def get_investigation(investigation_id: str) -> dict | None: @@ -355,24 +357,28 @@ def get_graph_data(investigation_id: str) -> dict | None: nodes = [] for e in entities: - nodes.append({ - "id": e["id"], - "label": e["value"][:60], - "type": e["entity_type"], - "platform": e["platform"], - "confidence": e["confidence"], - }) + nodes.append( + { + "id": e["id"], + "label": e["value"][:60], + "type": e["entity_type"], + "platform": e["platform"], + "confidence": e["confidence"], + } + ) links = [] entity_ids = {e["id"] for e in entities} for r in relationships: if r["source_entity_id"] in entity_ids and r["target_entity_id"] in entity_ids: - links.append({ - "source": r["source_entity_id"], - "target": r["target_entity_id"], - "type": r["relationship_type"], - "confidence": r["confidence"], - }) + links.append( + { + "source": r["source_entity_id"], + "target": r["target_entity_id"], + "type": r["relationship_type"], + "confidence": r["confidence"], + } + ) return {"nodes": nodes, "links": links} diff --git a/ghost/backend/server.py b/ghost/backend/server.py index b2a0d00..7814728 100644 --- a/ghost/backend/server.py +++ b/ghost/backend/server.py @@ -29,6 +29,7 @@ # ── Static dashboard ──────────────────────────────────────────────── + @app.route("/") def index(): return send_from_directory(app.static_folder, "dashboard.html") @@ -41,6 +42,7 @@ def ui_static(filename): # ── API endpoints ─────────────────────────────────────────────────── + @app.route("/api/investigate", methods=["POST"]) def start_investigation(): """Start a new investigation. @@ -67,10 +69,12 @@ def start_investigation(): return jsonify({"error": "Provide at least one of: target, name, email, phone, username"}), 400 if data.get("authorized_use") is not True: - return jsonify({ - "error": "authorized_use must be true for API investigations", - "detail": "Only run Ghost for authorized security research, journalism, law enforcement, or self-audits.", - }), 400 + return jsonify( + { + "error": "authorized_use must be true for API investigations", + "detail": "Only run Ghost for authorized security research, journalism, law enforcement, or self-audits.", + } + ), 400 modules = data.get("modules") scope = data.get("scope", "authorized API investigation") @@ -140,6 +144,7 @@ def get_entity_graph(investigation_id): # ── Run ───────────────────────────────────────────────────────────── + def main(): app.run(host=config.host, port=config.port, debug=config.debug) diff --git a/ghost/core/config.py b/ghost/core/config.py index 93231d8..ae04074 100644 --- a/ghost/core/config.py +++ b/ghost/core/config.py @@ -47,9 +47,9 @@ class Config: rate_limit_period: int = int(os.getenv("RATE_LIMIT_PERIOD", "60")) # Module toggles - enabled_modules: list = field(default_factory=lambda: [ - "username", "email", "phone", "social", "domain", "image", "darkweb", "geolocation" - ]) + enabled_modules: list = field( + default_factory=lambda: ["username", "email", "phone", "social", "domain", "image", "darkweb", "geolocation"] + ) # Request settings request_timeout: int = 30 diff --git a/ghost/core/correlator.py b/ghost/core/correlator.py index 3c228fa..9be9842 100644 --- a/ghost/core/correlator.py +++ b/ghost/core/correlator.py @@ -58,21 +58,25 @@ def _correlate_identities(self, findings: dict) -> list[dict]: for name, sources in seen_names.items(): if len(sources) > 1: - identities.append({ - "type": "name", - "value": name, - "sources": sources, - "confidence": min(0.9, 0.5 + 0.1 * len(sources)), - }) + identities.append( + { + "type": "name", + "value": name, + "sources": sources, + "confidence": min(0.9, 0.5 + 0.1 * len(sources)), + } + ) for email, sources in seen_emails.items(): if len(sources) > 1: - identities.append({ - "type": "email", - "value": email, - "sources": sources, - "confidence": 0.95, - }) + identities.append( + { + "type": "email", + "value": email, + "sources": sources, + "confidence": 0.95, + } + ) return identities @@ -113,12 +117,14 @@ def _find_connections(self, findings: dict) -> list[dict]: url = p.get("url", "") for uname in all_usernames: if uname in url.lower() and uname != p.get("username", "").lower(): - connections.append({ - "type": "username_link", - "from": uname, - "to": p.get("platform", module), - "evidence": url, - }) + connections.append( + { + "type": "username_link", + "from": uname, + "to": p.get("platform", module), + "evidence": url, + } + ) return connections @@ -133,31 +139,37 @@ def _build_timeline(self, findings: dict) -> list[dict]: # Account creation dates created = data.get("created_at") or data.get("creation_date") or data.get("registered") if created: - events.append({ - "date": str(created), - "event": f"Account/entity created on {module}", - "source": module, - }) + events.append( + { + "date": str(created), + "event": f"Account/entity created on {module}", + "source": module, + } + ) # Breach dates breaches = data.get("breaches", []) if isinstance(breaches, list): for breach in breaches: if isinstance(breach, dict) and breach.get("date"): - events.append({ - "date": breach["date"], - "event": f"Data breach: {breach.get('name', 'Unknown')}", - "source": "darkweb", - }) + events.append( + { + "date": breach["date"], + "event": f"Data breach: {breach.get('name', 'Unknown')}", + "source": "darkweb", + } + ) # Posts/activity last_active = data.get("last_active") or data.get("last_post") if last_active: - events.append({ - "date": str(last_active), - "event": f"Last activity on {module}", - "source": module, - }) + events.append( + { + "date": str(last_active), + "event": f"Last activity on {module}", + "source": module, + } + ) events.sort(key=lambda x: x.get("date", "")) return events @@ -173,23 +185,27 @@ def _correlate_locations(self, findings: dict) -> list[dict]: for key in ("location", "city", "country", "region", "geo"): val = data.get(key) if val and isinstance(val, str): - locations.append({ - "value": val, - "source": module, - "field": key, - }) + locations.append( + { + "value": val, + "source": module, + "field": key, + } + ) # GPS coordinates lat = data.get("latitude") or data.get("lat") lon = data.get("longitude") or data.get("lon") or data.get("lng") if lat and lon: - locations.append({ - "value": f"{lat}, {lon}", - "lat": lat, - "lon": lon, - "source": module, - "field": "coordinates", - }) + locations.append( + { + "value": f"{lat}, {lon}", + "lat": lat, + "lon": lon, + "source": module, + "field": "coordinates", + } + ) return locations @@ -208,8 +224,7 @@ async def _ai_correlate(self, findings: dict) -> dict: for module, data in findings.items(): if isinstance(data, dict) and "error" not in data: summary[module] = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool, list)) and k != "raw" + k: v for k, v in data.items() if isinstance(v, (str, int, float, bool, list)) and k != "raw" } prompt = f"""Analyze the following OSINT investigation findings and identify: @@ -227,7 +242,10 @@ async def _ai_correlate(self, findings: dict) -> dict: response = await client.chat.completions.create( model=self.config.openai_model, messages=[ - {"role": "system", "content": "You are an OSINT analyst. Analyze data objectively and identify connections. Always respond with valid JSON."}, + { + "role": "system", + "content": "You are an OSINT analyst. Analyze data objectively and identify connections. Always respond with valid JSON.", + }, {"role": "user", "content": prompt}, ], temperature=0.3, diff --git a/ghost/core/doctor.py b/ghost/core/doctor.py index 3d95d32..76067a9 100644 --- a/ghost/core/doctor.py +++ b/ghost/core/doctor.py @@ -33,11 +33,13 @@ def run_doctor_checks(config_override: Config | None = None) -> list[DoctorCheck except Exception as exc: checks.append(DoctorCheck("database", False, str(exc), "error")) - checks.append(DoctorCheck( - "OpenAI key", - cfg.has_api_key("openai_api_key"), - "set" if cfg.has_api_key("openai_api_key") else "missing; fallback summaries still work", - )) + checks.append( + DoctorCheck( + "OpenAI key", + cfg.has_api_key("openai_api_key"), + "set" if cfg.has_api_key("openai_api_key") else "missing; fallback summaries still work", + ) + ) for package, label in [ ("aiohttp", "HTTP collection"), diff --git a/ghost/core/investigator.py b/ghost/core/investigator.py index de50936..7d70fb3 100644 --- a/ghost/core/investigator.py +++ b/ghost/core/investigator.py @@ -151,9 +151,7 @@ async def investigate_async( # Phase 2: Correlate findings self._report_progress("correlator", "start", "Correlating findings") try: - investigation.correlations = await self.correlator.correlate( - investigation.findings - ) + investigation.correlations = await self.correlator.correlate(investigation.findings) except Exception as e: investigation.errors.append(f"Correlation error: {e}") @@ -180,9 +178,7 @@ async def investigate_async( return investigation - async def _run_module( - self, name: str, target: str, input_type: str, investigation: Investigation - ): + async def _run_module(self, name: str, target: str, input_type: str, investigation: Investigation): """Run a single OSINT module with error handling.""" self._report_progress(name, "running", f"Scanning {name}") try: @@ -195,8 +191,6 @@ async def _run_module( investigation.findings[name] = {"error": str(e)} self._report_progress(name, "error", str(e)) - def generate_report( - self, investigation: Investigation, format: str = "html", output_path: str = None - ) -> str: + def generate_report(self, investigation: Investigation, format: str = "html", output_path: str = None) -> str: """Generate a report from an investigation.""" return self.report_generator.generate(investigation, format, output_path) diff --git a/ghost/core/report_generator.py b/ghost/core/report_generator.py index 19b2220..5efec66 100644 --- a/ghost/core/report_generator.py +++ b/ghost/core/report_generator.py @@ -64,9 +64,7 @@ def _generate_html(self, inv: dict, output_path: str) -> str: def _generate_json(self, inv: dict, output_path: str) -> str: """Generate a JSON report.""" Path(output_path).parent.mkdir(parents=True, exist_ok=True) - Path(output_path).write_text( - json.dumps(inv, indent=2, default=str), encoding="utf-8" - ) + Path(output_path).write_text(json.dumps(inv, indent=2, default=str), encoding="utf-8") return output_path def _generate_pdf(self, inv: dict, output_path: str) -> str: @@ -75,6 +73,7 @@ def _generate_pdf(self, inv: dict, output_path: str) -> str: self._generate_html(inv, html_path) try: from weasyprint import HTML + HTML(filename=html_path).write_pdf(output_path) return output_path except ImportError: @@ -100,9 +99,7 @@ def collect_urls(value): collect_urls(findings) module_errors = { - module: data["error"] - for module, data in findings.items() - if isinstance(data, dict) and data.get("error") + module: data["error"] for module, data in findings.items() if isinstance(data, dict) and data.get("error") } return { diff --git a/ghost/modules/darkweb.py b/ghost/modules/darkweb.py index 15b45a7..764bba4 100644 --- a/ghost/modules/darkweb.py +++ b/ghost/modules/darkweb.py @@ -42,6 +42,7 @@ async def _search_ahmia(self, query: str) -> dict: text = await resp.text() # Parse results from HTML from bs4 import BeautifulSoup + soup = BeautifulSoup(text, "html.parser") results = [] for item in soup.select("li.result"): @@ -49,11 +50,13 @@ async def _search_ahmia(self, query: str) -> dict: link_el = item.select_one("a") desc_el = item.select_one("p") if title_el and link_el: - results.append({ - "title": title_el.get_text(strip=True), - "url": link_el.get("href", ""), - "description": desc_el.get_text(strip=True) if desc_el else "", - }) + results.append( + { + "title": title_el.get_text(strip=True), + "url": link_el.get("href", ""), + "description": desc_el.get_text(strip=True) if desc_el else "", + } + ) return {"results": results[:20], "count": len(results)} return {"results": [], "error": f"Status {resp.status}"} except Exception as e: @@ -81,14 +84,16 @@ async def _check_breach_databases(self, target: str) -> dict: if resp.status == 200: data = await resp.json() for breach in data: - breaches.append({ - "name": breach.get("Name"), - "title": breach.get("Title"), - "date": breach.get("BreachDate"), - "pwn_count": breach.get("PwnCount"), - "data_classes": breach.get("DataClasses", []), - "verified": breach.get("IsVerified"), - }) + breaches.append( + { + "name": breach.get("Name"), + "title": breach.get("Title"), + "date": breach.get("BreachDate"), + "pwn_count": breach.get("PwnCount"), + "data_classes": breach.get("DataClasses", []), + "verified": breach.get("IsVerified"), + } + ) # Check pastes if "@" in target: @@ -124,9 +129,9 @@ async def _check_paste_sites(self, target: str) -> dict: # Search public paste indices paste_searches = [ - ("PasteBin (Google)", f"site:pastebin.com \"{target}\""), - ("GitHub Gists", f"site:gist.github.com \"{target}\""), - ("Ghostbin", f"site:ghostbin.com \"{target}\""), + ("PasteBin (Google)", f'site:pastebin.com "{target}"'), + ("GitHub Gists", f'site:gist.github.com "{target}"'), + ("Ghostbin", f'site:ghostbin.com "{target}"'), ] # Use Google Custom Search if available @@ -144,12 +149,14 @@ async def _check_paste_sites(self, target: str) -> dict: if resp.status == 200: data = await resp.json() for item in data.get("items", [])[:5]: - pastes.append({ - "source": name, - "title": item.get("title"), - "url": item.get("link"), - "snippet": item.get("snippet"), - }) + pastes.append( + { + "source": name, + "title": item.get("title"), + "url": item.get("link"), + "snippet": item.get("snippet"), + } + ) except Exception: pass diff --git a/ghost/modules/domain.py b/ghost/modules/domain.py index dabf794..2863a6d 100644 --- a/ghost/modules/domain.py +++ b/ghost/modules/domain.py @@ -45,6 +45,7 @@ async def _whois_lookup(self, domain: str) -> dict: """Get WHOIS information.""" try: import whois + w = whois.whois(domain) return { "registrar": w.registrar, @@ -99,10 +100,34 @@ async def _subdomain_enum(self, domain: str) -> dict: # Common subdomain brute-force list common = [ - "www", "mail", "ftp", "admin", "api", "dev", "staging", "test", - "blog", "shop", "store", "app", "portal", "secure", "vpn", - "remote", "webmail", "ns1", "ns2", "cdn", "media", "static", - "docs", "help", "support", "status", "m", "mobile", + "www", + "mail", + "ftp", + "admin", + "api", + "dev", + "staging", + "test", + "blog", + "shop", + "store", + "app", + "portal", + "secure", + "vpn", + "remote", + "webmail", + "ns1", + "ns2", + "cdn", + "media", + "static", + "docs", + "help", + "support", + "status", + "m", + "mobile", ] for sub in common: @@ -137,10 +162,17 @@ async def _tech_stack(self, domain: str) -> dict: # Analyze headers resp_headers = dict(resp.headers) tech["headers"] = { - k: v for k, v in resp_headers.items() - if k.lower() in ( - "server", "x-powered-by", "x-generator", "x-aspnet-version", - "x-frame-options", "content-security-policy", "strict-transport-security", + k: v + for k, v in resp_headers.items() + if k.lower() + in ( + "server", + "x-powered-by", + "x-generator", + "x-aspnet-version", + "x-frame-options", + "content-security-policy", + "strict-transport-security", ) } diff --git a/ghost/modules/email.py b/ghost/modules/email.py index 54af9d8..3378e44 100644 --- a/ghost/modules/email.py +++ b/ghost/modules/email.py @@ -40,7 +40,7 @@ async def run(self, target: str, input_type: str = "email") -> dict[str, Any]: async def _validate_email(self, email: str) -> dict: """Validate email format and check MX records.""" # Format check - pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" is_valid_format = bool(re.match(pattern, email)) domain = email.split("@")[1] @@ -49,11 +49,9 @@ async def _validate_email(self, email: str) -> dict: mx_records = [] try: import dns.resolver + answers = dns.resolver.resolve(domain, "MX") - mx_records = [ - {"priority": r.preference, "host": str(r.exchange).rstrip(".")} - for r in answers - ] + mx_records = [{"priority": r.preference, "host": str(r.exchange).rstrip(".")} for r in answers] except Exception: pass @@ -124,11 +122,13 @@ async def _discover_accounts(self, email: str) -> dict: if resp.status == 200: data = await resp.json() # Each API has different response structure - accounts.append({ - "service": name, - "exists": True, - "response_code": resp.status, - }) + accounts.append( + { + "service": name, + "exists": True, + "response_code": resp.status, + } + ) except Exception: pass @@ -142,13 +142,14 @@ async def _discover_accounts(self, email: str) -> dict: if resp.status == 200: data = await resp.json() emails = data.get("data", {}).get("emails", []) - accounts.append({ - "service": "Hunter.io", - "related_emails": [ - {"email": e["value"], "type": e.get("type")} - for e in emails[:10] - ], - }) + accounts.append( + { + "service": "Hunter.io", + "related_emails": [ + {"email": e["value"], "type": e.get("type")} for e in emails[:10] + ], + } + ) except Exception: pass @@ -161,15 +162,18 @@ async def _domain_info(self, email: str) -> dict: try: import whois + w = whois.whois(domain) - info.update({ - "registrar": w.registrar, - "creation_date": str(w.creation_date) if w.creation_date else None, - "expiration_date": str(w.expiration_date) if w.expiration_date else None, - "name_servers": w.name_servers if w.name_servers else [], - "org": w.org, - "country": w.country, - }) + info.update( + { + "registrar": w.registrar, + "creation_date": str(w.creation_date) if w.creation_date else None, + "expiration_date": str(w.expiration_date) if w.expiration_date else None, + "name_servers": w.name_servers if w.name_servers else [], + "org": w.org, + "country": w.country, + } + ) except Exception as e: info["whois_error"] = str(e) @@ -178,6 +182,7 @@ async def _domain_info(self, email: str) -> dict: async def _gravatar_check(self, email: str) -> dict: """Check for Gravatar profile.""" import hashlib + email_hash = hashlib.md5(email.strip().lower().encode()).hexdigest() profile_url = f"https://en.gravatar.com/{email_hash}.json" avatar_url = f"https://www.gravatar.com/avatar/{email_hash}" @@ -196,8 +201,7 @@ async def _gravatar_check(self, email: str) -> dict: "about": entry.get("aboutMe"), "location": entry.get("currentLocation"), "accounts": [ - {"service": a.get("shortname"), "url": a.get("url")} - for a in entry.get("accounts", []) + {"service": a.get("shortname"), "url": a.get("url")} for a in entry.get("accounts", []) ], } return {"exists": False, "avatar_url": avatar_url + "?d=404"} @@ -226,9 +230,17 @@ def _detect_provider(self, domain: str) -> str: def _is_disposable(self, domain: str) -> bool: disposable_domains = { - "tempmail.com", "throwaway.email", "guerrillamail.com", - "mailinator.com", "10minutemail.com", "trashmail.com", - "yopmail.com", "sharklasers.com", "guerrillamailblock.com", - "dispostable.com", "maildrop.cc", "temp-mail.org", + "tempmail.com", + "throwaway.email", + "guerrillamail.com", + "mailinator.com", + "10minutemail.com", + "trashmail.com", + "yopmail.com", + "sharklasers.com", + "guerrillamailblock.com", + "dispostable.com", + "maildrop.cc", + "temp-mail.org", } return domain in disposable_domains diff --git a/ghost/modules/geolocation.py b/ghost/modules/geolocation.py index 9ac9b79..33d636f 100644 --- a/ghost/modules/geolocation.py +++ b/ghost/modules/geolocation.py @@ -112,8 +112,10 @@ async def _ip_geolocation(self, ip: str) -> dict: "city": best.get("city"), "region": best.get("region") or best.get("regionName"), "country": best.get("country"), - "latitude": best.get("latitude") or (best.get("location", ",").split(",")[0] if best.get("location") else None), - "longitude": best.get("longitude") or (best.get("location", ",").split(",")[1] if best.get("location") else None), + "latitude": best.get("latitude") + or (best.get("location", ",").split(",")[0] if best.get("location") else None), + "longitude": best.get("longitude") + or (best.get("location", ",").split(",")[1] if best.get("location") else None), "isp": best.get("isp") or best.get("org"), } diff --git a/ghost/modules/image.py b/ghost/modules/image.py index efb31dc..005be2c 100644 --- a/ghost/modules/image.py +++ b/ghost/modules/image.py @@ -65,6 +65,7 @@ async def _extract_exif(self, target: str, image_data: bytes) -> dict: """Extract EXIF metadata from image.""" try: import exifread + tags = exifread.process_file(io.BytesIO(image_data), details=False) exif = {} @@ -101,6 +102,7 @@ async def _extract_exif(self, target: str, image_data: bytes) -> dict: try: from PIL import Image from PIL.ExifTags import TAGS + img = Image.open(io.BytesIO(image_data)) exif_data = img._getexif() if exif_data: @@ -129,13 +131,12 @@ async def _reverse_image_search(self, target: str, image_data: bytes) -> dict: if resp.status == 200: data = await resp.json() items = data.get("items", []) - results["engines"].append({ - "engine": "Google", - "matches": [ - {"title": i["title"], "url": i["link"]} - for i in items[:10] - ], - }) + results["engines"].append( + { + "engine": "Google", + "matches": [{"title": i["title"], "url": i["link"]} for i in items[:10]], + } + ) except Exception as e: results["engines"].append({"engine": "Google", "error": str(e)}) @@ -166,11 +167,13 @@ async def _detect_faces(self, image_data: bytes) -> dict: faces = [] for i, (location, encoding) in enumerate(zip(face_locations, face_encodings)): top, right, bottom, left = location - faces.append({ - "id": i, - "bounding_box": {"top": top, "right": right, "bottom": bottom, "left": left}, - "encoding_hash": hashlib.md5(encoding.tobytes()).hexdigest(), - }) + faces.append( + { + "id": i, + "bounding_box": {"top": top, "right": right, "bottom": bottom, "left": left}, + "encoding_hash": hashlib.md5(encoding.tobytes()).hexdigest(), + } + ) return {"face_count": len(faces), "faces": faces} except ImportError: @@ -182,6 +185,7 @@ async def _extract_geolocation(self, target: str, image_data: bytes) -> dict: """Extract GPS coordinates from EXIF and resolve to address.""" try: import exifread + tags = exifread.process_file(io.BytesIO(image_data)) lat = tags.get("GPS GPSLatitude") @@ -204,6 +208,7 @@ async def _extract_geolocation(self, target: str, image_data: bytes) -> dict: # Reverse geocode try: from geopy.geocoders import Nominatim + geolocator = Nominatim(user_agent="ghost-osint") location = geolocator.reverse(f"{lat_decimal}, {lon_decimal}") if location: diff --git a/ghost/modules/phone.py b/ghost/modules/phone.py index a98345d..855bcbc 100644 --- a/ghost/modules/phone.py +++ b/ghost/modules/phone.py @@ -108,13 +108,15 @@ async def _check_spam_databases(self, number: str) -> dict: if resp.status == 200: data = await resp.json() if data.get("valid"): - reports.append({ - "source": "numverify", - "valid": data.get("valid"), - "line_type": data.get("line_type"), - "carrier": data.get("carrier"), - "location": data.get("location"), - }) + reports.append( + { + "source": "numverify", + "valid": data.get("valid"), + "line_type": data.get("line_type"), + "carrier": data.get("carrier"), + "location": data.get("location"), + } + ) except Exception: pass @@ -145,11 +147,13 @@ async def _check_social_media(self, number: str) -> dict: headers = {"User-Agent": self.config.user_agent} async with session.head(url, headers=headers, allow_redirects=True) as resp: if resp.status == 200: - linked_accounts.append({ - "service": service, - "url": url, - "status": "possibly_linked", - }) + linked_accounts.append( + { + "service": service, + "url": url, + "status": "possibly_linked", + } + ) except Exception: pass diff --git a/ghost/modules/social.py b/ghost/modules/social.py index 464f15b..782673c 100644 --- a/ghost/modules/social.py +++ b/ghost/modules/social.py @@ -29,9 +29,7 @@ async def run(self, target: str, input_type: str = "username") -> dict[str, Any] ) platforms = {} - names = [ - "instagram", "twitter", "reddit", "tiktok", "github", "linkedin" - ] + names = ["instagram", "twitter", "reddit", "tiktok", "github", "linkedin"] for name, result in zip(names, results): if isinstance(result, Exception): platforms[name] = {"error": str(result)} @@ -42,18 +40,20 @@ async def run(self, target: str, input_type: str = "username") -> dict[str, Any] profiles = [] for name, data in platforms.items(): if isinstance(data, dict) and data.get("found"): - profiles.append({ - "platform": name, - "username": data.get("username", username), - "url": data.get("url", ""), - "display_name": data.get("display_name", ""), - "bio": data.get("bio", ""), - "followers": data.get("followers"), - "following": data.get("following"), - "posts": data.get("posts"), - "verified": data.get("verified", False), - "created_at": data.get("created_at"), - }) + profiles.append( + { + "platform": name, + "username": data.get("username", username), + "url": data.get("url", ""), + "display_name": data.get("display_name", ""), + "bio": data.get("bio", ""), + "followers": data.get("followers"), + "following": data.get("following"), + "posts": data.get("posts"), + "verified": data.get("verified", False), + "created_at": data.get("created_at"), + } + ) return { "username": username, diff --git a/ghost/modules/username.py b/ghost/modules/username.py index e49d00a..d483f13 100644 --- a/ghost/modules/username.py +++ b/ghost/modules/username.py @@ -123,11 +123,13 @@ async def check_platform( allow_redirects=allow_redirects, ) as resp: if resp.status == expected_status: - found.append({ - "platform": name, - "url": url, - "status": resp.status, - }) + found.append( + { + "platform": name, + "url": url, + "status": resp.status, + } + ) else: not_found.append(name) except asyncio.TimeoutError: @@ -135,10 +137,7 @@ async def check_platform( except Exception as e: errors.append({"platform": name, "error": str(e)}) - tasks = [ - check_platform(name, url_tpl, status, redirects) - for name, url_tpl, status, redirects in PLATFORMS - ] + tasks = [check_platform(name, url_tpl, status, redirects) for name, url_tpl, status, redirects in PLATFORMS] await asyncio.gather(*tasks) # Also try sherlock/maigret if available @@ -158,7 +157,11 @@ async def _run_sherlock(self, username: str) -> dict: """Attempt to run sherlock for additional coverage.""" try: proc = await asyncio.create_subprocess_exec( - "sherlock", username, "--print-found", "--timeout", "15", + "sherlock", + username, + "--print-found", + "--timeout", + "15", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/ghost/ui/cli.py b/ghost/ui/cli.py index f5488d7..4a6484d 100644 --- a/ghost/ui/cli.py +++ b/ghost/ui/cli.py @@ -3,6 +3,7 @@ import asyncio import json import sys +from pathlib import Path import click from rich.console import Console @@ -17,7 +18,13 @@ from ghost.core.investigator import GhostInvestigator from ghost.core.config import config from ghost.core.doctor import run_doctor_checks -from ghost.backend.db import get_graph_data, get_investigation, list_investigations +from ghost.backend.db import ( + delete_investigation, + get_graph_data, + get_investigation, + list_investigations, + save_investigation, +) console = Console() @@ -39,11 +46,13 @@ def print_banner(): - console.print(Panel( - Text(BANNER, style="bold green", justify="center"), - border_style="green", - padding=(0, 2), - )) + console.print( + Panel( + Text(BANNER, style="bold green", justify="center"), + border_style="green", + padding=(0, 2), + ) + ) console.print(DISCLAIMER, justify="center") console.print() @@ -66,7 +75,9 @@ def create_progress(): @click.option("--output", "-o", help="Output file path") @click.option("--format", "-f", "fmt", default="html", help="Report format: html/pdf/json") @click.option("--no-ai", is_flag=True, help="Disable OpenAI calls and use deterministic heuristic analysis") -@click.option("--scope", default="authorized CLI investigation", help="Authorized scope/purpose recorded in report provenance") +@click.option( + "--scope", default="authorized CLI investigation", help="Authorized scope/purpose recorded in report provenance" +) @click.option("--authorized", is_flag=True, help="Acknowledge this investigation is authorized") @click.pass_context def cli(ctx, target, input_type, modules, output, fmt, no_ai, scope, authorized): @@ -92,7 +103,9 @@ def cli(ctx, target, input_type, modules, output, fmt, no_ai, scope, authorized) @click.option("--output", "-o") @click.option("--format", "-f", "fmt", default="html") @click.option("--no-ai", is_flag=True, help="Disable OpenAI calls and use deterministic heuristic analysis") -@click.option("--scope", default="authorized CLI investigation", help="Authorized scope/purpose recorded in report provenance") +@click.option( + "--scope", default="authorized CLI investigation", help="Authorized scope/purpose recorded in report provenance" +) @click.option("--authorized", is_flag=True, help="Acknowledge this investigation is authorized") def investigate(target, input_type, modules, output, fmt, no_ai, scope, authorized): """Run an investigation on a target.""" @@ -180,35 +193,97 @@ def show(investigation_id, as_json): click.echo(json.dumps(investigation, indent=2, default=str)) return - console.print(Panel( - f"[bold green]ID:[/bold green] {investigation['id']}\n" - f"[bold green]Target:[/bold green] {investigation['target']}\n" - f"[bold green]Type:[/bold green] {investigation['input_type']}\n" - f"[bold green]Status:[/bold green] {investigation['status']}\n" - f"[bold green]Risk:[/bold green] {float(investigation.get('risk_score') or 0):.0%}\n" - f"[bold green]Authorized:[/bold green] {'yes' if investigation.get('authorized_use') else 'no'}\n" - f"[bold green]Scope:[/bold green] {investigation.get('scope', '')}", - title="[bold green]CASE FILE[/bold green]", - border_style="green", - )) + console.print( + Panel( + f"[bold green]ID:[/bold green] {investigation['id']}\n" + f"[bold green]Target:[/bold green] {investigation['target']}\n" + f"[bold green]Type:[/bold green] {investigation['input_type']}\n" + f"[bold green]Status:[/bold green] {investigation['status']}\n" + f"[bold green]Risk:[/bold green] {float(investigation.get('risk_score') or 0):.0%}\n" + f"[bold green]Authorized:[/bold green] {'yes' if investigation.get('authorized_use') else 'no'}\n" + f"[bold green]Scope:[/bold green] {investigation.get('scope', '')}", + title="[bold green]CASE FILE[/bold green]", + border_style="green", + ) + ) if investigation.get("summary"): console.print(Panel(investigation["summary"], title="Summary", border_style="green")) graph = get_graph_data(investigation["id"]) or {"nodes": [], "links": []} - console.print(f"[dim]Findings modules:[/dim] {', '.join(sorted(investigation.get('findings', {}).keys())) or 'none'}") + console.print( + f"[dim]Findings modules:[/dim] {', '.join(sorted(investigation.get('findings', {}).keys())) or 'none'}" + ) console.print(f"[dim]Graph:[/dim] {len(graph['nodes'])} nodes, {len(graph['links'])} links") +@cli.command(name="export") +@click.argument("investigation_id") +@click.option("--output", "-o", type=click.Path(dir_okay=False, path_type=Path), help="Destination JSON file") +def export_case(investigation_id, output): + """Export one saved investigation as a portable JSON case file.""" + investigation = _find_investigation_by_id_or_prefix(investigation_id) + if investigation is None: + raise click.ClickException(f"No investigation found for '{investigation_id}'") + if investigation == "ambiguous": + raise click.ClickException(f"Investigation prefix '{investigation_id}' is ambiguous") + + output = output or Path(f"ghost-case-{investigation['id']}.json") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(investigation, indent=2, default=str), encoding="utf-8") + console.print(f"[green]Exported case[/green] {investigation['id']} [dim]to[/dim] {output}") + + +@cli.command(name="import") +@click.argument("case_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--replace", is_flag=True, help="Replace an existing case with the same investigation ID") +def import_case(case_file, replace): + """Import a portable JSON case file into the local SQLite store.""" + try: + case_data = json.loads(case_file.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise click.ClickException(f"Invalid JSON case file: {exc}") from exc + + required = {"id", "target", "input_type", "status", "started_at"} + missing = sorted(required - set(case_data)) + if missing: + raise click.ClickException(f"Invalid Ghost case file; missing: {', '.join(missing)}") + + existing = get_investigation(case_data["id"]) + if existing and not replace: + raise click.ClickException(f"Case {case_data['id']} already exists. Re-run with --replace to overwrite it.") + + save_investigation(case_data) + console.print(f"[green]Imported case[/green] {case_data['id']} [dim]from[/dim] {case_file}") + + +@cli.command(name="delete") +@click.argument("investigation_id") +@click.option("--yes", is_flag=True, help="Skip confirmation prompt") +def delete_case(investigation_id, yes): + """Delete one saved investigation case file.""" + investigation = _find_investigation_by_id_or_prefix(investigation_id) + if investigation is None: + raise click.ClickException(f"No investigation found for '{investigation_id}'") + if investigation == "ambiguous": + raise click.ClickException(f"Investigation prefix '{investigation_id}' is ambiguous") + + if not yes and not Confirm.ask(f"Delete case {investigation['id']} for {investigation['target']}?", default=False): + console.print("[dim]Delete cancelled.[/dim]") + return + + if delete_investigation(investigation["id"]): + console.print(f"[green]Deleted case[/green] {investigation['id']}") + else: + raise click.ClickException(f"Could not delete case {investigation['id']}") + + def _find_investigation_by_id_or_prefix(investigation_id: str): investigation = get_investigation(investigation_id) if investigation: return investigation - matches = [ - item for item in list_investigations(limit=500) - if item.get("id", "").startswith(investigation_id) - ] + matches = [item for item in list_investigations(limit=500) if item.get("id", "").startswith(investigation_id)] if len(matches) == 1: return get_investigation(matches[0]["id"]) if len(matches) > 1: @@ -220,11 +295,13 @@ def interactive_menu(): """Interactive menu for investigations.""" while True: console.print() - console.print(Panel( - "[bold green]INVESTIGATION MENU[/bold green]", - border_style="green", - padding=(0, 2), - )) + console.print( + Panel( + "[bold green]INVESTIGATION MENU[/bold green]", + border_style="green", + padding=(0, 2), + ) + ) table = Table(show_header=False, box=box.SIMPLE, border_style="green") table.add_column(style="bold green", width=6) @@ -292,16 +369,18 @@ def run_investigation( investigator = GhostInvestigator() console.print() - console.print(Panel( - f"[bold green]TARGET:[/bold green] {target}\n" - f"[bold green]TYPE:[/bold green] {input_type}\n" - f"[bold green]MODULES:[/bold green] {', '.join(modules) if modules else 'auto'}\n" - f"[bold green]AI:[/bold green] {'disabled' if no_ai else 'enabled when configured'}\n" - f"[bold green]SCOPE:[/bold green] {scope}\n" - f"[bold green]AUTHORIZED:[/bold green] {'yes' if authorized else 'not acknowledged'}", - title="[bold green]INVESTIGATION STARTING[/bold green]", - border_style="green", - )) + console.print( + Panel( + f"[bold green]TARGET:[/bold green] {target}\n" + f"[bold green]TYPE:[/bold green] {input_type}\n" + f"[bold green]MODULES:[/bold green] {', '.join(modules) if modules else 'auto'}\n" + f"[bold green]AI:[/bold green] {'disabled' if no_ai else 'enabled when configured'}\n" + f"[bold green]SCOPE:[/bold green] {scope}\n" + f"[bold green]AUTHORIZED:[/bold green] {'yes' if authorized else 'not acknowledged'}", + title="[bold green]INVESTIGATION STARTING[/bold green]", + border_style="green", + ) + ) console.print() # Progress tracking @@ -316,9 +395,7 @@ def progress_callback(module, status, detail): return if module not in module_tasks: - module_tasks[module] = progress.add_task( - f" {module}", total=100, status=status - ) + module_tasks[module] = progress.add_task(f" {module}", total=100, status=status) if status == "done": progress.update(module_tasks[module], completed=100, status=detail) @@ -330,7 +407,11 @@ def progress_callback(module, status, detail): # Update main progress done_count = sum(1 for m in module_tasks.values() if progress.tasks[m].completed >= 100) total_modules = max(len(module_tasks), 1) - progress.update(main_task, completed=int(done_count / total_modules * 90), status=f"{done_count}/{total_modules} modules") + progress.update( + main_task, + completed=int(done_count / total_modules * 90), + status=f"{done_count}/{total_modules} modules", + ) investigator.set_progress_callback(progress_callback) @@ -347,10 +428,12 @@ def progress_callback(module, status, detail): # Generate report report_path = investigator.generate_report(investigation, fmt, output) console.print() - console.print(Panel( - f"[bold green]Report saved to:[/bold green] {report_path}", - border_style="green", - )) + console.print( + Panel( + f"[bold green]Report saved to:[/bold green] {report_path}", + border_style="green", + ) + ) finally: if no_ai: config.openai_api_key = original_openai_key @@ -362,19 +445,23 @@ def display_results(investigation): # Summary if inv.get("summary"): - console.print(Panel( - inv["summary"], - title="[bold green]EXECUTIVE SUMMARY[/bold green]", - border_style="green", - )) + console.print( + Panel( + inv["summary"], + title="[bold green]EXECUTIVE SUMMARY[/bold green]", + border_style="green", + ) + ) # Risk Score risk = inv.get("risk_score", 0) risk_color = "green" if risk < 0.4 else "yellow" if risk < 0.7 else "red" - console.print(Panel( - f"[bold {risk_color}]Risk Score: {risk:.0%}[/bold {risk_color}]", - border_style=risk_color, - )) + console.print( + Panel( + f"[bold {risk_color}]Risk Score: {risk:.0%}[/bold {risk_color}]", + border_style=risk_color, + ) + ) # Findings tree tree = Tree("[bold green]Investigation Findings[/bold green]") diff --git a/tests/test_investigator.py b/tests/test_investigator.py index 81cae2e..75d77fb 100644 --- a/tests/test_investigator.py +++ b/tests/test_investigator.py @@ -11,11 +11,19 @@ _detect_input_type, INPUT_TYPE_MODULES, ) -from ghost.backend.db import init_db, save_investigation, get_investigation, list_investigations, get_graph_data, delete_investigation +from ghost.backend.db import ( + init_db, + save_investigation, + get_investigation, + list_investigations, + get_graph_data, + delete_investigation, +) # ── Input detection ───────────────────────────────────────────────── + class TestInputDetection: def test_email(self): assert _detect_input_type("user@example.com") == "email" @@ -41,6 +49,7 @@ def test_username(self): # ── Investigation model ───────────────────────────────────────────── + class TestInvestigationModel: def test_create(self): inv = Investigation("johndoe", "username") @@ -71,6 +80,7 @@ def test_to_dict_serializable(self): # ── Module routing ────────────────────────────────────────────────── + class TestModuleRouting: def test_username_modules(self): assert "username" in INPUT_TYPE_MODULES["username"] @@ -90,6 +100,7 @@ def test_domain_modules(self): # ── Database layer ────────────────────────────────────────────────── + @pytest.fixture(autouse=True) def _setup_test_db(tmp_path, monkeypatch): """Use a temporary database for each test.""" @@ -179,12 +190,8 @@ def test_entities_stored(self): } } inv.correlations = { - "identities": [ - {"type": "email", "value": "test@example.com", "confidence": 0.9} - ], - "locations": [ - {"value": "New York", "source": "email"} - ], + "identities": [{"type": "email", "value": "test@example.com", "confidence": 0.9}], + "locations": [{"value": "New York", "source": "email"}], } save_investigation(inv.to_dict()) @@ -196,6 +203,7 @@ def test_entities_stored(self): # ── Database configuration ────────────────────────────────────────── + class TestDatabaseConfiguration: def test_sqlite_database_url_resolves_absolute_path(self, tmp_path): from ghost.backend.db import resolve_database_path @@ -229,6 +237,7 @@ def test_doctor_checks_return_structured_results(self): # ── Report provenance ─────────────────────────────────────────────── + class TestReportProvenance: def test_json_report_includes_provenance(self, tmp_path): from ghost.core.report_generator import ReportGenerator @@ -259,6 +268,7 @@ def test_json_report_includes_provenance(self, tmp_path): # ── API safety gates ──────────────────────────────────────────────── + class TestApiSafetyGates: def test_api_requires_authorized_use_acknowledgement(self): from ghost.backend.server import app @@ -292,6 +302,7 @@ def test_api_accepts_authorized_scope(self, mock_thread): # ── CLI saved case commands ──────────────────────────────────────── + class TestCliCaseCommands: def test_cli_list_outputs_saved_case(self): from click.testing import CliRunner @@ -324,9 +335,72 @@ def test_cli_show_accepts_unique_prefix(self): assert data["id"] == inv.id assert data["summary"] == "Found public profiles." + def test_cli_export_writes_portable_case_file(self, tmp_path): + from click.testing import CliRunner + from ghost.ui.cli import cli + + inv = Investigation("exportme", "username", scope="client-owned audit", authorized_use=True) + inv.summary = "Portable case." + save_investigation(inv.to_dict()) + output = tmp_path / "case.json" + + result = CliRunner().invoke(cli, ["export", inv.id[:8], "--output", str(output)]) + + assert result.exit_code == 0 + data = json.loads(output.read_text()) + assert data["id"] == inv.id + assert data["target"] == "exportme" + assert data["scope"] == "client-owned audit" + + def test_cli_import_rejects_existing_case_without_replace(self, tmp_path): + from click.testing import CliRunner + from ghost.ui.cli import cli + + inv = Investigation("importme", "username") + case_file = tmp_path / "case.json" + case_file.write_text(json.dumps(inv.to_dict()), encoding="utf-8") + + save_investigation(inv.to_dict()) + result = CliRunner().invoke(cli, ["import", str(case_file)]) + + assert result.exit_code != 0 + assert "already exists" in result.output + + def test_cli_import_with_replace_updates_case(self, tmp_path): + from click.testing import CliRunner + from ghost.ui.cli import cli + + inv = Investigation("old-target", "username") + save_investigation(inv.to_dict()) + exported = inv.to_dict() + exported["target"] = "new-target" + exported["summary"] = "Replaced case." + case_file = tmp_path / "case.json" + case_file.write_text(json.dumps(exported), encoding="utf-8") + + result = CliRunner().invoke(cli, ["import", str(case_file), "--replace"]) + + assert result.exit_code == 0 + loaded = get_investigation(inv.id) + assert loaded["target"] == "new-target" + assert loaded["summary"] == "Replaced case." + + def test_cli_delete_removes_case_with_yes(self): + from click.testing import CliRunner + from ghost.ui.cli import cli + + inv = Investigation("deleteme", "username") + save_investigation(inv.to_dict()) + + result = CliRunner().invoke(cli, ["delete", inv.id[:8], "--yes"]) + + assert result.exit_code == 0 + assert get_investigation(inv.id) is None + # ── GhostInvestigator (mocked modules) ───────────────────────────── + class TestGhostInvestigator: @patch("ghost.core.investigator.UsernameModule") @patch("ghost.core.investigator.EmailModule") @@ -340,17 +414,28 @@ class TestGhostInvestigator: @patch("ghost.core.investigator.AIAnalyzer") @patch("ghost.core.investigator.Summarizer") def test_investigate_username( - self, MockSummarizer, MockAnalyzer, MockCorrelator, - MockGeo, MockDarkweb, MockImage, MockDomain, - MockSocial, MockPhone, MockEmail, MockUsername, + self, + MockSummarizer, + MockAnalyzer, + MockCorrelator, + MockGeo, + MockDarkweb, + MockImage, + MockDomain, + MockSocial, + MockPhone, + MockEmail, + MockUsername, ): # Set up mocks mock_username = MagicMock() - mock_username.run = AsyncMock(return_value={ - "username": "johndoe", - "profiles": [{"platform": "GitHub", "url": "https://github.com/johndoe", "status": "found"}], - "found_count": 1, - }) + mock_username.run = AsyncMock( + return_value={ + "username": "johndoe", + "profiles": [{"platform": "GitHub", "url": "https://github.com/johndoe", "status": "found"}], + "found_count": 1, + } + ) MockUsername.return_value = mock_username mock_social = MagicMock() @@ -368,16 +453,23 @@ def test_investigate_username( Mock.return_value = m mock_correlator = MagicMock() - mock_correlator.correlate = AsyncMock(return_value={ - "identities": [], "connections": [], "timeline": [], "locations": [], - }) + mock_correlator.correlate = AsyncMock( + return_value={ + "identities": [], + "connections": [], + "timeline": [], + "locations": [], + } + ) MockCorrelator.return_value = mock_correlator mock_analyzer = MagicMock() - mock_analyzer.analyze = AsyncMock(return_value={ - "risk_score": 0.3, - "risk_assessment": {"risk_level": "low"}, - }) + mock_analyzer.analyze = AsyncMock( + return_value={ + "risk_score": 0.3, + "risk_assessment": {"risk_level": "low"}, + } + ) MockAnalyzer.return_value = mock_analyzer mock_summarizer = MagicMock() @@ -385,9 +477,7 @@ def test_investigate_username( MockSummarizer.return_value = mock_summarizer investigator = GhostInvestigator() - result = asyncio.run( - investigator.investigate_async("johndoe", "username") - ) + result = asyncio.run(investigator.investigate_async("johndoe", "username")) assert result.status == "completed" assert result.target == "johndoe" diff --git a/tests/test_modules.py b/tests/test_modules.py index e2d9eb4..b971082 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -138,6 +138,7 @@ class TestCorrelator: def test_identity_correlation(self): from ghost.core.correlator import Correlator + cfg = Config() cfg.openai_api_key = "" # Disable AI correlation correlator = Correlator(cfg) @@ -165,6 +166,7 @@ def test_identity_correlation(self): def test_timeline_extraction(self): from ghost.core.correlator import Correlator + cfg = Config() correlator = Correlator(cfg) @@ -188,6 +190,7 @@ def test_timeline_extraction(self): def test_location_correlation(self): from ghost.core.correlator import Correlator + cfg = Config() correlator = Correlator(cfg) @@ -208,6 +211,7 @@ class TestAIAnalyzerFallback: def test_fallback_analysis_basic(self): from ghost.ai.analyzer import AIAnalyzer + cfg = Config() cfg.openai_api_key = "" analyzer = AIAnalyzer(cfg) @@ -229,6 +233,7 @@ def test_fallback_analysis_basic(self): def test_fallback_no_findings(self): from ghost.ai.analyzer import AIAnalyzer + cfg = Config() cfg.openai_api_key = "" analyzer = AIAnalyzer(cfg) @@ -242,6 +247,7 @@ class TestSummarizerFallback: async def test_heuristic_summary(self): from ghost.ai.summarizer import Summarizer + cfg = Config() cfg.openai_api_key = "" summarizer = Summarizer(cfg)