diff --git a/.env.production.example b/.env.production.example index c656281..75f1664 100644 --- a/.env.production.example +++ b/.env.production.example @@ -1,4 +1,5 @@ POSTGRES_PASSWORD=replace-with-a-long-random-password API_AUTH_TOKEN=replace-with-a-long-random-bearer-token -STIX_FEED_URL=https://example.invalid/approved-stix-bundle.json +ABUSECH_AUTH_KEY=replace-with-your-abuse-ch-auth-key +THREATFOX_DAYS=7 PUBLIC_PORT=8080 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..39cffda --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + +jobs: + backend-tests: + name: Backend tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.13"] + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: backend/pyproject.toml + + - name: Install package with dev dependencies + run: python -m pip install -e '.[dev]' + + - name: Verify Alembic migrations apply cleanly (SQLite) + env: + DATABASE_URL: sqlite:///./ci.db + run: alembic upgrade head + + - name: Run test suite + run: pytest + + backend-image: + name: Build backend image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build API/worker image + run: docker build ./backend diff --git a/.gitignore b/.gitignore index 9f13b55..c8896a5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ htmlcov/ *.db *.sqlite *.sqlite3 +*.bak +*.backup diff --git a/BACKEND.md b/BACKEND.md index fab7d6e..d4c3ee2 100644 --- a/BACKEND.md +++ b/BACKEND.md @@ -30,18 +30,12 @@ docker compose up --build The API is available at `http://127.0.0.1:8001`. -## MITRE and Malpedia sync +## Feed synchronization -Run both source adapters once: +Run the actor adapters independently or together: ```bash cd backend -python -m app.worker --once --source all -``` - -Run a single adapter: - -```bash python -m app.worker --once --source mitre python -m app.worker --once --source malpedia ``` @@ -59,18 +53,40 @@ Malpedia actor endpoints do not require registration. If a token is provided, the backend sends `Authorization: apitoken ` as documented by Malpedia. Never put this token in `index.html` or another browser-delivered file. -The identity merge order is: +The actor identity merge order is source external ID, MITRE group ID, then +case-insensitive name or alias. Each actor can retain multiple source profiles. + +### ThreatFox IOC feed + +Obtain a free Auth-Key from and configure: + +```dotenv +THREATFOX_API_URL=https://threatfox-api.abuse.ch/api/v1/ +ABUSECH_AUTH_KEY=your-auth-key +THREATFOX_DAYS=7 +``` + +Then run: + +```bash +alembic upgrade head +python -m app.worker --once --source threatfox +``` + +The adapter requests recent IOCs, maps IP/port, domain, URL, and hash types, +and upserts by type and value. It preserves per-source IDs, confidence score, +malware name, threat type, tags, reference, first/last-seen times, last-sync +time, and a six-month expiration date. API responses exclude inactive or +expired indicators unless `include_inactive=true` is requested. + +Run every configured feed with: -1. Existing source external ID -2. MITRE ATT&CK group ID -3. Case-insensitive canonical name or alias -4. New actor profile +```bash +python -m app.worker --once --source all +``` -Each actor may have multiple `actor_sources` records. These retain the source -name, external ID, public profile URL, and last successful sync time. MITRE -contributes descriptions and ATT&CK technique relationships; Malpedia enriches -aliases, country/sponsor metadata when existing fields are unknown, and its -actor profile link. Empty source fields never erase populated actor fields. +If `ABUSECH_AUTH_KEY` is absent, an `all` run logs a warning and continues with +MITRE and Malpedia. An explicit `--source threatfox` run fails clearly instead. To run scheduled syncs in Docker: diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index d8ab703..ef3faaf 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,8 +1,8 @@ # Production deployment The production Compose stack runs four services: Nginx for the dashboard, -FastAPI for the API, PostgreSQL for persistence, and a scheduled STIX ingestion -worker. +FastAPI for the API, PostgreSQL for persistence, and a scheduled intelligence +ingestion worker for MITRE ATT&CK, Malpedia, and ThreatFox. ## Prepare secrets @@ -11,7 +11,9 @@ cp .env.production.example .env.production ``` Replace every placeholder. Use long random values for `POSTGRES_PASSWORD` and -`API_AUTH_TOKEN`, and configure only an approved STIX bundle URL. +`API_AUTH_TOKEN`. Obtain `ABUSECH_AUTH_KEY` from +. Keep all credentials in the backend environment or a +managed secret store; never place them in `index.html`. ## Start the stack @@ -26,8 +28,11 @@ authenticated API request and stores it only for the current browser session. ## Operational requirements - Terminate TLS in front of the published port before internet exposure. +- Confirm the abuse.ch community API fair-use terms fit the deployment; obtain + a commercial subscription when required. - Use a managed secret store instead of an environment file where available. - Back up the PostgreSQL volume and test restoration. - Pin and scan container images in the deployment environment. -- Restrict the STIX source allowlist and outbound network access. +- Restrict feed destinations and outbound network access. - Monitor `/api/health`, container restarts, and ingestion run failures. +- Validate confidence, age, and internal telemetry before blocking an IOC. diff --git a/README.md b/README.md index 15e4904..73e1eed 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ operational reporting. - Browse threat-actor profiles, aliases, confidence, and attribution - Merge MITRE ATT&CK and Malpedia metadata into one actor profile - Preserve per-source external IDs, profile links, and last-sync timestamps +- Ingest recent ThreatFox IOCs with confidence, malware, tags, and expiration - Explore campaigns by actor, sector, and status - Filter IP, domain, hash, and URL indicators - Review ATT&CK technique coverage and report metadata @@ -27,7 +28,7 @@ operational reporting. ## Architecture ```text -MITRE ATT&CK STIX Malpedia actor API +MITRE ATT&CK STIX Malpedia actor API ThreatFox IOC API \ / v v Source-aware ingestion and identity merge @@ -65,24 +66,42 @@ python3 -m http.server 8000 Open [http://127.0.0.1:8000](http://127.0.0.1:8000). API documentation is available at [http://127.0.0.1:8001/docs](http://127.0.0.1:8001/docs). -## Sync threat actors +## Sync live intelligence -With the backend environment active, fetch and merge both official sources: +MITRE and Malpedia can be synchronized without credentials: ```bash cd backend -python -m app.worker --once --source all +python -m app.worker --once --source mitre +python -m app.worker --once --source malpedia ``` -You can sync only one source with `--source mitre` or `--source malpedia`. -The default MITRE URL is the latest Enterprise ATT&CK STIX 2.1 bundle. The -Malpedia actor metadata endpoints are public; an optional `MALPEDIA_API_TOKEN` -is supported and is sent only by the backend. +ThreatFox requires a free abuse.ch Auth-Key. Create one through the +[abuse.ch Authentication Portal](https://auth.abuse.ch/), then place it in +`backend/.env` without committing it: + +```dotenv +ABUSECH_AUTH_KEY=your-auth-key +THREATFOX_DAYS=7 +``` + +Apply migrations and synchronize recent IOCs: + +```bash +alembic upgrade head +python -m app.worker --once --source threatfox +``` + +`--source all` synchronizes MITRE, Malpedia, and ThreatFox when the Auth-Key is +configured. Without the key, the worker safely skips ThreatFox. The IOC API +hides records after their six-month freshness window by default; use +`include_inactive=true` only for historical review. Source documentation: - [MITRE ATT&CK STIX data](https://github.com/mitre-attack/attack-stix-data) - [Malpedia API](https://malpedia.caad.fkie.fraunhofer.de/usage/api) +- [ThreatFox API](https://threatfox.abuse.ch/api/) ## Docker development @@ -90,7 +109,7 @@ Source documentation: docker compose up --build ``` -Include the scheduled MITRE and Malpedia sync worker with: +Include the scheduled actor and IOC synchronization worker with: ```bash docker compose --profile ingestion up --build @@ -118,7 +137,7 @@ pytest ``` The suite covers the API, seed collections, filters, authentication, frontend -contract, STIX normalization, source-aware actor merging, audit records, and +contract, STIX normalization, source-aware actor and IOC merging, freshness filtering, audit records, and idempotent ingestion. ## Project structure @@ -142,7 +161,8 @@ idempotent ingestion. - Validate attribution, timestamps, confidence, and provenance before use. - Do not automatically visit indicators or execute referenced files. - Keep API tokens and database credentials out of Git and in a secret store. -- Malpedia credentials are backend-only and are never sent to the dashboard. +- Feed credentials are backend-only and are never sent to the dashboard. +- Review confidence and age before operationally blocking any indicator. - Enable authentication, TLS, restricted CORS, and outbound allowlists in production. - Set `SEED_ON_STARTUP=false` before loading approved production data. - Back up PostgreSQL and monitor health and failed ingestion runs. diff --git a/backend/.env.example b/backend/.env.example index c041b77..d462f8a 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -8,6 +8,9 @@ API_AUTH_TOKEN= MITRE_STIX_URL=https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json MALPEDIA_BASE_URL=https://malpedia.caad.fkie.fraunhofer.de MALPEDIA_API_TOKEN= +THREATFOX_API_URL=https://threatfox-api.abuse.ch/api/v1/ +ABUSECH_AUTH_KEY= +THREATFOX_DAYS=7 STIX_FEED_URL= INGESTION_INTERVAL_SECONDS=3600 REQUEST_TIMEOUT_SECONDS=120 diff --git a/backend/alembic/versions/0004_threatfox_iocs.py b/backend/alembic/versions/0004_threatfox_iocs.py new file mode 100644 index 0000000..59dc78c --- /dev/null +++ b/backend/alembic/versions/0004_threatfox_iocs.py @@ -0,0 +1,75 @@ +"""Add live IOC context and multi-source provenance.""" + +from alembic import op +import sqlalchemy as sa + +revision = "0004_threatfox_iocs" +down_revision = "0003_actor_sources" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("indicators") as batch: + batch.add_column(sa.Column("confidence_score", sa.Integer(), nullable=True)) + batch.add_column(sa.Column("threat_type", sa.String(120), nullable=True)) + batch.add_column(sa.Column("malware", sa.String(160), nullable=True)) + batch.add_column(sa.Column("tags", sa.JSON(), nullable=False, server_default=sa.text("'[]'"))) + batch.add_column(sa.Column("reference_url", sa.String(1000), nullable=True)) + batch.add_column(sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True)) + batch.add_column(sa.Column("last_synced_at", sa.DateTime(timezone=True), nullable=True)) + batch.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true())) + op.create_index("ix_indicators_expires_at", "indicators", ["expires_at"]) + op.create_index("ix_indicators_is_active", "indicators", ["is_active"]) + + op.create_table( + "indicator_sources", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("indicator_id", sa.Integer(), sa.ForeignKey("indicators.id", ondelete="CASCADE"), nullable=False), + sa.Column("source_id", sa.Integer(), sa.ForeignKey("sources.id", ondelete="CASCADE"), nullable=False), + sa.Column("external_id", sa.String(160), nullable=False), + sa.Column("reference_url", sa.String(1000), nullable=True), + sa.Column("confidence_score", sa.Integer(), nullable=True), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_synced_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.UniqueConstraint("source_id", "external_id", name="uq_indicator_sources_external_identity"), + sa.UniqueConstraint("indicator_id", "source_id", name="uq_indicator_sources_indicator_source"), + ) + op.create_index("ix_indicator_sources_indicator_id", "indicator_sources", ["indicator_id"]) + op.create_index("ix_indicator_sources_source_id", "indicator_sources", ["source_id"]) + + connection = op.get_bind() + connection.execute( + sa.text( + """ + INSERT INTO indicator_sources ( + indicator_id, source_id, external_id, reference_url, + first_seen, last_seen, last_synced_at, is_active + ) + SELECT indicators.id, indicators.source_id, indicators.external_id, sources.url, + indicators.first_seen, indicators.last_seen, CURRENT_TIMESTAMP, TRUE + FROM indicators JOIN sources ON sources.id = indicators.source_id + WHERE indicators.source_id IS NOT NULL AND indicators.external_id IS NOT NULL + """ + ) + ) + + +def downgrade() -> None: + op.drop_index("ix_indicator_sources_source_id", table_name="indicator_sources") + op.drop_index("ix_indicator_sources_indicator_id", table_name="indicator_sources") + op.drop_table("indicator_sources") + op.drop_index("ix_indicators_is_active", table_name="indicators") + op.drop_index("ix_indicators_expires_at", table_name="indicators") + with op.batch_alter_table("indicators") as batch: + batch.drop_column("is_active") + batch.drop_column("last_synced_at") + batch.drop_column("expires_at") + batch.drop_column("reference_url") + batch.drop_column("tags") + batch.drop_column("malware") + batch.drop_column("threat_type") + batch.drop_column("confidence_score") diff --git a/backend/app/api.py b/backend/app/api.py index 9f8378b..c9785e5 100644 --- a/backend/app/api.py +++ b/backend/app/api.py @@ -1,10 +1,12 @@ +from datetime import datetime, timezone + from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import select, text +from sqlalchemy import or_, select, text from sqlalchemy.orm import Session, selectinload from .auth import require_api_token from .database import get_db -from .models import Actor, ActorSource, Campaign, Indicator, IngestionRun, Report, Technique +from .models import Actor, ActorSource, Campaign, Indicator, IndicatorSource, IngestionRun, Report, Technique from .schemas import ( ActorDetail, ActorSummary, @@ -30,6 +32,21 @@ def _actor_loading(): ) +def _indicator_loading(): + return ( + selectinload(Indicator.source), + selectinload(Indicator.source_profiles).selectinload(IndicatorSource.source), + ) + + +def _active_indicator_filter(): + now = datetime.now(timezone.utc) + return ( + Indicator.is_active.is_(True), + or_(Indicator.expires_at.is_(None), Indicator.expires_at > now), + ) + + @router.get("/health", response_model=HealthOut) def health(session: Session = Depends(get_db)) -> HealthOut: session.execute(text("SELECT 1")) @@ -77,19 +94,23 @@ def list_campaigns( def _indicator_out(item: Indicator) -> IndicatorOut: - return IndicatorOut.model_validate(item).model_copy( - update={"source_name": item.source.name if item.source else None} - ) + source_name = item.source.name if item.source else None + if item.source_profiles: + source_name = ", ".join(sorted({profile.source.name for profile in item.source_profiles})) + return IndicatorOut.model_validate(item).model_copy(update={"source_name": source_name}) @data_router.get("/iocs", response_model=list[IndicatorOut]) def list_indicators( indicator_type: str | None = Query(default=None, alias="type"), actor_id: str | None = None, + include_inactive: bool = False, limit: int = Query(default=100, ge=1, le=1000), session: Session = Depends(get_db), ) -> list[IndicatorOut]: - statement = select(Indicator).options(selectinload(Indicator.source)).order_by(Indicator.id.desc()).limit(limit) + statement = select(Indicator).options(*_indicator_loading()).order_by(Indicator.first_seen.desc()).limit(limit) + if not include_inactive: + statement = statement.where(*_active_indicator_filter()) if indicator_type: statement = statement.where(Indicator.type == indicator_type.upper()) if actor_id: @@ -113,7 +134,10 @@ def dashboard_bootstrap(session: Session = Depends(get_db)) -> DashboardBootstra campaigns = list(session.scalars(select(Campaign).order_by(Campaign.name))) indicators = list( session.scalars( - select(Indicator).options(selectinload(Indicator.source)).order_by(Indicator.id.desc()) + select(Indicator) + .options(*_indicator_loading()) + .where(*_active_indicator_filter()) + .order_by(Indicator.first_seen.desc()) ) ) techniques = list(session.scalars(select(Technique).order_by(Technique.tactic, Technique.id))) diff --git a/backend/app/config.py b/backend/app/config.py index cf0c9d1..1768cbb 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -19,6 +19,9 @@ class Settings(BaseSettings): ) malpedia_base_url: str = "https://malpedia.caad.fkie.fraunhofer.de" malpedia_api_token: str | None = None + threatfox_api_url: str = "https://threatfox-api.abuse.ch/api/v1/" + abusech_auth_key: str | None = None + threatfox_days: int = 7 stix_feed_url: str | None = None ingestion_interval_seconds: int = 3600 request_timeout_seconds: float = 120.0 diff --git a/backend/app/ingestion.py b/backend/app/ingestion.py index 08011ab..7017f1b 100644 --- a/backend/app/ingestion.py +++ b/backend/app/ingestion.py @@ -1,11 +1,11 @@ import re -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Iterable from sqlalchemy import func, or_, select from sqlalchemy.orm import Session -from .models import Actor, ActorAlias, ActorSource, Indicator, IngestionRun, Source, Technique +from .models import Actor, ActorAlias, ActorSource, Indicator, IndicatorSource, IngestionRun, Source, Technique COUNTRY_NAMES = { @@ -398,3 +398,182 @@ def ingest_malpedia_actors( except Exception as exc: _fail_run(session, run_id, exc) raise + + +THREATFOX_SOURCE_NAME = "ThreatFox" +THREATFOX_SOURCE_URL = "https://threatfox.abuse.ch/" +THREATFOX_RETENTION_DAYS = 180 + + +def _threatfox_type(ioc_type: Any) -> str | None: + normalized = str(ioc_type or "").strip().lower() + if normalized in {"ip", "ip:port"}: + return "IP" + if normalized in {"domain", "hostname"}: + return "DOMAIN" + if normalized == "url": + return "URL" + if normalized in {"md5_hash", "sha1_hash", "sha256_hash", "sha512_hash", "hash"}: + return "HASH" + return None + + +def _threatfox_time(value: Any) -> datetime | None: + if not isinstance(value, str) or not value.strip(): + return None + cleaned = value.strip() + try: + if cleaned.endswith(" UTC"): + return datetime.strptime(cleaned, "%Y-%m-%d %H:%M:%S UTC").replace(tzinfo=timezone.utc) + parsed = datetime.fromisoformat(cleaned.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + except ValueError: + return None + + +def _utc(value: datetime) -> datetime: + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) + + +def _confidence(score: int | None) -> str: + if score is None: + return "Medium" + if score >= 80: + return "High" + if score >= 50: + return "Medium" + return "Low" + + +def _safe_reference(value: Any) -> str | None: + if not isinstance(value, str): + return None + reference = value.strip()[:1000] + return reference if reference.startswith(("https://", "http://")) else None + + +def _record_indicator_source( + session: Session, + *, + indicator: Indicator, + source: Source, + external_id: str, + reference_url: str | None, + confidence_score: int | None, + first_seen: datetime | None, + last_seen: datetime | None, + expires_at: datetime, + synced_at: datetime, +) -> IndicatorSource: + profile = session.scalar( + select(IndicatorSource).where( + IndicatorSource.indicator_id == indicator.id, + IndicatorSource.source_id == source.id, + ) + ) + if profile is None: + profile = IndicatorSource(indicator=indicator, source=source, external_id=external_id, last_synced_at=synced_at) + session.add(profile) + profile.external_id = external_id + profile.reference_url = reference_url + profile.confidence_score = confidence_score + profile.first_seen = first_seen + profile.last_seen = last_seen + profile.expires_at = expires_at + profile.last_synced_at = synced_at + profile.is_active = True + return profile + + +def ingest_threatfox_iocs( + session: Session, + payload: dict[str, Any], + *, + source_name: str = THREATFOX_SOURCE_NAME, + source_url: str = THREATFOX_SOURCE_URL, +) -> IngestionRun: + records = payload.get("data") + if payload.get("query_status") != "ok" or not isinstance(records, list): + raise ValueError(f"ThreatFox query failed: {payload.get('query_status', 'invalid response')}") + + source, run_id = _source_and_run( + session, source_name=source_name, source_url=source_url, items_seen=len(records) + ) + written = 0 + synced_at = datetime.now(timezone.utc) + try: + for record in records: + if not isinstance(record, dict): + continue + external_id = str(record.get("id") or "").strip()[:160] + value = str(record.get("ioc") or "").strip()[:1000] + indicator_type = _threatfox_type(record.get("ioc_type")) + if not external_id or not value or not indicator_type: + continue + + profile = session.scalar( + select(IndicatorSource).where( + IndicatorSource.source_id == source.id, + IndicatorSource.external_id == external_id, + ) + ) + indicator = profile.indicator if profile else session.scalar( + select(Indicator).where(Indicator.type == indicator_type, Indicator.value == value) + ) + score_value = record.get("confidence_level") + try: + confidence_score = max(0, min(int(score_value), 100)) if score_value is not None else None + except (TypeError, ValueError): + confidence_score = None + first_seen = _threatfox_time(record.get("first_seen")) + last_seen = _threatfox_time(record.get("last_seen")) + expires_at = (first_seen or synced_at) + timedelta(days=THREATFOX_RETENTION_DAYS) + reference_url = _safe_reference(record.get("reference")) + tags = _clean_names(_as_list(record.get("tags"))) + malware = str(record.get("malware_printable") or record.get("malware") or "").strip()[:160] or None + threat_type = str(record.get("threat_type") or "").strip()[:120] or None + + if indicator is None: + indicator = Indicator( + type=indicator_type, + value=value, + external_id=f"threatfox:{external_id}", + source=source, + confidence=_confidence(confidence_score), + tags=[], + ) + session.add(indicator) + session.flush() + elif not indicator.external_id or indicator.external_id.startswith("demo:"): + indicator.external_id = f"threatfox:{external_id}" + + indicator.confidence = _confidence(confidence_score) + indicator.confidence_score = confidence_score + indicator.threat_type = threat_type or indicator.threat_type + indicator.malware = malware or indicator.malware + indicator.tags = sorted(set([*(indicator.tags or []), *tags]), key=str.casefold) + indicator.reference_url = reference_url or indicator.reference_url + if first_seen and (indicator.first_seen is None or _utc(first_seen) < _utc(indicator.first_seen)): + indicator.first_seen = first_seen + if last_seen and (indicator.last_seen is None or _utc(last_seen) > _utc(indicator.last_seen)): + indicator.last_seen = last_seen + indicator.expires_at = expires_at + indicator.last_synced_at = synced_at + indicator.is_active = True + _record_indicator_source( + session, + indicator=indicator, + source=source, + external_id=external_id, + reference_url=reference_url, + confidence_score=confidence_score, + first_seen=first_seen, + last_seen=last_seen, + expires_at=expires_at, + synced_at=synced_at, + ) + written += 1 + return _finish_run(session, run_id, written) + except Exception as exc: + _fail_run(session, run_id, exc) + raise diff --git a/backend/app/models.py b/backend/app/models.py index 26d91fe..697ddc2 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,6 +1,6 @@ from datetime import date, datetime -from sqlalchemy import Column, Date, DateTime, ForeignKey, String, Table, Text, UniqueConstraint +from sqlalchemy import Boolean, Column, Date, DateTime, ForeignKey, Integer, JSON, String, Table, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from .database import Base @@ -96,6 +96,9 @@ class Source(Base): actor_profiles: Mapped[list["ActorSource"]] = relationship( back_populates="source", cascade="all, delete-orphan" ) + indicator_profiles: Mapped[list["IndicatorSource"]] = relationship( + back_populates="source", cascade="all, delete-orphan" + ) class ActorSource(Base): @@ -131,10 +134,48 @@ class Indicator(Base): actor_id: Mapped[str | None] = mapped_column(ForeignKey("actors.id", ondelete="SET NULL"), index=True) source_id: Mapped[int | None] = mapped_column(ForeignKey("sources.id", ondelete="SET NULL")) confidence: Mapped[str] = mapped_column(String(20), nullable=False) + confidence_score: Mapped[int | None] = mapped_column(Integer) + threat_type: Mapped[str | None] = mapped_column(String(120)) + malware: Mapped[str | None] = mapped_column(String(160)) + tags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + reference_url: Mapped[str | None] = mapped_column(String(1000)) first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) + last_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) actor: Mapped[Actor | None] = relationship(back_populates="indicators") source: Mapped[Source | None] = relationship(back_populates="indicators") + source_profiles: Mapped[list["IndicatorSource"]] = relationship( + back_populates="indicator", cascade="all, delete-orphan" + ) + + +class IndicatorSource(Base): + __tablename__ = "indicator_sources" + __table_args__ = ( + UniqueConstraint("source_id", "external_id", name="uq_indicator_sources_external_identity"), + UniqueConstraint("indicator_id", "source_id", name="uq_indicator_sources_indicator_source"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + indicator_id: Mapped[int] = mapped_column(ForeignKey("indicators.id", ondelete="CASCADE"), index=True) + source_id: Mapped[int] = mapped_column(ForeignKey("sources.id", ondelete="CASCADE"), index=True) + external_id: Mapped[str] = mapped_column(String(160), nullable=False) + reference_url: Mapped[str | None] = mapped_column(String(1000)) + confidence_score: Mapped[int | None] = mapped_column(Integer) + first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_synced_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + + indicator: Mapped[Indicator] = relationship(back_populates="source_profiles") + source: Mapped[Source] = relationship(back_populates="indicator_profiles") + + @property + def source_name(self) -> str: + return self.source.name class Report(Base): diff --git a/backend/app/schemas.py b/backend/app/schemas.py index e04b413..4717158 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -60,15 +60,36 @@ class CampaignOut(APIModel): last_seen: date | None +class IndicatorSourceOut(APIModel): + source_name: str + external_id: str + reference_url: str | None + confidence_score: int | None + first_seen: datetime | None + last_seen: datetime | None + expires_at: datetime | None + last_synced_at: datetime + is_active: bool + + class IndicatorOut(APIModel): id: int type: str value: str actor_id: str | None confidence: str + confidence_score: int | None = None + threat_type: str | None = None + malware: str | None = None + tags: list[str] = Field(default_factory=list) + reference_url: str | None = None first_seen: datetime | None last_seen: datetime | None + expires_at: datetime | None = None + last_synced_at: datetime | None = None + is_active: bool = True source_name: str | None = None + source_profiles: list[IndicatorSourceOut] = Field(default_factory=list) class ReportOut(APIModel): diff --git a/backend/app/seed.py b/backend/app/seed.py index a266f33..68d05ba 100644 --- a/backend/app/seed.py +++ b/backend/app/seed.py @@ -3,7 +3,7 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session -from .models import Actor, ActorAlias, ActorEvent, Campaign, Indicator, Report, Source, Technique +from .models import Actor, ActorAlias, ActorEvent, Campaign, Indicator, IndicatorSource, Report, Source, Technique ACTORS = [ @@ -141,18 +141,29 @@ def seed_database(session: Session) -> bool: observed = datetime(2026, 7, 30, tzinfo=timezone.utc) for number, (indicator_type, value, actor_id, confidence, source_name) in enumerate(INDICATORS, start=1): - session.add( - Indicator( - type=indicator_type, - value=value, - external_id=f"demo:ioc:{number}", - actor_id=actor_id, + external_id = f"demo:ioc:{number}" + indicator = Indicator( + type=indicator_type, + value=value, + external_id=external_id, + actor_id=actor_id, + source=sources[source_name], + confidence=confidence, + tags=[], + first_seen=observed, + last_seen=observed, + ) + indicator.source_profiles.append( + IndicatorSource( source=sources[source_name], - confidence=confidence, + external_id=external_id, first_seen=observed, last_seen=observed, + last_synced_at=observed, + is_active=True, ) ) + session.add(indicator) for title, report_type, related, published, report_format in REPORTS: session.add( diff --git a/backend/app/worker.py b/backend/app/worker.py index cf3c354..b738b82 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -9,7 +9,7 @@ from .config import settings from .database import Base, SessionLocal, engine -from .ingestion import ingest_malpedia_actors, ingest_stix_bundle +from .ingestion import ingest_malpedia_actors, ingest_stix_bundle, ingest_threatfox_iocs logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("argus.ingestion") @@ -29,6 +29,23 @@ def fetch_json(url: str, *, api_token: str | None = None) -> Any: return response.json() +def fetch_threatfox_payload() -> dict[str, Any]: + if not settings.abusech_auth_key: + raise ValueError("Set ABUSECH_AUTH_KEY before syncing ThreatFox") + response = httpx.post( + settings.threatfox_api_url, + headers={"Accept": "application/json", "Auth-Key": settings.abusech_auth_key}, + json={"query": "get_iocs", "days": max(1, min(settings.threatfox_days, 7))}, + timeout=settings.request_timeout_seconds, + follow_redirects=True, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("ThreatFox returned an unexpected response") + return payload + + def load_bundle(*, file_path: str | None, url: str | None) -> tuple[dict[str, Any], str, str | None]: if file_path: path = Path(file_path) @@ -54,6 +71,13 @@ def run_malpedia() -> None: logger.info("ingestion completed source=Malpedia seen=%s written=%s", run.items_seen, run.items_written) +def run_threatfox() -> None: + payload = fetch_threatfox_payload() + with SessionLocal() as session: + run = ingest_threatfox_iocs(session, payload) + logger.info("ingestion completed source=ThreatFox seen=%s written=%s", run.items_seen, run.items_written) + + def run_selected(source: str, *, file_path: str | None = None, url: str | None = None) -> None: if file_path or source == "stix": run_stix(file_path=file_path, url=url or settings.stix_feed_url) @@ -63,6 +87,10 @@ def run_selected(source: str, *, file_path: str | None = None, url: str | None = tasks.append(("MITRE ATT&CK", lambda: run_stix(url=url or settings.mitre_stix_url, source_name="MITRE ATT&CK"))) if source in {"all", "malpedia"}: tasks.append(("Malpedia", run_malpedia)) + if source == "threatfox" or (source == "all" and settings.abusech_auth_key): + tasks.append(("ThreatFox", run_threatfox)) + elif source == "all": + logger.warning("skipping ThreatFox because ABUSECH_AUTH_KEY is not configured") failures = [] for name, task in tasks: try: @@ -76,10 +104,14 @@ def run_selected(source: str, *, file_path: str | None = None, url: str | None = def main() -> None: - parser = argparse.ArgumentParser(description="Sync MITRE ATT&CK and Malpedia actor data into ARGUS TI") - parser.add_argument("--source", choices=("all", "mitre", "malpedia", "stix"), default="all") + parser = argparse.ArgumentParser(description="Sync actor and IOC intelligence into ARGUS TI") + parser.add_argument( + "--source", + choices=("all", "mitre", "malpedia", "threatfox", "stix"), + default="all", + ) parser.add_argument("--file", help="Local STIX bundle JSON file") - parser.add_argument("--url", help="Override the selected remote URL") + parser.add_argument("--url", help="Override the selected STIX URL") parser.add_argument("--once", action="store_true", help="Run once instead of on an interval") parser.add_argument("--interval", type=int, default=settings.ingestion_interval_seconds) args = parser.parse_args() diff --git a/backend/tests/test_frontend_contract.py b/backend/tests/test_frontend_contract.py index b6bba4a..d2bc0b6 100644 --- a/backend/tests/test_frontend_contract.py +++ b/backend/tests/test_frontend_contract.py @@ -7,3 +7,7 @@ def test_dashboard_loads_the_backend_bootstrap_endpoint(): assert "/dashboard/bootstrap" in source assert "Authorization" in source assert "backendStatus" in source + assert "indicator.threat_type" in source + assert "indicator.last_synced_at" in source + assert "Expired indicators are hidden by the API" in source + assert "ABUSECH_AUTH_KEY" not in source diff --git a/backend/tests/test_ioc_multi_source.py b/backend/tests/test_ioc_multi_source.py new file mode 100644 index 0000000..3af90d2 --- /dev/null +++ b/backend/tests/test_ioc_multi_source.py @@ -0,0 +1,34 @@ +from sqlalchemy import select + +from app.ingestion import ingest_threatfox_iocs +from app.models import Indicator, IndicatorSource + + +def test_threatfox_enriches_existing_indicator_without_losing_provenance(db_session): + payload = { + "query_status": "ok", + "data": [ + { + "id": "existing-demo-ioc", + "ioc": "185.220.101.42", + "ioc_type": "ip", + "threat_type": "botnet_cc", + "malware_printable": "Example RAT", + "confidence_level": 90, + "first_seen": "2026-08-04 00:00:00 UTC", + "tags": ["C2"], + } + ], + } + + ingest_threatfox_iocs(db_session, payload) + matches = db_session.scalars( + select(Indicator).where(Indicator.type == "IP", Indicator.value == "185.220.101.42") + ).all() + assert len(matches) == 1 + profiles = db_session.scalars( + select(IndicatorSource).where(IndicatorSource.indicator_id == matches[0].id) + ).all() + assert {profile.source.name for profile in profiles} == {"Internal Telemetry", "ThreatFox"} + assert matches[0].malware == "Example RAT" + assert matches[0].confidence_score == 90 diff --git a/backend/tests/test_threatfox.py b/backend/tests/test_threatfox.py new file mode 100644 index 0000000..1bb3090 --- /dev/null +++ b/backend/tests/test_threatfox.py @@ -0,0 +1,113 @@ +from datetime import datetime, timezone + +from sqlalchemy import select + +from app.config import settings +from app.ingestion import ingest_threatfox_iocs +from app.models import Indicator, IndicatorSource, IngestionRun +from app.worker import fetch_threatfox_payload + + +THREATFOX_PAYLOAD = { + "query_status": "ok", + "data": [ + { + "id": "90001", + "ioc": "198.51.100.42:443", + "threat_type": "botnet_cc", + "threat_type_desc": "Botnet command-and-control server", + "ioc_type": "ip:port", + "malware": "win.cobalt_strike", + "malware_printable": "Cobalt Strike", + "confidence_level": 75, + "first_seen": "2026-08-03 12:00:00 UTC", + "last_seen": None, + "reference": "https://example.test/report/90001", + "tags": ["C2", "test"], + }, + { + "id": "90002", + "ioc": "a" * 64, + "threat_type": "payload_delivery", + "ioc_type": "sha256_hash", + "malware": "win.test", + "malware_printable": "Test Malware", + "confidence_level": 100, + "first_seen": "2026-08-04 01:02:03 UTC", + "last_seen": "2026-08-04 02:03:04 UTC", + "reference": None, + "tags": None, + }, + ], +} + + +def test_threatfox_ingestion_is_contextual_and_idempotent(db_session): + first = ingest_threatfox_iocs(db_session, THREATFOX_PAYLOAD) + assert first.status == "completed" + assert first.items_seen == 2 + assert first.items_written == 2 + + ip = db_session.scalar(select(Indicator).where(Indicator.value == "198.51.100.42:443")) + assert ip is not None + assert ip.type == "IP" + assert ip.confidence == "Medium" + assert ip.confidence_score == 75 + assert ip.threat_type == "botnet_cc" + assert ip.malware == "Cobalt Strike" + assert ip.tags == ["C2", "test"] + assert ip.reference_url == "https://example.test/report/90001" + assert ip.expires_at > ip.first_seen + assert ip.is_active is True + + profiles = db_session.scalars(select(IndicatorSource).where(IndicatorSource.indicator_id == ip.id)).all() + assert len(profiles) == 1 + assert profiles[0].source.name == "ThreatFox" + assert profiles[0].external_id == "90001" + + second = ingest_threatfox_iocs(db_session, THREATFOX_PAYLOAD) + assert second.status == "completed" + assert len(db_session.scalars(select(Indicator).where(Indicator.value == "198.51.100.42:443")).all()) == 1 + assert len(db_session.scalars(select(IndicatorSource).where(IndicatorSource.indicator_id == ip.id)).all()) == 1 + assert len(db_session.scalars(select(IngestionRun).where(IngestionRun.source.has(name="ThreatFox"))).all()) == 2 + + +def test_ioc_api_returns_threatfox_context_and_hides_expired(client, db_session): + ingest_threatfox_iocs(db_session, THREATFOX_PAYLOAD) + response = client.get("/api/iocs", params={"type": "ip"}) + assert response.status_code == 200 + item = next(row for row in response.json() if row["value"] == "198.51.100.42:443") + assert item["malware"] == "Cobalt Strike" + assert item["confidence_score"] == 75 + assert item["source_name"] == "ThreatFox" + assert item["source_profiles"][0]["external_id"] == "90001" + + indicator = db_session.scalar(select(Indicator).where(Indicator.value == "198.51.100.42:443")) + indicator.expires_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + db_session.commit() + assert all(row["value"] != indicator.value for row in client.get("/api/iocs").json()) + inactive = client.get("/api/iocs", params={"include_inactive": "true"}).json() + assert any(row["value"] == indicator.value for row in inactive) + + +def test_threatfox_client_keeps_auth_key_in_backend(monkeypatch): + observed = {} + + class Response: + def raise_for_status(self): + return None + + def json(self): + return THREATFOX_PAYLOAD + + def fake_post(url, **kwargs): + observed["url"] = url + observed.update(kwargs) + return Response() + + monkeypatch.setattr(settings, "abusech_auth_key", "test-auth-key") + monkeypatch.setattr(settings, "threatfox_days", 99) + monkeypatch.setattr("app.worker.httpx.post", fake_post) + assert fetch_threatfox_payload() == THREATFOX_PAYLOAD + assert observed["headers"]["Auth-Key"] == "test-auth-key" + assert observed["json"] == {"query": "get_iocs", "days": 7} diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 479f3e9..33962ec 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -38,6 +38,9 @@ services: MITRE_STIX_URL: ${MITRE_STIX_URL:-https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json} MALPEDIA_BASE_URL: ${MALPEDIA_BASE_URL:-https://malpedia.caad.fkie.fraunhofer.de} MALPEDIA_API_TOKEN: ${MALPEDIA_API_TOKEN:-} + THREATFOX_API_URL: ${THREATFOX_API_URL:-https://threatfox-api.abuse.ch/api/v1/} + ABUSECH_AUTH_KEY: ${ABUSECH_AUTH_KEY:?set ABUSECH_AUTH_KEY} + THREATFOX_DAYS: ${THREATFOX_DAYS:-7} INGESTION_INTERVAL_SECONDS: ${INGESTION_INTERVAL_SECONDS:-3600} depends_on: api: diff --git a/docker-compose.yml b/docker-compose.yml index d3d64d9..dc6c0ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,9 @@ services: MITRE_STIX_URL: ${MITRE_STIX_URL:-https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json} MALPEDIA_BASE_URL: ${MALPEDIA_BASE_URL:-https://malpedia.caad.fkie.fraunhofer.de} MALPEDIA_API_TOKEN: ${MALPEDIA_API_TOKEN:-} + THREATFOX_API_URL: ${THREATFOX_API_URL:-https://threatfox-api.abuse.ch/api/v1/} + ABUSECH_AUTH_KEY: ${ABUSECH_AUTH_KEY:-} + THREATFOX_DAYS: ${THREATFOX_DAYS:-7} INGESTION_INTERVAL_SECONDS: ${INGESTION_INTERVAL_SECONDS:-3600} depends_on: api: diff --git a/index.html b/index.html index f182a28..0d656b5 100644 --- a/index.html +++ b/index.html @@ -752,38 +752,51 @@ return `
${t}
`; }).join(''); const rows = iocsGlobal.filter(i=>state.iocTypeFilter==='All'||i.type===state.iocTypeFilter); - const bodyHtml = rows.length ? rows.map(i=>` -
+ const latestSync = iocsGlobal.map(i=>i.syncedAt).filter(Boolean).sort().at(-1); + const freshness = latestSync ? `Last synchronized ${displayDateTime(latestSync)}` : 'Bundled demonstration data'; + const bodyHtml = rows.length ? rows.map(i=>{ + const expired = i.expiresAt && new Date(i.expiresAt).getTime() <= Date.now(); + const active = i.isActive!==false && !expired; + const context = i.malware||i.threatType||'No malware attribution'; + const detail = [i.threatType, ...(i.tags||[]).slice(0,2)].filter(value=>value&&value!==context).join(' · '); + return ` +
${i.type}
-
${esc(i.value)}
-
${i.confidence}
-
${esc(i.actor)}
-
${esc(i.source)}
-
${esc(i.firstSeen)}
-
${esc(i.lastSeen)}
-
`).join('') : ` +
+
${esc(i.value)}
+
${esc(i.actor)}
+
+
+
${esc(context)}
+
${esc(detail)}
+
+
${i.confidence}${Number.isInteger(i.confidenceScore)?` · ${i.confidenceScore}`:''}
+
${esc(i.source)}
+
${esc(i.firstSeen||'Not reported')}
+
${active?'ACTIVE':'EXPIRED'}
+
`; + }).join('') : `
-
No indicators match this filter
-
Try a different indicator type or clear the search.
+
No active indicators match this filter
+
Try a different indicator type.
`; return `
-
Indicator Search
-
3,214 indicators tracked across all sources
+
Live Indicators
+
${iocsGlobal.length} active indicators · ${esc(freshness)}
-
- - Search by IP, domain, hash, or URL… +
+
${chips}
+
Expired indicators are hidden by the API
-
${chips}
-
-
-
Type
Indicator
Confidence
Associated Actor
Source
First Seen
Last Seen
+
+
+
Type
Indicator
Malware / Threat
Confidence
Source
First Seen
Status
${bodyHtml}
@@ -963,6 +976,11 @@ const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? String(value) : parsed.toLocaleDateString('en-US',{month:'short',day:'2-digit',year:'numeric'}); } +function displayDateTime(value){ + if(!value) return ''; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? String(value) : parsed.toLocaleString('en-US',{month:'short',day:'2-digit',hour:'numeric',minute:'2-digit'}); +} function storedApiToken(){ try { return globalThis.sessionStorage?.getItem('argusApiToken') || ''; } @@ -1042,9 +1060,17 @@ value:indicator.value, actor:actorNames.get(indicator.actor_id)||'Unattributed', confidence:indicator.confidence, + confidenceScore:indicator.confidence_score, + threatType:indicator.threat_type, + malware:indicator.malware, + tags:indicator.tags||[], source:indicator.source_name||'Unknown', + sourceProfiles:indicator.source_profiles||[], firstSeen:displayDate(indicator.first_seen), - lastSeen:displayDate(indicator.last_seen) + lastSeen:displayDate(indicator.last_seen), + expiresAt:indicator.expires_at, + syncedAt:indicator.last_synced_at, + isActive:indicator.is_active })); reportsList = data.reports.map(report=>({