Add prometheus metrics

.env.example

@@ -5,3 +5,10 @@ NOTIFICATION_TELEGRAM_BOT_TOKEN=secret
 NOTIFICATION_TELEGRAM_CHAT_ID=secret
 NOTIFICATION_SLACK_WEBHOOK=https://hooks.slack.com/services/secret/secret/secret
 NOTIFICATION_GENERIC_WEBHOOK=http://host:port/path
+
+# Prometheus metrics
+METRICS_ENABLED=false
+METRICS_PORT=8000
+METRICS_ADDRESS=0.0.0.0
+
+LOG_LEVEL=INFO

README.md

@@ -31,6 +31,9 @@ docker run \
     -e NOTIFICATION_TELEGRAM_CHAT_ID="secret" \
     -e NOTIFICATION_SLACK_WEBHOOK="https://hooks.slack.com/services/secret/secret/secret" \
     -e NOTIFICATION_GENERIC_WEBHOOK="http://host:port/path" \
+    -e METRICS_ENABLED="true" \
+    -e METRICS_PORT="8000" \
+    -e LOG_LEVEL="INFO" \
     ghcr.io/flare-foundation/fsp-observer:main
 ```
 
@@ -44,3 +47,46 @@ RPC_BASE_URL="https://flare-api.flare.network" \
   IDENTITY_ADDRESS="0x0000000000000000000000000000000000000000" \
   python main.py
 ```
+
+## Environment variables
+
+| Variable | Required | Default | Description |
+|---|---|---|---|
+| `RPC_BASE_URL` | yes | - | RPC base URL without `/ext/bc/C/rpc` suffix |
+| `IDENTITY_ADDRESS` | yes | - | Identity address of the observed entity |
+| `FEE_THRESHOLD` | no | `25` | Balance threshold in FLR to trigger low balance warning |
+| `NOTIFICATION_DISCORD_WEBHOOK` | no | - | Discord webhook URL (comma-separated for multiple) |
+| `NOTIFICATION_DISCORD_EMBED_WEBHOOK` | no | - | Discord embed webhook URL |
+| `NOTIFICATION_SLACK_WEBHOOK` | no | - | Slack webhook URL |
+| `NOTIFICATION_TELEGRAM_BOT_TOKEN` | no | - | Telegram bot token (comma-separated for multiple) |
+| `NOTIFICATION_TELEGRAM_CHAT_ID` | no | - | Telegram chat ID (comma-separated, paired with bot tokens) |
+| `NOTIFICATION_GENERIC_WEBHOOK` | no | - | Generic HTTP POST webhook URL |
+| `METRICS_ENABLED` | no | `false` | Enable Prometheus metrics endpoint |
+| `METRICS_PORT` | no | `8000` | Prometheus metrics server port |
+| `METRICS_ADDRESS` | no | `0.0.0.0` | Prometheus metrics server bind address |
+| `LOG_LEVEL` | no | `INFO` | Logging level (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
+
+## Prometheus metrics
+
+When `METRICS_ENABLED=true`, metrics are exposed at `http://host:METRICS_PORT/metrics`.
+
+| Metric | Type | Description |
+|---|---|---|
+| `flare_fsp_submit_ok_total` | Counter | Successful submissions per protocol/phase |
+| `flare_fsp_submit_late_total` | Counter | Late submissions per protocol/phase |
+| `flare_fsp_submit_early_total` | Counter | Early submissions per protocol/phase |
+| `flare_fsp_submit_missing_total` | Counter | Missing submissions per protocol/phase |
+| `flare_fsp_address_balance_wei` | Gauge | Address balance in wei per role |
+| `flare_fsp_registered_current_epoch` | Gauge | 1 if registered in current reward epoch |
+| `flare_fsp_registered_next_epoch` | Gauge | 1 if registered for next reward epoch |
+| `flare_fsp_voting_round` | Gauge | Current voting round ID |
+| `flare_fsp_reward_epoch` | Gauge | Current reward epoch ID |
+| `flare_fsp_node_uptime_ratio` | Gauge | Node uptime ratio per node ID |
+| `flare_fsp_fast_update_blocks_since_last` | Gauge | Blocks since last fast update submission |
+| `flare_fsp_ftso_anchor_feeds_success_rate_bips` | Gauge | FTSO anchor feeds success rate in bips |
+| `flare_fsp_fdc_participation_rate_bips` | Gauge | FDC participation rate in bips |
+| `flare_fsp_reveal_offence_total` | Counter | Reveal offences per protocol |
+| `flare_fsp_signature_grace_period_missed_total` | Counter | Signature submissions past grace period |
+| `flare_fsp_signature_mismatch_total` | Counter | Signature mismatches per protocol |
+| `flare_fsp_contract_address_wrong_total` | Counter | Wrong contract address detections |
+| `flare_fsp_unclaimed_rewards_wei` | Gauge | Unclaimed reward amount in wei |

configuration/config.py

@@ -8,6 +8,7 @@
     Configuration,
     Contracts,
     Epoch,
+    MetricsConfig,
     Notification,
     NotificationDiscord,
     NotificationGeneric,
@@ -132,6 +133,13 @@ def get_notification_config() -> Notification:
     )
 
 
+def get_metrics_config() -> MetricsConfig:
+    enabled = os.environ.get("METRICS_ENABLED", "false").lower() == "true"
+    port = int(os.environ.get("METRICS_PORT", "8000"))
+    address = os.environ.get("METRICS_ADDRESS", "0.0.0.0")
+    return MetricsConfig(enabled=enabled, port=port, address=address)
+
+
 def get_config() -> Configuration:
     rpc_base_url = os.environ.get("RPC_BASE_URL")
     if rpc_base_url is None:
@@ -155,6 +163,8 @@ def get_config() -> Configuration:
     _fee_threshold = os.environ.get("FEE_THRESHOLD", "25")
     fee_threshold = int(_fee_threshold)
 
+    log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
+
     config = Configuration(
         rpc_url=rpc_url,
         p_chain_rpc_url=p_chain_rpc_url,
@@ -164,6 +174,8 @@ def get_config() -> Configuration:
         epoch=get_epoch(chain_id),
         notification=get_notification_config(),
         fee_threshold=fee_threshold,
+        metrics=get_metrics_config(),
+        log_level=log_level,
     )
 
     return config

configuration/types.py

@@ -1,6 +1,7 @@
 import json
 import re
-from typing import Callable, Self
+from collections.abc import Callable
+from typing import Self
 
 from attrs import field, frozen
 from eth_typing import ABI, ABIEvent, ABIFunction, ChecksumAddress
@@ -250,6 +251,13 @@ class Notification:
     generic: NotificationGeneric
 
 
+@frozen
+class MetricsConfig:
+    enabled: bool
+    port: int
+    address: str
+
+
 @frozen
 class Configuration:
     identity_address: ChecksumAddress
@@ -260,3 +268,5 @@ class Configuration:
     epoch: Epoch
     notification: Notification
     fee_threshold: int
+    metrics: MetricsConfig
+    log_level: str

observer/address.py

@@ -4,6 +4,7 @@
 from web3 import AsyncWeb3
 
 from configuration.types import Configuration
+from observer import metrics
 from observer.message import Message, MessageLevel
 
 
@@ -20,6 +21,9 @@ async def check_addresses(
 
         for name, addr in address_list:
             balance = await w.eth.get_balance(addr, "latest")
+            metrics.ADDRESS_BALANCE.labels(
+                identity_address=metrics._ia, address=addr, role=name
+            ).set(balance)
             if balance < config.fee_threshold * 1e18:
                 level = MessageLevel.WARNING
                 if balance <= 5e18:

observer/contract_manager.py

@@ -3,6 +3,7 @@
 from attrs import define
 
 from configuration.types import Contract, Contracts
+from observer import metrics
 from observer.message import Message, MessageLevel
 
 
@@ -21,6 +22,9 @@ def check_submission_address(self, address) -> Sequence[Message]:
         mb = Message.builder()
         messages = []
         if address != self.contracts.Submission.address:
+            metrics.CONTRACT_ADDRESS_WRONG.labels(
+                identity_address=metrics._ia, contract="submission"
+            ).inc()
             messages.append(
                 mb.build(MessageLevel.CRITICAL, "Incorrect Submmission address")
             )
@@ -30,5 +34,8 @@ def check_relay_address(self, address) -> Sequence[Message]:
         mb = Message.builder()
         messages = []
         if address != self.contracts.Relay.address:
+            metrics.CONTRACT_ADDRESS_WRONG.labels(
+                identity_address=metrics._ia, contract="relay"
+            ).inc()
             messages.append(mb.build(MessageLevel.CRITICAL, "Incorrect Relay address"))
         return messages

observer/metrics.py

@@ -0,0 +1,176 @@
+from prometheus_client import Counter, Gauge, start_http_server
+
+# Identity address set once at startup via setup()
+_ia: str = ""
+
+
+def setup(identity_address: str) -> None:
+    global _ia
+    _ia = identity_address
+
+
+# ---------------------------------------------------------------------------
+# State
+# ---------------------------------------------------------------------------
+
+VOTING_ROUND = Gauge(
+    "flare_fsp_voting_round_current",
+    "Current voting round ID",
+)
+
+REWARD_EPOCH = Gauge(
+    "flare_fsp_reward_epoch_current",
+    "Current reward epoch ID",
+)
+
+REGISTERED_CURRENT_EPOCH = Gauge(
+    "flare_fsp_registered_current_epoch",
+    "Whether the entity is in the active signing policy for the current epoch (0 or 1)",
+    ["identity_address"],
+)
+
+REGISTERED_NEXT_EPOCH = Gauge(
+    "flare_fsp_registered_next_epoch",
+    "Whether the entity has registered for the next epoch (0 or 1)",
+    ["identity_address"],
+)
+
+# ---------------------------------------------------------------------------
+# Submissions — Counters per protocol (ftso, fdc) and phase
+# (submit1, submit2, signatures)
+# ---------------------------------------------------------------------------
+
+SUBMIT_OK = Counter(
+    "flare_fsp_submit_ok_total",
+    "Total rounds where submission was present and valid",
+    ["identity_address", "protocol", "phase"],
+)
+
+SUBMIT_MISSING = Counter(
+    "flare_fsp_submit_missing_total",
+    "Total rounds where submission was absent",
+    ["identity_address", "protocol", "phase"],
+)
+
+SUBMIT_LATE = Counter(
+    "flare_fsp_submit_late_total",
+    "Total rounds where submission was sent after the allowed window",
+    ["identity_address", "protocol", "phase"],
+)
+
+SUBMIT_EARLY = Counter(
+    "flare_fsp_submit_early_total",
+    "Total rounds where submission was sent before the allowed window",
+    ["identity_address", "protocol", "phase"],
+)
+
+# ---------------------------------------------------------------------------
+# Minimal conditions
+# ---------------------------------------------------------------------------
+
+FTSO_ANCHOR_FEEDS_SUCCESS_RATE = Gauge(
+    "flare_fsp_ftso_anchor_feeds_success_rate_bips",
+    "FTSO anchor feeds success rate in bips (0-10000) over the last 2 hours",
+    ["identity_address"],
+)
+
+FAST_UPDATE_BLOCKS_SINCE_LAST = Gauge(
+    "flare_fsp_fast_update_blocks_since_last",
+    "Number of blocks elapsed since the last fast update submission",
+    ["identity_address"],
+)
+
+NODE_UPTIME_RATIO = Gauge(
+    "flare_fsp_node_uptime_ratio",
+    "Node uptime ratio over the sliding window (0.0 to 1.0)",
+    ["identity_address", "node_id"],
+)
+
+FDC_PARTICIPATION_RATE = Gauge(
+    "flare_fsp_fdc_participation_rate_bips",
+    "FDC participation rate in bips (0-10000) over the last 2 hours",
+    ["identity_address"],
+)
+
+# ---------------------------------------------------------------------------
+# Balance
+# ---------------------------------------------------------------------------
+
+ADDRESS_BALANCE = Gauge(
+    "flare_fsp_address_balance_wei",
+    "Address balance in wei",
+    ["identity_address", "address", "role"],
+)
+
+# ---------------------------------------------------------------------------
+# Validation issues
+# ---------------------------------------------------------------------------
+
+REVEAL_OFFENCE = Counter(
+    "flare_fsp_reveal_offence_total",
+    "Total rounds where a reveal offence occurred"
+    " (missing reveal after commit, or hash mismatch)",
+    ["identity_address", "protocol"],
+)
+
+SIGNATURE_GRACE_PERIOD_MISSED = Counter(
+    "flare_fsp_signature_grace_period_missed_total",
+    "Total rounds where submitSignatures was sent after the grace period deadline",
+    ["identity_address", "protocol"],
+)
+
+SIGNATURE_MISMATCH = Counter(
+    "flare_fsp_signature_mismatch_total",
+    "Total rounds where submitSignatures signature did not match finalization",
+    ["identity_address", "protocol"],
+)
+
+# ---------------------------------------------------------------------------
+# Contract address issues
+# ---------------------------------------------------------------------------
+
+CONTRACT_ADDRESS_WRONG = Counter(
+    "flare_fsp_contract_address_wrong_total",
+    "Total times a wrong contract address was detected (submission or relay)",
+    ["identity_address", "contract"],
+)
+
+# ---------------------------------------------------------------------------
+# Unclaimed rewards
+# ---------------------------------------------------------------------------
+
+UNCLAIMED_REWARDS = Gauge(
+    "flare_fsp_unclaimed_rewards_wei",
+    "Unclaimed reward amount in wei per address/epoch/claim_type",
+    ["identity_address", "address", "reward_epoch", "claim_type"],
+)
+
+
+def initialize_labels(node_ids: list[str] | None = None) -> None:
+    """Pre-initialize all label combinations so time series appear immediately at 0."""
+    for protocol, phases in [
+        ("ftso", ["submit1", "submit2", "signatures"]),
+        ("fdc", ["submit2", "signatures"]),
+    ]:
+        for phase in phases:
+            SUBMIT_OK.labels(identity_address=_ia, protocol=protocol, phase=phase)
+            SUBMIT_MISSING.labels(identity_address=_ia, protocol=protocol, phase=phase)
+            SUBMIT_LATE.labels(identity_address=_ia, protocol=protocol, phase=phase)
+            SUBMIT_EARLY.labels(identity_address=_ia, protocol=protocol, phase=phase)
+
+        REVEAL_OFFENCE.labels(identity_address=_ia, protocol=protocol)
+        SIGNATURE_GRACE_PERIOD_MISSED.labels(identity_address=_ia, protocol=protocol)
+        SIGNATURE_MISMATCH.labels(identity_address=_ia, protocol=protocol)
+
+    for contract in ["submission", "relay"]:
+        CONTRACT_ADDRESS_WRONG.labels(identity_address=_ia, contract=contract)
+
+    FAST_UPDATE_BLOCKS_SINCE_LAST.labels(identity_address=_ia).set(0)
+
+    if node_ids:
+        for node_id in node_ids:
+            NODE_UPTIME_RATIO.labels(identity_address=_ia, node_id=node_id).set(0)
+
+
+def start_metrics_server(port: int, address: str = "0.0.0.0") -> None:
+    start_http_server(port, addr=address)