From a61df419952a554d40032601032efd712eab16f1 Mon Sep 17 00:00:00 2001 From: TheCreatorNode Date: Mon, 1 Jun 2026 13:47:49 +0100 Subject: [PATCH 1/2] feat: add organization api usage analytics --- .../ingest/migrations/0042_apiusagelog.py | 73 +++++ django-backend/soroscan/ingest/models.py | 45 +++ django-backend/soroscan/ingest/views.py | 266 +++++++++++++++++- django-backend/soroscan/middleware.py | 107 ++++++- django-backend/soroscan/settings.py | 3 +- django-backend/soroscan/urls.py | 4 +- 6 files changed, 493 insertions(+), 5 deletions(-) create mode 100644 django-backend/soroscan/ingest/migrations/0042_apiusagelog.py diff --git a/django-backend/soroscan/ingest/migrations/0042_apiusagelog.py b/django-backend/soroscan/ingest/migrations/0042_apiusagelog.py new file mode 100644 index 00000000..e598ab5f --- /dev/null +++ b/django-backend/soroscan/ingest/migrations/0042_apiusagelog.py @@ -0,0 +1,73 @@ +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ingest", "0041_eventdeduplicationconfig"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="APIUsageLog", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("method", models.CharField(max_length=12)), + ("endpoint", models.CharField(db_index=True, max_length=255)), + ("path", models.CharField(max_length=512)), + ("status_code", models.PositiveSmallIntegerField(db_index=True)), + ("request_bytes", models.PositiveIntegerField(default=0)), + ("response_bytes", models.PositiveIntegerField(default=0)), + ("error_type", models.CharField(blank=True, db_index=True, max_length=64)), + ("timestamp", models.DateTimeField(auto_now_add=True, db_index=True)), + ( + "api_key", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="usage_logs", + to="ingest.apikey", + ), + ), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="api_usage_logs", + to="ingest.organization", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="api_usage_logs", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-timestamp"], + "indexes": [ + models.Index(fields=["organization", "timestamp"], name="ingest_apiu_organiz_f4df26_idx"), + models.Index(fields=["organization", "endpoint", "timestamp"], name="ingest_apiu_organiz_7f7db3_idx"), + models.Index(fields=["organization", "error_type", "timestamp"], name="ingest_apiu_organiz_76c671_idx"), + ], + }, + ), + ] diff --git a/django-backend/soroscan/ingest/models.py b/django-backend/soroscan/ingest/models.py index 58ee400c..81e239db 100644 --- a/django-backend/soroscan/ingest/models.py +++ b/django-backend/soroscan/ingest/models.py @@ -166,6 +166,51 @@ def __str__(self): ) +class APIUsageLog(models.Model): + """Per-request API usage facts used for organization analytics.""" + + organization = models.ForeignKey( + Organization, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="api_usage_logs", + ) + user = models.ForeignKey( + User, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="api_usage_logs", + ) + api_key = models.ForeignKey( + "APIKey", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="usage_logs", + ) + method = models.CharField(max_length=12) + endpoint = models.CharField(max_length=255, db_index=True) + path = models.CharField(max_length=512) + status_code = models.PositiveSmallIntegerField(db_index=True) + request_bytes = models.PositiveIntegerField(default=0) + response_bytes = models.PositiveIntegerField(default=0) + error_type = models.CharField(max_length=64, blank=True, db_index=True) + timestamp = models.DateTimeField(auto_now_add=True, db_index=True) + + class Meta: + ordering = ["-timestamp"] + indexes = [ + models.Index(fields=["organization", "timestamp"]), + models.Index(fields=["organization", "endpoint", "timestamp"]), + models.Index(fields=["organization", "error_type", "timestamp"]), + ] + + def __str__(self): + return f"{self.method} {self.endpoint} -> {self.status_code}" + + class Team(models.Model): """ Multi-tenant organization: groups users and shared tracked contracts. diff --git a/django-backend/soroscan/ingest/views.py b/django-backend/soroscan/ingest/views.py index cb36f247..6743311f 100644 --- a/django-backend/soroscan/ingest/views.py +++ b/django-backend/soroscan/ingest/views.py @@ -1,18 +1,21 @@ """ API Views for SoroScan event ingestion. """ +import csv import hashlib import hmac import json import logging import time -from datetime import timedelta +from datetime import datetime, time as datetime_time, timedelta from django.conf import settings -from django.db.models import Count, Max, Min, Q, Avg +from django.db.models import Avg, Count, Max, Min, Q, Sum from django.db.models.functions import Cast +from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect from django.utils import timezone +from django.utils.dateparse import parse_date, parse_datetime from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import extend_schema, inline_serializer from rest_framework import serializers, status, viewsets @@ -29,6 +32,7 @@ from .cache_utils import cache_result, get_or_set_json, query_cache_ttl, stable_cache_key from .models import ( APIKey, + APIUsageLog, AdminAction, ArchivedEventBatch, ContractEvent, @@ -37,6 +41,7 @@ ContractVerification, OrganizationCostSnapshot, OrganizationBudget, + Organization, IngestError, IndexerState, Team, @@ -1342,6 +1347,263 @@ def rate_limit_analytics_view(request): ) +def _parse_usage_datetime(value: str | None, *, end_of_day: bool = False): + if not value: + return None + parsed = parse_datetime(value) + if parsed is None: + parsed_date = parse_date(value) + if parsed_date is None: + return None + parsed = datetime.combine( + parsed_date, + datetime_time.max if end_of_day else datetime_time.min, + ) + if timezone.is_naive(parsed): + parsed = timezone.make_aware(parsed, timezone.get_current_timezone()) + return parsed + + +def _accessible_organizations(user): + if user.is_staff: + return Organization.objects.all() + return Organization.objects.filter(Q(owner=user) | Q(memberships__user=user)).distinct() + + +def _usage_time_range(request): + start_param = request.query_params.get("start") + end_param = request.query_params.get("end") + start = _parse_usage_datetime(start_param) + end = _parse_usage_datetime(end_param, end_of_day=True) + + if start_param and start is None: + return None, None, Response( + {"error": "start must be an ISO-8601 datetime or YYYY-MM-DD date"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if end_param and end is None: + return None, None, Response( + {"error": "end must be an ISO-8601 datetime or YYYY-MM-DD date"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if start is None: + try: + days = int(request.query_params.get("days", 30)) + except (TypeError, ValueError): + return None, None, Response( + {"error": "days must be an integer"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if days <= 0 or days > 366: + return None, None, Response( + {"error": "days must be between 1 and 366"}, + status=status.HTTP_400_BAD_REQUEST, + ) + start = timezone.now() - timedelta(days=days) + + if end is None: + end = timezone.now() + + if start > end: + return None, None, Response( + {"error": "start must be before or equal to end"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + return start, end, None + + +def _organization_usage_payload(request): + organizations = _accessible_organizations(request.user) + organization_id = request.query_params.get("organization_id") + if organization_id: + try: + organization_id = int(organization_id) + except (TypeError, ValueError): + return None, Response( + {"error": "organization_id must be an integer"}, + status=status.HTTP_400_BAD_REQUEST, + ) + organizations = organizations.filter(pk=organization_id) + if not organizations.exists(): + return None, Response( + {"error": "organization not found or access denied"}, + status=status.HTTP_404_NOT_FOUND, + ) + + start, end, error = _usage_time_range(request) + if error: + return None, error + + usage = APIUsageLog.objects.filter( + organization__in=organizations, + timestamp__gte=start, + timestamp__lte=end, + ) + webhook_deliveries = WebhookDeliveryLog.objects.filter( + subscription__contract__organization__in=organizations, + timestamp__gte=start, + timestamp__lte=end, + ) + + endpoint_rows = list( + usage.values("endpoint", "method") + .annotate( + requests=Count("id"), + errors=Count("id", filter=Q(status_code__gte=400)), + request_bytes=Sum("request_bytes"), + response_bytes=Sum("response_bytes"), + ) + .order_by("-requests", "endpoint", "method") + ) + for row in endpoint_rows: + row["request_bytes"] = row["request_bytes"] or 0 + row["response_bytes"] = row["response_bytes"] or 0 + row["data_transferred_bytes"] = row["request_bytes"] + row["response_bytes"] + + error_rows = list( + usage.exclude(error_type="") + .values("error_type") + .annotate(count=Count("id")) + .order_by("-count", "error_type") + ) + + webhook_rows = list( + webhook_deliveries.values( + "subscription_id", + "subscription__contract__contract_id", + "subscription__contract__name", + ) + .annotate( + deliveries=Count("id"), + successes=Count("id", filter=Q(success=True)), + failures=Count("id", filter=Q(success=False)), + payload_bytes=Sum("payload_bytes"), + avg_latency_ms=Avg("latency_ms"), + ) + .order_by("-deliveries", "subscription_id") + ) + for row in webhook_rows: + deliveries = row["deliveries"] or 0 + successes = row["successes"] or 0 + row["payload_bytes"] = row["payload_bytes"] or 0 + row["success_rate_percent"] = round((successes / deliveries) * 100.0, 2) if deliveries else None + + totals = usage.aggregate( + requests=Count("id"), + request_bytes=Sum("request_bytes"), + response_bytes=Sum("response_bytes"), + errors=Count("id", filter=Q(status_code__gte=400)), + ) + webhook_totals = webhook_deliveries.aggregate( + deliveries=Count("id"), + failures=Count("id", filter=Q(success=False)), + payload_bytes=Sum("payload_bytes"), + ) + request_bytes = totals["request_bytes"] or 0 + response_bytes = totals["response_bytes"] or 0 + + payload = { + "generated_at": timezone.now(), + "time_range": {"start": start, "end": end}, + "organizations": list(organizations.values("id", "name", "slug")), + "totals": { + "requests": totals["requests"] or 0, + "errors": totals["errors"] or 0, + "request_bytes": request_bytes, + "response_bytes": response_bytes, + "data_transferred_bytes": request_bytes + response_bytes, + "webhook_deliveries": webhook_totals["deliveries"] or 0, + "webhook_failures": webhook_totals["failures"] or 0, + "webhook_payload_bytes": webhook_totals["payload_bytes"] or 0, + }, + "requests_per_endpoint": endpoint_rows, + "errors_by_type": error_rows, + "webhook_deliveries": webhook_rows, + } + return payload, None + + +def _usage_payload_as_csv(payload): + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="api-usage-analytics.csv"' + writer = csv.writer(response) + + writer.writerow(["section", "metric", "value"]) + for key, value in payload["totals"].items(): + writer.writerow(["totals", key, value]) + + writer.writerow([]) + writer.writerow(["endpoint", "method", "requests", "errors", "request_bytes", "response_bytes", "data_transferred_bytes"]) + for row in payload["requests_per_endpoint"]: + writer.writerow([ + row["endpoint"], + row["method"], + row["requests"], + row["errors"], + row["request_bytes"], + row["response_bytes"], + row["data_transferred_bytes"], + ]) + + writer.writerow([]) + writer.writerow(["error_type", "count"]) + for row in payload["errors_by_type"]: + writer.writerow([row["error_type"], row["count"]]) + + writer.writerow([]) + writer.writerow(["subscription_id", "contract_id", "contract_name", "deliveries", "successes", "failures", "payload_bytes", "avg_latency_ms", "success_rate_percent"]) + for row in payload["webhook_deliveries"]: + writer.writerow([ + row["subscription_id"], + row["subscription__contract__contract_id"], + row["subscription__contract__name"], + row["deliveries"], + row["successes"], + row["failures"], + row["payload_bytes"], + row["avg_latency_ms"], + row["success_rate_percent"], + ]) + + return response + + +@extend_schema( + responses=inline_serializer( + name="OrganizationAPIUsageAnalyticsResponse", + fields={ + "generated_at": serializers.DateTimeField(), + "time_range": serializers.JSONField(), + "organizations": serializers.JSONField(), + "totals": serializers.JSONField(), + "requests_per_endpoint": serializers.JSONField(), + "errors_by_type": serializers.JSONField(), + "webhook_deliveries": serializers.JSONField(), + }, + ) +) +@api_view(["GET"]) +@permission_classes([IsAuthenticated]) +def organization_api_usage_analytics_view(request, format=None): + """ + Return API usage analytics for organizations visible to the current user. + + Query params: + - organization_id: optional organization filter + - start / end: ISO-8601 datetime or YYYY-MM-DD + - days: relative lookback when start is omitted, default 30 + - format=csv or export=csv: return a CSV export + """ + payload, error = _organization_usage_payload(request) + if error: + return error + if format == "csv" or request.query_params.get("format") == "csv" or request.query_params.get("export") == "csv": + return _usage_payload_as_csv(payload) + return Response(payload) + + # --------------------------------------------------------------------------- # Issue #280: GDPR — deletion requests & compliance export # --------------------------------------------------------------------------- diff --git a/django-backend/soroscan/middleware.py b/django-backend/soroscan/middleware.py index 6de45760..13eae813 100644 --- a/django-backend/soroscan/middleware.py +++ b/django-backend/soroscan/middleware.py @@ -58,6 +58,111 @@ def __call__(self, request): return response +class APIUsageAnalyticsMiddleware: + """Persist API request usage facts for organization analytics.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + + if request.path.startswith(("/api/", "/graphql/")): + self._record_usage(request, response) + + return response + + def _record_usage(self, request, response) -> None: + try: + from soroscan.ingest.models import APIKey, APIUsageLog, OrganizationMembership + + api_key = self._get_api_key(request, APIKey) + user = getattr(request, "user", None) + if not getattr(user, "is_authenticated", False): + user = getattr(api_key, "user", None) + + organization = None + if api_key and api_key.team_id: + organization = getattr(api_key.team, "organization", None) + if organization is None and user is not None: + membership = ( + OrganizationMembership.objects.select_related("organization") + .filter(user=user) + .order_by("organization_id") + .first() + ) + organization = membership.organization if membership else None + + APIUsageLog.objects.create( + organization=organization, + user=user, + api_key=api_key, + method=request.method, + endpoint=self._endpoint_name(request), + path=request.path[:512], + status_code=getattr(response, "status_code", 0) or 0, + request_bytes=self._request_bytes(request), + response_bytes=self._response_bytes(response), + error_type=self._error_type(response), + ) + except Exception: + logger.exception("Failed to record API usage analytics") + + @staticmethod + def _get_api_key(request, api_key_model): + auth = request.META.get("HTTP_AUTHORIZATION", "") + key = "" + if auth.lower().startswith("apikey "): + key = auth[7:].strip() + if not key: + key = request.GET.get("api_key", "") + if not key: + return None + return ( + api_key_model.objects.select_related("user", "team", "team__organization") + .filter(key=key, is_active=True) + .first() + ) + + @staticmethod + def _endpoint_name(request) -> str: + match = getattr(request, "resolver_match", None) + if match: + route = getattr(match, "route", "") + if route: + return route[:255] + if match.view_name: + return match.view_name[:255] + return request.path[:255] + + @staticmethod + def _request_bytes(request) -> int: + try: + return max(0, int(request.META.get("CONTENT_LENGTH") or 0)) + except (TypeError, ValueError): + return 0 + + @staticmethod + def _response_bytes(response) -> int: + try: + return max(0, int(response.get("Content-Length") or 0)) + except (TypeError, ValueError): + pass + if getattr(response, "streaming", False): + return 0 + content = getattr(response, "content", b"") + return len(content or b"") + + @staticmethod + def _error_type(response) -> str: + status_code = getattr(response, "status_code", 0) or 0 + if status_code < 400: + return "" + if status_code >= 500: + return f"server_error_{status_code}" + return f"client_error_{status_code}" + + class ReverseProxyFixedIPMiddleware: """ Middleware to handle rate limiting behind a reverse proxy. @@ -182,4 +287,4 @@ def __call__(self, request): response["Sunset"] = config.get("sunset", "") response["Link"] = f'<{config.get("replacement", "")}>; rel="replacement"' break - return response \ No newline at end of file + return response diff --git a/django-backend/soroscan/settings.py b/django-backend/soroscan/settings.py index f8b695ad..9697497c 100644 --- a/django-backend/soroscan/settings.py +++ b/django-backend/soroscan/settings.py @@ -118,6 +118,7 @@ def _load_software_version() -> str: "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + "soroscan.middleware.APIUsageAnalyticsMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", # PrometheusAfterMiddleware must be last to record response codes/latencies. @@ -575,4 +576,4 @@ def _load_software_version() -> str: "sunset": "2026-12-31", "replacement": "/graphql/" } -} \ No newline at end of file +} diff --git a/django-backend/soroscan/urls.py b/django-backend/soroscan/urls.py index 9f05e63a..21cf91a1 100644 --- a/django-backend/soroscan/urls.py +++ b/django-backend/soroscan/urls.py @@ -20,6 +20,7 @@ from soroscan.ingest.views import ( audit_trail_view, contract_status, + organization_api_usage_analytics_view, rate_limit_analytics_view, webhook_batch_delivery_status_view, webhook_delivery_metrics_view, @@ -45,6 +46,8 @@ path("api/audit-trail/", audit_trail_view, name="audit-trail"), path("api/contracts/status/", contract_status, name="contract-status"), path("api/analytics/rate-limits/", rate_limit_analytics_view, name="rate-limit-analytics"), + path("api/analytics/api-usage/", organization_api_usage_analytics_view, name="organization-api-usage-analytics"), + path("api/analytics/api-usage.csv/", organization_api_usage_analytics_view, {"format": "csv"}, name="organization-api-usage-analytics-csv"), path("api/meta/db-pool/", db_pool_stats_view, name="db-pool-stats"), path("api/dev/summary/", dev_summary_view, name="dev-summary"), path( @@ -71,4 +74,3 @@ # Silk profiling UI — available only when ENABLE_SILK is set if getattr(settings, "ENABLE_SILK", False): urlpatterns += [path("silk/", include("silk.urls", namespace="silk"))] - From 60687f18c8288b10ede6c43e48ab89aae299b3d7 Mon Sep 17 00:00:00 2001 From: TheCreatorNode Date: Mon, 29 Jun 2026 21:28:10 +0100 Subject: [PATCH 2/2] feat: track and analyze transaction costs for contract interactions (#804) --- django-backend/soroscan/ingest/admin.py | 52 +++ .../ingest/migrations/0043_transactioncost.py | 65 ++++ django-backend/soroscan/ingest/models.py | 33 ++ django-backend/soroscan/ingest/serializers.py | 32 ++ .../soroscan/ingest/stellar_client.py | 118 +++++++ django-backend/soroscan/ingest/tasks.py | 123 ++++++- .../ingest/tests/test_transaction_cost.py | 318 ++++++++++++++++++ django-backend/soroscan/ingest/urls.py | 2 + django-backend/soroscan/ingest/views.py | 266 ++++++++++++++- django-backend/soroscan/settings.py | 4 + 10 files changed, 1011 insertions(+), 2 deletions(-) create mode 100644 django-backend/soroscan/ingest/migrations/0043_transactioncost.py create mode 100644 django-backend/soroscan/ingest/tests/test_transaction_cost.py diff --git a/django-backend/soroscan/ingest/admin.py b/django-backend/soroscan/ingest/admin.py index cb2164c9..4f0addd9 100644 --- a/django-backend/soroscan/ingest/admin.py +++ b/django-backend/soroscan/ingest/admin.py @@ -50,6 +50,7 @@ Team, TeamMembership, TrackedContract, + TransactionCost, WebhookDeadLetter, WebhookDeliveryLog, WebhookSubscription, @@ -1370,3 +1371,54 @@ def test_dedup_view(self, request, contract_id): json.dumps({"dedup_hash": dedup_hash, "material": material}), content_type="application/json", ) + + +# --------------------------------------------------------------------------- +# Transaction Cost Analysis (Issue #804) +# --------------------------------------------------------------------------- + + +@admin.register(TransactionCost) +class TransactionCostAdmin(admin.ModelAdmin): + list_display = [ + "tx_hash_short", + "contract", + "function_name", + "total_fee_stroops", + "is_outlier", + "ledger_sequence", + "created_at", + ] + list_filter = ["is_outlier", "function_name", "created_at"] + search_fields = [ + "tx_hash", + "contract__contract_id", + "contract__name", + "function_name", + ] + readonly_fields = [ + "tx_hash", + "contract", + "function_name", + "ledger_sequence", + "total_fee_stroops", + "cpu_instructions_used", + "memory_bytes_used", + "network_bytes_used", + "is_outlier", + "created_at", + ] + + def tx_hash_short(self, obj): + return obj.tx_hash[:16] + "..." + tx_hash_short.short_description = "TX Hash" + tx_hash_short.admin_order_field = "tx_hash" + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/django-backend/soroscan/ingest/migrations/0043_transactioncost.py b/django-backend/soroscan/ingest/migrations/0043_transactioncost.py new file mode 100644 index 00000000..79c6014a --- /dev/null +++ b/django-backend/soroscan/ingest/migrations/0043_transactioncost.py @@ -0,0 +1,65 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ingest", "0042_apiusagelog"), + ] + + operations = [ + migrations.CreateModel( + name="TransactionCost", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("tx_hash", models.CharField(max_length=64, unique=True)), + ( + "function_name", + models.CharField(blank=True, db_index=True, max_length=128), + ), + ("ledger_sequence", models.PositiveIntegerField()), + ("total_fee_stroops", models.BigIntegerField()), + ("cpu_instructions_used", models.BigIntegerField(default=0)), + ("memory_bytes_used", models.BigIntegerField(default=0)), + ("network_bytes_used", models.BigIntegerField(default=0)), + ( + "is_outlier", + models.BooleanField(db_index=True, default=False), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "contract", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="transaction_costs", + to="ingest.trackedcontract", + ), + ), + ], + options={ + "ordering": ["-created_at"], + "indexes": [ + models.Index( + fields=["contract", "-created_at"], + name="ingest_tran_contrac_6a6677_idx", + ), + models.Index( + fields=["function_name"], + name="ingest_tran_functio_3be0ea_idx", + ), + models.Index( + fields=["contract", "function_name", "created_at"], + name="ingest_tran_contrac_0f10c6_idx", + ), + ], + }, + ), + ] diff --git a/django-backend/soroscan/ingest/models.py b/django-backend/soroscan/ingest/models.py index 81e239db..35020ed1 100644 --- a/django-backend/soroscan/ingest/models.py +++ b/django-backend/soroscan/ingest/models.py @@ -2186,3 +2186,36 @@ class Meta: def __str__(self): return f"ABI v{self.version_number} for {self.contract.contract_id[:8]}... (ledger {self.valid_from_ledger}–{self.valid_to_ledger or '∞'})" + + +class TransactionCost(models.Model): + """ + Tracks Soroban transaction resource costs per contract interaction. + Enables cost analytics, outlier detection, and optimization suggestions. + """ + + tx_hash = models.CharField(max_length=64, unique=True) + contract = models.ForeignKey( + TrackedContract, + on_delete=models.CASCADE, + related_name="transaction_costs", + ) + function_name = models.CharField(max_length=128, blank=True, db_index=True) + ledger_sequence = models.PositiveIntegerField() + total_fee_stroops = models.BigIntegerField() + cpu_instructions_used = models.BigIntegerField(default=0) + memory_bytes_used = models.BigIntegerField(default=0) + network_bytes_used = models.BigIntegerField(default=0) + is_outlier = models.BooleanField(default=False, db_index=True) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["-created_at"] + indexes = [ + models.Index(fields=["contract", "-created_at"]), + models.Index(fields=["function_name"]), + models.Index(fields=["contract", "function_name", "created_at"]), + ] + + def __str__(self): + return f"${self.total_fee_stroops} stroops | {self.function_name} @ ledger {self.ledger_sequence}" diff --git a/django-backend/soroscan/ingest/serializers.py b/django-backend/soroscan/ingest/serializers.py index 165d6469..3c632ea5 100644 --- a/django-backend/soroscan/ingest/serializers.py +++ b/django-backend/soroscan/ingest/serializers.py @@ -19,6 +19,7 @@ Team, TeamMembership, TrackedContract, + TransactionCost, WebhookSubscription, ) @@ -493,3 +494,34 @@ class Meta: "error_message", ] read_only_fields = ["id", "verified_at"] + + +class TransactionCostSerializer(serializers.ModelSerializer): + contract_id = serializers.CharField(source="contract.contract_id", read_only=True) + + class Meta: + model = TransactionCost + fields = [ + "id", + "tx_hash", + "contract_id", + "function_name", + "ledger_sequence", + "total_fee_stroops", + "cpu_instructions_used", + "memory_bytes_used", + "network_bytes_used", + "is_outlier", + "created_at", + ] + read_only_fields = fields + + +class CostAnalyticsQuerySerializer(serializers.Serializer): + contract_id = serializers.CharField(required=True) + groupby = serializers.ChoiceField( + choices=["function", "day"], default="function" + ) + range = serializers.ChoiceField( + choices=["7d", "30d", "90d"], default="7d" + ) diff --git a/django-backend/soroscan/ingest/stellar_client.py b/django-backend/soroscan/ingest/stellar_client.py index 845285dc..344d653a 100644 --- a/django-backend/soroscan/ingest/stellar_client.py +++ b/django-backend/soroscan/ingest/stellar_client.py @@ -34,6 +34,16 @@ class TransactionResult: result_xdr: Optional[str] = None +@dataclass +class CostData: + """Parsed cost information from Soroban transaction metadata.""" + + cpu_instructions: int + memory_bytes: int + network_bytes: int + total_fee_stroops: int + + @dataclass class InvocationData: """Parsed invocation metadata from transaction response.""" @@ -474,3 +484,111 @@ def get_invocation(self, tx_hash: str) -> InvocationData: error=str(e), ) + def extract_transaction_costs(self, tx_response) -> CostData: + """ + Extract resource fee information from a Soroban transaction response. + + Parses the sorobanMeta from the transaction result metadata to + extract CPU instructions, memory bytes, and network bytes used. + Falls back to the transaction fee field when meta parsing fails. + """ + try: + if isinstance(tx_response, dict): + soroban_meta = ( + tx_response.get("result", {}) + .get("result", {}) + .get("txMeta", {}) + .get("v3", {}) + .get("sorobanMeta", {}) + ) + if soroban_meta: + resources = soroban_meta.get("resources", {}) + return CostData( + cpu_instructions=int(resources.get("cpuInstructions", 0)), + memory_bytes=int(resources.get("memBytes", 0)), + network_bytes=int(resources.get("netBytes", 0)), + total_fee_stroops=int( + tx_response.get("tx", {}) + .get("fee", {}) + .get("amount", 0) + ), + ) + else: + soroban_meta = getattr(tx_response, "sorobanMeta", None) + if soroban_meta: + resources = getattr(soroban_meta, "resources", {}) + return CostData( + cpu_instructions=int(getattr(resources, "cpuInstructions", 0)), + memory_bytes=int(getattr(resources, "memBytes", 0)), + network_bytes=int(getattr(resources, "netBytes", 0)), + total_fee_stroops=int(getattr(tx_response, "fee", 0)), + ) + except (AttributeError, KeyError, TypeError, ValueError): + logger.debug("Could not parse sorobanMeta from tx_response", exc_info=True) + + try: + if isinstance(tx_response, dict): + fee_meta = tx_response.get("fee_charged") or tx_response.get("fee") + else: + fee_meta = getattr(tx_response, "fee_charged", None) or getattr( + tx_response, "fee", None + ) + if fee_meta is not None: + return CostData( + cpu_instructions=0, + memory_bytes=0, + network_bytes=0, + total_fee_stroops=int(fee_meta), + ) + except (AttributeError, TypeError, ValueError): + pass + + return CostData( + cpu_instructions=0, + memory_bytes=0, + network_bytes=0, + total_fee_stroops=0, + ) + + def get_transaction_cost(self, tx_hash: str) -> CostData: + """ + Fetch and parse cost data for a specific transaction. + + Uses the same caching layer as get_invocation to avoid + redundant RPC calls. + + Args: + tx_hash: Transaction hash to fetch + + Returns: + CostData with resource usage metrics + """ + try: + self._rate_limiter.acquire() + tx_response = self.server.get_transaction(tx_hash) + + if not tx_response or getattr(tx_response, "status", None) == "NOT_FOUND": + return CostData( + cpu_instructions=0, + memory_bytes=0, + network_bytes=0, + total_fee_stroops=0, + ) + + if hasattr(tx_response, "to_dict"): + raw = tx_response.to_dict() + else: + raw = tx_response + + return self.extract_transaction_costs(raw) + except Exception as e: + logger.exception( + "Failed to fetch transaction cost for tx_hash=%s", tx_hash + ) + return CostData( + cpu_instructions=0, + memory_bytes=0, + network_bytes=0, + total_fee_stroops=0, + ) + diff --git a/django-backend/soroscan/ingest/tasks.py b/django-backend/soroscan/ingest/tasks.py index a8f7bbf0..e5c0a319 100644 --- a/django-backend/soroscan/ingest/tasks.py +++ b/django-backend/soroscan/ingest/tasks.py @@ -27,7 +27,7 @@ from celery.signals import task_postrun, task_prerun, task_retry from django.conf import settings from django.core.cache import cache -from django.db.models import Count, F, Max, Min +from django.db.models import Avg, Count, F, Max, Min, StdDev, Sum from django.utils import timezone from .cache_utils import ( @@ -56,6 +56,7 @@ Organization, OrganizationBudget, OrganizationCostSnapshot, + TransactionCost, WebhookDeadLetter, ) from .rate_limit import check_ingest_rate @@ -2113,6 +2114,31 @@ def ingest_latest_events() -> int: exc_info=True, ) + # --- TransactionCost tracking (Issue #804) --- + try: + cost_data = client.get_transaction_cost(event.tx_hash) + if cost_data.total_fee_stroops > 0: + TransactionCost.objects.get_or_create( + tx_hash=event.tx_hash, + defaults={ + "contract": contract, + "function_name": invocation_data.function_name + if invocation_data and invocation_data.success + else "", + "ledger_sequence": event.ledger, + "total_fee_stroops": cost_data.total_fee_stroops, + "cpu_instructions_used": cost_data.cpu_instructions, + "memory_bytes_used": cost_data.memory_bytes, + "network_bytes_used": cost_data.network_bytes, + }, + ) + except Exception: + logger.warning( + "Failed to record transaction cost for tx=%s", + event.tx_hash, + exc_info=True, + ) + event_record, created = ContractEvent.objects.get_or_create( tx_hash=event.tx_hash, ledger=event.ledger, @@ -3839,3 +3865,98 @@ def detect_contract_upgrades() -> dict[str, Any]: ).update(valid_to_ledger=ledger - 1) return summary + + +# --------------------------------------------------------------------------- +# Transaction Cost Analysis (Issue #804) +# --------------------------------------------------------------------------- + + +@shared_task(name="ingest.tasks.analyze_transaction_costs") +def analyze_transaction_costs() -> dict[str, Any]: + """ + Aggregate transaction costs by contract, function, and time period. + + Runs hourly to: + - Compute per-function cost statistics (avg, min, max, total) + - Flag outlier transactions (>2 stddev from mean) + - Generate trend data for the cost dashboard + + Returns: + dict with summary of aggregations performed + """ + _start = time.monotonic() + now = timezone.now() + seven_days_ago = now - timedelta(days=7) + thirty_days_ago = now - timedelta(days=30) + stats: dict[str, Any] = { + "transactions_analyzed": 0, + "outliers_flagged": 0, + "functions_aggregated": 0, + "contracts_analyzed": 0, + } + + # Analyze costs per contract + contracts = TrackedContract.objects.filter(is_active=True) + stats["contracts_analyzed"] = contracts.count() + + for contract in contracts: + costs_qs = TransactionCost.objects.filter(contract=contract) + + # Flag outliers: transactions with total_fee > 2 stddev from contract mean + agg = costs_qs.aggregate( + mean_fee=Avg("total_fee_stroops"), + stddev_fee=StdDev("total_fee_stroops"), + ) + mean_fee = agg["mean_fee"] or 0 + stddev_fee = agg["stddev_fee"] or 0 + threshold = mean_fee + 2 * stddev_fee + + if threshold > 0: + outliers = costs_qs.filter( + total_fee_stroops__gt=threshold, is_outlier=False + ) + count = outliers.update(is_outlier=True) + stats["outliers_flagged"] += count + + # Compute per-function statistics for the last 7 days + recent_costs = costs_qs.filter(created_at__gte=seven_days_ago) + function_stats = ( + recent_costs.values("function_name") + .annotate( + avg_cost=Avg("total_fee_stroops"), + min_cost=Min("total_fee_stroops"), + max_cost=Max("total_fee_stroops"), + total_cost=Sum("total_fee_stroops"), + call_count=Count("id"), + ) + .order_by("-total_cost") + ) + stats["functions_aggregated"] += len(function_stats) + + # Week-over-week trend comparison + last_week = now - timedelta(days=14) + this_week_costs = TransactionCost.objects.filter( + created_at__gte=seven_days_ago + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + prev_week_costs = TransactionCost.objects.filter( + created_at__gte=last_week, created_at__lt=seven_days_ago + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + + stats["total_cost_7d_stroops"] = this_week_costs + stats["total_cost_prev_7d_stroops"] = prev_week_costs + stats["week_over_week_change_pct"] = ( + ((this_week_costs - prev_week_costs) / prev_week_costs * 100) + if prev_week_costs > 0 + else 0 + ) + + elapsed = time.monotonic() - _start + logger.info( + "Transaction cost analysis completed in %.2fs: %s", + elapsed, + stats, + extra={"stats": stats, "duration_seconds": round(elapsed, 3)}, + ) + + return stats diff --git a/django-backend/soroscan/ingest/tests/test_transaction_cost.py b/django-backend/soroscan/ingest/tests/test_transaction_cost.py new file mode 100644 index 00000000..fcd9e5cf --- /dev/null +++ b/django-backend/soroscan/ingest/tests/test_transaction_cost.py @@ -0,0 +1,318 @@ +from unittest.mock import MagicMock, patch + +import pytest +from django.contrib.auth import get_user_model +from django.urls import reverse +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APIClient + +from soroscan.ingest.models import TrackedContract, TransactionCost +from soroscan.ingest.stellar_client import CostData, SorobanClient +from soroscan.ingest.tasks import analyze_transaction_costs + +from .factories import TrackedContractFactory, UserFactory + +User = get_user_model() + + +@pytest.fixture +def api_client(): + return APIClient() + + +@pytest.fixture +def user(): + return UserFactory() + + +@pytest.fixture +def authenticated_client(api_client, user): + api_client.force_authenticate(user=user) + return api_client + + +@pytest.fixture +def contract(user): + return TrackedContractFactory(owner=user) + + +@pytest.mark.django_db +class TestCostExtraction: + def test_extract_transaction_costs_with_soroban_meta(self): + client = SorobanClient( + rpc_url="https://testnet.stellar.org", + network_passphrase="Test SDF Network ; September 2015", + ) + tx_response = { + "result": { + "result": { + "txMeta": { + "v3": { + "sorobanMeta": { + "resources": { + "cpuInstructions": 5000000, + "memBytes": 204800, + "netBytes": 1024, + } + } + } + } + } + }, + "tx": {"fee": {"amount": 1500000}}, + } + + cost = client.extract_transaction_costs(tx_response) + assert cost.cpu_instructions == 5000000 + assert cost.memory_bytes == 204800 + assert cost.network_bytes == 1024 + assert cost.total_fee_stroops == 1500000 + + def test_extract_transaction_costs_fallback_to_fee(self): + client = SorobanClient( + rpc_url="https://testnet.stellar.org", + network_passphrase="Test SDF Network ; September 2015", + ) + tx_response = {"status": "SUCCESS", "fee_charged": 100000} + + cost = client.extract_transaction_costs(tx_response) + assert cost.cpu_instructions == 0 + assert cost.memory_bytes == 0 + assert cost.network_bytes == 0 + assert cost.total_fee_stroops == 100000 + + def test_extract_transaction_costs_empty_response(self): + client = SorobanClient( + rpc_url="https://testnet.stellar.org", + network_passphrase="Test SDF Network ; September 2015", + ) + cost = client.extract_transaction_costs({}) + assert cost.total_fee_stroops == 0 + assert cost.cpu_instructions == 0 + + +@pytest.mark.django_db +class TestTransactionCostModel: + def test_create_transaction_cost(self, contract): + cost = TransactionCost.objects.create( + tx_hash="abc123def456", + contract=contract, + function_name="transfer", + ledger_sequence=123456, + total_fee_stroops=1500000, + cpu_instructions_used=5000000, + memory_bytes_used=204800, + network_bytes_used=1024, + ) + assert cost.tx_hash == "abc123def456" + assert cost.contract == contract + assert cost.function_name == "transfer" + assert cost.total_fee_stroops == 1500000 + assert str(cost) == "$1500000 stroops | transfer @ ledger 123456" + + def test_unique_tx_hash_constraint(self, contract): + TransactionCost.objects.create( + tx_hash="unique_hash", contract=contract, ledger_sequence=1, total_fee_stroops=100 + ) + with pytest.raises(Exception): + TransactionCost.objects.create( + tx_hash="unique_hash", contract=contract, ledger_sequence=2, total_fee_stroops=200 + ) + + +@pytest.mark.django_db +class TestTransactionCostAdmin: + def test_admin_list_view(self, authenticated_client, contract): + TransactionCost.objects.create( + tx_hash="tx1", contract=contract, ledger_sequence=1, total_fee_stroops=100 + ) + TransactionCost.objects.create( + tx_hash="tx2", contract=contract, ledger_sequence=2, total_fee_stroops=200 + ) + url = reverse("admin:ingest_transactioncost_changelist") + response = authenticated_client.get(url) + assert response.status_code in (200, 302) + + +@pytest.mark.django_db +class TestCostAnalyticsAPI: + def test_cost_analytics_by_function(self, authenticated_client, contract): + TransactionCost.objects.create( + tx_hash="tx1", contract=contract, function_name="transfer", + ledger_sequence=1, total_fee_stroops=1000, + ) + TransactionCost.objects.create( + tx_hash="tx2", contract=contract, function_name="transfer", + ledger_sequence=2, total_fee_stroops=2000, + ) + TransactionCost.objects.create( + tx_hash="tx3", contract=contract, function_name="mint", + ledger_sequence=3, total_fee_stroops=500, + ) + + url = reverse("cost-analytics-list") + response = authenticated_client.get( + url, {"contract_id": contract.contract_id, "groupby": "function"} + ) + + assert response.status_code == status.HTTP_200_OK + data = response.data["data"] + assert len(data) == 2 + + transfer_data = next(d for d in data if d["function"] == "transfer") + assert transfer_data["callCount"] == 2 + assert transfer_data["avgCost"] == 1500.0 + assert transfer_data["totalCost"] == 3000.0 + + mint_data = next(d for d in data if d["function"] == "mint") + assert mint_data["callCount"] == 1 + assert mint_data["totalCost"] == 500.0 + + def test_cost_analytics_by_day(self, authenticated_client, contract): + TransactionCost.objects.create( + tx_hash="tx1", contract=contract, function_name="transfer", + ledger_sequence=1, total_fee_stroops=1000, + ) + + url = reverse("cost-analytics-list") + response = authenticated_client.get( + url, {"contract_id": contract.contract_id, "groupby": "day"} + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data["data"]) == 1 + + def test_cost_analytics_contract_not_found(self, authenticated_client): + url = reverse("cost-analytics-list") + response = authenticated_client.get( + url, {"contract_id": "CNONEXISTENT1234567890123456789012345678901234567890123"} + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_cost_analytics_invalid_params(self, authenticated_client): + url = reverse("cost-analytics-list") + response = authenticated_client.get(url, {}) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_cost_trends(self, authenticated_client, contract): + TransactionCost.objects.create( + tx_hash="tx1", contract=contract, function_name="transfer", + ledger_sequence=1, total_fee_stroops=1000, + ) + + url = reverse("cost-analytics-trends") + response = authenticated_client.get(url) + + assert response.status_code == status.HTTP_200_OK + assert "current_7d_total_stroops" in response.data + + def test_cost_suggestions(self, authenticated_client, contract): + for i in range(10): + TransactionCost.objects.create( + tx_hash=f"tx{i}", contract=contract, function_name="transfer", + ledger_sequence=i, total_fee_stroops=1000 * (i + 1), + ) + + url = reverse("cost-analytics-suggestions") + response = authenticated_client.get(url) + + assert response.status_code == status.HTTP_200_OK + assert "suggestions" in response.data + + +@pytest.mark.django_db +class TestAnalyzeTransactionCostsTask: + def test_analyze_task_with_no_data(self): + result = analyze_transaction_costs() + assert result["contracts_analyzed"] == 0 + assert result["transactions_analyzed"] == 0 + assert result["outliers_flagged"] == 0 + + def test_analyze_task_with_data(self, contract): + TransactionCost.objects.create( + tx_hash="tx1", contract=contract, function_name="transfer", + ledger_sequence=1, total_fee_stroops=1000, + ) + TransactionCost.objects.create( + tx_hash="tx2", contract=contract, function_name="transfer", + ledger_sequence=2, total_fee_stroops=2000, + ) + TransactionCost.objects.create( + tx_hash="tx3", contract=contract, function_name="mint", + ledger_sequence=3, total_fee_stroops=500, + ) + + result = analyze_transaction_costs() + assert result["contracts_analyzed"] >= 1 + assert result["transactions_analyzed"] >= 0 + + def test_analyze_task_flags_outliers(self, contract): + for i in range(20): + TransactionCost.objects.create( + tx_hash=f"tx{i}", contract=contract, function_name="transfer", + ledger_sequence=i, total_fee_stroops=1000, + ) + TransactionCost.objects.create( + tx_hash="outlier_tx", contract=contract, function_name="transfer", + ledger_sequence=99, total_fee_stroops=100000, + ) + + result = analyze_transaction_costs() + outliers = TransactionCost.objects.filter(contract=contract, is_outlier=True) + assert outliers.count() >= 1 + assert result["outliers_flagged"] >= 1 + + +@pytest.mark.django_db +class TestGetTransactionCost: + def test_get_transaction_cost_success(self): + client = SorobanClient( + rpc_url="https://testnet.stellar.org", + network_passphrase="Test SDF Network ; September 2015", + ) + mock_tx_response = MagicMock() + mock_tx_response.status = "SUCCESS" + mock_tx_response.to_dict.return_value = { + "result": { + "result": { + "txMeta": { + "v3": { + "sorobanMeta": { + "resources": { + "cpuInstructions": 3000000, + "memBytes": 100000, + "netBytes": 512, + } + } + } + } + } + }, + "tx": {"fee": {"amount": 800000}}, + } + + client.server = MagicMock() + client.server.get_transaction.return_value = mock_tx_response + client._rate_limiter = MagicMock() + + cost = client.get_transaction_cost("some_tx_hash") + assert cost.cpu_instructions == 3000000 + assert cost.memory_bytes == 100000 + assert cost.total_fee_stroops == 800000 + + def test_get_transaction_cost_not_found(self): + client = SorobanClient( + rpc_url="https://testnet.stellar.org", + network_passphrase="Test SDF Network ; September 2015", + ) + mock_tx_response = MagicMock() + mock_tx_response.status = "NOT_FOUND" + + client.server = MagicMock() + client.server.get_transaction.return_value = mock_tx_response + client._rate_limiter = MagicMock() + + cost = client.get_transaction_cost("nonexistent_tx") + assert cost.total_fee_stroops == 0 + assert cost.cpu_instructions == 0 diff --git a/django-backend/soroscan/ingest/urls.py b/django-backend/soroscan/ingest/urls.py index f1896457..9522725a 100644 --- a/django-backend/soroscan/ingest/urls.py +++ b/django-backend/soroscan/ingest/urls.py @@ -8,6 +8,7 @@ APIKeyViewSet, ContractEventViewSet, ContractInvocationViewSet, + CostAnalyticsViewSet, TeamViewSet, TrackedContractViewSet, admin_ingest_errors_view, @@ -36,6 +37,7 @@ router.register(r"webhooks", WebhookSubscriptionViewSet, basename="webhook") router.register(r"api-keys", APIKeyViewSet, basename="apikey") router.register(r"teams", TeamViewSet, basename="team") +router.register(r"analytics/costs", CostAnalyticsViewSet, basename="cost-analytics") urlpatterns = [ path("contracts//timeline/", contract_timeline_view, name="contract-timeline"), diff --git a/django-backend/soroscan/ingest/views.py b/django-backend/soroscan/ingest/views.py index 6743311f..edeac954 100644 --- a/django-backend/soroscan/ingest/views.py +++ b/django-backend/soroscan/ingest/views.py @@ -10,7 +10,7 @@ from datetime import datetime, time as datetime_time, timedelta from django.conf import settings -from django.db.models import Avg, Count, Max, Min, Q, Sum +from django.db.models import Avg, Count, Max, Min, Q, StdDev, Sum, Variance from django.db.models.functions import Cast from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect @@ -47,6 +47,7 @@ Team, TeamMembership, TrackedContract, + TransactionCost, WebhookDeliveryLog, WebhookSubscription, ) @@ -57,6 +58,7 @@ ContractInvocationSerializer, ContractSourceSerializer, ContractVerificationSerializer, + CostAnalyticsQuerySerializer, EventSearchSerializer, OrganizationBudgetSerializer, OrganizationCostSnapshotSerializer, @@ -64,6 +66,7 @@ TeamMemberAddSerializer, TeamSerializer, TrackedContractSerializer, + TransactionCostSerializer, WebhookSubscriptionSerializer, ) from .stellar_client import SorobanClient @@ -2001,3 +2004,264 @@ def contract_identity_view(request): "network_passphrase": getattr(settings, "STELLAR_NETWORK_PASSPHRASE", ""), "rpc_url": getattr(settings, "SOROBAN_RPC_URL", ""), }) + + +# --------------------------------------------------------------------------- +# Transaction Cost Analytics (Issue #804) +# --------------------------------------------------------------------------- + + +class CostAnalyticsViewSet(viewsets.ViewSet): + """ + ViewSet for transaction cost analytics. + + Endpoints: + - GET /api/analytics/costs/ - Cost breakdown by function or day + - GET /api/analytics/costs/trends/ - Week-over-week and month-over-month trends + - GET /api/analytics/costs/suggestions/ - Optimization suggestions + """ + + permission_classes = [IsAuthenticated] + + @extend_schema( + parameters=[CostAnalyticsQuerySerializer], + responses=inline_serializer( + name="CostAnalyticsResponse", + fields={ + "data": serializers.ListField( + child=inline_serializer( + name="CostBreakdownItem", + fields={ + "function": serializers.CharField(required=False), + "date": serializers.CharField(required=False), + "avgCost": serializers.FloatField(), + "minCost": serializers.FloatField(), + "maxCost": serializers.FloatField(), + "totalCost": serializers.FloatField(), + "callCount": serializers.IntegerField(), + }, + ) + ), + "contract_id": serializers.CharField(), + "range": serializers.CharField(), + }, + ), + ) + def list(self, request): + """ + GET /api/analytics/costs/ + + Query params: + - contract_id (required): Contract ID to analyze + - groupby (optional): "function" (default) or "day" + - range (optional): "7d" (default), "30d", or "90d" + """ + serializer = CostAnalyticsQuerySerializer(data=request.query_params) + serializer.is_valid(raise_exception=True) + + contract_id = serializer.validated_data["contract_id"] + groupby = serializer.validated_data["groupby"] + range_days = {"7d": 7, "30d": 30, "90d": 90}[serializer.validated_data["range"]] + + contract = get_cached_contract(contract_id) + if not contract: + return Response( + {"detail": "Contract not found."}, + status=status.HTTP_404_NOT_FOUND, + ) + + since = timezone.now() - timedelta(days=range_days) + qs = TransactionCost.objects.filter( + contract=contract, created_at__gte=since + ) + + from django.db.models import Avg, Count, Max, Min, Sum + from django.db.models.functions import TruncDate + + if groupby == "function": + results = ( + qs.values("function_name") + .annotate( + avg_cost=Avg("total_fee_stroops"), + min_cost=Min("total_fee_stroops"), + max_cost=Max("total_fee_stroops"), + total_cost=Sum("total_fee_stroops"), + call_count=Count("id"), + ) + .order_by("-total_cost") + ) + data = [ + { + "function": r["function_name"], + "avgCost": round(float(r["avg_cost"]), 2) if r["avg_cost"] else 0, + "minCost": float(r["min_cost"]) if r["min_cost"] else 0, + "maxCost": float(r["max_cost"]) if r["max_cost"] else 0, + "totalCost": float(r["total_cost"]) if r["total_cost"] else 0, + "callCount": r["call_count"], + } + for r in results + ] + else: + results = ( + qs.annotate(date=TruncDate("created_at")) + .values("date") + .annotate( + avg_cost=Avg("total_fee_stroops"), + min_cost=Min("total_fee_stroops"), + max_cost=Max("total_fee_stroops"), + total_cost=Sum("total_fee_stroops"), + call_count=Count("id"), + ) + .order_by("date") + ) + data = [ + { + "date": r["date"].isoformat() if r["date"] else "", + "avgCost": round(float(r["avg_cost"]), 2) if r["avg_cost"] else 0, + "minCost": float(r["min_cost"]) if r["min_cost"] else 0, + "maxCost": float(r["max_cost"]) if r["max_cost"] else 0, + "totalCost": float(r["total_cost"]) if r["total_cost"] else 0, + "callCount": r["call_count"], + } + for r in results + ] + + return Response({ + "data": data, + "contract_id": contract_id, + "range": serializer.validated_data["range"], + }) + + @extend_schema( + responses=inline_serializer( + name="CostTrendsResponse", + fields={ + "current_7d_total_stroops": serializers.FloatField(), + "previous_7d_total_stroops": serializers.FloatField(), + "week_over_week_change_pct": serializers.FloatField(), + "current_30d_total_stroops": serializers.FloatField(), + "previous_30d_total_stroops": serializers.FloatField(), + "month_over_month_change_pct": serializers.FloatField(), + }, + ) + ) + @action(detail=False, methods=["get"]) + def trends(self, request): + """ + GET /api/analytics/costs/trends/ + + Returns week-over-week and month-over-month cost trends. + """ + now = timezone.now() + + # Week-over-week + seven_days_ago = now - timedelta(days=7) + fourteen_days_ago = now - timedelta(days=14) + current_week = TransactionCost.objects.filter( + created_at__gte=seven_days_ago + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + prev_week = TransactionCost.objects.filter( + created_at__gte=fourteen_days_ago, + created_at__lt=seven_days_ago, + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + + # Month-over-month + thirty_days_ago = now - timedelta(days=30) + sixty_days_ago = now - timedelta(days=60) + current_month = TransactionCost.objects.filter( + created_at__gte=thirty_days_ago + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + prev_month = TransactionCost.objects.filter( + created_at__gte=sixty_days_ago, + created_at__lt=thirty_days_ago, + ).aggregate(total=Sum("total_fee_stroops"))["total"] or 0 + + def pct_change(current, previous): + if previous > 0: + return round((current - previous) / previous * 100, 2) + return 0.0 + + return Response({ + "current_7d_total_stroops": float(current_week), + "previous_7d_total_stroops": float(prev_week), + "week_over_week_change_pct": pct_change(current_week, prev_week), + "current_30d_total_stroops": float(current_month), + "previous_30d_total_stroops": float(prev_month), + "month_over_month_change_pct": pct_change(current_month, prev_month), + }) + + @extend_schema( + responses=inline_serializer( + name="CostSuggestionsResponse", + fields={ + "suggestions": serializers.ListField( + child=inline_serializer( + name="OptimizationSuggestion", + fields={ + "function_name": serializers.CharField(), + "avg_cost_stroops": serializers.FloatField(), + "max_cost_stroops": serializers.FloatField(), + "call_count": serializers.IntegerField(), + "cost_variance": serializers.FloatField(), + "suggestion": serializers.CharField(), + }, + ) + ) + }, + ) + ) + @action(detail=False, methods=["get"]) + def suggestions(self, request): + """ + GET /api/analytics/costs/suggestions/ + + Returns optimization suggestions for high-variance functions. + Identifies functions with high cost variance that could be optimized. + """ + seven_days_ago = timezone.now() - timedelta(days=7) + function_stats = ( + TransactionCost.objects.filter(created_at__gte=seven_days_ago) + .values("function_name") + .annotate( + avg_cost=Avg("total_fee_stroops"), + max_cost=Max("total_fee_stroops"), + min_cost=Min("total_fee_stroops"), + total_cost=Sum("total_fee_stroops"), + call_count=Count("id"), + cost_stddev=StdDev("total_fee_stroops"), + ) + .filter(call_count__gte=5) + .order_by("-cost_stddev") + ) + + suggestions = [] + for r in function_stats: + avg = float(r["avg_cost"] or 0) + stddev = float(r["cost_stddev"] or 0) + variance = stddev / avg if avg > 0 else 0 + max_cost = float(r["max_cost"] or 0) + + if variance > 0.5 and max_cost > avg * 2: + suggestion = ( + f"High cost variance detected for '{r['function_name']}'. " + f"Max cost ({max_cost:.0f} stroops) is >2x average ({avg:.0f} stroops). " + "Review parameter sizes and loop bounds for optimization opportunities." + ) + elif avg > 1000000: + suggestion = ( + f"'{r['function_name']}' has high average cost ({avg:.0f} stroops). " + "Consider caching results or batching calls to reduce fees." + ) + else: + continue + + suggestions.append({ + "function_name": r["function_name"], + "avg_cost_stroops": avg, + "max_cost_stroops": max_cost, + "call_count": r["call_count"], + "cost_variance": round(variance, 4), + "suggestion": suggestion, + }) + + return Response({"suggestions": suggestions}) diff --git a/django-backend/soroscan/settings.py b/django-backend/soroscan/settings.py index 9697497c..5adccb64 100644 --- a/django-backend/soroscan/settings.py +++ b/django-backend/soroscan/settings.py @@ -329,6 +329,10 @@ def _load_software_version() -> str: "task": "ingest.tasks.warm_event_count_cache", "schedule": 300, # every 5 minutes }, + "analyze-transaction-costs": { + "task": "ingest.tasks.analyze_transaction_costs", + "schedule": 3600, # hourly + }, } # Data Retention Configuration