From fb2d6eeee76f450a5393880f78eccc11abea2d34 Mon Sep 17 00:00:00 2001 From: Jordan123321 Date: Wed, 27 May 2026 11:57:59 +0100 Subject: [PATCH] Add snooker private challenge v0 --- .../snooker-private-challenge-v0-brief.md | 112 +++++ documentation/snooker-private-challenge-v0.md | 114 +++++ scorevision/__init__.py | 27 +- scorevision/cli/benchmark.py | 6 +- scorevision/miner/private_track/MINER.md | 10 +- .../miner/private_track/SNOOKER_MINER_SPEC.md | 118 +++++ scorevision/miner/private_track/predictor.py | 35 +- scorevision/miner/private_track/routes.py | 26 +- scorevision/miner/private_track/security.py | 30 +- scorevision/utils/bittensor_helpers.py | 22 +- scorevision/utils/cloudflare_helpers.py | 10 +- scorevision/utils/manifest.py | 2 + scorevision/utils/miner_registry.py | 3 +- scorevision/utils/schemas.py | 45 +- scorevision/validator/__init__.py | 53 ++- scorevision/validator/audit/__init__.py | 32 +- .../validator/audit/open_source/storage.py | 3 +- .../audit/private_track/spotcheck.py | 31 +- scorevision/validator/central/__init__.py | 20 +- .../central/private_track/challenges.py | 55 ++- .../validator/central/private_track/miners.py | 1 + .../central/private_track/registry.py | 6 +- .../validator/central/private_track/runner.py | 32 +- .../central/private_track/scoring.py | 414 +++++++++++++++++- scorevision/validator/core/__init__.py | 29 +- scorevision/validator/core/weights.py | 38 +- tests/private/test_private_miner_routes.py | 63 ++- tests/private/test_schemas.py | 47 ++ tests/validator/test_private_challenges.py | 89 +++- tests/validator/test_private_miners.py | 101 +++++ tests/validator/test_private_runner.py | 93 +++- .../test_private_runner_integration.py | 144 +++++- tests/validator/test_private_scoring.py | 395 ++++++++++++++++- tests/validator/test_private_spotcheck.py | 119 ++++- tests/validator/test_weights.py | 26 ++ 35 files changed, 2240 insertions(+), 111 deletions(-) create mode 100644 documentation/snooker-private-challenge-v0-brief.md create mode 100644 documentation/snooker-private-challenge-v0.md create mode 100644 scorevision/miner/private_track/SNOOKER_MINER_SPEC.md create mode 100644 tests/validator/test_private_miners.py diff --git a/documentation/snooker-private-challenge-v0-brief.md b/documentation/snooker-private-challenge-v0-brief.md new file mode 100644 index 0000000..4c54572 --- /dev/null +++ b/documentation/snooker-private-challenge-v0-brief.md @@ -0,0 +1,112 @@ +# Snooker Private Challenge V0 Brief + +## Decision + +Use the existing TurboVision private-track architecture and add a narrow snooker element: + +```text +manako/DetectSnookerBallState +``` + +This is not a redesign. It is the smallest rigorous path to a live snooker CV challenge: validators send a short clip plus explicit target frame ids, miners return ball states for those frames, and scoring stays inside the existing private runner/weights/audit flow. + +## What This Branch Provides + +- Public request contract: `video_url` plus `target_frames` +- Miner response contract: `prediction.type = "snooker_ball_state"` with `frames[].balls[]` +- Snooker manifest support: `groundtruth_type = "snooker_ball_state"` and `metrics.pillars.snooker_ball_state` +- Parser support for backend `/api/challenge/v3` snooker tasks +- Miner forwarding of `target_frames` +- Runner scoring/upload support for snooker +- Spotcheck rescoring support for snooker +- Score-scaled private weighting for snooker, matching cricket behavior +- Hardened ball-state scorer with target-frame-only evaluation and red Hungarian matching +- Miner-facing spec and backend/data handoff docs +- Full local validator/private test coverage + +## V0 Challenge Shape + +The backend returns a normalized 20 second clip and fixed target frames: + +```json +{ + "task_id": "12345", + "payload": { + "clip_url": "https://assets.example.com/snooker/window-abc.mp4", + "target_frames": [50, 150, 250, 350, 450] + } +} +``` + +The miner returns table-normalized ball states: + +```json +{ + "challenge_id": "12345", + "prediction": { + "type": "snooker_ball_state", + "frames": [ + { + "frame": 50, + "balls": [ + {"label": "cue", "x": 0.52, "y": 0.33, "state": "on_table"}, + {"label": "black", "state": "occluded"} + ] + } + ] + }, + "processing_time": 0.82 +} +``` + +## Scoring Summary + +Only requested target frames are scored. Missing requested frames score zero; extra non-target frames are ignored. + +The scorer rewards coordinate accuracy, identity accuracy, red count accuracy, state accuracy, and low false-positive rate. It penalizes duplicate unique colours, wrong red counts, missing balls, extra balls, invalid labels, empty predictions, and `on_table` balls without coordinates. + +Canonical labels are `cue`, `red`, `yellow`, `green`, `brown`, `blue`, `pink`, `black`. Reds are matched as an unordered set using Hungarian assignment. + +Canonical states are `on_table`, `potted`, `occluded`, `unknown`. + +## What Is Still External + +The branch deliberately does not implement the external backend or publish the live manifest. The remaining launch work is: + +- create 6 public bootstrap windows +- create 20 hidden scoring windows +- normalize clips to 20 seconds, 25 fps, max 480p +- annotate hidden ground truth for frames `[50, 150, 250, 350, 450]` +- host private clips and keep hidden source ids/timestamps unpublished +- implement backend `/api/challenge/v3` and `/api/tasks/{id}/ground-truth` responses using the branch contract +- publish the manifest element after backend/data are ready + +## Manifest Snippet + +```yaml +id: manako/DetectSnookerBallState +track: private +groundtruth_type: snooker_ball_state +weight: 0.05 +window_block: 300 +eval_window: 4 +metrics: + pillars: + snooker_ball_state: 1.0 +targets: + snooker_ball_state: 0.85 +baselines: + snooker_ball_state: 0.0 +first_block: +``` + +## Verification + +Current branch verification: + +```text +python -m compileall -q scorevision tests +python -m pytest tests/private tests/validator +``` + +Result: `155 passed`. diff --git a/documentation/snooker-private-challenge-v0.md b/documentation/snooker-private-challenge-v0.md new file mode 100644 index 0000000..9b58b7f --- /dev/null +++ b/documentation/snooker-private-challenge-v0.md @@ -0,0 +1,114 @@ +# Snooker Private Challenge V0 + +This branch defines the TurboVision-side contract for `manako/DetectSnookerBallState`. It does not implement the external challenge backend or publish the live manifest. + +## Challenge Shape + +Validators request one short video window and a fixed frame set: + +```json +{ + "task_id": "12345", + "payload": { + "clip_url": "https://assets.example.com/snooker/window-abc.mp4", + "target_frames": [50, 150, 250, 350, 450] + } +} +``` + +`video_url` may be provided at top level instead of `payload.clip_url`. For snooker, a video URL and non-empty `target_frames` are required. Frame ids are relative to the supplied 20 second clip. + +## Ground Truth Shape + +`GET /api/tasks/{id}/ground-truth` must return: + +```json +{ + "ground_truth": { + "frames": [ + { + "frame": 50, + "balls": [ + { + "label": "cue", + "x": 0.52, + "y": 0.33, + "state": "on_table", + "confidence": 1.0, + "bbox": [120, 90, 135, 105] + } + ] + } + ] + } +} +``` + +Hidden annotation storage should also retain table corners and orientation for QA. V0 miner responses do not require table-corner output. + +## Data Target + +Prototype dataset target: + +- 6 public bootstrap windows +- 20 hidden/private scoring windows +- Candidate source footage may be YouTube prototype footage for dry run +- Hidden clip ids, source timestamps, and ground truth must not be published + +Normalize every scoring clip before annotation: + +- 20 seconds +- 25 fps +- max height 480p +- target frames `[50, 150, 250, 350, 450]` + +## Manifest Snippet + +Use this as the private manifest element once the backend and hidden data are ready: + +```yaml +id: manako/DetectSnookerBallState +track: private +groundtruth_type: snooker_ball_state +weight: 0.05 +window_block: 300 +eval_window: 4 +metrics: + pillars: + snooker_ball_state: 1.0 +targets: + snooker_ball_state: 0.85 +baselines: + snooker_ball_state: 0.0 +first_block: +``` + +## V0 Scoring Contract + +The validator scores only requested `target_frames`. Missing requested frames score zero for that frame, and extra non-target frames are ignored. Duplicate response objects for a requested target frame are treated as contract violations: their ball lists are merged for scoring and the final frame-set score is penalized. + +Labels are canonical: `cue`, `red`, `yellow`, `green`, `brown`, `blue`, `pink`, `black`. Unique colours match by label. Reds match as an unordered set using Hungarian assignment in normalized table coordinates. + +States are canonical: `on_table`, `potted`, `occluded`, `unknown`. Invalid state strings are not normalized to `unknown`; they are scored as invalid predictions. `on_table` balls require `x` and `y`; `potted`, `occluded`, and `unknown` may omit coordinates. + +On-table coordinate scoring uses normalized table distance with a `0.05` tolerance. A perfect coordinate match receives full coordinate credit; coordinate credit decays linearly to zero at or beyond the tolerance. On-table predictions outside that tolerance are not counted as matched balls. Red-count accuracy only counts valid red predictions, so invalid red entries cannot earn red-count credit. + +Breakdown keys remain: + +- `coordinate_accuracy` +- `identity_accuracy` +- `red_count_accuracy` +- `state_accuracy` +- `false_positive_score` +- `snooker_ball_state` + +## External Dependencies + +Backend work still required: + +- `/api/challenge/v3` must emit snooker tasks with a clip URL and target frames +- `/api/tasks/{id}/ground-truth` must emit `ground_truth.frames` +- private data hosting must provide short normalized clips +- hidden annotation and QA must be produced outside this repo + +Rights/leakage note: public YouTube footage is acceptable only for dry-run examples. Hidden scoring should avoid exposed timestamps and ideally use rights-cleared or non-public assets. diff --git a/scorevision/__init__.py b/scorevision/__init__.py index 7eda989..da5e6a4 100644 --- a/scorevision/__init__.py +++ b/scorevision/__init__.py @@ -1,20 +1,23 @@ import asyncio from asyncio import run +import importlib from logging import getLogger from pathlib import Path import click -from scorevision.cli.audit_validator import audit_validator -from scorevision.cli.benchmark import benchmark_cli -from scorevision.cli.central_validator import central_validator -from scorevision.cli.elements import elements_cli -from scorevision.cli.index_maintenance import index_cli -from scorevision.cli.manifest import manifest_cli from scorevision.utils.logging import setup_logging from scorevision.utils.settings import get_settings logger = getLogger(__name__) +def _add_optional_command(module_name: str, command_name: str) -> None: + try: + module = importlib.import_module(module_name) + app.add_command(getattr(module, command_name)) + except Exception as e: + logger.debug("Optional CLI command %s.%s unavailable: %s", module_name, command_name, e) + + @click.group(name="sv") @click.option( "-v", @@ -126,9 +129,9 @@ def validate_cmd(tail: int, m_min: int, tempo: int, manifest_path): ) -app.add_command(audit_validator) -app.add_command(benchmark_cli) -app.add_command(central_validator) -app.add_command(index_cli) -app.add_command(manifest_cli) -app.add_command(elements_cli) +_add_optional_command("scorevision.cli.audit_validator", "audit_validator") +_add_optional_command("scorevision.cli.benchmark", "benchmark_cli") +_add_optional_command("scorevision.cli.central_validator", "central_validator") +_add_optional_command("scorevision.cli.index_maintenance", "index_cli") +_add_optional_command("scorevision.cli.manifest", "manifest_cli") +_add_optional_command("scorevision.cli.elements", "elements_cli") diff --git a/scorevision/cli/benchmark.py b/scorevision/cli/benchmark.py index fed9d0c..a61d984 100644 --- a/scorevision/cli/benchmark.py +++ b/scorevision/cli/benchmark.py @@ -3,11 +3,9 @@ from json import dumps, loads from pathlib import Path -import bittensor as bt import click from scorevision.miner.open_source.chute_template.schemas import TVPredictInput -from scorevision.utils.bittensor_helpers import get_subtensor from scorevision.utils.cloudflare_helpers import get_s3_client from scorevision.utils.manifest import get_current_manifest, load_manifest_from_public_index from scorevision.utils.miner_registry import get_miners_from_registry @@ -134,6 +132,8 @@ async def _resolve_private_target( element_id: str, winner_entry: dict, ): + from scorevision.utils.bittensor_helpers import get_subtensor + settings = get_settings() winner_hotkey = str(winner_entry.get("winner_hotkey") or "").strip() if not winner_hotkey: @@ -186,6 +186,8 @@ async def run_top_performer_benchmark( success_count = 0 if track == "private": + import bittensor as bt + target_miner = await _resolve_private_target(selected_element_id, winner_entry) wallet = bt.wallet( name=settings.BITTENSOR_WALLET_COLD, diff --git a/scorevision/miner/private_track/MINER.md b/scorevision/miner/private_track/MINER.md index 73ddde9..f290b7d 100644 --- a/scorevision/miner/private_track/MINER.md +++ b/scorevision/miner/private_track/MINER.md @@ -192,11 +192,12 @@ Current private elements are served as separate tracks from the miner point of v - Football private element (`groundtruth_type=soccer_action`) - Cricket private element (`groundtruth_type=cricket_delivery`) +- Snooker ball-state private element (`groundtruth_type=snooker_ball_state`) The template miner uses a static mode switch in `scorevision/miner/private_track/routes.py`: ```python -MINER_MODE = "soccer_action" # or "cricket_delivery" +MINER_MODE = "soccer_action" # or "cricket_delivery" or "snooker_ball_state" ``` Keep one deployment per mode. Do not expect one running container to handle both private element types automatically. @@ -238,6 +239,7 @@ Keep one deployment per mode. Do not expect one running container to handle both ``` For full cricket field guidance, see `scorevision/miner/CRICKET_MINER_SPEC.md`. +For snooker ball-state guidance, see `scorevision/miner/private_track/SNOOKER_MINER_SPEC.md`. ### 1.3 On-Chain Commitment Must Match Element @@ -255,7 +257,7 @@ Example commitment shape (conceptual): } ``` -If you serve cricket, use the cricket element id instead. +If you serve cricket or snooker, use that element id instead. ### 2. Test Locally (Without Wallet) @@ -318,12 +320,12 @@ The CLI will: After deployment, **share with Score** (see [GHCR Setup](#ghcr-setup) step 4). -## Private Miner Checklist (Football / Cricket) +## Private Miner Checklist (Football / Cricket / Snooker) Before declaring your miner live: 1. Set `MINER_MODE` to the intended private element type. -2. Ensure predictor output matches the expected response payload (`soccer_action.items` or `cricket_delivery.item`). +2. Ensure predictor output matches the expected response payload (`soccer_action.items`, `cricket_delivery.item`, or `snooker_ball_state.frames`). 3. Deploy image and commit with the correct private `element_id`. 4. Verify your axon IP/port is reachable. 5. Confirm Score has GHCR read access to your package. diff --git a/scorevision/miner/private_track/SNOOKER_MINER_SPEC.md b/scorevision/miner/private_track/SNOOKER_MINER_SPEC.md new file mode 100644 index 0000000..c72b00b --- /dev/null +++ b/scorevision/miner/private_track/SNOOKER_MINER_SPEC.md @@ -0,0 +1,118 @@ +# Snooker Private Track Miner Spec + +This is the v0 miner-facing contract for: + +```text +manako/DetectSnookerBallState +``` + +V0 is a narrow ball-state challenge. Miners receive a normalized short clip and explicit target frame ids, then return ball state only for those requested frames. + +## Request Envelope + +The validator sends `/challenge` requests using `ChallengeRequest`. + +```json +{ + "challenge_id": "12345", + "video_url": "https://assets.example.com/snooker/window-abc.mp4", + "target_frames": [50, 150, 250, 350, 450] +} +``` + +For snooker, `video_url` and non-empty `target_frames` are required. Frame ids are relative to the provided 20 second clip, not to the source match. + +## Response Envelope + +Return one `snooker_ball_state` frame object per requested target frame. + +```json +{ + "challenge_id": "12345", + "prediction": { + "type": "snooker_ball_state", + "frames": [ + { + "frame": 50, + "balls": [ + { + "label": "cue", + "x": 0.52, + "y": 0.33, + "state": "on_table", + "confidence": 0.98 + }, + { + "label": "red", + "x": 0.61, + "y": 0.41, + "state": "on_table" + }, + { + "label": "black", + "state": "occluded" + } + ] + } + ] + }, + "processing_time": 0.82 +} +``` + +Extra non-target frames are ignored by scoring. Missing target frames score zero for that frame. Return at most one object per requested frame; duplicate target-frame objects are merged and penalized. + +## Coordinates + +`x` and `y` are normalized table coordinates: + +- `x=0.0`: baulk end +- `x=1.0`: black-spot end +- `y=0.0`: left cushion in the canonical broadcast-overhead orientation +- `y=1.0`: right cushion in the canonical broadcast-overhead orientation + +`x` and `y` are required for balls with `state: "on_table"`. Coordinates may be omitted for `potted`, `occluded`, or `unknown` balls. + +On-table coordinate credit decays linearly to zero at a normalized table distance of `0.05`. Predictions farther away than that are treated as unmatched for that ball. + +Optional `bbox` fields may be included for audit/debugging, but table-space coordinates are the scoring target. + +## Labels + +Accepted canonical labels: + +- `cue` +- `red` +- `yellow` +- `green` +- `brown` +- `blue` +- `pink` +- `black` + +Reds are returned as multiple entries with `label: "red"` and are matched as an unordered set in normalized table coordinates. + +## States + +Accepted canonical states: + +- `on_table` +- `potted` +- `occluded` +- `unknown` + +Invalid state strings are scored as invalid predictions. They are not treated as `unknown`. + +## Scoring Intent + +The scorer rewards: + +- coordinate accuracy for matched on-table balls +- identity/class accuracy +- red count and red-set matching quality +- state accuracy +- low false-positive, duplicate, invalid-label, and extra-ball rate + +Duplicate unique colours, wrong red counts, missing balls, invalid labels, empty predictions, and `on_table` balls without coordinates are penalized. + +V0 deliberately excludes shot boundaries, shot outcomes, cue geometry, table-boundary scoring, and tactical classification. diff --git a/scorevision/miner/private_track/predictor.py b/scorevision/miner/private_track/predictor.py index cc88425..99d9558 100644 --- a/scorevision/miner/private_track/predictor.py +++ b/scorevision/miner/private_track/predictor.py @@ -1,6 +1,13 @@ from pathlib import Path from scorevision.miner.private_track.video import get_frame_count -from scorevision.utils.schemas import ChallengeRequest, CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + ChallengeRequest, + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) def predict_actions(video_path: Path) -> list[FramePrediction]: @@ -42,3 +49,29 @@ def predict_cricket_delivery(request: ChallengeRequest) -> CricketDeliveryPredic runs=-1, wickets=-1, ) + + +def predict_snooker_ball_state(request: ChallengeRequest) -> SnookerBallStatePrediction: + # TODO: Replace this stub with actual snooker ball-state inference. + frame_id = 0 + if request.target_frames: + frame_id = request.target_frames[0] + elif request.frames: + frame_id = request.frames[0].frame_id + + return SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=frame_id, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="unknown", + confidence=0.0, + ) + ], + ) + ] + ) diff --git a/scorevision/miner/private_track/routes.py b/scorevision/miner/private_track/routes.py index 510ecbf..50e116b 100644 --- a/scorevision/miner/private_track/routes.py +++ b/scorevision/miner/private_track/routes.py @@ -1,11 +1,16 @@ import time from fastapi import HTTPException -from scorevision.miner.private_track.predictor import predict_actions, predict_cricket_delivery +from scorevision.miner.private_track.predictor import ( + predict_actions, + predict_cricket_delivery, + predict_snooker_ball_state, +) from scorevision.miner.private_track.video import delete_video, download_video from scorevision.utils.schemas import ChallengeRequest, ChallengeResponse, PredictionPayload from scorevision.miner.private_track.logging import logger -# TODO: choose a single mode per miner deployment: "soccer_action" or "cricket_delivery". +# TODO: choose a single mode per miner deployment: +# "soccer_action", "cricket_delivery", or "snooker_ball_state". MINER_MODE = "soccer_action" @@ -29,6 +34,23 @@ async def handle_challenge(request: ChallengeRequest) -> ChallengeResponse: processing_time=processing_time, ) + if MINER_MODE == "snooker_ball_state": + prediction = predict_snooker_ball_state(request) + processing_time = time.perf_counter() - start_time + logger.info( + "Snooker ball-state challenge completed: %s, time: %.1fs", + request.challenge_id, + processing_time, + ) + return ChallengeResponse( + challenge_id=request.challenge_id, + prediction=PredictionPayload( + type="snooker_ball_state", + frames=prediction.frames, + ), + processing_time=processing_time, + ) + video_path = await download_video(request.video_url) predictions = predict_actions(video_path) processing_time = time.perf_counter() - start_time diff --git a/scorevision/miner/private_track/security.py b/scorevision/miner/private_track/security.py index df22273..bdc55e8 100644 --- a/scorevision/miner/private_track/security.py +++ b/scorevision/miner/private_track/security.py @@ -1,14 +1,28 @@ import os +from types import SimpleNamespace from fastapi import Depends, Header, HTTPException, Request -from fiber import constants as cst -from fiber import utils -from fiber.chain import signatures -from fiber.miner.security.nonce_management import NonceManager +try: + from fiber import constants as cst + from fiber import utils + from fiber.chain import signatures + from fiber.miner.security.nonce_management import NonceManager + _FIBER_AVAILABLE = True +except ImportError: + cst = SimpleNamespace( + VALIDATOR_HOTKEY="X-Validator-Hotkey", + SIGNATURE="X-Signature", + MINER_HOTKEY="X-Miner-Hotkey", + NONCE="X-Nonce", + ) + utils = None + signatures = None + NonceManager = None + _FIBER_AVAILABLE = False BLACKLIST_ENABLED = os.environ.get("BLACKLIST_ENABLED", "true").lower() in ("true", "1", "yes") VERIFY_ENABLED = os.environ.get("VERIFY_ENABLED", "true").lower() in ("true", "1", "yes") -_nonce_manager = NonceManager() +_nonce_manager = NonceManager() if NonceManager is not None else None async def verify_request( @@ -18,6 +32,8 @@ async def verify_request( miner_hotkey: str = Header(..., alias=cst.MINER_HOTKEY), nonce: str = Header(..., alias=cst.NONCE), ): + if not _FIBER_AVAILABLE or _nonce_manager is None or signatures is None or utils is None: + raise HTTPException(status_code=503, detail="fiber security dependency is not installed") if not _nonce_manager.nonce_is_valid(nonce): raise HTTPException(status_code=401, detail="Invalid nonce") @@ -41,10 +57,14 @@ def get_security_dependencies() -> list: deps = [] if BLACKLIST_ENABLED: + if not _FIBER_AVAILABLE: + raise RuntimeError("fiber security dependency is required when BLACKLIST_ENABLED=true") from fiber.miner.dependencies import blacklist_low_stake deps.append(Depends(blacklist_low_stake)) if VERIFY_ENABLED: + if not _FIBER_AVAILABLE: + raise RuntimeError("fiber security dependency is required when VERIFY_ENABLED=true") deps.append(Depends(verify_request)) return deps diff --git a/scorevision/utils/bittensor_helpers.py b/scorevision/utils/bittensor_helpers.py index 0e05cbd..47e49f6 100644 --- a/scorevision/utils/bittensor_helpers.py +++ b/scorevision/utils/bittensor_helpers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import os from pathlib import Path @@ -5,10 +7,10 @@ from json import load, dumps, loads from logging import getLogger from traceback import print_exc -from typing import Optional +from typing import Optional, TYPE_CHECKING -from substrateinterface import Keypair -from bittensor import wallet, async_subtensor +if TYPE_CHECKING: + from substrateinterface import Keypair from scorevision.utils.settings import get_settings from scorevision.utils.huggingface_helpers import get_huggingface_repo_name @@ -118,6 +120,8 @@ def get_last_update_for_hotkey( def load_hotkey_keypair(wallet_name: str, hotkey_name: str) -> Keypair: + from substrateinterface import Keypair + settings = get_settings() wallet_dir = Path(settings.BITTENSOR_WALLET_PATH).expanduser() @@ -149,6 +153,8 @@ async def get_subtensor(): init_timeout = float(os.getenv("SUBTENSOR_INIT_TIMEOUT_S", "15.0")) async def _init(ep: str): + from bittensor import async_subtensor + st = async_subtensor(ep) await asyncio.wait_for(st.initialize(), timeout=init_timeout) return st @@ -175,6 +181,8 @@ async def on_chain_commit( chute_slug: str | None, element_id: str | None, ) -> None: + from bittensor import wallet + settings = get_settings() repo_name = get_huggingface_repo_name() w = wallet( @@ -314,6 +322,8 @@ async def _set_weights_with_confirmation( async def on_chain_commit_validator(index_url: str) -> None: """ """ + from bittensor import wallet + settings = get_settings() w = wallet( name=settings.BITTENSOR_WALLET_COLD, @@ -388,6 +398,8 @@ async def _already_committed_same_index(netuid: int, index_url: str) -> bool: meta = await st.metagraph(netuid, mechid=settings.SCOREVISION_MECHID) commits = await st.get_all_revealed_commitments(netuid) + from bittensor import wallet + w = wallet( name=settings.BITTENSOR_WALLET_COLD, hotkey=settings.BITTENSOR_WALLET_HOT, @@ -486,6 +498,8 @@ async def _first_commit_block_by_miner( ): st_archive = None try: + from bittensor import async_subtensor + st_archive = async_subtensor(_TIEBREAK_COMMIT_BACKFILL_ARCHIVE_ENDPOINT) await asyncio.wait_for(st_archive.initialize(), timeout=20.0) sem = asyncio.Semaphore(_TIEBREAK_COMMIT_BACKFILL_CONCURRENCY) @@ -631,6 +645,8 @@ async def on_chain_commit_validator_retry( max_retries: int | None = None, ) -> bool: """ """ + from bittensor import wallet + settings = get_settings() w = wallet( name=settings.BITTENSOR_WALLET_COLD, diff --git a/scorevision/utils/cloudflare_helpers.py b/scorevision/utils/cloudflare_helpers.py index 7a5010a..6c37292 100644 --- a/scorevision/utils/cloudflare_helpers.py +++ b/scorevision/utils/cloudflare_helpers.py @@ -8,8 +8,12 @@ from time import time from urllib.parse import urljoin, urlparse import aiohttp -from async_substrate_interface.errors import SubstrateRequestException -from substrateinterface import Keypair +try: + from async_substrate_interface.errors import SubstrateRequestException +except ImportError: + class SubstrateRequestException(Exception): + pass + from scorevision.utils.bittensor_helpers import get_subtensor, reset_subtensor from scorevision.utils.data_models import SVChallenge, SVEvaluation, SVRunOutput from scorevision.utils.prometheus import ( @@ -88,6 +92,8 @@ def _get_cache_dir(): def _verify_signature(hk_ss58: str, payload: str, sig_hex: str) -> bool: try: + from substrateinterface import Keypair + if not hk_ss58 or not sig_hex: return False sig_hex = sig_hex[2:] if sig_hex.startswith("0x") else sig_hex diff --git a/scorevision/utils/manifest.py b/scorevision/utils/manifest.py index d316dd4..30e6789 100644 --- a/scorevision/utils/manifest.py +++ b/scorevision/utils/manifest.py @@ -63,6 +63,7 @@ class PillarName(str, Enum): RECALL = "recall" SOCCER_ACTION = "soccer_action" CRICKET_SCORING = "cricket_scoring" + SNOOKER_BALL_STATE = "snooker_ball_state" FALSE_POSITIVE = "false_positive" @@ -92,6 +93,7 @@ class ChallengeType(str, Enum): class GroundTruthType(str, Enum): SOCCER_ACTION = "soccer_action" CRICKET_DELIVERY = "cricket_delivery" + SNOOKER_BALL_STATE = "snooker_ball_state" # ------------------------------------------------------------ diff --git a/scorevision/utils/miner_registry.py b/scorevision/utils/miner_registry.py index 8370bfe..eb5ef0d 100644 --- a/scorevision/utils/miner_registry.py +++ b/scorevision/utils/miner_registry.py @@ -8,7 +8,6 @@ import aiohttp from huggingface_hub import HfApi -from bittensor import async_subtensor from scorevision.utils.bittensor_helpers import ( get_subtensor, @@ -629,6 +628,8 @@ async def get_miners_from_registry( ) st_archive = None try: + from bittensor import async_subtensor + st_archive = async_subtensor(_REGISTRY_COMMIT_BACKFILL_ARCHIVE_ENDPOINT) await asyncio.wait_for(st_archive.initialize(), timeout=20.0) sem = asyncio.Semaphore(_REGISTRY_COMMIT_BACKFILL_CONCURRENCY) diff --git a/scorevision/utils/schemas.py b/scorevision/utils/schemas.py index feff9c6..8bd30bc 100644 --- a/scorevision/utils/schemas.py +++ b/scorevision/utils/schemas.py @@ -52,10 +52,29 @@ class CricketDeliveryPrediction(BaseModel): wickets: int | None = Field(default=None, validation_alias=AliasChoices("wickets", "wkts")) +class SnookerBallPrediction(BaseModel): + label: str + x: float | None = Field(default=None, ge=0.0, le=1.0) + y: float | None = Field(default=None, ge=0.0, le=1.0) + state: str = "on_table" + confidence: float = Field(default=1.0, ge=0.0, le=1.0) + bbox: list[int] | None = None + + +class SnookerBallStateFrame(BaseModel): + frame: int = Field(ge=0) + balls: list[SnookerBallPrediction] + + +class SnookerBallStatePrediction(BaseModel): + frames: list[SnookerBallStateFrame] + + class PredictionPayload(BaseModel): type: str items: list[FramePrediction] | None = None item: CricketDeliveryPrediction | None = None + frames: list[SnookerBallStateFrame] | None = None @model_validator(mode="after") def validate_payload(self): @@ -64,12 +83,22 @@ def validate_payload(self): raise ValueError("soccer_action prediction requires items") if self.item is not None: raise ValueError("soccer_action prediction must not include item") + if self.frames is not None: + raise ValueError("soccer_action prediction must not include frames") return self if self.type == "cricket_delivery": if self.item is None: raise ValueError("cricket_delivery prediction requires item") if self.items is not None: raise ValueError("cricket_delivery prediction must not include items") + if self.frames is not None: + raise ValueError("cricket_delivery prediction must not include frames") + return self + if self.type == "snooker_ball_state": + if self.frames is None: + raise ValueError("snooker_ball_state prediction requires frames") + if self.items is not None or self.item is not None: + raise ValueError("snooker_ball_state prediction must not include items or item") return self raise ValueError(f"Unsupported prediction type: {self.type}") @@ -90,9 +119,12 @@ class ChallengeRequest(BaseModel): challenge_id: str video_url: str | None = None frames: list[ChallengeFrame] | None = None + target_frames: list[int] | None = None @model_validator(mode="after") def validate_payload(self): + if self.target_frames is not None and any(frame < 0 for frame in self.target_frames): + raise ValueError("target_frames must be non-negative frame ids") if self.video_url: return self if self.frames: @@ -103,13 +135,18 @@ def validate_payload(self): class ChallengeResponse(BaseModel): challenge_id: str predictions: list[FramePrediction] | None = None - prediction: PredictionPayload | CricketDeliveryPrediction | None = None + prediction: PredictionPayload | CricketDeliveryPrediction | SnookerBallStatePrediction | None = None processing_time: float @model_validator(mode="after") def validate_payload(self): if isinstance(self.prediction, CricketDeliveryPrediction): self.prediction = PredictionPayload(type="cricket_delivery", item=self.prediction) + if isinstance(self.prediction, SnookerBallStatePrediction): + self.prediction = PredictionPayload( + type="snooker_ball_state", + frames=self.prediction.frames, + ) has_legacy_predictions = self.predictions is not None has_prediction = self.prediction is not None @@ -128,6 +165,8 @@ def validate_payload(self): def prediction_count(self) -> int: if isinstance(self.prediction, PredictionPayload) and self.prediction.type == "cricket_delivery": return 1 + if isinstance(self.prediction, PredictionPayload) and self.prediction.type == "snooker_ball_state": + return sum(len(frame.balls) for frame in (self.prediction.frames or [])) if isinstance(self.prediction, PredictionPayload) and self.prediction.type == "soccer_action": return len(self.prediction.items or []) return len(self.predictions or []) @@ -136,6 +175,10 @@ def prediction_count(self) -> int: def is_cricket(self) -> bool: return isinstance(self.prediction, PredictionPayload) and self.prediction.type == "cricket_delivery" + @property + def is_snooker_ball_state(self) -> bool: + return isinstance(self.prediction, PredictionPayload) and self.prediction.type == "snooker_ball_state" + @model_serializer(mode="wrap") def serialize_model(self, handler): data = handler(self) diff --git a/scorevision/validator/__init__.py b/scorevision/validator/__init__.py index 39f5195..1b442b1 100644 --- a/scorevision/validator/__init__.py +++ b/scorevision/validator/__init__.py @@ -19,30 +19,37 @@ stake_of, weighted_median, ) -from scorevision.validator.winner import get_winner_for_element -from scorevision.validator.core import ( - weights_loop, - set_weights_via_signer, - load_manifest_for_block, - commit_validator_on_start, - get_validator_hotkey_ss58, - run_signer, -) -from scorevision.validator.central import ( - runner, - runner_loop, -) -from scorevision.validator.audit import ( - spotcheck_loop, - run_single_spotcheck, - run_spotcheck, - fetch_random_challenge_record, - load_challenge_record_from_mock_dir, - calculate_match_percentage, - scores_match, - calculate_next_spotcheck_delay, -) +_LAZY_EXPORTS = { + "get_winner_for_element": ("scorevision.validator.winner", "get_winner_for_element"), + "weights_loop": ("scorevision.validator.core", "weights_loop"), + "set_weights_via_signer": ("scorevision.validator.core", "set_weights_via_signer"), + "load_manifest_for_block": ("scorevision.validator.core", "load_manifest_for_block"), + "commit_validator_on_start": ("scorevision.validator.core", "commit_validator_on_start"), + "get_validator_hotkey_ss58": ("scorevision.validator.core", "get_validator_hotkey_ss58"), + "run_signer": ("scorevision.validator.core", "run_signer"), + "runner": ("scorevision.validator.central", "runner"), + "runner_loop": ("scorevision.validator.central", "runner_loop"), + "spotcheck_loop": ("scorevision.validator.audit", "spotcheck_loop"), + "run_single_spotcheck": ("scorevision.validator.audit", "run_single_spotcheck"), + "run_spotcheck": ("scorevision.validator.audit", "run_spotcheck"), + "fetch_random_challenge_record": ("scorevision.validator.audit", "fetch_random_challenge_record"), + "load_challenge_record_from_mock_dir": ("scorevision.validator.audit", "load_challenge_record_from_mock_dir"), + "calculate_match_percentage": ("scorevision.validator.audit", "calculate_match_percentage"), + "scores_match": ("scorevision.validator.audit", "scores_match"), + "calculate_next_spotcheck_delay": ("scorevision.validator.audit", "calculate_next_spotcheck_delay"), +} + + +def __getattr__(name: str): + if name not in _LAZY_EXPORTS: + raise AttributeError(name) + import importlib + + module_name, attr_name = _LAZY_EXPORTS[name] + value = getattr(importlib.import_module(module_name), attr_name) + globals()[name] = value + return value __all__ = [ "ChallengeRecord", diff --git a/scorevision/validator/audit/__init__.py b/scorevision/validator/audit/__init__.py index 31119c9..157952f 100644 --- a/scorevision/validator/audit/__init__.py +++ b/scorevision/validator/audit/__init__.py @@ -1,13 +1,25 @@ -from scorevision.validator.audit.open_source.spotcheck import ( - spotcheck_loop, - run_single_spotcheck, - run_spotcheck, - fetch_random_challenge_record, - load_challenge_record_from_mock_dir, - calculate_match_percentage, - scores_match, - calculate_next_spotcheck_delay, -) +_LAZY_EXPORTS = { + "spotcheck_loop": ("scorevision.validator.audit.open_source.spotcheck", "spotcheck_loop"), + "run_single_spotcheck": ("scorevision.validator.audit.open_source.spotcheck", "run_single_spotcheck"), + "run_spotcheck": ("scorevision.validator.audit.open_source.spotcheck", "run_spotcheck"), + "fetch_random_challenge_record": ("scorevision.validator.audit.open_source.spotcheck", "fetch_random_challenge_record"), + "load_challenge_record_from_mock_dir": ("scorevision.validator.audit.open_source.spotcheck", "load_challenge_record_from_mock_dir"), + "calculate_match_percentage": ("scorevision.validator.audit.open_source.spotcheck", "calculate_match_percentage"), + "scores_match": ("scorevision.validator.audit.open_source.spotcheck", "scores_match"), + "calculate_next_spotcheck_delay": ("scorevision.validator.audit.open_source.spotcheck", "calculate_next_spotcheck_delay"), +} + + +def __getattr__(name: str): + if name not in _LAZY_EXPORTS: + raise AttributeError(name) + import importlib + + module_name, attr_name = _LAZY_EXPORTS[name] + value = getattr(importlib.import_module(module_name), attr_name) + globals()[name] = value + return value + __all__ = [ "spotcheck_loop", diff --git a/scorevision/validator/audit/open_source/storage.py b/scorevision/validator/audit/open_source/storage.py index 5c433b3..06dee98 100644 --- a/scorevision/validator/audit/open_source/storage.py +++ b/scorevision/validator/audit/open_source/storage.py @@ -3,7 +3,6 @@ from json import dumps from logging import getLogger from time import time -from bittensor import wallet from scorevision.utils.bittensor_helpers import get_subtensor, reset_subtensor from scorevision.utils.r2 import ( add_index_key_if_new, @@ -124,6 +123,8 @@ async def commit_audit_index_on_start() -> None: async def _commit_audit_index(index_url: str) -> bool: + from bittensor import wallet + s = get_settings() w = wallet( name=s.BITTENSOR_WALLET_COLD, diff --git a/scorevision/validator/audit/private_track/spotcheck.py b/scorevision/validator/audit/private_track/spotcheck.py index ba1b6d0..e493f02 100644 --- a/scorevision/validator/audit/private_track/spotcheck.py +++ b/scorevision/validator/audit/private_track/spotcheck.py @@ -9,13 +9,18 @@ from scorevision.utils.r2 import audit_r2_config, create_s3_client, ensure_index_exists, add_index_key_if_new, is_configured from scorevision.utils.r2_public import fetch_index_keys, filter_keys_by_tail, fetch_shard_lines from scorevision.utils.request_signing import build_signed_headers -from scorevision.utils.schemas import CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + CricketDeliveryPrediction, + FramePrediction, + SnookerBallStatePrediction, +) from scorevision.utils.settings import get_settings from scorevision.utils.signing import _sign_batch from scorevision.validator.audit.open_source.spotcheck import calculate_match_percentage from scorevision.validator.central.private_track.challenges import fetch_ground_truth from scorevision.validator.central.private_track.scoring import ( score_cricket_prediction_with_breakdown, + score_snooker_ball_state_with_breakdown, score_predictions, ) from scorevision.validator.models import SpotcheckResult @@ -107,21 +112,28 @@ def _prediction_looks_cricket(prediction: dict) -> bool: return bool(keys & _CRICKET_HINT_FIELDS) +def _prediction_looks_snooker(prediction: dict) -> bool: + return "frame" in prediction and isinstance(prediction.get("balls"), list) + + def _infer_groundtruth_type( challenge_results: list[dict], miner_responses: dict[str, list[dict]], ) -> str: for entry in challenge_results: gt = str(entry.get("groundtruth_type") or "").strip() - if gt in {"soccer_action", "cricket_delivery"}: + if gt in {"soccer_action", "cricket_delivery", "snooker_ball_state"}: return gt for predictions_raw in miner_responses.values(): if not predictions_raw: continue first = predictions_raw[0] - if isinstance(first, dict) and _prediction_looks_cricket(first): - return "cricket_delivery" + if isinstance(first, dict): + if _prediction_looks_snooker(first): + return "snooker_ball_state" + if _prediction_looks_cricket(first): + return "cricket_delivery" return "soccer_action" @@ -156,6 +168,15 @@ def rescore_miner_cricket( return score +def rescore_miner_snooker( + predictions_raw: list[dict], + ground_truth: SnookerBallStatePrediction, +) -> float: + prediction_obj = SnookerBallStatePrediction(frames=predictions_raw) + score, _ = score_snooker_ball_state_with_breakdown(prediction_obj, ground_truth) + return score + + async def run_private_spotcheck( challenge_id: str, challenge_results: list[dict], @@ -184,6 +205,8 @@ async def run_private_spotcheck( if groundtruth_type == "cricket_delivery": audit_score = rescore_miner_cricket(predictions_raw, ground_truth) + elif groundtruth_type == "snooker_ball_state": + audit_score = rescore_miner_snooker(predictions_raw, ground_truth) else: audit_score = rescore_miner_soccer(predictions_raw, ground_truth) match_pct = calculate_match_percentage(central_score, audit_score) diff --git a/scorevision/validator/central/__init__.py b/scorevision/validator/central/__init__.py index 19169e6..09f4509 100644 --- a/scorevision/validator/central/__init__.py +++ b/scorevision/validator/central/__init__.py @@ -1,7 +1,19 @@ -from scorevision.validator.central.open_source.runner import ( - runner, - runner_loop, -) +_LAZY_EXPORTS = { + "runner": ("scorevision.validator.central.open_source.runner", "runner"), + "runner_loop": ("scorevision.validator.central.open_source.runner", "runner_loop"), +} + + +def __getattr__(name: str): + if name not in _LAZY_EXPORTS: + raise AttributeError(name) + import importlib + + module_name, attr_name = _LAZY_EXPORTS[name] + value = getattr(importlib.import_module(module_name), attr_name) + globals()[name] = value + return value + __all__ = [ "runner", diff --git a/scorevision/validator/central/private_track/challenges.py b/scorevision/validator/central/private_track/challenges.py index 1fac30b..b181b51 100644 --- a/scorevision/validator/central/private_track/challenges.py +++ b/scorevision/validator/central/private_track/challenges.py @@ -3,7 +3,12 @@ import httpx from scorevision.utils.challenges import get_next_challenge_v3, _coerce_payload_frames from scorevision.utils.signing import build_validator_query_params -from scorevision.utils.schemas import ChallengeFrame, CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + ChallengeFrame, + CricketDeliveryPrediction, + FramePrediction, + SnookerBallStatePrediction, +) from scorevision.utils.settings import get_settings logger = getLogger(__name__) @@ -12,15 +17,41 @@ @dataclass class Challenge: challenge_id: str - ground_truth: list[FramePrediction] | CricketDeliveryPrediction + ground_truth: list[FramePrediction] | CricketDeliveryPrediction | SnookerBallStatePrediction groundtruth_type: str = "soccer_action" video_url: str | None = None payload_frames: list[ChallengeFrame] | None = None + target_frames: list[int] | None = None -def has_sufficient_actions(ground_truth: list[FramePrediction] | CricketDeliveryPrediction, groundtruth_type: str) -> bool: +def _coerce_target_frames(raw: object) -> list[int]: + if raw is None: + return [] + if not isinstance(raw, list): + return [] + + frames: list[int] = [] + seen: set[int] = set() + for item in raw: + try: + frame = int(item) + except (TypeError, ValueError): + continue + if frame < 0 or frame in seen: + continue + seen.add(frame) + frames.append(frame) + return frames + + +def has_sufficient_actions( + ground_truth: list[FramePrediction] | CricketDeliveryPrediction | SnookerBallStatePrediction, + groundtruth_type: str, +) -> bool: if groundtruth_type == "cricket_delivery": return True + if groundtruth_type == "snooker_ball_state": + return bool(getattr(ground_truth, "frames", None)) return len(ground_truth) >= get_settings().PRIVATE_MIN_ACTIONS_FOR_CHALLENGE @@ -36,7 +67,7 @@ async def fetch_ground_truth( keypair, element_id: str | None = None, groundtruth_type: str = "soccer_action", -) -> list[FramePrediction] | CricketDeliveryPrediction: +) -> list[FramePrediction] | CricketDeliveryPrediction | SnookerBallStatePrediction: settings = get_settings() api_url = settings.PRIVATE_GT_API_URL or settings.SCOREVISION_API if not api_url: @@ -63,6 +94,14 @@ async def fetch_ground_truth( raise ValueError("Cricket ground truth missing meta payload") return CricketDeliveryPrediction(**meta) + if groundtruth_type == "snooker_ball_state": + raw = data.get("ground_truth") or {} + if isinstance(raw, dict) and "frames" in raw: + return SnookerBallStatePrediction(**raw) + if isinstance(raw, list): + return SnookerBallStatePrediction(frames=raw) + raise ValueError("Snooker ground truth missing frames payload") + ground_truth: list[FramePrediction] = [] for gt in data.get("ground_truth", []): action = gt.get("type") or gt.get("action") @@ -118,12 +157,19 @@ async def get_challenge_with_ground_truth( or payload.get("video_url") or payload.get("clip_url") ) + target_frames = _coerce_target_frames(chal.get("target_frames") or payload.get("target_frames")) payload_frames_raw = _coerce_payload_frames(payload) payload_frames = [ChallengeFrame(**frame) for frame in payload_frames_raw] or None if not challenge_id or (not video_url and not payload_frames): logger.warning("Challenge missing task_id or challenge asset (video_url/frames), retrying") continue + if groundtruth_type == "snooker_ball_state" and (not video_url or not target_frames): + logger.warning( + "Snooker challenge %s missing required video_url or target_frames, retrying", + challenge_id, + ) + continue try: await complete_task_assignment( @@ -153,6 +199,7 @@ async def get_challenge_with_ground_truth( challenge_id=str(challenge_id), video_url=video_url, payload_frames=payload_frames, + target_frames=target_frames or None, ground_truth=ground_truth, groundtruth_type=groundtruth_type, ) diff --git a/scorevision/validator/central/private_track/miners.py b/scorevision/validator/central/private_track/miners.py index 71a4fff..4f4ceb8 100644 --- a/scorevision/validator/central/private_track/miners.py +++ b/scorevision/validator/central/private_track/miners.py @@ -29,6 +29,7 @@ async def send_challenge( challenge_id=challenge.challenge_id, video_url=challenge.video_url, frames=challenge.payload_frames, + target_frames=challenge.target_frames, ) start = perf_counter() diff --git a/scorevision/validator/central/private_track/registry.py b/scorevision/validator/central/private_track/registry.py index b3b2d9b..1a16e30 100644 --- a/scorevision/validator/central/private_track/registry.py +++ b/scorevision/validator/central/private_track/registry.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import json import logging from dataclasses import dataclass -import bittensor as bt +from typing import Any from scorevision.utils.settings import get_settings logger = logging.getLogger(__name__) @@ -58,7 +60,7 @@ def _pick_latest_private_commit_for_element( async def get_registered_miners( - subtensor: bt.AsyncSubtensor, + subtensor: Any, metagraph, blacklist: set[str], element_id: str | None = None, diff --git a/scorevision/validator/central/private_track/runner.py b/scorevision/validator/central/private_track/runner.py index 1afc197..6716431 100644 --- a/scorevision/validator/central/private_track/runner.py +++ b/scorevision/validator/central/private_track/runner.py @@ -31,6 +31,7 @@ from scorevision.validator.central.private_track.scoring import ( PRIVATE_SCORING_VERSION, score_cricket_prediction_with_breakdown, + score_snooker_ball_state_with_breakdown, score_predictions_with_breakdown, ) from scorevision.validator.central.private_track.spotcheck import PendingSpotcheck @@ -42,6 +43,7 @@ update_element_state, ) from scorevision.validator.models import PrivateEvaluationResult +from scorevision.utils.schemas import SnookerBallStatePrediction logger = logging.getLogger(__name__) @@ -71,6 +73,8 @@ def _ground_truth_count(challenge: Challenge) -> int: ground_truth = challenge.ground_truth if isinstance(ground_truth, list): return len(ground_truth) + if isinstance(ground_truth, SnookerBallStatePrediction): + return sum(len(frame.balls) for frame in ground_truth.frames) return 1 if ground_truth is not None else 0 @@ -140,7 +144,7 @@ async def _upload_private_response_blob( block: int, response_predictions: list[dict] | None, ) -> str | None: - if not response_predictions: + if response_predictions is None: return None prefix = (get_settings().PRIVATE_RESPONSES_R2_PREFIX or "private_responses").strip().strip("/") @@ -155,6 +159,7 @@ async def _upload_private_response_blob( "challenge_id": challenge.challenge_id, "video_url": challenge.video_url, "frames": [frame.model_dump(mode="json") for frame in (challenge.payload_frames or [])] or None, + "target_frames": challenge.target_frames, "miner_hotkey": miner.hotkey, "miner_uid": miner.uid, "predictions": response_predictions, @@ -335,6 +340,24 @@ async def _challenge_miner( [cricket_prediction.model_dump(mode="json")] if cricket_prediction else [] ) benchmark_result = None + elif challenge.groundtruth_type == "snooker_ball_state": + snooker_prediction = None + if response.prediction is not None and hasattr(response.prediction, "frames"): + snooker_prediction = SnookerBallStatePrediction( + frames=response.prediction.frames or [] + ) + score, score_breakdown = score_snooker_ball_state_with_breakdown( + snooker_prediction, + challenge.ground_truth, + target_frames=challenge.target_frames, + ) + pred_count = response.prediction_count + response_predictions = ( + [frame.model_dump(mode="json") for frame in snooker_prediction.frames] + if snooker_prediction + else [] + ) + benchmark_result = None else: score, score_breakdown = score_predictions_with_breakdown( response.predictions or [], @@ -432,11 +455,15 @@ async def _emit_private_score_to_public_db( payload=TVPredictInput( url=challenge.video_url, frames=payload_frames, - meta={"track": "private"}, + meta={ + "track": "private", + "target_frames": challenge.target_frames, + }, ), meta={ "source": "private_track", "task_id": challenge.challenge_id, + "target_frames": challenge.target_frames, }, prompt="private-track challenge", challenge_id=challenge.challenge_id, @@ -471,6 +498,7 @@ async def _emit_private_score_to_public_db( "timed_out": timed_out, "prediction_count": result.get("prediction_count", 0), "ground_truth_count": result.get("ground_truth_count", 0), + "target_frames": challenge.target_frames, }, latency_p95_ms=latency_ms, latency_pass=not timed_out, diff --git a/scorevision/validator/central/private_track/scoring.py b/scorevision/validator/central/private_track/scoring.py index e5831e8..c4b1cd3 100644 --- a/scorevision/validator/central/private_track/scoring.py +++ b/scorevision/validator/central/private_track/scoring.py @@ -2,10 +2,16 @@ from typing import Callable from scorevision.utils.actions import ACTION_CONFIGS, Action -from scorevision.utils.schemas import CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) from scorevision.utils.settings import get_settings -PRIVATE_SCORING_VERSION = 2 +PRIVATE_SCORING_VERSION = 4 PillarScorer = Callable[[list[FramePrediction], list[FramePrediction]], float] @@ -181,6 +187,27 @@ def _soccer_action_scorer( "wickets", } +_SNOOKER_UNIQUE_LABELS = { + "cue", + "yellow", + "green", + "brown", + "blue", + "pink", + "black", +} +_SNOOKER_VALID_LABELS = _SNOOKER_UNIQUE_LABELS | {"red"} +_SNOOKER_VALID_STATES = {"on_table", "potted", "occluded", "unknown"} +_SNOOKER_COORDINATE_TOLERANCE = 0.05 +_SNOOKER_DUPLICATE_FRAME_PENALTY = 0.10 +_SNOOKER_COMPONENT_WEIGHTS = { + "coordinate_accuracy": 0.45, + "identity_accuracy": 0.25, + "red_count_accuracy": 0.10, + "state_accuracy": 0.15, + "false_positive_score": 0.05, +} + def register_pillar_scorer(pillar: str, scorer: PillarScorer) -> None: key = _normalize_pillar_key(pillar) @@ -247,6 +274,389 @@ def score_cricket_prediction_with_breakdown( return max(0.0, min(1.0, weighted_score)), breakdown +def _normalize_snooker_label(label: str) -> str: + value = " ".join(str(label or "").strip().lower().split()) + aliases = { + "cue ball": "cue", + "white": "cue", + "white ball": "cue", + "red ball": "red", + "yellow ball": "yellow", + "green ball": "green", + "brown ball": "brown", + "blue ball": "blue", + "pink ball": "pink", + "black ball": "black", + } + return aliases.get(value, value) + + +def _normalize_snooker_state(state: str) -> str: + value = " ".join(str(state or "").strip().lower().split()) + return value + + +def _snooker_distance_score( + prediction: SnookerBallPrediction, + ground_truth: SnookerBallPrediction, + *, + tolerance: float = _SNOOKER_COORDINATE_TOLERANCE, +) -> float | None: + if prediction.x is None or prediction.y is None: + return None + if ground_truth.x is None or ground_truth.y is None: + return None + distance = ( + (float(prediction.x) - float(ground_truth.x)) ** 2 + + (float(prediction.y) - float(ground_truth.y)) ** 2 + ) ** 0.5 + if tolerance <= 0: + return 1.0 if distance == 0 else 0.0 + return max(0.0, 1.0 - min(1.0, distance / tolerance)) + + +def _empty_snooker_breakdown() -> dict[str, float]: + return { + "coordinate_accuracy": 0.0, + "identity_accuracy": 0.0, + "red_count_accuracy": 0.0, + "state_accuracy": 0.0, + "false_positive_score": 0.0, + "snooker_ball_state": 0.0, + } + + +def _requires_snooker_coordinates(ball: SnookerBallPrediction) -> bool: + return ball.state == "on_table" + + +def _has_snooker_coordinates(ball: SnookerBallPrediction) -> bool: + return ball.x is not None and ball.y is not None + + +def _is_valid_snooker_prediction_ball(ball: SnookerBallPrediction) -> bool: + if ball.label not in _SNOOKER_VALID_LABELS: + return False + if ball.state not in _SNOOKER_VALID_STATES: + return False + if _requires_snooker_coordinates(ball) and not _has_snooker_coordinates(ball): + return False + return True + + +def _is_valid_snooker_ground_truth_ball(ball: SnookerBallPrediction) -> bool: + if ball.label not in _SNOOKER_VALID_LABELS: + return False + if ball.state not in _SNOOKER_VALID_STATES: + return False + if _requires_snooker_coordinates(ball) and not _has_snooker_coordinates(ball): + return False + return True + + +def _match_score_for_assignment( + prediction: SnookerBallPrediction, + ground_truth: SnookerBallPrediction, +) -> float: + if _requires_snooker_coordinates(ground_truth): + coordinate_score = _snooker_distance_score(prediction, ground_truth) or 0.0 + if coordinate_score <= 0.0: + return 0.0 + if prediction.state == ground_truth.state: + return coordinate_score + return 0.5 * coordinate_score + return 1.0 if prediction.state == ground_truth.state else 0.0 + + +def _weighted_snooker_score(breakdown: dict[str, float]) -> float: + score = 0.0 + for key, weight in _SNOOKER_COMPONENT_WEIGHTS.items(): + score += weight * breakdown.get(key, 0.0) + return max(0.0, min(1.0, score)) + + +def _linear_assignment(scores: list[list[float]]) -> list[tuple[int, int]]: + if not scores or not scores[0]: + return [] + + def _solve_min_assignment(cost: list[list[float]]) -> list[tuple[int, int]]: + row_count = len(cost) + col_count = len(cost[0]) + potentials_row = [0.0] * (row_count + 1) + potentials_col = [0.0] * (col_count + 1) + matching = [0] * (col_count + 1) + parent_col = [0] * (col_count + 1) + + for row in range(1, row_count + 1): + matching[0] = row + current_col = 0 + min_values = [float("inf")] * (col_count + 1) + used = [False] * (col_count + 1) + + while True: + used[current_col] = True + current_row = matching[current_col] + delta = float("inf") + next_col = 0 + + for col in range(1, col_count + 1): + if used[col]: + continue + candidate = ( + cost[current_row - 1][col - 1] + - potentials_row[current_row] + - potentials_col[col] + ) + if candidate < min_values[col]: + min_values[col] = candidate + parent_col[col] = current_col + if min_values[col] < delta: + delta = min_values[col] + next_col = col + + for col in range(col_count + 1): + if used[col]: + potentials_row[matching[col]] += delta + potentials_col[col] -= delta + else: + min_values[col] -= delta + + current_col = next_col + if matching[current_col] == 0: + break + + while True: + previous_col = parent_col[current_col] + matching[current_col] = matching[previous_col] + current_col = previous_col + if current_col == 0: + break + + return [ + (matching[col] - 1, col - 1) + for col in range(1, col_count + 1) + if matching[col] != 0 + ] + + row_count = len(scores) + col_count = len(scores[0]) + normalized_scores = [ + [max(0.0, min(1.0, score)) for score in row] + for row in scores + ] + + if row_count <= col_count: + cost_matrix = [[1.0 - score for score in row] for row in normalized_scores] + return _solve_min_assignment(cost_matrix) + + transposed_scores = [ + [normalized_scores[row][col] for row in range(row_count)] + for col in range(col_count) + ] + cost_matrix = [[1.0 - score for score in row] for row in transposed_scores] + return [(col, row) for row, col in _solve_min_assignment(cost_matrix)] + + +def _score_snooker_frame( + prediction_frame: SnookerBallStateFrame | None, + ground_truth_frame: SnookerBallStateFrame, +) -> dict[str, float]: + pred_balls = prediction_frame.balls if prediction_frame is not None else [] + gt_balls = ground_truth_frame.balls + if not gt_balls: + return _empty_snooker_breakdown() + if not pred_balls: + return _empty_snooker_breakdown() + + normalized_pred = [ + ball.model_copy( + update={ + "label": _normalize_snooker_label(ball.label), + "state": _normalize_snooker_state(ball.state), + } + ) + for ball in pred_balls + ] + normalized_gt = [ + ball.model_copy( + update={ + "label": _normalize_snooker_label(ball.label), + "state": _normalize_snooker_state(ball.state), + } + ) + for ball in gt_balls + ] + normalized_gt = [ + ball for ball in normalized_gt + if _is_valid_snooker_ground_truth_ball(ball) + ] + gt_count = len(normalized_gt) + if gt_count == 0: + return _empty_snooker_breakdown() + + invalid_pred_count = sum( + 1 for ball in normalized_pred if not _is_valid_snooker_prediction_ball(ball) + ) + valid_pred = [ + (idx, ball) + for idx, ball in enumerate(normalized_pred) + if _is_valid_snooker_prediction_ball(ball) + ] + + used_pred: set[int] = set() + matches: list[tuple[SnookerBallPrediction, SnookerBallPrediction]] = [] + + for gt in normalized_gt: + if gt.label == "red": + continue + if gt.label not in _SNOOKER_UNIQUE_LABELS: + continue + best_idx = None + best_score = -1.0 + for idx, pred in valid_pred: + if idx in used_pred or pred.label != gt.label: + continue + candidate_score = _match_score_for_assignment(pred, gt) + if candidate_score > best_score: + best_idx = idx + best_score = candidate_score + if best_idx is not None and best_score > 0.0: + used_pred.add(best_idx) + matches.append((normalized_pred[best_idx], gt)) + + gt_reds = [ball for ball in normalized_gt if ball.label == "red"] + pred_reds = [ + (idx, ball) + for idx, ball in valid_pred + if ball.label == "red" and idx not in used_pred + ] + red_scores = [ + [_match_score_for_assignment(pred, gt) for _pred_idx, pred in pred_reds] + for gt in gt_reds + ] + for red_idx, pred_pos in _linear_assignment(red_scores): + if red_idx >= len(gt_reds) or pred_pos >= len(pred_reds): + continue + if red_scores[red_idx][pred_pos] <= 0.0: + continue + pred_idx, _pred = pred_reds[pred_pos] + if pred_idx in used_pred: + continue + used_pred.add(pred_idx) + matches.append((normalized_pred[pred_idx], gt_reds[red_idx])) + + coordinate_targets = [ + ball + for ball in normalized_gt + if _requires_snooker_coordinates(ball) and _has_snooker_coordinates(ball) + ] + coordinate_target_ids = {id(ball) for ball in coordinate_targets} + coordinate_sum = 0.0 + for pred, gt in matches: + if id(gt) not in coordinate_target_ids: + continue + coordinate_sum += _snooker_distance_score(pred, gt) or 0.0 + coordinate_accuracy = ( + coordinate_sum / len(coordinate_targets) + if coordinate_targets + else len(matches) / gt_count + ) + identity_accuracy = len(matches) / gt_count + state_accuracy = ( + sum(1.0 for pred, gt in matches if pred.state == gt.state) / gt_count + ) + red_count_accuracy = 1.0 + valid_pred_red_count = sum( + 1 for _idx, ball in valid_pred + if ball.label == "red" + ) + if gt_reds: + red_count_accuracy = max( + 0.0, + 1.0 - abs(valid_pred_red_count - len(gt_reds)) / len(gt_reds), + ) + else: + red_count_accuracy = 1.0 if valid_pred_red_count == 0 and matches else 0.0 + + false_positive_count = invalid_pred_count + max(0, len(valid_pred) - len(matches)) + false_positive_score = max(0.0, 1.0 - false_positive_count / max(1, len(normalized_pred))) + + breakdown = { + "coordinate_accuracy": max(0.0, min(1.0, coordinate_accuracy)), + "identity_accuracy": max(0.0, min(1.0, identity_accuracy)), + "red_count_accuracy": max(0.0, min(1.0, red_count_accuracy)), + "state_accuracy": max(0.0, min(1.0, state_accuracy)), + "false_positive_score": max(0.0, min(1.0, false_positive_score)), + } + breakdown["snooker_ball_state"] = _weighted_snooker_score(breakdown) + return breakdown + + +def score_snooker_ball_state_with_breakdown( + prediction: SnookerBallStatePrediction | None, + ground_truth: SnookerBallStatePrediction, + *, + target_frames: list[int] | None = None, +) -> tuple[float, dict[str, float]]: + if not ground_truth.frames: + return 0.0, _empty_snooker_breakdown() + + gt_by_frame = {frame.frame: frame for frame in ground_truth.frames} + if target_frames is None: + frame_ids = [frame.frame for frame in ground_truth.frames] + else: + frame_ids = [] + seen: set[int] = set() + for raw_frame in target_frames: + try: + frame_id = int(raw_frame) + except (TypeError, ValueError): + continue + if frame_id < 0 or frame_id in seen: + continue + seen.add(frame_id) + frame_ids.append(frame_id) + + if not frame_ids: + return 0.0, _empty_snooker_breakdown() + + target_frame_set = set(frame_ids) + pred_by_frame: dict[int, SnookerBallStateFrame] = {} + duplicate_target_frame_count = 0 + for frame in prediction.frames if prediction is not None else []: + if frame.frame not in target_frame_set: + continue + if frame.frame in pred_by_frame: + duplicate_target_frame_count += 1 + pred_by_frame[frame.frame] = SnookerBallStateFrame( + frame=frame.frame, + balls=[*pred_by_frame[frame.frame].balls, *frame.balls], + ) + continue + pred_by_frame[frame.frame] = frame + + frame_breakdowns = [ + _score_snooker_frame(pred_by_frame.get(frame_id), gt_by_frame[frame_id]) + if frame_id in gt_by_frame + else _empty_snooker_breakdown() + for frame_id in frame_ids + ] + aggregate: dict[str, float] = {} + for key in frame_breakdowns[0].keys(): + aggregate[key] = sum(frame[key] for frame in frame_breakdowns) / len(frame_breakdowns) + if duplicate_target_frame_count: + duplicate_penalty = max( + 0.0, + 1.0 - (_SNOOKER_DUPLICATE_FRAME_PENALTY * duplicate_target_frame_count), + ) + aggregate["false_positive_score"] *= duplicate_penalty + aggregate["snooker_ball_state"] = _weighted_snooker_score(aggregate) * duplicate_penalty + else: + aggregate["snooker_ball_state"] = _weighted_snooker_score(aggregate) + score = aggregate.get("snooker_ball_state", 0.0) + return max(0.0, min(1.0, score)), aggregate + + def score_predictions_for_pillar( *, pillar: str, diff --git a/scorevision/validator/core/__init__.py b/scorevision/validator/core/__init__.py index 6bbde81..eae1c8d 100644 --- a/scorevision/validator/core/__init__.py +++ b/scorevision/validator/core/__init__.py @@ -1,11 +1,23 @@ -from scorevision.validator.core.weights import ( - weights_loop, - set_weights_via_signer, - load_manifest_for_block, - commit_validator_on_start, - get_validator_hotkey_ss58, -) -from scorevision.validator.core.signer import run_signer +_LAZY_EXPORTS = { + "weights_loop": ("scorevision.validator.core.weights", "weights_loop"), + "set_weights_via_signer": ("scorevision.validator.core.weights", "set_weights_via_signer"), + "load_manifest_for_block": ("scorevision.validator.core.weights", "load_manifest_for_block"), + "commit_validator_on_start": ("scorevision.validator.core.weights", "commit_validator_on_start"), + "get_validator_hotkey_ss58": ("scorevision.validator.core.weights", "get_validator_hotkey_ss58"), + "run_signer": ("scorevision.validator.core.signer", "run_signer"), +} + + +def __getattr__(name: str): + if name not in _LAZY_EXPORTS: + raise AttributeError(name) + import importlib + + module_name, attr_name = _LAZY_EXPORTS[name] + value = getattr(importlib.import_module(module_name), attr_name) + globals()[name] = value + return value + __all__ = [ "weights_loop", @@ -15,4 +27,3 @@ "get_validator_hotkey_ss58", "run_signer", ] - diff --git a/scorevision/validator/core/weights.py b/scorevision/validator/core/weights.py index e4236c3..7b46022 100644 --- a/scorevision/validator/core/weights.py +++ b/scorevision/validator/core/weights.py @@ -9,7 +9,6 @@ from logging import getLogger from pathlib import Path import aiohttp -import bittensor as bt from scorevision.utils.settings import get_settings from scorevision.utils.windows import get_current_window_id from scorevision.utils.prometheus import ( @@ -66,6 +65,21 @@ "5Gj9pWjksQXkuaoVxHRaKN1pmgQYiUddrNweY3SFBWMGo2QD", } +SCORE_SCALED_PRIVATE_GROUNDTRUTH_TYPES = {"cricket_delivery", "snooker_ball_state"} + + +def _private_element_weight_share( + *, + elem_weight: float, + is_private: bool, + groundtruth_type: str, + winner_score: float, +) -> float: + if not is_private or groundtruth_type not in SCORE_SCALED_PRIVATE_GROUNDTRUTH_TYPES: + return float(elem_weight) + clamped_score = max(0.0, min(1.0, float(winner_score))) + return float(elem_weight) * clamped_score + def _top_rows( rows: list[dict[str, float | int | str]], @@ -88,6 +102,8 @@ def _top_rows( @lru_cache(maxsize=1) def get_validator_hotkey_ss58() -> str: + import bittensor as bt + settings = get_settings() wallet = bt.wallet( name=settings.BITTENSOR_WALLET_COLD, @@ -253,6 +269,8 @@ async def weights_loop( if commit_on_start: await commit_validator_on_start(netuid) + import bittensor as bt + wallet = bt.wallet( name=settings.BITTENSOR_WALLET_COLD, hotkey=settings.BITTENSOR_WALLET_HOT, @@ -418,22 +436,26 @@ async def weights_loop( logger.warning("[weights] No winner for element_id=%s", element_id) continue - share = float(elem_weight) raw_groundtruth_type = getattr(elem, "groundtruth_type", None) if elem is not None else None if hasattr(raw_groundtruth_type, "value"): groundtruth_type = str(raw_groundtruth_type.value or "").strip().lower() else: groundtruth_type = str(raw_groundtruth_type or "").strip().lower() - if is_private and groundtruth_type == "cricket_delivery": - winner_score_raw = float(winner_scores_by_uid.get(winner_uid, 0.0) or 0.0) - winner_score = max(0.0, min(1.0, winner_score_raw)) - share = share * winner_score + winner_score_raw = float(winner_scores_by_uid.get(winner_uid, 0.0) or 0.0) + share = _private_element_weight_share( + elem_weight=elem_weight, + is_private=is_private, + groundtruth_type=groundtruth_type, + winner_score=winner_score_raw, + ) + if is_private and groundtruth_type in SCORE_SCALED_PRIVATE_GROUNDTRUTH_TYPES: logger.info( - "[weights] Cricket private weighting enabled element=%s winner_uid=%d elem_weight=%.6f winner_score=%.6f share=%.6f", + "[weights] Score-scaled private weighting enabled element=%s groundtruth_type=%s winner_uid=%d elem_weight=%.6f winner_score=%.6f share=%.6f", element_id, + groundtruth_type, winner_uid, elem_weight, - winner_score, + max(0.0, min(1.0, winner_score_raw)), share, ) elif is_private: diff --git a/tests/private/test_private_miner_routes.py b/tests/private/test_private_miner_routes.py index 599a9ec..773d3de 100644 --- a/tests/private/test_private_miner_routes.py +++ b/tests/private/test_private_miner_routes.py @@ -4,7 +4,14 @@ import pytest from scorevision.miner.private_track.routes import handle_challenge -from scorevision.utils.schemas import ChallengeRequest, CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + ChallengeRequest, + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) @pytest.mark.asyncio @@ -47,3 +54,57 @@ async def test_handle_challenge_cricket_mode_returns_prediction(): assert response.prediction.type == "cricket_delivery" assert response.prediction.item is not None assert response.prediction.item.kph == 130.0 + + +@pytest.mark.asyncio +async def test_handle_challenge_snooker_mode_returns_prediction(): + request = ChallengeRequest( + challenge_id="c1", + video_url="https://example.com/v.mp4", + target_frames=[50, 150], + ) + + with ( + patch("scorevision.miner.private_track.routes.MINER_MODE", "snooker_ball_state"), + patch( + "scorevision.miner.private_track.routes.predict_snooker_ball_state", + return_value=SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ), + ), + ): + response = await handle_challenge(request) + + assert response.predictions is None + assert response.prediction is not None + assert response.prediction.type == "snooker_ball_state" + assert response.prediction.frames is not None + assert response.prediction.frames[0].balls[0].label == "cue" + + +@pytest.mark.asyncio +async def test_handle_challenge_snooker_stub_uses_target_frame(): + request = ChallengeRequest( + challenge_id="c1", + video_url="https://example.com/v.mp4", + target_frames=[50, 150], + ) + + with patch("scorevision.miner.private_track.routes.MINER_MODE", "snooker_ball_state"): + response = await handle_challenge(request) + + assert response.prediction is not None + assert response.prediction.frames is not None + assert response.prediction.frames[0].frame == 50 diff --git a/tests/private/test_schemas.py b/tests/private/test_schemas.py index 684e4b8..39e0081 100644 --- a/tests/private/test_schemas.py +++ b/tests/private/test_schemas.py @@ -6,6 +6,10 @@ ChallengeRequest, ChallengeResponse, FramePrediction, + PredictionPayload, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, ) @@ -25,6 +29,24 @@ def test_challenge_request_valid(): assert req.challenge_id == "abc" +def test_challenge_request_valid_with_target_frames(): + req = ChallengeRequest( + challenge_id="abc", + video_url="https://example.com/snooker.mp4", + target_frames=[50, 150, 250, 350, 450], + ) + assert req.target_frames == [50, 150, 250, 350, 450] + + +def test_challenge_request_rejects_negative_target_frame(): + with pytest.raises(ValidationError): + ChallengeRequest( + challenge_id="abc", + video_url="https://example.com/snooker.mp4", + target_frames=[50, -1], + ) + + def test_challenge_request_valid_with_frames_only(): req = ChallengeRequest( challenge_id="abc", @@ -59,6 +81,31 @@ def test_challenge_response_cricket_valid(): assert resp.prediction_count == 1 +def test_challenge_response_snooker_ball_state_valid(): + resp = ChallengeResponse( + challenge_id="abc", + prediction=SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=10, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ), + processing_time=0.2, + ) + assert isinstance(resp.prediction, PredictionPayload) + assert resp.prediction.type == "snooker_ball_state" + assert resp.prediction_count == 1 + + def test_challenge_response_requires_exactly_one_payload(): with pytest.raises(ValidationError): ChallengeResponse( diff --git a/tests/validator/test_private_challenges.py b/tests/validator/test_private_challenges.py index 5b28931..1b11718 100644 --- a/tests/validator/test_private_challenges.py +++ b/tests/validator/test_private_challenges.py @@ -4,7 +4,13 @@ from scorevision.validator.central.private_track.challenges import ( get_challenge_with_ground_truth, ) -from scorevision.utils.schemas import CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) _FAKE_SETTINGS = SimpleNamespace( @@ -116,3 +122,84 @@ async def test_get_challenge_accepts_single_cricket_ground_truth(): assert challenge is not None assert challenge.groundtruth_type == "cricket_delivery" + + +@pytest.mark.asyncio +async def test_get_challenge_accepts_snooker_ball_state_ground_truth(): + fake_chal = { + "task_id": "c123", + "payload": { + "clip_url": "https://example.com/v1.mp4", + "target_frames": [50, 150, 250, 350, 450], + }, + } + snooker_gt = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ) + + with _patch_settings(), \ + patch(f"{_MODULE}.fetch_next_challenge", new_callable=AsyncMock, return_value=fake_chal), \ + patch(f"{_MODULE}.complete_task_assignment", new_callable=AsyncMock), \ + patch(f"{_MODULE}.fetch_ground_truth", new_callable=AsyncMock, return_value=snooker_gt): + challenge = await get_challenge_with_ground_truth( + manifest_hash="abc123", + element_id="elem1", + keypair=None, + groundtruth_type="snooker_ball_state", + max_retries=1, + ) + + assert challenge is not None + assert challenge.groundtruth_type == "snooker_ball_state" + assert challenge.video_url == "https://example.com/v1.mp4" + assert challenge.target_frames == [50, 150, 250, 350, 450] + + +@pytest.mark.asyncio +async def test_get_challenge_rejects_snooker_without_target_frames(): + fake_chal = {"task_id": "c123", "video_url": "https://example.com/v1.mp4"} + snooker_gt = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ) + complete_mock = AsyncMock() + fetch_gt_mock = AsyncMock(return_value=snooker_gt) + + with _patch_settings(), \ + patch(f"{_MODULE}.fetch_next_challenge", new_callable=AsyncMock, return_value=fake_chal), \ + patch(f"{_MODULE}.complete_task_assignment", complete_mock), \ + patch(f"{_MODULE}.fetch_ground_truth", fetch_gt_mock): + challenge = await get_challenge_with_ground_truth( + manifest_hash="abc123", + element_id="elem1", + keypair=None, + groundtruth_type="snooker_ball_state", + max_retries=1, + ) + + assert challenge is None + complete_mock.assert_not_awaited() + fetch_gt_mock.assert_not_awaited() diff --git a/tests/validator/test_private_miners.py b/tests/validator/test_private_miners.py new file mode 100644 index 0000000..2cca723 --- /dev/null +++ b/tests/validator/test_private_miners.py @@ -0,0 +1,101 @@ +from unittest.mock import patch + +import pytest + +from scorevision.utils.schemas import ( + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) +from scorevision.validator.central.private_track.challenges import Challenge +from scorevision.validator.central.private_track.miners import send_challenge +from scorevision.validator.central.private_track.registry import RegisteredMiner + + +def _miner() -> RegisteredMiner: + return RegisteredMiner( + uid=7, + hotkey="5MinerHotkey", + ip="127.0.0.1", + port=8000, + image_repo="org/repo", + image_tag="v1", + commit_block=123, + image_digest="sha256:abc123", + ) + + +@pytest.mark.asyncio +async def test_send_challenge_forwards_target_frames(): + captured: dict = {} + + class _FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return { + "challenge_id": "snooker-1", + "prediction": { + "type": "snooker_ball_state", + "frames": [ + { + "frame": 50, + "balls": [ + { + "label": "cue", + "x": 0.5, + "y": 0.5, + "state": "on_table", + } + ], + } + ], + }, + "processing_time": 0.05, + } + + class _FakeClient: + def __init__(self, timeout): + self.timeout = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def post(self, url, *, json, headers): + captured["url"] = url + captured["json"] = json + captured["headers"] = headers + return _FakeResponse() + + challenge = Challenge( + challenge_id="snooker-1", + video_url="https://example.com/snooker.mp4", + target_frames=[50, 150, 250], + ground_truth=SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5)], + ) + ] + ), + groundtruth_type="snooker_ball_state", + ) + + with ( + patch("scorevision.validator.central.private_track.miners.httpx.AsyncClient", _FakeClient), + patch( + "scorevision.validator.central.private_track.miners.build_signed_headers", + return_value={"x-signature": "ok"}, + ), + ): + attempt = await send_challenge(_miner(), challenge, hotkey=object(), timeout=5.0) + + assert attempt.response is not None + assert captured["url"] == "http://127.0.0.1:8000/challenge" + assert captured["json"]["video_url"] == "https://example.com/snooker.mp4" + assert captured["json"]["target_frames"] == [50, 150, 250] diff --git a/tests/validator/test_private_runner.py b/tests/validator/test_private_runner.py index 5fe0111..a748b9c 100644 --- a/tests/validator/test_private_runner.py +++ b/tests/validator/test_private_runner.py @@ -1,6 +1,13 @@ from unittest.mock import AsyncMock, patch import pytest -from scorevision.utils.schemas import ChallengeResponse, FramePrediction +from scorevision.utils.schemas import ( + ChallengeResponse, + FramePrediction, + PredictionPayload, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) from scorevision.validator.central.private_track.challenges import Challenge from scorevision.validator.central.private_track.miners import ChallengeAttempt from scorevision.validator.central.private_track.registry import RegisteredMiner @@ -31,6 +38,30 @@ def _challenge() -> Challenge: ) +def _snooker_challenge() -> Challenge: + return Challenge( + challenge_id="snooker-1", + video_url="https://example.com/snooker.mp4", + target_frames=[50], + ground_truth=SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ), + groundtruth_type="snooker_ball_state", + ) + + @pytest.mark.asyncio async def test_challenge_miner_scores_when_response_is_on_time(): attempt = ChallengeAttempt( @@ -107,6 +138,66 @@ async def test_challenge_miner_excludes_timeout_from_weights(): assert benchmark_result is None +@pytest.mark.asyncio +async def test_challenge_miner_scores_snooker_only_on_target_frames(): + attempt = ChallengeAttempt( + response=ChallengeResponse( + challenge_id="snooker-1", + prediction=PredictionPayload( + type="snooker_ball_state", + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ), + SnookerBallStateFrame( + frame=999, + balls=[ + SnookerBallPrediction( + label="alien", + x=0.1, + y=0.1, + state="on_table", + ) + ], + ), + ], + ), + processing_time=1.2, + ), + elapsed_s=2.5, + timed_out=False, + ) + + with patch( + "scorevision.validator.central.private_track.runner.send_challenge", + new=AsyncMock(return_value=attempt), + ): + result, response_predictions, benchmark_result = await _challenge_miner( + miner=_miner(), + challenge=_snooker_challenge(), + keypair=None, + timeout=30.0, + block=1234, + element_id="manako/DetectSnookerBallState", + pillar_weights={"snooker_ball_state": 1.0}, + image_digest="sha256:abc123", + ) + + assert result["score"] == pytest.approx(1.0) + assert result["score_breakdown"]["snooker_ball_state"] == pytest.approx(1.0) + assert response_predictions is not None + assert len(response_predictions) == 2 + assert benchmark_result is None + + def test_is_weight_eligible_result_defaults_to_true(): assert _is_weight_eligible_result({"miner_hotkey": "hk"}) is True diff --git a/tests/validator/test_private_runner_integration.py b/tests/validator/test_private_runner_integration.py index b1d6499..0752f1f 100644 --- a/tests/validator/test_private_runner_integration.py +++ b/tests/validator/test_private_runner_integration.py @@ -5,7 +5,14 @@ import pytest from scorevision.utils.manifest import Element, Manifest, Metrics, PillarName, Tee -from scorevision.utils.schemas import ChallengeResponse, CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + ChallengeResponse, + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) from scorevision.validator.central.private_track.challenges import Challenge from scorevision.validator.central.private_track.registry import RegisteredMiner from scorevision.validator.central.private_track.runner import ( @@ -35,12 +42,22 @@ def _private_manifest() -> Manifest: groundtruth_type="cricket_delivery", metrics=Metrics(pillars={PillarName.CRICKET_SCORING: 1.0}), ) + snooker = Element( + id="manako/DetectSnookerBallState", + track="private", + weight=0.05, + window_block=300, + eval_window=4, + beta=1.0, + groundtruth_type="snooker_ball_state", + metrics=Metrics(pillars={PillarName.SNOOKER_BALL_STATE: 1.0}), + ) return Manifest( window_id="2025-10-27", version=1.3, expiry_block=12345678910, tee=Tee(trusted_share_gamma=0.2), - elements=[soccer, cricket], + elements=[soccer, cricket, snooker], ) @@ -75,8 +92,32 @@ def _cricket_challenge() -> Challenge: ) +def _snooker_challenge() -> Challenge: + return Challenge( + challenge_id="snooker-1", + video_url="https://example.com/snooker.mp4", + target_frames=[50, 150, 250, 350, 450], + ground_truth=SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction( + label="cue", + x=0.5, + y=0.5, + state="on_table", + ) + ], + ) + ] + ), + groundtruth_type="snooker_ball_state", + ) + + @pytest.mark.asyncio -async def test_trigger_scheduled_runners_launches_two_private_elements_in_parallel(): +async def test_trigger_scheduled_runners_launches_three_private_elements_in_parallel(): manifest = _private_manifest() block = 600 keypair = object() @@ -84,6 +125,7 @@ async def test_trigger_scheduled_runners_launches_two_private_elements_in_parall element_state = { "manako/DetectFootballEvent": {"tempo": 300, "anchor": 0, "task": None}, "manako/DetectCricketDelivery": {"tempo": 300, "anchor": 0, "task": None}, + "manako/DetectSnookerBallState": {"tempo": 300, "anchor": 0, "task": None}, } started: list[str] = [] @@ -99,7 +141,11 @@ async def _fake_run(element_id, *_args, **_kwargs): await asyncio.gather(*[entry["task"] for entry in element_state.values()]) assert sorted(started) == sorted( - ["manako/DetectFootballEvent", "manako/DetectCricketDelivery"] + [ + "manako/DetectFootballEvent", + "manako/DetectCricketDelivery", + "manako/DetectSnookerBallState", + ] ) @@ -259,6 +305,96 @@ async def test_run_challenge_for_element_cricket_uses_cricket_pillars_and_upload assert upload_shard_mock.await_count == 1 +@pytest.mark.asyncio +async def test_run_challenge_for_element_snooker_uses_snooker_pillars_and_uploads_results(): + manifest = _private_manifest() + miner = _miner(9, "hk-snooker") + settings = SimpleNamespace( + BLACKLIST_API_URL="", + SCOREVISION_NETUID=44, + PRIVATE_MINER_TIMEOUT_S=30.0, + ) + subtensor = SimpleNamespace(metagraph=AsyncMock(return_value=SimpleNamespace())) + + challenge_miner_mock = AsyncMock( + return_value=( + { + "challenge_id": "snooker-1", + "element_id": "manako/DetectSnookerBallState", + "miner_hotkey": miner.hotkey, + "miner_uid": miner.uid, + "score": 0.88, + "prediction_count": 1, + "ground_truth_count": 1, + "processing_time": 2.1, + "response_time_s": 2.1, + "timed_out": False, + "image_digest": miner.image_digest, + "score_breakdown": {"snooker_ball_state": 0.88}, + }, + [ + { + "frame": 50, + "balls": [ + { + "label": "cue", + "x": 0.5, + "y": 0.5, + "state": "on_table", + } + ], + } + ], + None, + ) + ) + upload_shard_mock = AsyncMock(return_value="privatevision_results/shard.json") + + with ( + patch("scorevision.validator.central.private_track.runner.get_settings", return_value=settings), + patch( + "scorevision.validator.central.private_track.runner.get_registered_miners", + new=AsyncMock(return_value=[miner]), + ), + patch( + "scorevision.validator.central.private_track.runner.get_challenge_with_ground_truth", + new=AsyncMock(return_value=_snooker_challenge()), + ), + patch( + "scorevision.validator.central.private_track.runner._challenge_miner", + new=challenge_miner_mock, + ), + patch( + "scorevision.validator.central.private_track.runner._upload_private_response_blob", + new=AsyncMock(return_value="private_responses/key.json"), + ), + patch( + "scorevision.validator.central.private_track.runner._emit_private_score_to_public_db", + new=AsyncMock(), + ), + patch( + "scorevision.validator.central.private_track.runner._upload_benchmark_result", + new=AsyncMock(), + ), + patch( + "scorevision.validator.central.private_track.runner._upload_shard", + new=upload_shard_mock, + ), + ): + await _run_challenge_for_element( + element_id="manako/DetectSnookerBallState", + manifest=manifest, + block=9002, + keypair=SimpleNamespace(ss58_address="validator-hk"), + subtensor=subtensor, + ) + + assert challenge_miner_mock.await_count == 1 + args = challenge_miner_mock.await_args.args + assert args[6] == {"snooker_ball_state": 1.0} + assert upload_shard_mock.await_count == 1 + + @pytest.mark.asyncio async def test_run_challenge_for_element_skips_when_no_registered_miners(): manifest = _private_manifest() diff --git a/tests/validator/test_private_scoring.py b/tests/validator/test_private_scoring.py index 4585ba9..a10558b 100644 --- a/tests/validator/test_private_scoring.py +++ b/tests/validator/test_private_scoring.py @@ -8,11 +8,18 @@ frame_to_seconds, register_pillar_scorer, score_cricket_prediction_with_breakdown, + score_snooker_ball_state_with_breakdown, score_predictions, score_predictions_for_pillar, score_predictions_with_breakdown, ) -from scorevision.utils.schemas import CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) _FAKE_SETTINGS = SimpleNamespace(PRIVATE_FRAME_RATE=25) @@ -168,3 +175,389 @@ def test_cricket_scoring_top6_fields_perfect_match(): # Their total weight is 0.74, so a perfect match on those fields yields 0.74. assert score == pytest.approx(0.74) assert breakdown["kph"] == 1.0 + + +def test_snooker_ball_state_perfect_match(): + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="red", x=0.6, y=0.6, state="on_table"), + ], + ) + ] + ) + score, breakdown = score_snooker_ball_state_with_breakdown(prediction, prediction) + + assert score == pytest.approx(1.0) + assert breakdown["snooker_ball_state"] == pytest.approx(1.0) + + +def test_snooker_ball_state_missing_target_frame_scores_zero(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=999, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(0.0) + assert breakdown["snooker_ball_state"] == pytest.approx(0.0) + + +def test_snooker_ball_state_empty_prediction_scores_zero(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction(frames=[SnookerBallStateFrame(frame=50, balls=[])]) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(0.0) + assert breakdown["false_positive_score"] == pytest.approx(0.0) + + +def test_snooker_ball_state_ignores_extra_non_target_frames(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ), + SnookerBallStateFrame( + frame=999, + balls=[SnookerBallPrediction(label="invalid", x=0.1, y=0.1, state="on_table")], + ), + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(1.0) + assert breakdown["snooker_ball_state"] == pytest.approx(1.0) + + +def test_snooker_ball_state_matches_reds_by_hungarian_assignment(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="red", x=0.2, y=0.2, state="on_table"), + SnookerBallPrediction(label="red", x=0.8, y=0.8, state="on_table"), + ], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="red", x=0.8, y=0.8, state="on_table"), + SnookerBallPrediction(label="red", x=0.2, y=0.2, state="on_table"), + ], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(1.0) + assert breakdown["coordinate_accuracy"] == pytest.approx(1.0) + + +def test_snooker_ball_state_penalizes_wrong_red_count(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="red", x=0.6, y=0.6, state="on_table"), + SnookerBallPrediction(label="red", x=0.7, y=0.7, state="on_table"), + ], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="red", x=0.6, y=0.6, state="on_table"), + ], + ) + ] + ) + score, breakdown = score_snooker_ball_state_with_breakdown(prediction, ground_truth) + + assert score < 1.0 + assert breakdown["red_count_accuracy"] < 1.0 + + +def test_snooker_ball_state_penalizes_duplicate_unique_colour(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="blue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="blue", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="blue", x=0.6, y=0.6, state="on_table"), + ], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score < 1.0 + assert breakdown["false_positive_score"] < 1.0 + + +def test_snooker_ball_state_allows_potted_and_occluded_without_coordinates(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="cue", state="potted"), + SnookerBallPrediction(label="blue", state="occluded"), + ], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="cue", state="potted"), + SnookerBallPrediction(label="blue", state="occluded"), + ], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(1.0) + assert breakdown["state_accuracy"] == pytest.approx(1.0) + + +def test_snooker_ball_state_penalizes_invalid_labels_and_missing_on_table_coordinates(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="orange", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="cue", state="on_table"), + ], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score < 1.0 + assert breakdown["identity_accuracy"] == pytest.approx(0.0) + assert breakdown["false_positive_score"] == pytest.approx(0.0) + + +def test_snooker_ball_state_uses_tighter_coordinate_tolerance(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.50, y=0.50, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.56, y=0.50, state="on_table")], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(0.0) + assert breakdown["coordinate_accuracy"] == pytest.approx(0.0) + assert breakdown["identity_accuracy"] == pytest.approx(0.0) + + +def test_snooker_ball_state_does_not_normalize_invalid_state_to_unknown(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="blue", state="unknown")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="blue", state="hidden")], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(0.0) + assert breakdown["identity_accuracy"] == pytest.approx(0.0) + assert breakdown["false_positive_score"] == pytest.approx(0.0) + + +def test_snooker_ball_state_invalid_reds_do_not_earn_red_count_credit(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="red", x=0.30, y=0.30, state="on_table"), + SnookerBallPrediction(label="red", x=0.40, y=0.40, state="on_table"), + ], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[ + SnookerBallPrediction(label="red", state="on_table"), + SnookerBallPrediction(label="red", state="on_table"), + ], + ) + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score == pytest.approx(0.0) + assert breakdown["red_count_accuracy"] == pytest.approx(0.0) + assert breakdown["false_positive_score"] == pytest.approx(0.0) + + +def test_snooker_ball_state_penalizes_duplicate_target_frame_objects(): + ground_truth = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ) + ] + ) + prediction = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ), + SnookerBallStateFrame( + frame=50, + balls=[SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table")], + ), + ] + ) + + score, breakdown = score_snooker_ball_state_with_breakdown( + prediction, + ground_truth, + target_frames=[50], + ) + + assert score < 1.0 + assert breakdown["false_positive_score"] < 1.0 + assert breakdown["snooker_ball_state"] == pytest.approx(score) diff --git a/tests/validator/test_private_spotcheck.py b/tests/validator/test_private_spotcheck.py index a84bfe3..3bea5a2 100644 --- a/tests/validator/test_private_spotcheck.py +++ b/tests/validator/test_private_spotcheck.py @@ -3,7 +3,13 @@ import pytest -from scorevision.utils.schemas import CricketDeliveryPrediction, FramePrediction +from scorevision.utils.schemas import ( + CricketDeliveryPrediction, + FramePrediction, + SnookerBallPrediction, + SnookerBallStateFrame, + SnookerBallStatePrediction, +) from scorevision.validator.audit.private_track import spotcheck as spotcheck_mod @@ -16,6 +22,15 @@ def test_infer_groundtruth_type_from_explicit_field(): ) +def test_infer_groundtruth_type_from_explicit_snooker_field(): + challenge_results = [{"groundtruth_type": "snooker_ball_state"}] + miner_responses = {} + assert ( + spotcheck_mod._infer_groundtruth_type(challenge_results, miner_responses) + == "snooker_ball_state" + ) + + def test_infer_groundtruth_type_from_prediction_shape(): challenge_results = [{"miner_hotkey": "hk1"}] miner_responses = {"hk1": [{"kph": 129.2, "bounce_x": 5.8}]} @@ -25,6 +40,15 @@ def test_infer_groundtruth_type_from_prediction_shape(): ) +def test_infer_groundtruth_type_from_snooker_prediction_shape(): + challenge_results = [{"miner_hotkey": "hk1"}] + miner_responses = {"hk1": [{"frame": 10, "balls": []}]} + assert ( + spotcheck_mod._infer_groundtruth_type(challenge_results, miner_responses) + == "snooker_ball_state" + ) + + def test_rescore_miner_soccer(): predictions = [{"frame": 25, "action": "pass"}] gt = [FramePrediction(frame=25, action="pass")] @@ -39,6 +63,31 @@ def test_rescore_miner_cricket(): assert 0.0 <= score <= 1.0 +def test_rescore_miner_snooker(): + predictions = [ + { + "frame": 0, + "balls": [ + {"label": "cue", "x": 0.5, "y": 0.5, "state": "on_table"}, + {"label": "red", "x": 0.6, "y": 0.6, "state": "on_table"}, + ], + } + ] + gt = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table"), + SnookerBallPrediction(label="red", x=0.6, y=0.6, state="on_table"), + ], + ) + ] + ) + score = spotcheck_mod.rescore_miner_snooker(predictions, gt) + assert score == 1.0 + + @pytest.mark.asyncio async def test_run_private_spotcheck_cricket_passes_element_and_groundtruth_type(): keypair = object() @@ -86,3 +135,71 @@ async def test_run_private_spotcheck_cricket_passes_element_and_groundtruth_type groundtruth_type="cricket_delivery", ) + +@pytest.mark.asyncio +async def test_run_private_spotcheck_snooker_passes_element_and_groundtruth_type(): + keypair = object() + threshold = 0.95 + challenge_id = "chal-snooker-1" + challenge_results = [ + { + "element_id": "manako/DetectSnookerBallState", + "groundtruth_type": "snooker_ball_state", + "miner_hotkey": "hk1", + "score": 1.0, + } + ] + gt = SnookerBallStatePrediction( + frames=[ + SnookerBallStateFrame( + frame=0, + balls=[ + SnookerBallPrediction(label="cue", x=0.5, y=0.5, state="on_table"), + ], + ) + ] + ) + miner_responses = { + "hk1": [ + { + "frame": 0, + "balls": [ + {"label": "cue", "x": 0.5, "y": 0.5, "state": "on_table"}, + ], + } + ] + } + + with ( + patch.object( + spotcheck_mod, + "fetch_miner_responses", + new=AsyncMock(return_value=miner_responses), + ), + patch.object( + spotcheck_mod, + "fetch_ground_truth", + new=AsyncMock(return_value=gt), + ) as fetch_gt_mock, + patch.object( + spotcheck_mod, + "calculate_match_percentage", + return_value=1.0, + ), + ): + results = await spotcheck_mod.run_private_spotcheck( + challenge_id=challenge_id, + challenge_results=challenge_results, + keypair=keypair, + threshold=threshold, + ) + + assert len(results) == 1 + assert results[0].element_id == "manako/DetectSnookerBallState" + fetch_gt_mock.assert_awaited_once_with( + challenge_id, + keypair, + element_id="manako/DetectSnookerBallState", + groundtruth_type="snooker_ball_state", + ) + diff --git a/tests/validator/test_weights.py b/tests/validator/test_weights.py index 1333c6f..5e1dc1a 100644 --- a/tests/validator/test_weights.py +++ b/tests/validator/test_weights.py @@ -13,6 +13,7 @@ are_similar_by_challenges, ) from scorevision.validator.models import WeightsResult, OpenSourceMinerMeta +from scorevision.validator.core.weights import _private_element_weight_share def test_extract_miner_and_score_from_payload_valid(): @@ -150,3 +151,28 @@ def test_weights_result_dataclass(): assert result.element_id == "soccer_detect" assert result.winner_uid == 7 assert result.scores_by_uid[7] == 0.9 + + +@pytest.mark.parametrize("groundtruth_type", ["cricket_delivery", "snooker_ball_state"]) +def test_private_score_scaled_element_share_for_cricket_and_snooker(groundtruth_type): + assert _private_element_weight_share( + elem_weight=0.05, + is_private=True, + groundtruth_type=groundtruth_type, + winner_score=0.5, + ) == pytest.approx(0.025) + + +def test_private_element_share_does_not_score_scale_soccer_or_public_elements(): + assert _private_element_weight_share( + elem_weight=0.2, + is_private=True, + groundtruth_type="soccer_action", + winner_score=0.5, + ) == pytest.approx(0.2) + assert _private_element_weight_share( + elem_weight=0.05, + is_private=False, + groundtruth_type="snooker_ball_state", + winner_score=0.5, + ) == pytest.approx(0.05)