From 3ed674de754c82d269b9159bc459973dc3b288e0 Mon Sep 17 00:00:00 2001 From: ComBba Date: Mon, 9 Feb 2026 22:20:02 +0900 Subject: [PATCH 1/3] feat: enable free public repo evaluation with six_sommeliers mode - Add POST /api/evaluate/public endpoint for anonymous evaluations - Implement server-side public repo verification via GitHub API - Add IP-based rate limiting (5/hour) for anonymous requests - Force gemini-3-flash-preview model, six_sommeliers mode for free tier - Update stream/result/graph endpoints to allow anonymous access (user_id='anonymous') - Change frontend default tab to 'Enter URL' - Enable anonymous submit for public repos + six_sommeliers mode - Add 'Free evaluation' indicator banner in UI - Disable Grand Tasting mode for non-authenticated users --- backend/app/api/routes/evaluate.py | 214 ++++++++++++++++-- backend/app/api/routes/graph.py | 31 ++- backend/app/services/github_service.py | 40 ++++ frontend/src/app/evaluate/page.tsx | 11 +- frontend/src/components/EvaluationForm.tsx | 55 +++-- .../src/components/EvaluationModeSelector.tsx | 102 ++++++--- frontend/src/lib/api.ts | 20 ++ 7 files changed, 394 insertions(+), 79 deletions(-) diff --git a/backend/app/api/routes/evaluate.py b/backend/app/api/routes/evaluate.py index cf30eb3..2680ad0 100644 --- a/backend/app/api/routes/evaluate.py +++ b/backend/app/api/routes/evaluate.py @@ -2,6 +2,7 @@ This module provides endpoints for: - Starting new evaluations (POST /api/evaluate) +- Starting anonymous public evaluations (POST /api/evaluate/public) - Streaming evaluation progress (GET /api/evaluate/{evaluation_id}/stream) - Getting evaluation results (GET /api/evaluate/{evaluation_id}/result) """ @@ -11,9 +12,10 @@ import asyncio import json import logging +import time from typing import Any -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field @@ -32,6 +34,7 @@ SommelierProgressEvent, create_sommelier_event, ) +from app.services.github_service import verify_public_repo from app.services.task_registry import register_task from app.services.quota import check_quota from app.database.repositories.api_key import APIKeyRepository @@ -40,6 +43,48 @@ router = APIRouter(prefix="/evaluate", tags=["evaluate"]) +_anonymous_rate_limit_store: dict[str, list[float]] = {} +_ANONYMOUS_RATE_LIMIT = 5 +_ANONYMOUS_RATE_WINDOW = 3600 + + +def _check_anonymous_rate_limit(client_ip: str) -> bool: + """Check if the client IP has exceeded the anonymous evaluation rate limit. + + Args: + client_ip: The client's IP address. + + Returns: + True if the request is allowed, False if rate limited. + """ + now = time.time() + window_start = now - _ANONYMOUS_RATE_WINDOW + timestamps = _anonymous_rate_limit_store.get(client_ip, []) + timestamps = [ts for ts in timestamps if ts > window_start] + + if len(timestamps) >= _ANONYMOUS_RATE_LIMIT: + _anonymous_rate_limit_store[client_ip] = timestamps + return False + + timestamps.append(now) + _anonymous_rate_limit_store[client_ip] = timestamps + return True + + +def _get_client_ip(request: Request) -> str: + """Extract the client IP address from the request. + + Args: + request: The FastAPI request object. + + Returns: + The client's IP address. + """ + forwarded = request.headers.get("X-Forwarded-For") + if forwarded: + return forwarded.split(",")[0].strip() + return request.client.host if request.client else "unknown" + class EvaluateRequest(BaseModel): """Request model for starting an evaluation.""" @@ -220,10 +265,125 @@ async def run_in_background(): raise CorkedError(f"Failed to start evaluation: {e!s}") from e +@router.post("/public", response_model=EvaluateResponse) +async def create_public_evaluation( + request: EvaluateRequest, + req: Request, +) -> EvaluateResponse: + """Start anonymous evaluation for public repositories. + + This endpoint allows unauthenticated users to evaluate public GitHub repositories. + Constraints: + - Only six_sommeliers mode allowed + - Only public GitHub repos (verified server-side) + - Forced model: gemini-3-flash-preview + - No BYOK/provider/model/temperature overrides + - Rate limited by IP (5 evaluations per hour) + + Args: + request: The evaluation request containing repo_url and criteria. + req: The FastAPI request object for IP-based rate limiting. + + Returns: + EvaluateResponse with evaluation_id and status. + + Raises: + CorkedError: If rate limit exceeded, repo is private, or evaluation fails. + """ + client_ip = _get_client_ip(req) + logger.info( + f"[Evaluate Public] Request from {client_ip}: repo_url={request.repo_url}" + ) + + if not _check_anonymous_rate_limit(client_ip): + logger.warning(f"[Evaluate Public] Rate limit exceeded for IP: {client_ip}") + raise CorkedError( + "Rate limit exceeded. Please try again later or login for unlimited access." + ) + + await verify_public_repo(request.repo_url) + + try: + eval_id = await start_evaluation( + repo_url=request.repo_url, + criteria=request.criteria, + user_id="anonymous", + custom_criteria=request.custom_criteria, + evaluation_mode="six_sommeliers", + ) + + event_channel = get_event_channel() + await event_channel.create_channel(eval_id) + + async def run_in_background(): + try: + await run_evaluation_pipeline_with_events( + evaluation_id=eval_id, + repo_url=request.repo_url, + criteria=request.criteria, + user_id="anonymous", + evaluation_mode="six_sommeliers", + provider="gemini", + model="gemini-3-flash-preview", + temperature=None, + api_key=None, + github_token=None, + ) + except Exception as e: + logger.exception(f"Background evaluation failed: {eval_id}") + error_msg = str(e) + if "Resource not found" in error_msg or "404" in error_msg: + user_message = "Repository not found or is private. Please check the URL and try again." + elif "rate limit" in error_msg.lower(): + user_message = ( + "GitHub API rate limit exceeded. Please try again later." + ) + else: + user_message = f"Evaluation failed: {error_msg}" + await event_channel.emit( + eval_id, + create_sommelier_event( + evaluation_id=eval_id, + sommelier="system", + event_type=EventType.EVALUATION_ERROR.value, + progress_percent=-1, + message=user_message, + ), + ) + await handle_evaluation_error(eval_id, error_msg) + finally: + await event_channel.close_channel(eval_id) + + try: + task = asyncio.create_task(run_in_background()) + await register_task(eval_id, task) + except Exception as e: + await event_channel.close_channel(eval_id) + raise CorkedError(f"Failed to start background task: {e!s}") from e + + logger.info(f"[Evaluate Public] Background task started: {eval_id}") + + return EvaluateResponse( + evaluation_id=eval_id, + status="pending", + evaluation_mode="six_sommeliers", + estimated_time=30, + ) + except CorkedError as e: + logger.error(f"[Evaluate Public] CorkedError: {e.detail}") + raise e + except Exception as e: + logger.error(f"[Evaluate Public] Exception: {type(e).__name__}: {str(e)}") + import traceback + + logger.error(f"[Evaluate Public] Traceback: {traceback.format_exc()}") + raise CorkedError(f"Failed to start evaluation: {e!s}") from e + + @router.get("/{evaluation_id}/stream") async def stream_evaluation( evaluation_id: str, - user=Depends(get_current_user), + user=Depends(get_optional_user), ) -> Any: """Stream evaluation progress via Server-Sent Events. @@ -235,13 +395,22 @@ async def stream_evaluation( - evaluation_complete: When the entire evaluation is finished - evaluation_error: When the entire evaluation fails - heartbeat: Keep-alive signal (every 30 seconds) + + Anonymous evaluations (user_id == "anonymous") can be accessed without + authentication. Authenticated users can only access their own evaluations. """ try: progress = await get_evaluation_progress(evaluation_id) except EmptyCellarError: raise EmptyCellarError(f"Evaluation not found: {evaluation_id}") from None - if progress.get("user_id") != user.id: + eval_user_id = progress.get("user_id") + + # Allow access if: authenticated owner OR anonymous evaluation without auth + if user is None and eval_user_id != "anonymous": + raise CorkedError("Authentication required to view this evaluation") + + if user is not None and eval_user_id != user.id: raise CorkedError("Access denied: evaluation belongs to another user") event_channel = get_event_channel() @@ -304,12 +473,12 @@ async def get_result( This endpoint returns the complete evaluation results including the final verdict from Jean-Pierre and all sommelier outputs. - Public demo evaluations (in PUBLIC_DEMO_EVALUATIONS) can be accessed - without authentication. + Public demo evaluations (in PUBLIC_DEMO_EVALUATIONS) and anonymous + evaluations (user_id == "anonymous") can be accessed without authentication. Args: evaluation_id: The evaluation ID. - user: The authenticated user (optional for public demos). + user: The authenticated user (optional for public access). Returns: The evaluation results. @@ -318,24 +487,33 @@ async def get_result( EmptyCellarError: If the evaluation is not found. CorkedError: If the evaluation is still in progress. """ - # Check if this is a public demo evaluation is_public_demo = evaluation_id in PUBLIC_DEMO_EVALUATIONS - # Require auth for non-public evaluations - if not is_public_demo and user is None: - raise CorkedError("Authentication required to view this evaluation") + if is_public_demo: + result = await get_evaluation_result(evaluation_id) + if result is None: + raise EmptyCellarError(f"Evaluation result not found: {evaluation_id}") + return ResultResponse( + evaluation_id=result["evaluation_id"], + final_evaluation=result["final_evaluation"], + created_at=str(result["created_at"]), + ) try: - if not is_public_demo: - progress = await get_evaluation_progress(evaluation_id) - if progress.get("user_id") != user.id: - raise CorkedError("Access denied: evaluation belongs to another user") - - result = await get_evaluation_result(evaluation_id) + progress = await get_evaluation_progress(evaluation_id) except EmptyCellarError: raise EmptyCellarError(f"Evaluation not found: {evaluation_id}") from None - except CorkedError: - raise + + eval_user_id = progress.get("user_id") + + # Allow access if: authenticated owner OR anonymous evaluation without auth + if user is None and eval_user_id != "anonymous": + raise CorkedError("Authentication required to view this evaluation") + + if user is not None and eval_user_id != user.id: + raise CorkedError("Access denied: evaluation belongs to another user") + + result = await get_evaluation_result(evaluation_id) if result is None: raise EmptyCellarError(f"Evaluation result not found: {evaluation_id}") diff --git a/backend/app/api/routes/graph.py b/backend/app/api/routes/graph.py index 5b4ac4f..d73861d 100644 --- a/backend/app/api/routes/graph.py +++ b/backend/app/api/routes/graph.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, Depends -from app.api.deps import get_current_user, get_optional_user +from app.api.deps import get_optional_user # Demo evaluation IDs that can be accessed without authentication PUBLIC_DEMO_EVALUATIONS = { @@ -53,25 +53,28 @@ async def _get_evaluation(evaluation_id: str) -> dict[str, Any] | None: def _check_ownership(evaluation: dict[str, Any], user, evaluation_id: str) -> None: - """Verify user owns the evaluation or it's a public demo. + """Verify user owns the evaluation or it's publicly accessible. Args: evaluation: The evaluation document. - user: The current authenticated user (can be None for public demos). + user: The current authenticated user (can be None for public/anonymous). evaluation_id: The evaluation ID. Raises: CorkedError: If user does not own the evaluation and it's not public. """ - # Allow access to public demo evaluations without auth if evaluation_id in PUBLIC_DEMO_EVALUATIONS: return - # Require auth for non-public evaluations + eval_user_id = evaluation.get("user_id") + + if eval_user_id == "anonymous": + return + if user is None: raise CorkedError("Authentication required to view this evaluation") - if evaluation.get("user_id") != user.id: + if eval_user_id != user.id: raise CorkedError( "Access denied: evaluation belongs to another user", status_code=403 ) @@ -148,24 +151,26 @@ async def get_graph( @router.get("/{evaluation_id}/graph/structure", response_model=ReactFlowGraph) async def get_graph_structure( evaluation_id: str, - user=Depends(get_current_user), + user=Depends(get_optional_user), ) -> ReactFlowGraph: """Get static graph structure (topology only, no execution state). Returns the evaluation workflow topology without any runtime state. Use this for displaying the graph structure before execution starts. + + Public demo and anonymous evaluations can be accessed without authentication. """ logger.info(f"[Graph] Getting structure for evaluation: {evaluation_id}") evaluation = await _get_evaluation(evaluation_id) if not evaluation: raise EmptyCellarError(f"Evaluation not found: {evaluation_id}") - _check_ownership(evaluation, user) + _check_ownership(evaluation, user, evaluation_id) mode = _determine_mode(evaluation) if mode == EvaluationMode.FULL_TECHNIQUES: graph = build_full_techniques_topology() - else: # SIX_SOMMELIERS, GRAND_TASTING + else: graph = build_six_sommeliers_topology() return graph @@ -174,24 +179,26 @@ async def get_graph_structure( @router.get("/{evaluation_id}/graph/execution", response_model=ReactFlowGraph) async def get_graph_execution( evaluation_id: str, - user=Depends(get_current_user), + user=Depends(get_optional_user), ) -> ReactFlowGraph: """Get graph with execution state overlay (status, progress from trace). Returns the evaluation workflow with runtime state from methodology_trace. Node status reflects the execution progress. + + Public demo and anonymous evaluations can be accessed without authentication. """ logger.info(f"[Graph] Getting execution graph for evaluation: {evaluation_id}") evaluation = await _get_evaluation(evaluation_id) if not evaluation: raise EmptyCellarError(f"Evaluation not found: {evaluation_id}") - _check_ownership(evaluation, user) + _check_ownership(evaluation, user, evaluation_id) mode = _determine_mode(evaluation) if mode == EvaluationMode.FULL_TECHNIQUES: graph = build_full_techniques_topology() - else: # SIX_SOMMELIERS, GRAND_TASTING + else: graph = build_six_sommeliers_topology() methodology_trace = evaluation.get("methodology_trace", []) diff --git a/backend/app/services/github_service.py b/backend/app/services/github_service.py index 0a61d56..9c4c28f 100644 --- a/backend/app/services/github_service.py +++ b/backend/app/services/github_service.py @@ -328,3 +328,43 @@ async def get_full_repo_context( "file_tree": file_tree, "readme": readme, } + + +async def verify_public_repo(repo_url: str) -> bool: + """Verify that a repository is publicly accessible via GitHub API. + + Makes an unauthenticated request to the GitHub API to check if the + repository exists and is public. Private repositories will return 404 + or 403 when accessed without authentication. + + Args: + repo_url: The GitHub repository URL to verify. + + Returns: + True if the repository is public and accessible. + + Raises: + CorkedError: If the repository is not found, is private, or rate limited. + """ + try: + owner, repo = parse_github_url(repo_url) + except CorkedError: + raise CorkedError("Invalid GitHub repository URL") + + url = f"{GITHUB_API_BASE}/repos/{owner}/{repo}" + + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=10.0) + + if response.status_code == 200: + return True + elif response.status_code == 404: + raise CorkedError( + "Repository not found or is private. Please login to evaluate private repositories." + ) + elif response.status_code == 403: + raise CorkedError( + "Repository not found or is private. Please login to evaluate private repositories." + ) + else: + raise CorkedError(f"GitHub API error: {response.status_code} - {response.text}") diff --git a/frontend/src/app/evaluate/page.tsx b/frontend/src/app/evaluate/page.tsx index 1a39140..4ebba6e 100644 --- a/frontend/src/app/evaluate/page.tsx +++ b/frontend/src/app/evaluate/page.tsx @@ -7,9 +7,11 @@ import { EvaluationForm } from '../../components/EvaluationForm'; import { api } from '../../lib/api'; import { CriteriaType, EvaluationMode } from '../../types'; import { Sparkles } from 'lucide-react'; +import { useAuth } from '@/contexts/AuthContext'; export default function EvaluatePage() { const router = useRouter(); + const { isAuthenticated } = useAuth(); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); @@ -18,7 +20,14 @@ export default function EvaluatePage() { setError(null); try { - const { id } = await api.startEvaluation(repoUrl, criteria, evaluationMode); + let id; + if (isAuthenticated) { + const result = await api.startEvaluation(repoUrl, criteria, evaluationMode); + id = result.id; + } else { + const result = await api.startPublicEvaluation(repoUrl); + id = result.id; + } router.push(`/progress/${id}?mode=${evaluationMode}`); } catch (err) { console.error('Evaluation failed:', err); diff --git a/frontend/src/components/EvaluationForm.tsx b/frontend/src/components/EvaluationForm.tsx index 2c07ea4..e9db766 100644 --- a/frontend/src/components/EvaluationForm.tsx +++ b/frontend/src/components/EvaluationForm.tsx @@ -19,7 +19,7 @@ interface EvaluationFormProps { type TabType = 'repos' | 'url'; export const EvaluationForm: React.FC = ({ onSubmit, isLoading = false, error = null }) => { - const [activeTab, setActiveTab] = useState('repos'); + const [activeTab, setActiveTab] = useState('url'); const [repoUrl, setRepoUrl] = useState(''); const [criteria, setCriteria] = useState('basic'); const [evaluationMode, setEvaluationMode] = useState('six_sommeliers'); @@ -85,22 +85,34 @@ export const EvaluationForm: React.FC = ({ onSubmit, isLoad return null; }; + const canSubmitAnonymously = !isAuthenticated && + activeTab === 'url' && + validation.status === 'valid' && + evaluationMode === 'six_sommeliers'; + + const canSubmit = isAuthenticated || canSubmitAnonymously; + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); setValidationError(null); - if (!isAuthenticated) { - setValidationError("Please login to submit an evaluation."); - return; - } - const error = validateUrl(repoUrl); if (error) { setValidationError(error); return; } - await onSubmit(repoUrl, criteria, evaluationMode); + if (canSubmitAnonymously) { + try { + await onSubmit(repoUrl, criteria, evaluationMode); + } catch (err) { + setValidationError(err instanceof Error ? err.message : 'Evaluation failed'); + } + } else if (isAuthenticated) { + await onSubmit(repoUrl, criteria, evaluationMode); + } else { + setValidationError("Please login to submit an evaluation."); + } }; const handleOAuthLogin = () => { @@ -176,10 +188,27 @@ export const EvaluationForm: React.FC = ({ onSubmit, isLoad ) : (
- {!isAuthenticated && ( -
- Login is required to submit a repository URL. -
+ {!isAuthenticated && activeTab === 'url' && ( + <> + {validation.status === 'valid' && evaluationMode === 'six_sommeliers' ? ( +
+ + Free evaluation available for public repositories +
+ ) : validation.status === 'private' ? ( +
+ Login required to evaluate private repositories. +
+ ) : evaluationMode !== 'six_sommeliers' ? ( +
+ Login required for Grand Tasting mode. +
+ ) : ( +
+ Login to access all features. +
+ )} + )}