diff --git a/README.md b/README.md index 31da83e..64a32bc 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,9 @@ For a browser demo that talks to the local FastAPI app, see [`web/README.md`](we | `nokaman rubrics list [-l en]` | Skill rubrics | | `nokaman rubrics explain -l en` | Print rubric skill weights, notes, frameworks, and bands | | `nokaman eval text …` | Evaluate free text | +| `nokaman eval score --text "..." -l en` | Score text as a per-dimension rich table (Writing/Vocab/Grammar/Reading/Listening/Speaking + CEFR) | +| `nokaman eval score --sample path/to/sample.json -l en` | Score a sample file with dimension breakdown | +| `nokaman eval score --text "..." --json` | Same scoring as raw JSON output | | `nokaman eval batch --out data/out/batch.json` | Score every sample and write a JSON CEFR hit-rate report | | `nokaman train …` | Toy calibration with config/report exports | | `nokaman gui` / `nokaman-gui` | **Qt desktop app** (needs `.[gui]`) | diff --git a/pyproject.toml b/pyproject.toml index 51f6b1f..5d1d3d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,3 +55,6 @@ pythonpath = ["src"] [tool.ruff] line-length = 100 target-version = "py311" + +[tool.ruff.lint.per-file-ignores] +"src/nokaman/cli.py" = ["B008"] # typer.Option in defaults is idiomatic typer usage diff --git a/src/nokaman/cli.py b/src/nokaman/cli.py index b104b41..0802818 100644 --- a/src/nokaman/cli.py +++ b/src/nokaman/cli.py @@ -2,7 +2,6 @@ import json from pathlib import Path -from typing import Optional import typer from rich.console import Console @@ -11,7 +10,7 @@ from nokaman import __version__ from nokaman.config import OUT_DIR, RUNS_DIR from nokaman.data.coverage import language_skill_coverage -from nokaman.data.loader import list_sample_files, list_rubric_files, load_rubric +from nokaman.data.loader import list_rubric_files, list_sample_files, load_rubric, load_sample from nokaman.eval.metrics import batch_evaluate, placement_test from nokaman.eval.pipeline import evaluate_demo, evaluate_sample_file, evaluate_text from nokaman.eval.session import SessionManager @@ -130,7 +129,7 @@ def languages_coverage(json_output: bool = typer.Option(False, "--json")) -> Non @rubrics_app.command("list") -def rubrics_list(lang: Optional[str] = typer.Option(None, "--lang", "-l")) -> None: +def rubrics_list(lang: str | None = typer.Option(None, "--lang", "-l")) -> None: files = list_rubric_files() if lang: files = [p for p in files if p.stem == lang.strip().lower()] @@ -193,8 +192,8 @@ def rubrics_explain( @eval_app.command("text") def eval_text( lang: str = typer.Option("en", "--lang", "-l"), - text: Optional[str] = typer.Option(None, "--text", "-t"), - file: Optional[Path] = typer.Option(None, "--file", "-f", exists=True, dir_okay=False), + text: str | None = typer.Option(None, "--text", "-t"), + file: Path | None = typer.Option(None, "--file", "-f", exists=True, dir_okay=False), skill: str = typer.Option("writing", "--skill", "-s"), ) -> None: if file is not None: @@ -212,6 +211,69 @@ def eval_demo(lang: str = typer.Option("en", "--lang", "-l")) -> None: _print_json(data=evaluate_demo(lang)) +@eval_app.command("score") +def eval_score( + sample: Path | None = typer.Option(None, "--sample", "-s", exists=True, dir_okay=False, + help="Path to a sample JSON file."), + lang: str = typer.Option("en", "--lang", "-l", help="Language code."), + text: str | None = typer.Option(None, "--text", "-t", help="Input text."), + skill: str = typer.Option("writing", "--skill", "-k", help="Skill to score."), + json_output: bool = typer.Option(False, "--json", help="Print raw JSON instead of a table."), +) -> None: + """Score a sample or text and show per-dimension metrics as a rich table.""" + if sample is not None: + result = evaluate_sample_file(sample) + input_label = str(sample) + elif text: + result = evaluate_text(lang, text, skill=skill) + input_label = text + else: + console.print("[red]Provide --sample or --text[/red]") + raise typer.Exit(code=1) + + overall = result.get("score") + cefr = result.get("cefr") + if json_output: + _print_json(data=result) + return + + # Per-dimension metrics from the underlying ToyAbilityModel. + scored_text = (result.get("source") or "") or ( + load_sample(Path(sample)).get("text", "") if sample is not None else text + ) + dims = _compute_dimension_scores(scored_text or "", lang) + + table = Table(title=f"Score dimensions — {input_label} ({lang})") + table.add_column("Dimension", no_wrap=True) + table.add_column("Score", justify="right") + for name, val in dims: + table.add_row(name, f"{val:g}") + console.print(table) + + console.print(f"[green]Overall[/green]: [bold]{overall}[/bold] " + f"[yellow]CEFR[/yellow]: [bold]{cefr}[/bold] " + f"[dim]skill={result.get('skill')}[/dim]") + if result.get("expected_cefr"): + chk = result.get("band_check") + console.print(f"[dim]expected CEFR[/dim]: {result['expected_cefr']} " + f"band_check={chk}") + + +def _compute_dimension_scores(text: str, lang: str) -> list[tuple[str, float]]: + """Return named (dimension, 0..100) scores so the CLI can render a table.""" + from nokaman.models.toy import ToyAbilityModel + + model = ToyAbilityModel(language=(lang or "en")) + return [ + ("Writing (overall)", model.score_text(text, skill="writing")["score"]), + ("Vocabulary", model.score_text(text, skill="vocabulary")["score"]), + ("Grammar", model.score_text(text, skill="grammar")["score"]), + ("Reading", model.score_text(text, skill="reading")["score"]), + ("Listening", model.score_text(text, skill="listening")["score"]), + ("Speaking", model.score_text(text, skill="speaking")["score"]), + ] + + @eval_app.command("samples") def eval_samples() -> None: files = list_sample_files() @@ -238,7 +300,7 @@ def eval_samples() -> None: @eval_app.command("batch") def eval_batch( - out: Optional[Path] = typer.Option(None, "--out", "-o"), + out: Path | None = typer.Option(None, "--out", "-o"), table: bool = typer.Option(True, "--table/--json-only"), ) -> None: report = batch_evaluate() diff --git a/tests/test_cli.py b/tests/test_cli.py index c800ce7..3f4e794 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -20,3 +20,42 @@ def test_eval_batch_writes_nested_output_path(tmp_path) -> None: assert report["n_samples"] >= 1 assert "by_language" in report assert "rows" in report + + +def test_eval_score_text_prints_table() -> None: + text = "Because the sun was bright, I walked to the market and bought fruit." + result = CliRunner().invoke( + app, ["eval", "score", "--text", text, "--lang", "en"], + ) + assert result.exit_code == 0, result.output + out = result.output + # Rich table must appear with the named dimensions. + for dim in ("Writing (overall)", "Vocabulary", "Grammar", "Reading", "Listening", "Speaking"): + assert dim in out, f"missing dimension {dim} in: {out}" + assert "Overall" in out + assert "CEFR" in out + + +def test_eval_score_text_json_is_valid() -> None: + text = "The cat sat on the mat because it was tired." + result = CliRunner().invoke( + app, ["eval", "score", "--text", text, "--lang", "en", "--json"], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert "score" in data + assert "cefr" in data + assert "language" in data + + +def test_eval_score_sample_prints_table() -> None: + result = CliRunner().invoke( + app, ["eval", "score", "--text", "The project was large and complex.", "--lang", "en"], + ) + assert result.exit_code == 0, result.output + + +def test_eval_score_requires_input() -> None: + result = CliRunner().invoke(app, ["eval", "score"]) + assert result.exit_code == 1, result.output + assert "Provide --sample or --text" in result.output