diff --git a/.gitignore b/.gitignore index a992906..62452bf 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ logs/* coverage/ .nyc_output/ +# Generated metrics data +scripts/metrics/*.json + # OpenTofu / Terraform infra/.terraform/ infra/*.tfstate diff --git a/docs/project-metrics-report.md b/docs/project-metrics-report.md new file mode 100644 index 0000000..687b50b --- /dev/null +++ b/docs/project-metrics-report.md @@ -0,0 +1,419 @@ +# AccountabilityAtlas Project Metrics Report + +> Generated: February 27, 2026 +> Project span: January 31 - February 27, 2026 (27 days) + +--- + +## Table of Contents + +1. [Project Size](#1-project-size) +2. [Complexity](#2-complexity) +3. [Quality Metrics](#3-quality-metrics) +4. [Developer Performance](#4-developer-performance) +5. [Methodology](#5-methodology) + +--- + +## 1. Project Size + +### 1.1 Lines of Code by Repository + +| Repository | Source | Tests | Config | Migrations | Build | Docs | Other | Total | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| **top-level** | - | - | 285 | - | 138 | 29,461 | 10,449 | 40,333 | +| **user-service** | 2,108 | 3,175 | 149 | 165 | 502 | 4,541 | 341 | 10,981 | +| **video-service** | 2,361 | 3,120 | 212 | 226 | 503 | 1,615 | 451 | 8,488 | +| **moderation-service** | 2,245 | 2,947 | 124 | 126 | 495 | 1,802 | 278 | 8,017 | +| **web-app** | 5,895 | 8,231 | 152 | - | 81 | 1,495 | 75 | 15,929 | +| **integration-tests** | - | 4,855 | 122 | - | 39 | 930 | 5,575 | 11,521 | +| **search-service** | 1,027 | 2,089 | 156 | 169 | 496 | 1,252 | 323 | 5,512 | +| **location-service** | 1,155 | 1,701 | 120 | 122 | 499 | 1,271 | 266 | 5,134 | +| **api-gateway** | 183 | 278 | 114 | - | 411 | 683 | 135 | 1,804 | +| **notification-service** | - | - | - | - | - | 412 | - | 412 | +| **Overall** | **14,974** | **26,396** | **1,434** | **808** | **3,164** | **43,462** | **17,893** | **108,131** | + +"Other" includes infrastructure (1,323), scripts (6,879), seed data fixtures, and miscellaneous files. + +### 1.2 Lines of Code by Category (Overall) + +| Category | Total Lines | Code Lines | Files | % of Project | +|---|---:|---:|---:|---:| +| Documentation | 43,462 | 34,242 | 99 | 40.2% | +| Tests | 26,396 | 20,321 | 155 | 24.4% | +| Production source | 14,974 | 12,304 | 236 | 13.8% | +| Other | 9,691 | 9,257 | 93 | 9.0% | +| Scripts & infra | 8,202 | 6,620 | 51 | 7.6% | +| Build files | 3,164 | 2,692 | 30 | 2.9% | +| Config | 1,434 | 1,319 | 43 | 1.3% | +| Migrations (SQL) | 808 | 719 | 26 | 0.7% | +| **Total** | **108,131** | **87,474** | **733** | **100%** | + +### 1.3 Language Breakdown + +| Language | Total Lines | Code Lines | Files | % of Code | +|---|---:|---:|---:|---:| +| Markdown | 39,190 | 30,060 | 95 | 34.4% | +| Java | 22,341 | 17,177 | 254 | 19.6% | +| TypeScript (JSX) | 11,866 | 9,899 | 86 | 11.3% | +| TypeScript | 7,281 | 5,647 | 53 | 6.5% | +| YAML | 6,981 | 6,580 | 70 | 7.5% | +| JSON | 6,574 | 6,574 | 10 | 7.5% | +| Shell | 3,882 | 2,971 | 22 | 3.4% | +| Python | 2,515 | 1,986 | 7 | 2.3% | +| SQL | 1,733 | 1,539 | 55 | 1.8% | +| Gradle (Groovy) | 1,507 | 1,203 | 14 | 1.4% | +| XML | 874 | 874 | 2 | 1.0% | +| Terraform/OpenTofu | 651 | 529 | 12 | 0.6% | +| Other | 2,736 | 2,435 | 53 | 2.8% | +| **Total** | **108,131** | **87,474** | **733** | **100%** | + +**Primary application languages**: Java (17,177 LOC) + TypeScript/TSX (15,546 LOC) = **32,723 LOC** of application code. + +### 1.4 File Counts by Repository + +| Repository | Source | Test | Config | Migration | Build | Docs | Other | Total | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| top-level | - | - | 2 | - | 3 | 58 | 62 | 125 | +| user-service | 52 | 25 | 5 | 9 | 4 | 8 | 17 | 120 | +| video-service | 36 | 14 | 5 | 7 | 4 | 4 | 14 | 84 | +| moderation-service | 36 | 17 | 5 | 3 | 4 | 4 | 10 | 79 | +| web-app | 63 | 43 | 7 | - | 2 | 8 | 9 | 132 | +| integration-tests | - | 29 | 4 | - | 1 | 4 | 5 | 43 | +| search-service | 22 | 10 | 6 | 2 | 4 | 4 | 9 | 57 | +| location-service | 22 | 13 | 5 | 5 | 4 | 4 | 12 | 65 | +| api-gateway | 5 | 4 | 4 | - | 4 | 3 | 6 | 26 | +| notification-service | - | - | - | - | - | 2 | - | 2 | +| **Overall** | **236** | **155** | **43** | **26** | **30** | **99** | **144** | **733** | + +--- + +## 2. Complexity + +### 2.1 API Endpoints (from OpenAPI Specifications) + +| Service | GET | POST | PUT | DELETE | Total | Paths | +|---|---:|---:|---:|---:|---:|---:| +| moderation-service | 6 | 6 | 1 | 1 | **14** | 13 | +| user-service | 2 | 7 | 2 | 0 | **11** | 10 | +| video-service | 6 | 2 | 1 | 2 | **11** | 7 | +| location-service | 5 | 1 | 0 | 0 | **6** | 5 | +| search-service | 4 | 0 | 0 | 0 | **4** | 4 | +| notification-service | 1 | 0 | 1 | 0 | **2** | 1 | +| **Backend total** | **24** | **16** | **5** | **3** | **48** | **40** | +| api-gateway (routing) | 9 | 4 | 3 | 1 | 17 | 9 | + +The API gateway exposes a subset of backend endpoints through unified routing. The 48 backend endpoints represent the true API surface area. No service uses PATCH. + +### 2.2 Domain Architecture + +**Java Backend Services** + +| Service | @Entity | @Repository | @Service | @RestController | @Configuration | @Bean | +|---|---:|---:|---:|---:|---:|---:| +| user-service | 7 | - | 6 | 3 | 4 | 6 | +| moderation-service | 3 | 3 | 5 | 2 | 3 | 6 | +| video-service | 2 | 2 | 4 | 2 | 2 | 4 | +| location-service | 2 | 2 | 4 | 3 | 2 | 2 | +| search-service | 1 | 1 | 3 | 2 | 2 | 2 | +| api-gateway | - | - | - | - | 2 | 2 | +| **Total** | **15** | **8** | **22** | **12** | **15** | **22** | + +**Web Frontend (Next.js)** + +| Metric | Count | +|---|---:| +| React components | 54 | +| Next.js pages | 8 | +| API routes | 0 | +| Runtime dependencies | 10 | +| Dev dependencies | 19 | + +### 2.3 Branching Complexity + +| Repository | if | else if | switch/case | catch | loops | && / \|\| | Est. CC | +|---|---:|---:|---:|---:|---:|---:|---:| +| web-app | 118 | 5 | 0 | 14 | 1 | 180 | **319** | +| integration-tests | 67 | 1 | 0 | 1 | 17 | 14 | **101** | +| video-service | 56 | 2 | 5 | 11 | 4 | 16 | **95** | +| moderation-service | 28 | 1 | 30 | 20 | 1 | 11 | **92** | +| user-service | 33 | 2 | 2 | 9 | 1 | 12 | **60** | +| search-service | 16 | 0 | 2 | 11 | 1 | 12 | **43** | +| location-service | 16 | 1 | 0 | 2 | 10 | 4 | **34** | +| api-gateway | 3 | 0 | 0 | 2 | 0 | 1 | **7** | +| **Overall** | **337** | **12** | **39** | **70** | **35** | **250** | **744** | + +*Estimated Cyclomatic Complexity (CC) = 1 + if + else_if + switch_cases + catch + loops + logical_operators* + +### 2.4 Dependencies + +| Repository | Compile | Test | Total | +|---|---:|---:|---:| +| web-app | 10 | 19 | **29** | +| video-service | 19 | 6 | **25** | +| user-service | 16 | 6 | **22** | +| moderation-service | 14 | 7 | **21** | +| search-service | 14 | 7 | **21** | +| location-service | 15 | 6 | **21** | +| api-gateway | 5 | 5 | **10** | +| integration-tests | 0 | 8 | **8** | +| **Total unique deps** | **93** | **64** | **157** | + +### 2.5 Estimated Function Points + +Using a simplified IFPUG methodology based on the system's functional capabilities: + +| Component Type | Count | Avg Weight | FP | +|---|---:|---:|---:| +| External Inputs (POST/PUT/DELETE endpoints) | 24 | 4 | 96 | +| External Outputs (GET endpoints) | 24 | 5 | 120 | +| Internal Logical Files (domain entities) | 15 | 10 | 150 | +| External Interface Files (YouTube, Mapbox, Nominatim, OAuth) | 4 | 7 | 28 | +| UI Pages (Next.js routes) | 8 | 5 | 40 | +| **Unadjusted Function Points** | | | **434** | + +Applying a value adjustment factor of 1.05 (microservice architecture adds some complexity): + +**Adjusted Function Points: ~456 FP** + +For context, this represents a medium-sized application. The COCOMO II model would estimate ~12-18 person-months of effort for this size. + +--- + +## 3. Quality Metrics + +### 3.1 Structural Quality + +| Metric | Value | +|---|---| +| **Test-to-source ratio** | 1.76:1 (26,396 test LOC / 14,974 source LOC) | +| **Avg source file size** | 63 lines (14,974 / 236 files) | +| **Avg test file size** | 170 lines (26,396 / 155 files) | +| **Documentation ratio** | 2.90:1 (43,462 doc LOC / 14,974 source LOC) | +| **Migration files** | 26 SQL migrations across 5 services | +| **Total test files** | 155 (unit + integration + E2E) | +| **CI/CD workflows** | Present in all active service repos | + +### 3.2 Test Coverage - Java Backend (JaCoCo) + +| Service | Line | Branch | Instruction | Method | Lines Covered | +|---|---:|---:|---:|---:|---:| +| user-service | **97.3%** | 87.4% | 97.6% | 95.0% | 639 / 657 | +| search-service | **96.9%** | 88.3% | 97.9% | 96.4% | 285 / 294 | +| video-service | **95.8%** | 72.1% | 95.4% | 93.8% | 711 / 742 | +| location-service | **93.5%** | 81.5% | 94.9% | 91.9% | 371 / 397 | +| api-gateway | **91.4%** | 75.0% | 94.2% | 78.6% | 53 / 58 | +| moderation-service | **84.4%** | 77.3% | 84.3% | 84.7% | 615 / 729 | +| **Weighted avg** | **92.9%** | **79.9%** | **93.3%** | **91.5%** | **2,674 / 2,877** | + +All services exceed the **80% coverage target**. Weighted average line coverage is **92.9%**. + +### 3.3 Test Coverage - Web Frontend (Jest) + +| Metric | Coverage | Covered / Total | +|---|---:|---:| +| Lines | **79.7%** | 1,009 / 1,266 | +| Statements | 78.6% | 1,053 / 1,339 | +| Functions | 75.8% | 238 / 314 | +| Branches | 71.2% | 389 / 546 | + +Web-app line coverage is just below the 80% target at 79.7%. + +### 3.4 Integration & E2E Tests + +| Test Suite | Test Files | Location | +|---|---:|---| +| API integration tests | 10 | `integration-tests/api/tests/` | +| E2E browser tests | 15 | `integration-tests/e2e/tests/` | +| Web-app unit tests | 43 suites (442 tests) | `web-app/src/__tests__/` | + +E2E tests cover **3 browsers**: Chromium, Firefox (via xvfb), and WebKit. + +### 3.5 Static Analysis + +All 6 active Java services pass the full quality gate: + +| Check | Tool | Status | +|---|---|---| +| Code formatting | Spotless (Google Java Style) | All pass | +| Static analysis | Error Prone | All pass | +| Coverage threshold | JaCoCo (80% minimum) | All pass | +| Web-app formatting | Prettier (via husky pre-commit) | Configured | + +### 3.6 Overall Quality Summary + +| Dimension | Rating | Notes | +|---|---|---| +| Test coverage (backend) | Excellent | 92.9% line coverage, all services > 80% | +| Test coverage (frontend) | Good | 79.7% line, 71.2% branch | +| Test-to-code ratio | Excellent | 1.76x more test code than source | +| Documentation ratio | Excellent | 2.90x more docs than source | +| Static analysis | Excellent | All checks pass, no violations | +| Architectural consistency | Excellent | All services follow same layered pattern | + +--- + +## 4. Developer Performance + +### 4.1 Project Timeline + +| Metric | Value | +|---|---| +| **Project start** | January 31, 2026 | +| **Latest activity** | February 27, 2026 | +| **Active development days** | 27 days | +| **Repositories** | 10 (1 parent + 7 services + 1 web-app + 1 integration-tests) | + +### 4.2 Commit Activity + +| Metric | Value | +|---|---:| +| **Total commits** | 308 | +| **Commits per day** (avg) | 11.4 | +| **Claude co-authored commits** | 96 (31.2%) | +| **Human-only commits** | 212 (68.8%) | + +**Commits by Repository:** + +| Repository | Commits | First Commit | Latest Commit | +|---|---:|---|---| +| top-level (AccountabilityAtlas) | 80 | Jan 31 | Feb 27 | +| integration-tests | 42 | Feb 7 | Feb 25 | +| video-service | 37 | Jan 31 | Feb 26 | +| user-service | 33 | Jan 31 | Feb 26 | +| web-app | 31 | Jan 31 | Feb 25 | +| search-service | 24 | Jan 31 | Feb 26 | +| moderation-service | 22 | Jan 31 | Feb 25 | +| location-service | 17 | Jan 31 | Feb 24 | +| api-gateway | 15 | Jan 31 | Feb 20 | +| notification-service | 7 | Jan 31 | Feb 5 | + +### 4.3 Code Volume + +| Metric | Value | +|---|---:| +| **Total lines added** | 118,550 | +| **Total lines removed** | 5,804 | +| **Net lines** | 112,746 | +| **Lines added per day** (avg) | 4,391 | +| **Churn rate** | 4.9% (lines removed / lines added) | + +**Lines by Repository:** + +| Repository | Added | Removed | Net | % of Total | +|---|---:|---:|---:|---:| +| top-level | 38,637 | 1,827 | 36,810 | 32.6% | +| web-app | 28,777 | 1,059 | 27,718 | 24.6% | +| user-service | 11,111 | 189 | 10,922 | 9.7% | +| integration-tests | 9,425 | 1,482 | 7,943 | 7.0% | +| video-service | 8,760 | 439 | 8,321 | 7.4% | +| moderation-service | 8,372 | 349 | 8,023 | 7.1% | +| search-service | 5,802 | 332 | 5,470 | 4.9% | +| location-service | 5,259 | 109 | 5,150 | 4.6% | +| api-gateway | 1,951 | 13 | 1,938 | 1.7% | +| notification-service | 456 | 5 | 451 | 0.4% | + +### 4.4 Pull Request Activity + +| Metric | Value | +|---|---:| +| **Total merged PRs** | 193 | +| **PRs per day** (avg) | 7.1 | +| **PR-to-commit ratio** | 0.63 (193 PRs / 308 commits) | + +**PRs by Repository:** + +| Repository | Merged PRs | +|---|---:| +| top-level | 37 | +| video-service | 28 | +| web-app | 27 | +| integration-tests | 27 | +| user-service | 23 | +| search-service | 16 | +| moderation-service | 15 | +| api-gateway | 10 | +| location-service | 10 | +| notification-service | 0 | + +### 4.5 Issue Tracking + +| Metric | Value | +|---|---:| +| **Total issues** | 213 | +| **Closed issues** | 147 (69.0%) | +| **Open issues** | 66 (31.0%) | +| **Issue closure rate** | 5.4 issues/day | + +**Issues by Repository:** + +| Repository | Total | Open | Closed | Closure Rate | +|---|---:|---:|---:|---:| +| top-level | 65 | 39 | 26 | 40.0% | +| web-app | 43 | 16 | 27 | 62.8% | +| user-service | 24 | 3 | 21 | 87.5% | +| video-service | 22 | 1 | 21 | 95.5% | +| integration-tests | 18 | 2 | 16 | 88.9% | +| search-service | 14 | 2 | 12 | 85.7% | +| moderation-service | 12 | 1 | 11 | 91.7% | +| location-service | 11 | 2 | 9 | 81.8% | +| api-gateway | 4 | 0 | 4 | 100.0% | +| notification-service | 0 | 0 | 0 | - | + +### 4.6 Velocity & Productivity Summary + +| Metric | Per Day | Per Week | Total | +|---|---:|---:|---:| +| Commits | 11.4 | 79.8 | 308 | +| Lines of code added | 4,391 | 30,737 | 118,550 | +| PRs merged | 7.1 | 49.7 | 193 | +| Issues closed | 5.4 | 37.8 | 147 | + +### 4.7 AI-Assisted Development + +| Metric | Value | +|---|---| +| Claude co-authored commits | 96 / 308 (31.2%) | +| Human-AI collaboration model | Human-directed, AI-implemented | +| Quality gate compliance | 100% - all services pass all checks | + +--- + +## 5. Methodology + +### Data Sources + +| Metric | Source | Tool | +|---|---|---| +| Lines of code | File system analysis | `collect_loc_metrics.py` | +| Endpoints | OpenAPI specification YAML files | `collect_endpoint_counts.py` | +| Domain annotations | Source code grep | `collect_loc_metrics.py` | +| Branching complexity | Source code regex matching | `collect_loc_metrics.py` | +| Function points | IFPUG simplified methodology | Manual estimation | +| Test coverage (Java) | JaCoCo XML/CSV reports | `collect_coverage.py` | +| Test coverage (JS) | Jest coverage reports | `npx jest --coverage` | +| Git metrics | Git log analysis | `collect_metrics.py` | +| GitHub metrics | GitHub API | `collect_metrics.py` | + +### Definitions + +- **Lines of code (LOC)**: All non-binary tracked files, including blank lines and comments +- **Code lines**: LOC minus blank lines and comments +- **Source**: Production application code (`src/main/java/**`, `src/**/*.{ts,tsx}`) +- **Tests**: Test code (`src/test/**`, `__tests__/**`, `*.spec.ts`) +- **Estimated CC**: Simplified cyclomatic complexity approximation, not equivalent to formal CC measurement +- **Function Points**: Estimated using simplified IFPUG methodology; actual FP analysis may differ +- **Coverage**: Measured on hand-written source code only; generated code (OpenAPI models/APIs) excluded +- **Comment detection**: Language-aware (C-style `//` `/* */` for Java/TS/CSS/Gradle; `#` for Python/Shell/YAML; no comments for Markdown/JSON/XML/SQL) + +### Scripts + +Collection scripts are located in `scripts/metrics/`: +- `collect_loc_metrics.py` - LOC, language, complexity, annotation analysis +- `collect_metrics.py` - Git and GitHub metrics +- `collect_coverage.py` - JaCoCo coverage extraction +- `collect_endpoint_counts.py` - API endpoint counts from OpenAPI specs + +--- + +*Report generated by Claude Code metrics analysis pipeline.* diff --git a/scripts/metrics/README.md b/scripts/metrics/README.md new file mode 100644 index 0000000..8da4d33 --- /dev/null +++ b/scripts/metrics/README.md @@ -0,0 +1,116 @@ +# Project Metrics Scripts + +Scripts for collecting size, complexity, quality, and developer performance metrics across all AccountabilityAtlas repos. Results feed into [docs/project-metrics-report.md](../../docs/project-metrics-report.md). + +## Prerequisites + +- **Python 3.x** (available as `python`) +- **Git** (for commit/log analysis) +- **GitHub CLI** (`gh`) authenticated with access to all `kelleyglenn/AcctAtlas-*` repos +- **Gradle wrapper** (`./gradlew`) in each Java service (for coverage reports) +- **Node.js / npm** (for web-app Jest coverage) + +## Scripts + +### 1. `collect_loc_metrics.py` - Size & Complexity + +Walks every repo's file tree and counts lines of code, categorized by type (source, test, config, migrations, build, docs). Also counts branching constructs, Java annotations, React components, and dependency counts. + +```bash +python scripts/metrics/collect_loc_metrics.py +``` + +**Output:** `scripts/metrics/loc_metrics.json` + +No build or network access required -- purely file-system analysis. + +### 2. `collect_metrics.py` - Git & GitHub Metrics + +Collects commit counts, lines added/removed, Claude co-authorship stats, merged PR counts, and issue counts across all repos. + +```bash +python scripts/metrics/collect_metrics.py +``` + +**Output:** `scripts/metrics/git_metrics.json` + +Requires `git` and `gh` CLI. GitHub API calls may be slow (~2 min for all repos). + +### 3. `collect_coverage.py` - Test Coverage (Java) + +Parses JaCoCo CSV reports from each Java service's `build/reports/jacoco/test/` directory. Reports must already exist from a prior `./gradlew check jacocoTestReport` run. + +```bash +python scripts/metrics/collect_coverage.py +``` + +**Output:** `scripts/metrics/coverage_data.json` + +**Prerequisite:** Run tests with coverage in each Java service first: + +```bash +# All services in parallel (from project root) +for svc in api-gateway user-service video-service location-service search-service moderation-service; do + (cd "AcctAtlas-$svc" && ./gradlew check jacocoTestReport) & +done +wait +``` + +### 4. Web-App Coverage (manual) + +Jest coverage for the web-app is collected separately: + +```bash +cd AcctAtlas-web-app +npx jest --coverage --silent +``` + +Results are written to `AcctAtlas-web-app/coverage/coverage-summary.json`. + +### 5. `collect_endpoint_counts.py` - API Endpoints + +Parses each service's `docs/api-specification.yaml` (OpenAPI 3.1) and counts HTTP methods (GET, POST, PUT, DELETE, PATCH) under the `paths:` section. Uses simple line-based YAML parsing -- no external YAML library required. + +```bash +python scripts/metrics/collect_endpoint_counts.py +``` + +**Output:** `scripts/metrics/endpoint_counts.json` + +## Running Everything + +To regenerate all metrics from scratch: + +```bash +# 1. LOC & structural metrics (fast, ~10s) +python scripts/metrics/collect_loc_metrics.py + +# 2. Endpoint counts (fast, ~1s) +python scripts/metrics/collect_endpoint_counts.py + +# 3. Git & GitHub metrics (~2 min) +python scripts/metrics/collect_metrics.py + +# 4. Build all Java services with coverage (~2-5 min) +for svc in api-gateway user-service video-service location-service search-service moderation-service; do + (cd "AcctAtlas-$svc" && ./gradlew check jacocoTestReport) & +done +wait + +# 5. Extract Java coverage data (fast, ~1s) +python scripts/metrics/collect_coverage.py + +# 6. Web-app Jest coverage (~30s) +cd AcctAtlas-web-app && npx jest --coverage --silent && cd .. +``` + +After running all scripts, update `docs/project-metrics-report.md` with the new data from the JSON output files. + +## Output Files + +| File | Contents | +|---|---| +| `loc_metrics.json` | LOC by repo/category/language, complexity, annotations, dependencies | +| `git_metrics.json` | Commits, lines added/removed, PRs, issues per repo | +| `coverage_data.json` | JaCoCo line/branch/instruction/method coverage per Java service | +| `endpoint_counts.json` | API endpoint counts by HTTP method per service | diff --git a/scripts/metrics/collect_coverage.py b/scripts/metrics/collect_coverage.py new file mode 100644 index 0000000..3603bf4 --- /dev/null +++ b/scripts/metrics/collect_coverage.py @@ -0,0 +1,150 @@ +""" +Extract JaCoCo test coverage data from all Java microservices. + +Parses JaCoCo CSV reports (preferred) or XML reports (fallback) and outputs +a JSON summary with instruction, branch, line, method, and complexity coverage +percentages for each service. + +Usage: + python scripts/metrics/collect_coverage.py + +Output: + scripts/metrics/coverage_data.json +""" + +import csv +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +# Use UTF-8 for stdout on Windows +sys.stdout.reconfigure(encoding="utf-8") + +BASE_DIR = Path(__file__).resolve().parent.parent.parent +OUTPUT_FILE = BASE_DIR / "scripts" / "metrics" / "coverage_data.json" + +SERVICES = { + "user-service": "AcctAtlas-user-service", + "video-service": "AcctAtlas-video-service", + "location-service": "AcctAtlas-location-service", + "search-service": "AcctAtlas-search-service", + "moderation-service": "AcctAtlas-moderation-service", + "api-gateway": "AcctAtlas-api-gateway", +} + +JACOCO_REPORT_PATH = Path("build", "reports", "jacoco", "test") +CSV_FILENAME = "jacocoTestReport.csv" +XML_FILENAME = "jacocoTestReport.xml" + +# Counter types we care about +COUNTER_TYPES = ["INSTRUCTION", "BRANCH", "LINE", "COMPLEXITY", "METHOD"] + + +def calc_percentage(covered: int, missed: int) -> float: + """Calculate coverage percentage, returning 0.0 if no data.""" + total = covered + missed + if total == 0: + return 0.0 + return round((covered / total) * 100, 1) + + +def parse_csv_report(csv_path: Path) -> dict: + """Parse a JaCoCo CSV report and aggregate coverage across all rows.""" + totals = {} + for ct in COUNTER_TYPES: + totals[f"{ct}_COVERED"] = 0 + totals[f"{ct}_MISSED"] = 0 + + with open(csv_path, encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + for ct in COUNTER_TYPES: + totals[f"{ct}_COVERED"] += int(row[f"{ct}_COVERED"]) + totals[f"{ct}_MISSED"] += int(row[f"{ct}_MISSED"]) + + return build_result(totals) + + +def parse_xml_report(xml_path: Path) -> dict: + """Parse a JaCoCo XML report and extract report-level counters.""" + tree = ET.parse(xml_path) + root = tree.getroot() + + totals = {} + for ct in COUNTER_TYPES: + totals[f"{ct}_COVERED"] = 0 + totals[f"{ct}_MISSED"] = 0 + + for counter in root.findall("counter"): + ctype = counter.attrib["type"] + if ctype in COUNTER_TYPES: + totals[f"{ctype}_COVERED"] += int(counter.attrib["covered"]) + totals[f"{ctype}_MISSED"] += int(counter.attrib["missed"]) + + return build_result(totals) + + +def build_result(totals: dict) -> dict: + """Build the coverage result dict from aggregated totals.""" + return { + "instruction_coverage": calc_percentage( + totals["INSTRUCTION_COVERED"], totals["INSTRUCTION_MISSED"] + ), + "branch_coverage": calc_percentage( + totals["BRANCH_COVERED"], totals["BRANCH_MISSED"] + ), + "line_coverage": calc_percentage( + totals["LINE_COVERED"], totals["LINE_MISSED"] + ), + "method_coverage": calc_percentage( + totals["METHOD_COVERED"], totals["METHOD_MISSED"] + ), + "complexity_coverage": calc_percentage( + totals["COMPLEXITY_COVERED"], totals["COMPLEXITY_MISSED"] + ), + "lines_covered": totals["LINE_COVERED"], + "lines_missed": totals["LINE_MISSED"], + "total_lines": totals["LINE_COVERED"] + totals["LINE_MISSED"], + } + + +def main(): + results = {} + + for display_name, dir_name in SERVICES.items(): + service_dir = BASE_DIR / dir_name + report_dir = service_dir / JACOCO_REPORT_PATH + + csv_path = report_dir / CSV_FILENAME + xml_path = report_dir / XML_FILENAME + + if csv_path.exists(): + print(f" {display_name}: parsing CSV report") + results[display_name] = parse_csv_report(csv_path) + elif xml_path.exists(): + print(f" {display_name}: parsing XML report") + results[display_name] = parse_xml_report(xml_path) + else: + print(f" {display_name}: NO REPORT FOUND (skipped)") + continue + + cov = results[display_name] + print( + f" -> line={cov['line_coverage']}% " + f"branch={cov['branch_coverage']}% " + f"instruction={cov['instruction_coverage']}% " + f"({cov['lines_covered']}/{cov['total_lines']} lines)" + ) + + output = {"services": results} + + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2) + + print(f"\nResults written to {OUTPUT_FILE}") + print(f"Services processed: {len(results)}/{len(SERVICES)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/metrics/collect_endpoint_counts.py b/scripts/metrics/collect_endpoint_counts.py new file mode 100644 index 0000000..ebe88de --- /dev/null +++ b/scripts/metrics/collect_endpoint_counts.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +"""Count REST API endpoints from OpenAPI specification files across all services. + +Parses the `paths:` section of each service's docs/api-specification.yaml and +counts HTTP methods (GET, POST, PUT, DELETE, PATCH) per service. + +Usage: + python scripts/metrics/collect_endpoint_counts.py + +Output: + scripts/metrics/endpoint_counts.json +""" + +import json +import sys +from pathlib import Path + +sys.stdout.reconfigure(encoding="utf-8") + +ROOT = Path(__file__).resolve().parent.parent.parent +OUTPUT = ROOT / "scripts" / "metrics" / "endpoint_counts.json" + +HTTP_METHODS = ("get", "post", "put", "delete", "patch") + +SERVICES = { + "user-service": ROOT / "AcctAtlas-user-service" / "docs" / "api-specification.yaml", + "video-service": ROOT / "AcctAtlas-video-service" / "docs" / "api-specification.yaml", + "location-service": ROOT / "AcctAtlas-location-service" / "docs" / "api-specification.yaml", + "search-service": ROOT / "AcctAtlas-search-service" / "docs" / "api-specification.yaml", + "moderation-service": ROOT / "AcctAtlas-moderation-service" / "docs" / "api-specification.yaml", + "notification-service": ROOT / "AcctAtlas-notification-service" / "docs" / "api-specification.yaml", + "api-gateway": ROOT / "AcctAtlas-api-gateway" / "docs" / "api-specification.yaml", +} + + +def count_endpoints(spec_path): + """Parse an OpenAPI YAML file and count endpoints by HTTP method. + + Uses simple line-based parsing to avoid requiring a YAML library. + Relies on the standard OpenAPI structure where paths are top-level keys + under `paths:` and HTTP methods are indented under each path. + """ + counts = {m.upper(): 0 for m in HTTP_METHODS} + paths = 0 + in_paths = False + current_indent = None + + try: + with open(spec_path, "r", encoding="utf-8") as f: + lines = f.readlines() + except FileNotFoundError: + print(f" WARNING: {spec_path} not found, skipping") + return None + + for line in lines: + stripped = line.rstrip() + if not stripped or stripped.lstrip().startswith("#"): + continue + + indent = len(line) - len(line.lstrip()) + + # Detect the `paths:` top-level section + if stripped == "paths:": + in_paths = True + current_indent = indent + continue + + # Detect leaving the paths section (another top-level key) + if in_paths and indent == current_indent and stripped.endswith(":"): + in_paths = False + continue + + if not in_paths: + continue + + # A path entry is typically indented one level under `paths:` + # and starts with / (e.g., " /users:" or " '/users/{id}':") + key = stripped.rstrip(":").strip().strip("'\"") + if key.startswith("/"): + paths += 1 + continue + + # HTTP method entries are indented under the path + method = stripped.strip().rstrip(":").lower() + if method in HTTP_METHODS: + counts[method.upper()] += 1 + + counts["total"] = sum(counts[m.upper()] for m in HTTP_METHODS) + counts["paths"] = paths + return counts + + +def main(): + print("Counting API endpoints from OpenAPI specifications...") + + results = {"services": {}, "overall": {}} + overall = {m.upper(): 0 for m in HTTP_METHODS} + overall["total"] = 0 + overall["paths"] = 0 + + for name, spec_path in SERVICES.items(): + print(f" {name}: {spec_path}") + counts = count_endpoints(spec_path) + if counts is None: + continue + + results["services"][name] = counts + for m in HTTP_METHODS: + overall[m.upper()] += counts[m.upper()] + overall["total"] += counts["total"] + overall["paths"] += counts["paths"] + + print(f" {counts['total']} endpoints across {counts['paths']} paths") + + results["overall"] = overall + + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + print(f"\nResults written to {OUTPUT}") + print(f"Overall: {overall['total']} endpoints across {overall['paths']} paths") + + +if __name__ == "__main__": + main() diff --git a/scripts/metrics/collect_loc_metrics.py b/scripts/metrics/collect_loc_metrics.py new file mode 100644 index 0000000..c904009 --- /dev/null +++ b/scripts/metrics/collect_loc_metrics.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python +"""Collect LOC, complexity, and structural metrics across all repos.""" + +import json +import os +import re +import sys +from pathlib import Path +from collections import defaultdict + +sys.stdout.reconfigure(encoding='utf-8') + +# Resolve project root relative to this script: scripts/metrics/ -> project root +ROOT = Path(__file__).resolve().parent.parent.parent +OUTPUT = ROOT / "scripts" / "metrics" / "loc_metrics.json" + +REPOS = { + "top-level": ROOT, + "api-gateway": ROOT / "AcctAtlas-api-gateway", + "user-service": ROOT / "AcctAtlas-user-service", + "video-service": ROOT / "AcctAtlas-video-service", + "location-service": ROOT / "AcctAtlas-location-service", + "search-service": ROOT / "AcctAtlas-search-service", + "moderation-service": ROOT / "AcctAtlas-moderation-service", + "notification-service": ROOT / "AcctAtlas-notification-service", + "web-app": ROOT / "AcctAtlas-web-app", + "integration-tests": ROOT / "AcctAtlas-integration-tests", +} + +SKIP_DIRS = { + ".git", "node_modules", "build", "out", ".gradle", ".next", + "coverage", "test-results", "playwright-report", ".idea", + ".vscode", ".worktrees", "__pycache__", "dist", "target", + ".husky", "seed-data", +} + +SKIP_EXTENSIONS = { + ".jar", ".class", ".png", ".jpg", ".jpeg", ".gif", ".ico", + ".svg", ".woff", ".woff2", ".ttf", ".eot", ".map", ".lock", + ".min.js", ".min.css", ".pyc", ".pyo", ".bin", ".dat", +} + + +def should_skip_dir(dirname): + return dirname in SKIP_DIRS or dirname.startswith(".") + + +def should_skip_file(filepath): + name = filepath.name.lower() + if name in ("package-lock.json", "gradlew.bat"): + return True + return filepath.suffix.lower() in SKIP_EXTENSIONS + + +# Languages where // and /* */ are comment syntax +C_STYLE_COMMENT_EXTS = {".java", ".ts", ".tsx", ".js", ".jsx", ".css", ".scss", ".gradle"} +# Languages where # is comment syntax +HASH_COMMENT_EXTS = {".py", ".sh", ".bash", ".yml", ".yaml", ".properties", ".tf", ".hcl", ".toml"} +# Languages with no comment counting (content is all "code") +NO_COMMENT_EXTS = {".md", ".json", ".xml", ".html", ".sql"} + + +def count_lines(filepath): + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + total = len(lines) + blank = sum(1 for line in lines if line.strip() == "") + comment = 0 + suffix = filepath.suffix.lower() + name = filepath.name.lower() + + if name == "dockerfile": + suffix = ".sh" # Dockerfile uses # comments + + if suffix in C_STYLE_COMMENT_EXTS: + in_block = False + for line in lines: + stripped = line.strip() + if in_block: + comment += 1 + if "*/" in stripped: + in_block = False + elif stripped.startswith("//"): + comment += 1 + elif stripped.startswith("/*"): + comment += 1 + if "*/" not in stripped: + in_block = True + elif stripped.startswith("* ") or stripped == "*": + comment += 1 + elif suffix in HASH_COMMENT_EXTS: + for line in lines: + stripped = line.strip() + if stripped.startswith("#"): + comment += 1 + # For NO_COMMENT_EXTS and unknown types, comment stays 0 + + code = total - blank - comment + return {"total": total, "code": max(0, code), "blank": blank, "comment": comment} + except Exception: + return {"total": 0, "code": 0, "blank": 0, "comment": 0} + + +def categorize_file(filepath, repo_name, repo_root): + rel = filepath.relative_to(repo_root) + parts = rel.parts + name = filepath.name.lower() + suffix = filepath.suffix.lower() + + if ".github" in parts: + return "ci_cd" + + # Java service categorization + if repo_name not in ("web-app", "integration-tests", "top-level"): + if "src" in parts: + src_idx = parts.index("src") + if src_idx + 1 < len(parts): + if parts[src_idx + 1] == "test": + return "test" + if parts[src_idx + 1] == "main": + if src_idx + 2 < len(parts): + if parts[src_idx + 2] == "resources": + if "migration" in parts or "db" in parts: + return "migration" + return "config" + if parts[src_idx + 2] == "java": + return "source" + return "source" + if name in ("build.gradle", "settings.gradle", "gradlew"): + return "build" + if name in ("docker-compose.yml", "dockerfile"): + return "config" + if suffix in (".md", ".yaml", ".yml") and ("docs" in parts or name == "readme.md"): + return "docs" + if "docker" in parts: + return "config" + if "gradle" in parts: + return "build" + return "other" + + # Web-app + if repo_name == "web-app": + if "__tests__" in parts or name.endswith(".test.ts") or name.endswith(".test.tsx") or name.endswith(".spec.ts"): + return "test" + if "src" in parts and suffix in (".ts", ".tsx", ".js", ".jsx", ".css"): + return "source" + if suffix in (".md",) or "docs" in parts or name == "readme.md": + return "docs" + if name in ("package.json", "tsconfig.json"): + return "build" + if "config" in name or name in ("next.config.js", "tailwind.config.ts", "postcss.config.mjs", + "jest.config.js", "playwright.config.js", ".env", + ".env.local", ".env.production"): + return "config" + if name == "dockerfile" or name == "docker-compose.yml": + return "config" + return "other" + + # Integration tests + if repo_name == "integration-tests": + if suffix in (".ts", ".js") and ("tests" in parts or "seeds" in parts or "fixtures" in parts): + return "test" + if "config" in name or name in ("playwright.config.ts",): + return "config" + if name in ("package.json", "tsconfig.json"): + return "build" + if suffix in (".md",) or name == "readme.md": + return "docs" + return "other" + + # Top-level + if repo_name == "top-level": + if "docs" in parts: + return "docs" + if "scripts" in parts: + return "scripts" + if "infra" in parts: + return "infra" + if name in ("build.gradle", "settings.gradle"): + return "build" + if name in ("docker-compose.yml", "sonar-project.properties"): + return "config" + if suffix == ".md": + return "docs" + if "gradle" in parts: + return "build" + return "other" + + return "other" + + +def get_language(filepath): + suffix = filepath.suffix.lower() + name = filepath.name.lower() + lang_map = { + ".java": "Java", + ".ts": "TypeScript", + ".tsx": "TypeScript (JSX)", + ".js": "JavaScript", + ".jsx": "JavaScript (JSX)", + ".css": "CSS", + ".scss": "SCSS", + ".html": "HTML", + ".sql": "SQL", + ".yml": "YAML", + ".yaml": "YAML", + ".json": "JSON", + ".xml": "XML", + ".properties": "Properties", + ".md": "Markdown", + ".sh": "Shell", + ".bash": "Shell", + ".tf": "Terraform/OpenTofu", + ".hcl": "HCL", + ".gradle": "Gradle (Groovy)", + ".toml": "TOML", + ".py": "Python", + } + if name == "dockerfile": + return "Dockerfile" + if name == "gradlew": + return "Shell" + return lang_map.get(suffix, "Other") + + +def count_complexity(filepath): + complexity = { + "if_statements": 0, "else_if": 0, "else_blocks": 0, + "switch_cases": 0, "catch_blocks": 0, "ternary": 0, + "loops": 0, "logical_operators": 0, + } + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + suffix = filepath.suffix.lower() + if suffix in (".java", ".ts", ".tsx", ".js", ".jsx"): + complexity["if_statements"] = len(re.findall(r"\bif\s*\(", content)) + complexity["else_if"] = len(re.findall(r"\belse\s+if\s*\(", content)) + complexity["else_blocks"] = len(re.findall(r"\}\s*else\s*\{", content)) + complexity["switch_cases"] = len(re.findall(r"\bcase\s+", content)) + complexity["catch_blocks"] = len(re.findall(r"\bcatch\s*\(", content)) + complexity["ternary"] = content.count(" ? ") + content.count("\t? ") + complexity["loops"] = len(re.findall(r"\b(for|while)\s*\(", content)) + complexity["logical_operators"] = len(re.findall(r"&&|\|\|", content)) + except Exception: + pass + return complexity + + +def count_java_annotations(filepath): + annotations = {} + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + for ann in ["@Entity", "@Repository", "@Service", "@RestController", + "@Controller", "@Component", "@Configuration", "@Bean", + "@EventListener", "@Scheduled"]: + count = len(re.findall(re.escape(ann) + r"(?:\b|$)", content)) + if count > 0: + annotations[ann] = count + except Exception: + pass + return annotations + + +def count_react_components(filepath): + count = 0 + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + count += len(re.findall(r"export\s+(default\s+)?function\s+\w+", content)) + count += len(re.findall(r"export\s+const\s+\w+\s*[:=]\s*(React\.)?FC", content)) + if filepath.name in ("page.tsx", "page.ts", "layout.tsx", "layout.ts"): + if "export default" in content: + count = max(count, 1) + except Exception: + pass + return count + + +def count_nextjs_routes(repo_root): + pages = 0 + api_routes = 0 + app_dir = repo_root / "src" / "app" + if app_dir.exists(): + for f in app_dir.rglob("page.tsx"): + pages += 1 + for f in app_dir.rglob("page.ts"): + pages += 1 + for f in app_dir.rglob("route.ts"): + api_routes += 1 + for f in app_dir.rglob("route.tsx"): + api_routes += 1 + return {"pages": pages, "api_routes": api_routes} + + +def parse_dependencies(repo_root): + deps = {"compile": 0, "test": 0, "total": 0} + build_gradle = repo_root / "build.gradle" + if build_gradle.exists(): + try: + with open(build_gradle, "r", encoding="utf-8") as f: + content = f.read() + compile_deps = len(re.findall(r"\b(implementation|api|compileOnly)\s+['\"]", content)) + test_deps = len(re.findall(r"\b(testImplementation|testCompileOnly|testRuntimeOnly)\s+['\"]", content)) + deps["compile"] = compile_deps + deps["test"] = test_deps + deps["total"] = compile_deps + test_deps + except Exception: + pass + pkg_json = repo_root / "package.json" + if pkg_json.exists(): + try: + with open(pkg_json, "r", encoding="utf-8") as f: + pkg = json.load(f) + compile_deps = len(pkg.get("dependencies", {})) + test_deps = len(pkg.get("devDependencies", {})) + deps["compile"] = compile_deps + deps["test"] = test_deps + deps["total"] = compile_deps + test_deps + except Exception: + pass + return deps + + +def analyze_repo(repo_name, repo_root): + print(f" Analyzing {repo_name}...") + if not repo_root.exists(): + return None + + result = { + "name": repo_name, + "loc": {}, + "languages": {}, + "complexity": { + "if_statements": 0, "else_if": 0, "else_blocks": 0, + "switch_cases": 0, "catch_blocks": 0, "ternary": 0, + "loops": 0, "logical_operators": 0, + }, + "annotations": {}, + "react_components": 0, + "dependencies": parse_dependencies(repo_root), + "file_count": 0, + "total_loc": {"total": 0, "code": 0, "blank": 0, "comment": 0}, + } + + loc_cats = defaultdict(lambda: {"total": 0, "code": 0, "blank": 0, "comment": 0, "files": 0}) + lang_stats = defaultdict(lambda: {"total": 0, "code": 0, "files": 0}) + ann_counts = defaultdict(int) + + skip_subdirs = set() + if repo_name == "top-level": + skip_subdirs = {p.name for name, p in REPOS.items() if name != "top-level"} + + for dirpath, dirnames, filenames in os.walk(repo_root): + dirnames[:] = [d for d in dirnames if not should_skip_dir(d) and d not in skip_subdirs] + + for filename in filenames: + filepath = Path(dirpath) / filename + if should_skip_file(filepath): + continue + + category = categorize_file(filepath, repo_name, repo_root) + language = get_language(filepath) + lines = count_lines(filepath) + + loc_cats[category]["total"] += lines["total"] + loc_cats[category]["code"] += lines["code"] + loc_cats[category]["blank"] += lines["blank"] + loc_cats[category]["comment"] += lines["comment"] + loc_cats[category]["files"] += 1 + + lang_stats[language]["total"] += lines["total"] + lang_stats[language]["code"] += lines["code"] + lang_stats[language]["files"] += 1 + + result["total_loc"]["total"] += lines["total"] + result["total_loc"]["code"] += lines["code"] + result["total_loc"]["blank"] += lines["blank"] + result["total_loc"]["comment"] += lines["comment"] + result["file_count"] += 1 + + if category == "source": + cx = count_complexity(filepath) + for key, val in cx.items(): + result["complexity"][key] += val + if filepath.suffix == ".java": + anns = count_java_annotations(filepath) + for ann, count in anns.items(): + ann_counts[ann] += count + if filepath.suffix in (".tsx", ".jsx"): + result["react_components"] += count_react_components(filepath) + + if category == "test": + cx = count_complexity(filepath) + for key, val in cx.items(): + result["complexity"][key] += val + + if repo_name == "web-app": + result["nextjs_routes"] = count_nextjs_routes(repo_root) + + result["loc"] = {k: dict(v) for k, v in loc_cats.items()} + result["languages"] = {k: dict(v) for k, v in lang_stats.items()} + result["annotations"] = dict(ann_counts) + + return result + + +def compute_estimated_cc(cx): + return (1 + + cx.get("if_statements", 0) + + cx.get("else_if", 0) + + cx.get("switch_cases", 0) + + cx.get("catch_blocks", 0) + + cx.get("loops", 0) + + cx.get("logical_operators", 0)) + + +def main(): + print("Collecting LOC and structural metrics...") + all_metrics = {"repos": {}, "overall": {}} + + for name, path in REPOS.items(): + result = analyze_repo(name, path) + if result: + result["estimated_cyclomatic_complexity"] = compute_estimated_cc(result["complexity"]) + all_metrics["repos"][name] = result + + # Overall + overall = { + "total_loc": {"total": 0, "code": 0, "blank": 0, "comment": 0}, + "file_count": 0, + "complexity": { + "if_statements": 0, "else_if": 0, "else_blocks": 0, + "switch_cases": 0, "catch_blocks": 0, "ternary": 0, + "loops": 0, "logical_operators": 0, + }, + "languages": {}, + "loc_by_category": {}, + } + + lang_agg = defaultdict(lambda: {"total": 0, "code": 0, "files": 0}) + cat_agg = defaultdict(lambda: {"total": 0, "code": 0, "blank": 0, "comment": 0, "files": 0}) + + for name, repo in all_metrics["repos"].items(): + overall["total_loc"]["total"] += repo["total_loc"]["total"] + overall["total_loc"]["code"] += repo["total_loc"]["code"] + overall["total_loc"]["blank"] += repo["total_loc"]["blank"] + overall["total_loc"]["comment"] += repo["total_loc"]["comment"] + overall["file_count"] += repo["file_count"] + for key in overall["complexity"]: + overall["complexity"][key] += repo["complexity"].get(key, 0) + for lang, data in repo["languages"].items(): + for k in ("total", "code", "files"): + lang_agg[lang][k] += data[k] + for cat, data in repo["loc"].items(): + for k in ("total", "code", "blank", "comment", "files"): + cat_agg[cat][k] += data.get(k, 0) + + overall["estimated_cyclomatic_complexity"] = compute_estimated_cc(overall["complexity"]) + overall["languages"] = {k: dict(v) for k, v in lang_agg.items()} + overall["loc_by_category"] = {k: dict(v) for k, v in cat_agg.items()} + all_metrics["overall"] = overall + + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(all_metrics, f, indent=2, default=str) + + print(f"\nMetrics written to {OUTPUT}") + print(f"\n{'='*50}") + print(f"Total files: {overall['file_count']}") + print(f"Total lines: {overall['total_loc']['total']:,}") + print(f"Code lines: {overall['total_loc']['code']:,}") + print(f"Blank lines: {overall['total_loc']['blank']:,}") + print(f"Comments: {overall['total_loc']['comment']:,}") + print(f"Est. CC: {overall['estimated_cyclomatic_complexity']:,}") + print(f"{'='*50}") + + # Per-repo summary + print(f"\n{'Repo':<25} {'Files':>6} {'Total':>8} {'Code':>8}") + print("-" * 50) + for name, repo in all_metrics["repos"].items(): + print(f"{name:<25} {repo['file_count']:>6} {repo['total_loc']['total']:>8,} {repo['total_loc']['code']:>8,}") + + +if __name__ == "__main__": + main() diff --git a/scripts/metrics/collect_metrics.py b/scripts/metrics/collect_metrics.py new file mode 100644 index 0000000..d391c83 --- /dev/null +++ b/scripts/metrics/collect_metrics.py @@ -0,0 +1,213 @@ +"""Collect git and GitHub metrics across all AccountabilityAtlas repos. + +Gathers commit counts, lines added/removed, Claude co-authorship stats, +merged PR counts, and issue counts for each repository. + +Usage: + python scripts/metrics/collect_metrics.py + +Output: + scripts/metrics/git_metrics.json +""" + +import subprocess +import json +import sys +import os +from datetime import datetime, timezone + +sys.stdout.reconfigure(encoding="utf-8") + +# Resolve project root relative to this script: scripts/metrics/ -> project root +BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +REPOS = [ + (BASE, "kelleyglenn/AccountabilityAtlas"), + (os.path.join(BASE, "AcctAtlas-api-gateway"), "kelleyglenn/AcctAtlas-api-gateway"), + (os.path.join(BASE, "AcctAtlas-user-service"), "kelleyglenn/AcctAtlas-user-service"), + (os.path.join(BASE, "AcctAtlas-video-service"), "kelleyglenn/AcctAtlas-video-service"), + (os.path.join(BASE, "AcctAtlas-location-service"), "kelleyglenn/AcctAtlas-location-service"), + (os.path.join(BASE, "AcctAtlas-search-service"), "kelleyglenn/AcctAtlas-search-service"), + (os.path.join(BASE, "AcctAtlas-moderation-service"), "kelleyglenn/AcctAtlas-moderation-service"), + (os.path.join(BASE, "AcctAtlas-notification-service"), "kelleyglenn/AcctAtlas-notification-service"), + (os.path.join(BASE, "AcctAtlas-web-app"), "kelleyglenn/AcctAtlas-web-app"), + (os.path.join(BASE, "AcctAtlas-integration-tests"), "kelleyglenn/AcctAtlas-integration-tests"), +] + +OUTPUT_FILE = os.path.join(BASE, "scripts", "metrics", "git_metrics.json") + + +def run(cmd, shell=True): + try: + result = subprocess.run( + cmd, shell=shell, capture_output=True, text=True, timeout=120, encoding="utf-8" + ) + if result.returncode != 0: + print(f" WARNING: command failed (rc={result.returncode}): {cmd}", file=sys.stderr) + print(f" stderr: {result.stderr.strip()}", file=sys.stderr) + return None + return result.stdout.strip() + except subprocess.TimeoutExpired: + print(f" WARNING: command timed out: {cmd}", file=sys.stderr) + return None + except Exception as e: + print(f" WARNING: command error: {cmd} -> {e}", file=sys.stderr) + return None + + +def collect_git_metrics(repo_path): + print(f"Collecting git metrics for {repo_path} ...") + metrics = {} + + out = run(f'git -C "{repo_path}" rev-list --count HEAD') + metrics["total_commits"] = int(out) if out else 0 + + out = run(f'git -C "{repo_path}" log --reverse --format=%aI') + if out: + metrics["first_commit_date"] = out.split("\n")[0].strip() + else: + metrics["first_commit_date"] = None + + out = run(f'git -C "{repo_path}" log -1 --format=%aI') + metrics["latest_commit_date"] = out if out else None + + out = run(f'git -C "{repo_path}" log --numstat --format=""') + la, lr = 0, 0 + if out: + for line in out.split("\n"): + parts = line.split("\t") + if len(parts) >= 2: + try: + la += int(parts[0]) + lr += int(parts[1]) + except ValueError: + pass + metrics["lines_added"] = la + metrics["lines_removed"] = lr + + out = run(f'git -C "{repo_path}" log --grep="Co-Authored-By: Claude" --oneline') + if out: + metrics["claude_coauthored_commits"] = len(out.strip().split("\n")) + else: + metrics["claude_coauthored_commits"] = 0 + + return metrics + + +def collect_github_metrics(gh_repo): + print(f"Collecting GitHub metrics for {gh_repo} ...") + metrics = {} + + out = run(f"gh pr list -R {gh_repo} --state merged --limit 999 --json number") + if out: + try: + metrics["merged_pr_count"] = len(json.loads(out)) + except json.JSONDecodeError: + metrics["merged_pr_count"] = 0 + else: + metrics["merged_pr_count"] = 0 + + out = run(f"gh issue list -R {gh_repo} --state all --limit 999 --json number,state") + if out: + try: + issues = json.loads(out) + metrics["total_issues"] = len(issues) + metrics["open_issues"] = sum(1 for i in issues if i.get("state") == "OPEN") + metrics["closed_issues"] = sum(1 for i in issues if i.get("state") == "CLOSED") + except json.JSONDecodeError: + metrics["total_issues"] = 0 + metrics["open_issues"] = 0 + metrics["closed_issues"] = 0 + else: + metrics["total_issues"] = 0 + metrics["open_issues"] = 0 + metrics["closed_issues"] = 0 + + return metrics + + +def main(): + all_metrics = { + "collected_at": datetime.now(timezone.utc).isoformat(), + "repos": {}, + "totals": {}, + } + + for repo_path, gh_repo in REPOS: + repo_name = os.path.basename(repo_path) + sep = "=" * 60 + print(f"\n{sep}") + print(f"Processing: {repo_name}") + print(sep) + + git_data = collect_git_metrics(repo_path) + gh_data = collect_github_metrics(gh_repo) + + all_metrics["repos"][repo_name] = { + "path": repo_path, + "github_repo": gh_repo, + "git": git_data, + "github": gh_data, + } + + totals = { + "total_commits": 0, + "total_lines_added": 0, + "total_lines_removed": 0, + "total_claude_coauthored_commits": 0, + "total_merged_prs": 0, + "total_issues": 0, + "total_open_issues": 0, + "total_closed_issues": 0, + "earliest_commit_date": None, + "latest_commit_date": None, + "repo_count": len(REPOS), + } + + for repo_name, data in all_metrics["repos"].items(): + g = data["git"] + h = data["github"] + totals["total_commits"] += g.get("total_commits", 0) + totals["total_lines_added"] += g.get("lines_added", 0) + totals["total_lines_removed"] += g.get("lines_removed", 0) + totals["total_claude_coauthored_commits"] += g.get("claude_coauthored_commits", 0) + totals["total_merged_prs"] += h.get("merged_pr_count", 0) + totals["total_issues"] += h.get("total_issues", 0) + totals["total_open_issues"] += h.get("open_issues", 0) + totals["total_closed_issues"] += h.get("closed_issues", 0) + + first = g.get("first_commit_date") + if first: + if totals["earliest_commit_date"] is None or first < totals["earliest_commit_date"]: + totals["earliest_commit_date"] = first + + latest = g.get("latest_commit_date") + if latest: + if totals["latest_commit_date"] is None or latest > totals["latest_commit_date"]: + totals["latest_commit_date"] = latest + + totals["total_net_lines"] = totals["total_lines_added"] - totals["total_lines_removed"] + all_metrics["totals"] = totals + + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + json.dump(all_metrics, f, indent=2, ensure_ascii=False) + + sep = "=" * 60 + print(f"\n{sep}") + print(f"Results written to {OUTPUT_FILE}") + print(sep) + t = totals + print(f"\nSUMMARY:") + print(f" Repos: {t['repo_count']}") + print(f" Total commits: {t['total_commits']}") + print(f" Lines added: {t['total_lines_added']:,}") + print(f" Lines removed: {t['total_lines_removed']:,}") + print(f" Net lines: {t['total_net_lines']:,}") + print(f" Claude co-authored: {t['total_claude_coauthored_commits']}") + print(f" Merged PRs: {t['total_merged_prs']}") + print(f" Total issues: {t['total_issues']} (open: {t['total_open_issues']}, closed: {t['total_closed_issues']})") + print(f" Earliest commit: {t['earliest_commit_date']}") + print(f" Latest commit: {t['latest_commit_date']}") + + +if __name__ == "__main__": + main()