diff --git a/CHANGELOG.md b/CHANGELOG.md index cbae6ca6..cd102326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,20 @@ - Initial CHANGELOG.md setup for tracking project history - Documentation structure for future contributor updates - Added .flake8 config file to enforce consistent 88-character line limit for all contributors +- Added `SKILL_SYNONYMS` dictionary to `utils/recommender.py` mapping 28 common tech abbreviations + (e.g. `js`, `reactjs`, `ts`, `node`, `py`) to their canonical lowercase names (#1116) +- Added 5 new unit tests in `tests/test_basic.py` to verify synonym normalization end-to-end (#1116) - DevPath Sentinel developer tool for repository health and dataset integrity validation (#1295) - Dataset validator to detect duplicate project IDs, duplicate project titles, missing required fields, empty required fields, and missing starter code references ### Changed - Contributors are now expected to document user-facing changes in CHANGELOG.md +- `parse_skills()` now normalizes skill abbreviations via `SKILL_SYNONYMS` before scoring, + so inputs like "JS, ReactJS, Node" correctly match projects tagged "JavaScript, React, Node.js" (#1116) ### Fixed -- Correct skills suggestions dropdown overlapping with available skill chips and resolve white background conflict in dark theme \ No newline at end of file +- Fixed missed skill matches when users enter common abbreviations (e.g. "JS" instead of "JavaScript"), + which previously caused skill coverage score to drop to 0 and returned poor recommendations (#1116) +- Correct skills suggestions dropdown overlapping with available skill chips and resolve white background conflict in dark theme diff --git a/src/utils/recommender.py b/src/utils/recommender.py index 087d2f50..3399b2d6 100644 --- a/src/utils/recommender.py +++ b/src/utils/recommender.py @@ -58,6 +58,53 @@ def clear_caches(): WEIGHT_INTEREST = SCORING_WEIGHTS["interest"] WEIGHT_TIME = SCORING_WEIGHTS["time"] +VALID_INTERESTS = { + "web", "data", "education", "automation", "games", + "cybersecurity", "devops", "mobile", "machine learning/ai", + "artificial intelligence", "cloud computing", "mobile app development", + "backend", "tools", "productivity", "business logic" +} +VALID_TIMES = {"low", "medium", "high"} + +# Canonical synonym map — maps common abbreviations / alternate names to the +# lowercase canonical skill name used throughout projects.json. +# Add new entries here; no other code changes are needed. +SKILL_SYNONYMS = { + # JavaScript ecosystem + "js": "javascript", + "javascript": "javascript", + "reactjs": "react", + "react.js": "react", + "vuejs": "vue", + "vue.js": "vue", + "nodejs": "node.js", + "node": "node.js", + "nextjs": "next.js", + "next": "next.js", + "expressjs": "express", + "ts": "typescript", + # Python ecosystem + "py": "python", + "django": "django", + "flask": "flask", + # Markup / styling + "html5": "html", + "css3": "css", + # Systems / low-level + "c++": "cpp", + "cplusplus": "cpp", + "c plus plus": "cpp", + "golang": "go", + # Databases + "postgres": "postgresql", + "psql": "postgresql", + "mongo": "mongodb", + # Misc + "web dev": "javascript", + "ml": "machine learning", + "ai": "artificial intelligence", + "k8s": "kubernetes", + "tf": "tensorflow", # Common aliases and abbreviations for skills # This improves recommendation accuracy by normalizing user input @@ -73,6 +120,30 @@ def _normalize_skill(s: str) -> str: """Normalize a skill string: strip surrounding whitespace and lowercase.""" return s.strip().lower() +# Keep the old name alive so score_single_project() and any external callers +# that reference SKILL_ALIASES continue to work without modification. +SKILL_ALIASES = SKILL_SYNONYMS + +def parse_skills(skills_string): + """ + Convert a raw skills string into a normalized, synonym-resolved lowercase list. + + Accepts either: + - A JSON array e.g. '["Python", "ReactJS"]' -> ["python", "react"] + - A comma-separated string e.g. "JS, TS, Node, " -> ["javascript", "typescript", "node.js"] + + Processing steps applied to every token: + 1. Strip surrounding whitespace. + 2. Convert to lowercase. + 3. Discard empty strings (handles trailing commas / double commas). + 4. Map through SKILL_SYNONYMS so abbreviations become canonical names. + """ + if not skills_string or not skills_string.strip(): + return [] + + stripped = skills_string.strip() + + # --- JSON array branch --- def parse_skill_entries(skills_string): """Parse skills with optional per-skill proficiency levels.""" @@ -82,6 +153,18 @@ def parse_skill_entries(skills_string): try: parsed = json.loads(stripped) if isinstance(parsed, list): + tokens = [str(s).strip().lower() for s in parsed if str(s).strip()] + return [SKILL_SYNONYMS.get(token, token) for token in tokens] + except (json.JSONDecodeError, ValueError): + pass # fall through to comma-splitting + + # --- Comma-separated branch --- + tokens = [ + s.strip().lower() + for s in skills_string.split(",") + if s.strip() # skip blanks produced by trailing / consecutive commas + ] + return [SKILL_SYNONYMS.get(token, token) for token in tokens] entries = [] for item in parsed: if isinstance(item, dict): diff --git a/tests/test_basic.py b/tests/test_basic.py index 24d3c9a0..a97834f9 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -15,6 +15,10 @@ SCORING_WEIGHTS, VALID_LEVELS, VALID_TIME_AVAILABILITY, + SKILL_SYNONYMS, + WEIGHT_LEVEL, + WEIGHT_INTEREST, + WEIGHT_TIME, ) from utils.roadmap_comparer import compare_roadmaps, load_all_career_roadmaps @@ -293,6 +297,63 @@ def test_validate_whitespace_only_fields(): " " ) +def test_skill_synonym_matching(): + """parse_skills must normalize common abbreviations to their canonical names. + + Verifies that: + - "JS" -> "javascript" + - "ReactJS" -> "react" + - " ts " -> "typescript" (extra whitespace stripped) + - trailing comma produces no empty entry + """ + result = parse_skills("JS, ReactJS, ts , ") + assert result == ["javascript", "react", "typescript"], ( + f"Expected ['javascript', 'react', 'typescript'] but got {result}" + ) + + +def test_skill_synonym_matching_node(): + """'node' and 'nodejs' must both resolve to 'node.js'.""" + assert parse_skills("node") == ["node.js"] + assert parse_skills("nodejs") == ["node.js"] + + +def test_skill_synonym_matching_unknown_passthrough(): + """Skills not present in SKILL_SYNONYMS must pass through unchanged.""" + result = parse_skills("flutter, solidity") + assert result == ["flutter", "solidity"], ( + f"Unknown skills should not be altered; got {result}" + ) + + +def test_skill_synonym_end_to_end_scoring(): + """A user who inputs 'JS' must score the same as one who inputs 'javascript'. + + This is the critical integration check: synonym normalisation in parse_skills + must translate all the way through score_single_project. + """ + project = { + "skills": ["JavaScript"], + "level": "Beginner", + "interest": "Web", + "time": "Low", + } + score_abbrev = score_single_project(project, parse_skills("JS"), "Beginner", "Web", "Low") + score_canonical = score_single_project(project, parse_skills("JavaScript"), "Beginner", "Web", "Low") + + assert score_abbrev > 0, "'JS' should score > 0 for a JavaScript project after synonym resolution" + assert score_abbrev == score_canonical, ( + f"'JS' ({score_abbrev}) and 'javascript' ({score_canonical}) should produce identical scores" + ) + + +def test_skill_synonyms_dict_has_minimum_entries(): + """SKILL_SYNONYMS must contain at least 10 entries to satisfy the issue requirement.""" + assert len(SKILL_SYNONYMS) >= 10, ( + f"Expected at least 10 synonym entries, found {len(SKILL_SYNONYMS)}" + ) + + assert len(errors) == 4 # ============================================================