feat: add job description scoring mode with weighted evaluation and semantic matching

.gitignore

@@ -6,6 +6,8 @@ run/*.pdf
 test_*.py
 cache/
 resume_evaluations.csv
+job_evaluations.csv
+package-lock.json
 greenhouse_resumes/*
 
 # Byte-compiled / optimized / DLL files
@@ -223,4 +225,7 @@ marimo/_lsp/
 __marimo__/
 
 # Streamlit
-.streamlit/secrets.toml
+.streamlit/secrets.toml
+
+
+CLAUDE.md

evaluator.py

@@ -1,6 +1,13 @@
 from typing import Dict, List, Optional, Tuple, Any
 from pydantic import BaseModel, Field, field_validator
-from models import JSONResume, EvaluationData
+from models import (
+    JSONResume,
+    EvaluationData,
+    JobDescriptionData,
+    JobScores,
+    LLMJobEvaluationResponse,
+    JobEvaluationData,
+)
 from llm_utils import initialize_llm_provider, extract_json_from_response
 import logging
 import json
@@ -89,3 +96,137 @@ def evaluate_resume(self, resume_text: str) -> EvaluationData:
         except Exception as e:
             logger.error(f"Error evaluating resume: {str(e)}")
             raise
+
+
+class JobDescriptionEvaluator:
+    WEIGHTS = {
+        "skills_match": 0.30,
+        "experience_match": 0.20,
+        "semantic_match": 0.15,
+        "job_title_alignment": 0.10,
+        "education": 0.10,
+        "resume_quality": 0.10,
+        "missing_critical_requirements": 0.05,
+    }
+
+    def __init__(self, job_description: str, model_name: str = DEFAULT_MODEL, model_params: dict = None):
+        if not job_description or not job_description.strip():
+            raise ValueError("Job description cannot be empty")
+        if not model_name:
+            raise ValueError("Model name cannot be empty")
+
+        self.job_description = job_description
+        self.model_name = model_name
+        self.model_params = model_params or MODEL_PARAMETERS.get(
+            model_name, {"temperature": 0.1, "top_p": 0.9}
+        )
+        self.template_manager = TemplateManager()
+        self.provider = initialize_llm_provider(model_name)
+        self._load_embedding_model()
+
+    def _load_embedding_model(self):
+        from sentence_transformers import SentenceTransformer
+        logger.info("Loading Sentence Transformers model (all-MiniLM-L6-v2)...")
+        self.embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+    def extract_job_requirements(self) -> JobDescriptionData:
+        prompt = self.template_manager.render_template(
+            "job_description_extraction", job_description=self.job_description
+        )
+        if prompt is None:
+            raise ValueError("Failed to render job_description_extraction template")
+
+        chat_params = {
+            "model": self.model_name,
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are an expert at extracting structured requirements from job descriptions. Return only valid JSON.",
+                },
+                {"role": "user", "content": prompt},
+            ],
+            "options": self.model_params,
+        }
+
+        response = self.provider.chat(**chat_params, format=JobDescriptionData.model_json_schema())
+        response_text = extract_json_from_response(response["message"]["content"])
+        return JobDescriptionData(**json.loads(response_text))
+
+    def compute_semantic_score(self, resume_text: str) -> float:
+        from sentence_transformers import util
+        job_embedding = self.embedding_model.encode(self.job_description, convert_to_tensor=True)
+        resume_embedding = self.embedding_model.encode(resume_text, convert_to_tensor=True)
+        similarity = util.cos_sim(job_embedding, resume_embedding).item()
+        return round(max(0.0, similarity) * 100, 1)
+
+    def _score_resume(self, resume_text: str, job_data: JobDescriptionData) -> LLMJobEvaluationResponse:
+        system_message = self.template_manager.render_template("job_evaluation_system_message")
+        if system_message is None:
+            raise ValueError("Failed to render job_evaluation_system_message template")
+
+        criteria_prompt = self.template_manager.render_template(
+            "job_evaluation_criteria",
+            job_description=self.job_description,
+            job_title=job_data.job_title,
+            required_skills=job_data.required_skills,
+            preferred_skills=job_data.preferred_skills,
+            years_of_experience=job_data.years_of_experience,
+            education_requirements=job_data.education_requirements,
+            must_have_qualifications=job_data.must_have_qualifications,
+            resume_text=resume_text,
+        )
+        if criteria_prompt is None:
+            raise ValueError("Failed to render job_evaluation_criteria template")
+
+        chat_params = {
+            "model": self.model_name,
+            "messages": [
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": criteria_prompt},
+            ],
+            "options": {
+                "stream": False,
+                "temperature": self.model_params.get("temperature", 0.1),
+                "top_p": self.model_params.get("top_p", 0.9),
+            },
+        }
+
+        response = self.provider.chat(**chat_params, format=LLMJobEvaluationResponse.model_json_schema())
+        response_text = extract_json_from_response(response["message"]["content"])
+        logger.info(f"Job evaluation LLM response: {response_text}")
+        return LLMJobEvaluationResponse(**json.loads(response_text))
+
+    def _compute_weighted_total(self, scores: JobScores, semantic_score: float) -> float:
+        total = (
+            scores.skills_match.score * self.WEIGHTS["skills_match"]
+            + scores.experience_match.score * self.WEIGHTS["experience_match"]
+            + semantic_score * self.WEIGHTS["semantic_match"]
+            + scores.job_title_alignment.score * self.WEIGHTS["job_title_alignment"]
+            + scores.education.score * self.WEIGHTS["education"]
+            + scores.resume_quality.score * self.WEIGHTS["resume_quality"]
+            + scores.missing_critical_requirements.score * self.WEIGHTS["missing_critical_requirements"]
+        )
+        return round(min(total, 100.0), 1)
+
+    def evaluate(self, resume_text: str) -> JobEvaluationData:
+        logger.info("Extracting requirements from job description...")
+        job_data = self.extract_job_requirements()
+        logger.info(f"Job title: {job_data.job_title} | Required skills: {job_data.required_skills}")
+
+        logger.info("Computing semantic similarity score...")
+        semantic_score = self.compute_semantic_score(resume_text)
+        logger.info(f"Semantic match score: {semantic_score}")
+
+        logger.info("Scoring resume against job requirements...")
+        llm_result = self._score_resume(resume_text, job_data)
+
+        weighted_total = self._compute_weighted_total(llm_result.scores, semantic_score)
+
+        return JobEvaluationData(
+            scores=llm_result.scores,
+            semantic_match_score=semantic_score,
+            weighted_total=weighted_total,
+            key_strengths=llm_result.key_strengths,
+            areas_for_improvement=llm_result.areas_for_improvement,
+            job_title=job_data.job_title,
+        )

models.py

@@ -249,6 +249,45 @@ class EvaluationData(BaseModel):
     areas_for_improvement: List[str] = Field(min_items=1, max_items=5)
 
 
+class JobDescriptionData(BaseModel):
+    job_title: str
+    required_skills: List[str]
+    preferred_skills: List[str] = []
+    years_of_experience: Optional[float] = None
+    education_requirements: Optional[str] = None
+    must_have_qualifications: List[str] = []
+    industry: Optional[str] = None
+
+
+class JobCategoryScore(BaseModel):
+    score: float = Field(ge=0, le=100, description="Score for this category out of 100")
+    evidence: str = Field(min_length=1, description="Evidence from the resume supporting this score")
+
+
+class JobScores(BaseModel):
+    skills_match: JobCategoryScore
+    experience_match: JobCategoryScore
+    job_title_alignment: JobCategoryScore
+    education: JobCategoryScore
+    resume_quality: JobCategoryScore
+    missing_critical_requirements: JobCategoryScore
+
+
+class LLMJobEvaluationResponse(BaseModel):
+    scores: JobScores
+    key_strengths: List[str] = Field(min_items=1, max_items=5)
+    areas_for_improvement: List[str] = Field(min_items=1, max_items=5)
+
+
+class JobEvaluationData(BaseModel):
+    scores: JobScores
+    semantic_match_score: float = Field(ge=0, le=100)
+    weighted_total: float = Field(ge=0, le=100)
+    key_strengths: List[str]
+    areas_for_improvement: List[str]
+    job_title: str
+
+
 class GitHubProfile(BaseModel):
     """Pydantic model for GitHub profile data."""
 

prompts/template_manager.py

@@ -45,6 +45,9 @@ def _load_templates(self):
             "github_project_selection": "github_project_selection.jinja",
             "resume_evaluation_criteria": "resume_evaluation_criteria.jinja",
             "resume_evaluation_system_message": "resume_evaluation_system_message.jinja",
+            "job_description_extraction": "job_description_extraction.jinja",
+            "job_evaluation_criteria": "job_evaluation_criteria.jinja",
+            "job_evaluation_system_message": "job_evaluation_system_message.jinja",
         }
 
         for section_name, filename in template_files.items():

prompts/templates/job_description_extraction.jinja

@@ -0,0 +1,24 @@
+Extract structured requirements from the following job description and return them as JSON.
+
+Job Description:
+{{ job_description }}
+
+Rules:
+- required_skills: Only skills explicitly stated as required, essential, or must-have
+- preferred_skills: Only skills explicitly stated as preferred, nice-to-have, or a bonus
+- years_of_experience: Minimum years required as a number (e.g. 2 for "2+ years"). Null if not specified.
+- education_requirements: The degree, field of study, or certification explicitly required. Null if not specified.
+- must_have_qualifications: Non-negotiable requirements such as work authorization, security clearance, licenses, or certifications explicitly stated as mandatory
+- industry: The primary industry or domain of the role. Null if not clear.
+
+Return ONLY this JSON structure, no other text:
+
+{
+    "job_title": "the target job title from the description",
+    "required_skills": ["skill1", "skill2"],
+    "preferred_skills": ["skill1", "skill2"],
+    "years_of_experience": null,
+    "education_requirements": null,
+    "must_have_qualifications": [],
+    "industry": null
+}

prompts/templates/job_evaluation_criteria.jinja

@@ -0,0 +1,130 @@
+You are evaluating how well a candidate's resume matches a job description. Score each category from 0 to 100.
+
+## FULL JOB DESCRIPTION
+
+{{ job_description }}
+
+---
+
+## EXTRACTED JOB REQUIREMENTS
+
+Job Title: {{ job_title }}
+
+Required Skills: {{ required_skills | join(", ") if required_skills else "None specified" }}
+
+Preferred Skills: {{ preferred_skills | join(", ") if preferred_skills else "None specified" }}
+
+Years of Experience Required: {{ years_of_experience if years_of_experience is not none else "Not specified" }}
+
+Education Requirements: {{ education_requirements if education_requirements else "Not specified" }}
+
+Must-Have Qualifications: {{ must_have_qualifications | join(", ") if must_have_qualifications else "None specified" }}
+
+---
+
+## SCORING CRITERIA
+
+### Skills Match (score 0-100)
+Compare the candidate's skills, work experience, and projects against the required and preferred skills.
+
+Required skills carry 80% of the weight, preferred skills carry 20%.
+
+Score bands:
+- 90-100: All required skills present, most preferred skills also present
+- 70-89: Most required skills present, minor gaps
+- 50-69: Around half of required skills present
+- 0-49: Few or no required skills present
+
+### Experience Match (score 0-100)
+Evaluate the relevance of work history and projects to the target role.
+
+Consider:
+- How closely past roles and responsibilities match the target role
+- Whether technologies used in work experience match the job requirements
+- Years of relevant experience vs years required (if years_of_experience is specified: score 100% if met, scale down proportionally if not)
+- Industry or domain similarity
+
+Score bands:
+- 90-100: Highly relevant experience that meets or exceeds all requirements
+- 70-89: Mostly relevant experience with minor gaps
+- 50-69: Somewhat relevant experience with notable gaps
+- 0-49: Little relevant experience or significant shortfall in years
+
+### Job Title Alignment (score 0-100)
+Compare the candidate's previous job titles to the target job title.
+
+Score bands:
+- 90-100: Previous titles directly match or are very similar (e.g. "Software Engineer" → "Software Engineer")
+- 70-89: Previous titles are closely related (e.g. "Software Engineer Intern" → "Software Engineer")
+- 50-69: Previous titles are somewhat related (e.g. "Backend Developer" → "Full Stack Engineer")
+- 20-49: Tangentially related titles
+- 0-19: No previous titles or completely unrelated titles
+
+### Education (score 0-100)
+Compare the candidate's education against the education requirements.
+
+Consider degree level, field of study, and any required certifications or licenses.
+
+Score bands:
+- 90-100: Education fully meets or exceeds requirements
+- 70-89: Education mostly meets requirements with minor gaps
+- 50-69: Education partially meets requirements (e.g. relevant field but wrong level)
+- 0-49: Education does not meet requirements or is absent
+
+If no education requirements are specified, score based on relevance of the candidate's education to the role.
+
+### Resume Quality (score 0-100)
+Evaluate the quality of the resume's writing and presentation.
+
+Assess:
+- Bullet points use strong action verbs: Built, Designed, Implemented, Optimized, Reduced, Automated, Architected, Led, Deployed, Improved
+- Achievements are quantified with numbers (e.g. "Reduced latency by 35%", "Processed 1M requests/day", "Served 20k users")
+- No vague filler statements (e.g. "Worked on APIs", "Helped with development", "Was responsible for")
+- All major sections present: work experience, skills, projects or education
+- Descriptions are clear and concise
+
+Score bands:
+- 90-100: Strong action verbs throughout, most achievements quantified, no vague statements
+- 70-89: Good action verbs, some quantified achievements, minor vague statements
+- 50-69: Mixed quality, noticeable vague statements, few quantified achievements
+- 0-49: Mostly vague, no quantified achievements, weak structure
+
+### Missing Critical Requirements (score 0-100)
+Start at 100 and penalize for missing must-have qualifications and required skills.
+
+Deductions:
+- -40 points for each missing must-have qualification (work authorization, clearance, license, etc.)
+- -15 points for each required skill completely absent from the resume
+
+If no must-have qualifications are specified, base this score only on required skill presence.
+Minimum score is 0.
+
+---
+
+## CANDIDATE RESUME
+
+{{ resume_text }}
+
+---
+
+## INSTRUCTIONS
+
+Score each category from 0 to 100 using only evidence from the resume above.
+Provide specific evidence for each score — reference actual content from the resume, not generic statements.
+Identify 1-5 key strengths relevant to this specific role.
+For areas_for_improvement, identify 1-5 SPECIFIC GAPS between the candidate's resume and the job requirements listed above. Each item must reference something the job requires that the candidate is missing or weak on. Do NOT give generic resume advice (e.g. "add more detail", "include links"). Every improvement must name a specific requirement from the job description that is absent or underdeveloped in the resume.
+
+Return ONLY this JSON structure, no other text:
+
+{
+    "scores": {
+        "skills_match": {"score": 0, "evidence": "string"},
+        "experience_match": {"score": 0, "evidence": "string"},
+        "job_title_alignment": {"score": 0, "evidence": "string"},
+        "education": {"score": 0, "evidence": "string"},
+        "resume_quality": {"score": 0, "evidence": "string"},
+        "missing_critical_requirements": {"score": 0, "evidence": "string"}
+    },
+    "key_strengths": ["strength1", "strength2"],
+    "areas_for_improvement": ["area1", "area2"]
+}