-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Expand file tree
/
Copy pathmodels.py
More file actions
379 lines (284 loc) · 11.7 KB
/
Copy pathmodels.py
File metadata and controls
379 lines (284 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
from typing import List, Optional, Dict, Tuple, Any, Type, Protocol, runtime_checkable
from pydantic import BaseModel, Field, create_model, field_validator
@runtime_checkable
class LLMProvider(Protocol):
"""Protocol for LLM providers."""
def chat(
self,
model: str,
messages: List[Dict[str, str]],
options: Dict[str, Any] = None,
**kwargs
) -> Dict[str, Any]:
"""Send a chat request to the LLM provider."""
...
class Location(BaseModel):
"""Location information for JSON Resume format."""
address: Optional[str] = None
postalCode: Optional[str] = None
city: Optional[str] = None
countryCode: Optional[str] = None
region: Optional[str] = None
class Profile(BaseModel):
"""Social profile information for JSON Resume format."""
network: Optional[str] = None
username: Optional[str] = None
url: str
class Basics(BaseModel):
"""Basic information for JSON Resume format."""
name: str
email: Optional[str] = None
phone: Optional[str] = None
url: Optional[str] = None
summary: Optional[str] = None
location: Optional[Location] = None
profiles: Optional[List[Profile]] = None
class Work(BaseModel):
"""Work experience for JSON Resume format."""
name: Optional[str] = None
position: Optional[str] = None
url: Optional[str] = None
startDate: Optional[str] = None
endDate: Optional[str] = None
summary: Optional[str] = None
highlights: Optional[List[str]] = None
class Volunteer(BaseModel):
"""Volunteer experience for JSON Resume format."""
organization: Optional[str] = None
position: Optional[str] = None
url: Optional[str] = None
startDate: Optional[str] = None
endDate: Optional[str] = None
summary: Optional[str] = None
highlights: Optional[List[str]] = None
class Education(BaseModel):
"""Education information for JSON Resume format."""
institution: Optional[str] = None
url: Optional[str] = None
area: Optional[str] = None
studyType: Optional[str] = None
startDate: Optional[str] = None
endDate: Optional[str] = None
score: Optional[str] = None
courses: Optional[List[str]] = None
class Award(BaseModel):
"""Award information for JSON Resume format."""
title: Optional[str] = None
date: Optional[str] = None
awarder: Optional[str] = None
summary: Optional[str] = None
class Certificate(BaseModel):
"""Certificate information for JSON Resume format."""
name: Optional[str] = None
date: Optional[str] = None
issuer: Optional[str] = None
url: Optional[str] = None
class Publication(BaseModel):
"""Publication information for JSON Resume format."""
name: Optional[str] = None
publisher: Optional[str] = None
releaseDate: Optional[str] = None
url: Optional[str] = None
summary: Optional[str] = None
class Skill(BaseModel):
"""Skill information for JSON Resume format."""
name: Optional[str] = None
level: Optional[str] = None
keywords: Optional[List[str]] = None
class Language(BaseModel):
"""Language information for JSON Resume format."""
language: Optional[str] = None
fluency: Optional[str] = None
class Interest(BaseModel):
"""Interest information for JSON Resume format."""
name: Optional[str] = None
keywords: Optional[List[str]] = None
class Reference(BaseModel):
"""Reference information for JSON Resume format."""
name: Optional[str] = None
reference: Optional[str] = None
class Project(BaseModel):
"""Project information for JSON Resume format."""
name: Optional[str] = None
startDate: Optional[str] = None
endDate: Optional[str] = None
description: Optional[str] = None
highlights: Optional[List[str]] = None
url: Optional[str] = None
technologies: Optional[List[str]] = None
skills: Optional[List[str]] = None
class BasicsSection(BaseModel):
"""Basics section containing basic information."""
basics: Optional[Basics] = None
class WorkSection(BaseModel):
"""Work section containing a list of work experiences."""
work: Optional[List[Work]] = None
class EducationSection(BaseModel):
"""Education section containing a list of education entries."""
education: Optional[List[Education]] = None
class SkillsSection(BaseModel):
"""Skills section containing a list of skill categories."""
skills: Optional[List[Skill]] = None
class ProjectsSection(BaseModel):
"""Projects section containing a list of projects."""
projects: Optional[List[Project]] = None
class AwardsSection(BaseModel):
"""Awards section containing a list of awards."""
awards: Optional[List[Award]] = None
class JSONResume(BaseModel):
"""Complete JSON Resume format model."""
basics: Optional[Basics] = None
work: Optional[List[Work]] = None
volunteer: Optional[List[Volunteer]] = None
education: Optional[List[Education]] = None
awards: Optional[List[Award]] = None
certificates: Optional[List[Certificate]] = None
publications: Optional[List[Publication]] = None
skills: Optional[List[Skill]] = None
languages: Optional[List[Language]] = None
interests: Optional[List[Interest]] = None
references: Optional[List[Reference]] = None
projects: Optional[List[Project]] = None
class CategoryScore(BaseModel):
score: float = Field(ge=0, description="Score achieved in this category")
max: int = Field(gt=0, description="Maximum possible score")
evidence: str = Field(min_length=1, description="Evidence supporting the score")
class Deductions(BaseModel):
total: float = Field(
ge=0,
description="Total deduction points (stored as positive, applied as negative)",
)
reasons: str = Field(description="Reasons for deductions")
def build_scores_model(categories) -> Type[BaseModel]:
"""Build a ``Scores`` model with one CategoryScore field per role category.
Using ``create_model`` (rather than a loose ``Dict[str, CategoryScore]``)
keeps the emitted JSON schema concrete — the exact category property names —
so the LLM's structured output stays as constrained as the old fixed schema.
"""
fields = {category.key: (CategoryScore, ...) for category in categories}
return create_model("Scores", **fields)
def build_evaluation_model(role) -> Type[BaseModel]:
"""Build the full ``EvaluationData`` model for a given role.
Categories/weights and the bonus cap come from the role definition, so each
role scores against its own rubric.
"""
scores_model = build_scores_model(role.categories)
bonus_model = create_model(
"BonusPoints",
total=(
float,
Field(ge=0, le=role.bonus_max, description="Total bonus points"),
),
breakdown=(str, Field(description="Breakdown of bonus points")),
)
return create_model(
"EvaluationData",
scores=(scores_model, ...),
bonus_points=(bonus_model, ...),
deductions=(Deductions, ...),
key_strengths=(List[str], Field(min_items=1, max_items=5)),
areas_for_improvement=(List[str], Field(min_items=1, max_items=5)),
)
class GitHubProfile(BaseModel):
"""Pydantic model for GitHub profile data."""
username: str
name: Optional[str] = None
bio: Optional[str] = None
location: Optional[str] = None
company: Optional[str] = None
public_repos: Optional[int] = None
followers: Optional[int] = None
following: Optional[int] = None
created_at: Optional[str] = None
updated_at: Optional[str] = None
avatar_url: Optional[str] = None
blog: Optional[str] = None
twitter_username: Optional[str] = None
hireable: Optional[bool] = None
class OpenAICompatibleProvider:
"""Generic OpenAI-chat-compatible LLM provider.
Works for Ollama (/v1), Gemini (/v1beta/openai), OpenAI, Groq, OpenRouter,
DeepSeek, LM Studio, vLLM, etc. via a configurable base_url. Adapts the
response to the {"message": {"content": ...}} shape the evaluator expects.
"""
def __init__(
self,
base_url: str,
api_key: Optional[str] = None,
structured_output: str = "json_schema",
extra_body: Optional[Dict[str, Any]] = None,
):
self.base_url = base_url.rstrip("/")
self.api_key = api_key
self.structured_output = structured_output
self.extra_body = extra_body or {}
def chat(
self,
model: str,
messages: List[Dict[str, str]],
options: Dict[str, Any] = None,
**kwargs
) -> Dict[str, Any]:
import requests
import time
import random
options = options or {}
body: Dict[str, Any] = {"model": model, "messages": messages, "stream": False}
if "temperature" in options:
body["temperature"] = options["temperature"]
if "top_p" in options:
body["top_p"] = options["top_p"]
# Structured-output translation: evaluator passes format=<json schema>.
if "format" in kwargs and self.structured_output != "none":
schema = kwargs["format"]
if self.structured_output == "json_schema":
body["response_format"] = {
"type": "json_schema",
"json_schema": {"name": "response", "schema": schema},
}
elif self.structured_output == "json_object":
body["response_format"] = {"type": "json_object"}
body.update(self.extra_body)
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
url = f"{self.base_url}/chat/completions"
MAX_RETRIES = 5
BASE_DELAY = 10.0 # seconds — base for exponential backoff
MAX_DELAY = 120.0 # cap so we never wait more than 2 minutes
# Transient server errors worth retrying with backoff. Unlike 429 these
# rarely carry a Retry-After header, so we always use exponential backoff.
RETRYABLE_SERVER_ERRORS = {500, 502, 503, 504}
for attempt in range(MAX_RETRIES):
response = requests.post(url, json=body, headers=headers, timeout=300)
if response.status_code == 429 and attempt < MAX_RETRIES - 1:
retry_after = response.headers.get("Retry-After")
exp_delay = min(BASE_DELAY * (2 ** attempt), MAX_DELAY)
delay = float(retry_after) if retry_after else exp_delay
sleep_time = round(delay * random.uniform(0.8, 1.2), 2)
print(
f"[OpenAICompatibleProvider] Rate limit hit "
f"(attempt {attempt + 1}/{MAX_RETRIES}). Retrying in {sleep_time}s..."
)
time.sleep(sleep_time)
continue
if (
response.status_code in RETRYABLE_SERVER_ERRORS
and attempt < MAX_RETRIES - 1
):
exp_delay = min(BASE_DELAY * (2 ** attempt), MAX_DELAY)
sleep_time = round(exp_delay * random.uniform(0.8, 1.2), 2)
print(
f"[OpenAICompatibleProvider] Transient server error "
f"{response.status_code} (attempt {attempt + 1}/{MAX_RETRIES}). "
f"Retrying in {sleep_time}s..."
)
time.sleep(sleep_time)
continue
response.raise_for_status()
data = response.json()
try:
content = data["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError):
raise ValueError(f"Unexpected response shape from {url}: {data}")
return {"message": {"role": "assistant", "content": content}}