Skip to content

Commit 6b737fd

Browse files
committed
Add timeout arg to SIS scraper
1 parent ad37f72 commit 6b737fd

File tree

1 file changed

+11
-2
lines changed

1 file changed

+11
-2
lines changed

app/scrapers/sis_scraper.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ async def get_course_data(
176176
subject_name_code_map: dict[str, str] = None,
177177
semaphore: asyncio.Semaphore = asyncio.Semaphore(1),
178178
limit_per_host: int = 5,
179+
timeout: int = 60,
179180
) -> dict[str, dict[str, Any]]:
180181
"""
181182
Gets all course data for a given term and subject.
@@ -200,7 +201,7 @@ async def get_course_data(
200201
async with semaphore:
201202
# Limit simultaneous connections to SIS server per session
202203
connector = aiohttp.TCPConnector(limit_per_host=limit_per_host)
203-
timeout = aiohttp.ClientTimeout(total=60)
204+
timeout = aiohttp.ClientTimeout(total=timeout)
204205

205206
async with aiohttp.ClientSession(
206207
connector=connector, timeout=timeout
@@ -230,6 +231,7 @@ async def get_term_course_data(
230231
subject_name_code_map: dict[str, str] = None,
231232
semaphore: asyncio.Semaphore = asyncio.Semaphore(10),
232233
limit_per_host: int = 5,
234+
timeout: int = 60,
233235
) -> None:
234236
"""
235237
Gets all course data for a given term, which includes all subjects in the term.
@@ -263,7 +265,12 @@ async def get_term_course_data(
263265
}
264266
task = tg.create_task(
265267
get_course_data(
266-
term, subject_code, subject_name_code_map, semaphore, limit_per_host
268+
term,
269+
subject_code,
270+
subject_name_code_map,
271+
semaphore,
272+
limit_per_host,
273+
timeout,
267274
)
268275
)
269276
tasks.append(task)
@@ -295,6 +302,7 @@ async def main(
295302
seasons: list[str] = None,
296303
semaphore_val: int = 10,
297304
limit_per_host: int = 5,
305+
timeout: int = 60,
298306
) -> bool:
299307
"""
300308
Runs the SIS scraper for the specified range of years and seasons.
@@ -360,6 +368,7 @@ async def main(
360368
subject_name_code_map,
361369
semaphore,
362370
limit_per_host,
371+
timeout,
363372
)
364373
)
365374

0 commit comments

Comments
 (0)