Integrate prerequisite parser into main scraper

This also uncomments the prereq code in the process_class_details function.

In addition, the subject_name_code_map was changed to use the new get_reverse_subject_map function, though I know that's not going to be the final implementation. It's just to get the prereq parser working here.

Author: jzgom067

app/scrapers/sis_scraper.py

@@ -9,6 +9,7 @@
 
 import aiohttp
 import bs4
+from prereq_parser import parse_prereq
 
 OUTPUT_DATA_DIR = "data"
 
@@ -326,12 +327,51 @@ async def get_class_restrictions(session: aiohttp.ClientSession, term: str, crn:
     return restrictions_data
 
 
-async def get_class_prerequisites(session: aiohttp.ClientSession, term: str, crn: str):
+async def get_class_prerequisites(
+    session: aiohttp.ClientSession,
+    term: str,
+    crn: str,
+    subject_name_code_map: dict[str, str],
+):
     """
     Fetches and parses data from the "Prerequisites" tab of a class details page.
     """
     url = "https://sis9.rpi.edu/StudentRegistrationSsb/ssb/searchResults/getSectionPrerequisites"
     params = {"term": term, "courseReferenceNumber": crn}
+    async with session.get(url, params=params) as response:
+        response.raise_for_status()
+        text = await response.text()
+    soup = bs4.BeautifulSoup(text, "html5lib")
+
+    data = ""
+    rows = soup.find_all("tr")
+    for row in rows:
+        cols = row.find_all("td")
+        if len(cols) == 0:
+            continue
+        data += (
+            " and " if cols[0].text == "And" else " or " if cols[0].text == "Or" else ""
+        )
+        data += " ( " if cols[1].text != "" else ""
+        if cols[2].text != "":
+            data += f" {cols[2].text} {cols[3].text} "
+        else:
+            if cols[4].text not in subject_name_code_map:
+                print(f"Unknown department in CRN {crn}: {cols[4].text}")
+                data += f" {cols[4].text} {cols[5].text} "
+            else:
+                data += f" {subject_name_code_map[cols[4].text]} {cols[5].text} "
+        data += " ) " if cols[8].text != "" else ""
+        data = data.replace("  ", " ").strip()
+        data = data.replace("  ", " ").strip()
+        data = data.replace("( ", "(").strip()
+        data = data.replace(" )", ")").strip()
+    if data:
+        try:
+            return parse_prereq(crn, data)
+        except Exception as e:
+            print(f"Error parsing prerequisites for CRN {crn} with data: {data} - {e}")
+    return {}
 
 
 async def get_class_corequisites(
@@ -480,7 +520,9 @@ async def process_class_details(
         description_task = tg.create_task(get_class_description(session, term, crn))
         attributes_task = tg.create_task(get_class_attributes(session, term, crn))
         restrictions_task = tg.create_task(get_class_restrictions(session, term, crn))
-        # prerequisites_task = tg.create_task(get_class_prerequisites(session, term, crn))
+        prerequisites_task = tg.create_task(
+            get_class_prerequisites(session, term, crn, subject_name_code_map)
+        )
         corequisites_task = tg.create_task(
             get_class_corequisites(session, term, crn, subject_name_code_map)
         )
@@ -490,7 +532,7 @@ async def process_class_details(
     description_data = description_task.result()
     attributes_data = attributes_task.result()
     restrictions_data = restrictions_task.result()
-    # prerequisites_data = prerequisites_task.result()
+    prerequisites_data = prerequisites_task.result()
     corequisites_data = corequisites_task.result()
     # crosslists_data = crosslists_task.result()
 
@@ -502,7 +544,7 @@ async def process_class_details(
             "course_detail": {
                 "description": description_data["description"],
                 "corequisite": corequisites_data,
-                "prerequisite": [],
+                "prerequisite": prerequisites_data,
                 "crosslist": [],
                 "attributes": attributes_data,
                 "restrictions": restrictions_data,
@@ -623,9 +665,7 @@ async def get_term_course_data(
     print(f"Processing {len(subjects)} subjects for term: {term}")
 
     # Create reverse mapping of subject names to codes
-    subject_name_code_map = {}
-    for subject in subjects:
-        subject_name_code_map[subject["description"]] = subject["code"]
+    subject_name_code_map = await get_reverse_subject_map(session)
 
     # Stores all course data for the term
     all_course_data = {}
@@ -730,7 +770,6 @@ async def main(start_year: int, end_year: int, seasons: list[str] = None) -> boo
 
 if __name__ == "__main__":
     start_year = 2023
-    start_year = 2025
     end_year = 2025
     start_time = time.time()
     asyncio.run(main(start_year, end_year))