Fix course restriction parsing

The reason for this change is detailed in the comment added to the code.

Basically, SIS data sucks and data for a single restriction can sometimes be split into multiple lines, causing each line to count as a separate restriction.

Author: ramonechen

app/scrapers/sis_api.py

@@ -382,6 +382,7 @@ async def get_class_restrictions(session: aiohttp.ClientSession, term: str, crn:
             key = f"not_{key_base}" if must_or_cannot == "Cannot" else key_base
             restriction_list = restrictions_data[key]
         i += 1
+        next_content_string = ""
         while i < len(restrictions_content):
             next_content = restrictions_content[i]
             if next_content.string is None:
@@ -390,13 +391,24 @@ async def get_class_restrictions(session: aiohttp.ClientSession, term: str, crn:
                 )
                 i += 1
                 continue
-            next_content_string = next_content.string.strip()
+            next_content_string += next_content.string.strip()
             # Stop if another restriction header is encountered
             if re.match(restriction_header_pattern, next_content_string) or re.match(
                 special_approvals_pattern, next_content_string
             ):
                 break
-            restriction_list.append(next_content_string)
+            # SIS separates one restriction item into multiple <span> tags if it contains
+            # commas, so a restriction item is only complete when parentheses are closed.
+            #
+            # For example, a restriction item "Communication, Media, & Design (COMD)"
+            # would be split into three <span> tags:
+            #
+            # <span>Communication</span>
+            # <span>Media</span>
+            # <span> & Design (COMD)</span>
+            if re.match(r".*\(.*\)", next_content_string):
+                restriction_list.append(next_content_string)
+            next_content_string = ""
             i += 1
     return restrictions_data
 

app/scrapers/sis_scraper.py

@@ -93,6 +93,8 @@ async def process_class_details(
         crosslists_data = crosslists_task.result()
 
         # Build attribute name to code map
+        # Attributes are known to be in the format "Attribute Name  CODE"
+        # Note the double space between name and code
         if attribute_name_code_map is not None:
             for attribute in attributes_data:
                 attribute_split = attribute.split()
@@ -113,6 +115,8 @@ async def process_class_details(
                 attribute_name_code_map[attribute_name] = attribute_code
 
         # Build restriction name to code map
+        # Restrictions are known to be in the format "Restriction Name (CODE)"
+        # Note the parentheses around the code
         if restriction_name_code_map is not None:
             restriction_pattern = r"(.*)\((.*)\)"
             for restriction_type in restrictions_data: