diff --git a/query_agnews/query4/validate.py b/query_agnews/query4/validate.py index 53642ee466..c792260139 100644 --- a/query_agnews/query4/validate.py +++ b/query_agnews/query4/validate.py @@ -1,9 +1,495 @@ +"""Recognize supported first-place claims for agnews/query4, including ties. + +This is a deterministic grammar, not a general semantic judge. Unsupported prose +does not establish a correct answer. Explicit answers, predicates, rankings and +counts provide evidence; incidental region or superlative words do not. +""" + +import re +import unicodedata + + +# Match qualified names as a whole: South Africa is not the region Africa. +REGION = ( + r"(?:(?:north|south)\s+america|" + r"(?:(?:north|south|east|west|central)(?:ern)?\s+)?africa|" + r"europe|asia|oceania|australia|antarctica)" +) +REGIONS = re.compile(rf"\b{REGION}\b") +REGION_LIST = rf"{REGION}(?:(?:\s*,\s*(?:and\s+)?|\s+(?:and|or|&)\s+){REGION})*" +SUBJECT = re.compile(rf"^({REGION_LIST})\b(.*)$") +NUMBER = r"(?:[0-9]{1,3}(?:,[0-9]{3})+|[0-9]+)" +ARTICLES = r"(?:(?:world(?:[ -]category)?|news)\s+)?articles" +MAXIMUM = ( + rf"(?:the\s+)?(?:most\s+{ARTICLES}|" + rf"(?:largest|highest|greatest|maximum)\s+(?:number|count|total)" + rf"(?:\s+(?:of\s+)?{ARTICLES})?)" +) +QUERY_CONTEXT = ( + r"(?:\s+(?:(?:in|for|during)\s+2015|" + r"(?:in|for)\s+(?:the\s+)?world(?:[ -]category)?))*" +) +FIRST = r"(?:first(?:\s+place)?|1st(?:\s+place)?|number\s+one|no\.?\s*1|#1)" +PUBLISH = r"(?:publish(?:ed|es)?|produc(?:e|ed|es)|record(?:ed|s)?)" +WINNING_PREDICATE = re.compile( + rf"^(?:(?:has|had|have|{PUBLISH})\s+{MAXIMUM}|" + rf"(?:{PUBLISH}|has|had|have)\s+more\s+{ARTICLES}\s+than\s+" + rf"(?:any|every|all)\s+other\s+regions?|" + rf"(?:rank(?:ed|s)?|place(?:d|s)?|finish(?:ed|es)?|came)\s+{FIRST}|" + rf"(?:is|was|were|are)\s+(?:ranked\s+)?(?:{FIRST}|(?:the\s+)?winner)|" + rf"came\s+out\s+on\s+top|" + rf"(?:is\s+|was\s+|were\s+|are\s+)?(?:tied|jointly\s+ranked)\s+" + rf"(?:for\s+)?(?:{FIRST}|{MAXIMUM}))\b" +) +LOWER_RANK = re.compile(r"\b(?:second|third|fourth|fifth|last|runner[ -]up|lowest|least|fewest)\b") +UNCERTAIN = re.compile( + r"\b(?:maybe|perhaps|might|may|could|possibly|probably|unclear|unsure|" + r"uncertain|whether|assuming|suppose|if|estimated|approximately|roughly)\b" +) +NEGATION = re.compile(r"\b(?:not|never|neither|cannot)\b") +LABEL = re.compile( + r"^(?:the\s+)?(?:final\s+)?(?:answer|region|winning region|top region|winner)" + r"\s*(?:is|was|:|=)\s*" +) +RANK_ROW = re.compile(r"(\d+)[.)]\s+(.+)") +COMPARISON = re.compile( + rf"^(?:has|had|have|{PUBLISH})\s+(more|fewer|less)\s+{ARTICLES}" + rf"\s+than\s+({REGION_LIST})\b" +) + + +def _names(text): + return {" ".join(match[0].split()) for match in REGIONS.finditer(text)} + + +def _normalize(text): + text = unicodedata.normalize("NFKC", text).lower() + text = "".join(char for char in text if unicodedata.category(char) != "Cf") + text = text.translate(str.maketrans({"’": "'", "‘": "'", "“": '"', "”": '"'})) + text = re.sub(r"[‐‑‒–—−]", "-", text) + text = re.sub(r"\b(north|south)[ -]*america\b", r"\1 america", text) + text = re.sub(r"[*`_]", "", text) + text = re.sub(r"\b(is|was|are|were|does|do|did|has|have|had)n['’]t\b", r"\1 not", text) + return text + + +def _years(text): + years = set(re.findall( + r"\b(?:in|for|during|year)\s+(?:the\s+year\s+)?((?:19|20)\d{2})\b", text + )) + heading_year = re.match(r"^((?:19|20)\d{2})\s+", text) + if heading_year: + years.add(heading_year[1]) + return years + + +def _mixed_scope(text): + years = _years(text) + return (("2015" in years and years != {"2015"}) + or (bool(re.search(r"\bworld\b", text)) + and bool(re.search(r"\b(?:sports|business|science|technology|population|gdp|height)\b", text)))) + + +def _in_scope(text): + """An explicitly different year/category/metric cannot establish this answer.""" + years = _years(text) + if years and years != {"2015"}: + return False + if re.search(r"\b(?:population|countries|gdp|revenue|land area|alphabetically|" + r"alphabetical|examined|discussed|mentioned|visited)\b", text): + return False + if re.search(r"\b(?:sports|business|science|technology)\b", text): + return False + return True + + +def _without_comparison(text): + parts = re.split( + r"\b(?:followed\s+(?:closely\s+)?by|ahead\s+of|compared\s+(?:to|with)|" + r"rather\s+than|instead\s+of)\b|,\s*not\b", text, maxsplit=1 + ) + if len(parts) == 2: + # Only discard a plain list of comparison regions (optionally counts). + # A trailing assertion or condition must remain visible to the parser. + item = rf"{REGION}(?:\s+(?:at|with)\s+{NUMBER}(?:\s+{ARTICLES})?)?" + tail = rf"{item}(?:(?:\s*,\s*(?:and\s+)?|\s+and\s+){item})*" + if not re.fullmatch(tail, parts[1].strip(" ,().")): + return text + return parts[0].rstrip(" ,(") + + +def _winner_predicate(text): + match = WINNING_PREDICATE.match(text) + if not match: + return False + return _answer_tail(text[match.end():]) + + +def _answer_tail(text): + """Only consume count, tie and question-scope adjuncts, not arbitrary prose.""" + adjunct = ( + rf"(?:(?:in|for|during)\s+2015|" + rf"(?:in|for)\s+(?:the\s+)?world(?:[ -]category)?|" + rf"(?:among|of)\s+(?:all\s+)?(?:the\s+)?regions|" + rf"tied\s+with\s+{REGION_LIST}|tied|" + rf"(?:each\s+)?(?:(?:with|at)\s+)?{NUMBER}\s+{ARTICLES}" + rf"(?:\s+out\s+of\s+{NUMBER}\s+(?:total\s+)?{ARTICLES})?" + rf"(?:\s+across\s+(?:all\s+)?{NUMBER}\s+articles(?:\s+published)?)?)\b" + ) + text = text.strip(" ,:()") + while text: + match = re.match(adjunct, text) + if not match: + return False + text = text[match.end():].strip(" ,:()") + return True + + +def _heading_in_scope(text): + return ( + _in_scope(text) + and not UNCERTAIN.search(text) + and not NEGATION.search(text) + and "?" not in text + and bool(re.search(r"\b(?:world|articles|counts?|rankings?|answer|results|breakdown)\b", text)) + ) + + +def _reverse_winner(prefix): + return bool(re.fullmatch( + rf"(?:(?:the\s+)?(?:region\s+(?:with|that\s+{PUBLISH})\s+{MAXIMUM}{QUERY_CONTEXT}|" + rf"winner|top\s+region)\s+(?:is|was)|" + rf"(?:in\s+2015,?\s+)?(?:two|three|several)\s+regions\s+tied\s+for\s+" + rf"(?:publishing\s+|having\s+)?{MAXIMUM}\s*:|" + rf"{MAXIMUM}\s+(?:was|were)\s+published\s+by)\s*", prefix + )) + + +def _prose_claim(clause): + """Return (winning regions, non-winners, ambiguous) for one assertion.""" + empty = (set(), set(), False) + clause = clause.strip(" \t\r\"'[]().!") + clause = re.sub(r"^(?:and|but|however)\b\s*,?\s*", "", clause) + if not clause: + return empty + if "?" in clause: + return set(), set(), bool(REGIONS.search(clause) or LABEL.match(clause)) + # Retractions may refer to an earlier sentence without repeating its region. + if clause == "no" or re.search(r"\b(?:retract|withdraw|guess|estimates|" + r"actually\s*,?\s*no|(?:that|this|it|the answer)\s+" + r"(?:is|was)\s+(?:not|incorrect|wrong|false|for)|" + r"(?:however|correction)\s*[:,].*\b(?:this|that|it)\b)", clause): + return set(), set(), True + clause = re.sub(r"^(?:in|for|during)\s+2015\s*[:,]?\s*", "", clause) + head = _without_comparison(clause) + if _mixed_scope(head) and REGIONS.search(head): + return set(), set(), True + if not _in_scope(head): + return empty + # An unparsed competing assertion must not silently disappear merely + # because a different sentence contains an accepted "Answer: Africa". + potential_rank = bool( + REGIONS.search(head) or LABEL.match(head) or UNCERTAIN.search(head) + or re.search(r"\b(?:first|most|largest|highest|greatest|winner|leader|led|tops?|" + r"second|third|fewer|more)\b", head) + ) + unsupported = (set(), set(), potential_rank) + + # Recognize negative short answers, including the form used after a label. + labelled = LABEL.match(head) + body = head[labelled.end():] if labelled else head + if re.fullmatch(rf"not\s+({REGION_LIST})", body): + return set(), _names(body), False + runner_up = re.fullmatch(rf"(?:the\s+)?runner[ -]up\s+(?:is|was)\s+({REGION})(.*)", body) + if runner_up and _answer_tail(runner_up[2]): + return set(), _names(runner_up[1]), False + + uncertain = bool(UNCERTAIN.search(head)) + body = re.sub(r"^(?:maybe|perhaps|possibly|probably)\s+", "", body) + subject = SUBJECT.match(body) + reverse = False + if not subject: + first = REGIONS.search(body) + if not first or not _reverse_winner(body[:first.start()]): + return unsupported + subject = SUBJECT.match(body[first.start():]) + reverse = True + regions = _names(subject[1]) + predicate = subject[2].strip() + alternatives = bool(re.search(r"\bor\b", subject[1])) + negative = bool(NEGATION.search(head)) + + # Comparative evidence can disprove a claimed winner without naming the + # global maximum: "North America had more articles than Africa" is enough. + comparison = COMPARISON.match(predicate) + if comparison and not negative: + losers = regions if comparison[1] in {"fewer", "less"} else _names(comparison[2]) + return set(), losers, uncertain or alternatives + + next_region = REGIONS.search(predicate) + local_predicate = predicate[:next_region.start()] if next_region else predicate + lower = bool(LOWER_RANK.search(local_predicate)) + if lower and not negative: + if re.search(rf"\b{FIRST}\b", local_predicate): + return set(), set(), True + return set(), regions, uncertain or alternatives + + positive_predicate = NEGATION.sub("", predicate) + positive_predicate = re.sub(r"^(?:did|does|do)\s+", "", positive_predicate) + positive_predicate = " ".join(positive_predicate.split()) + counted = bool(re.fullmatch( + rf"[,:(]\s*(?:with\s+)?{NUMBER}\s+{ARTICLES}\)?", predicate + )) + winner = ( + not predicate or ((bool(labelled) or reverse) and _answer_tail(predicate)) or counted + or _winner_predicate(positive_predicate) + ) + if not winner: + return unsupported + if uncertain or alternatives: + return set(), set(), True + if negative: + return set(), regions, False + + tie = re.search(rf"\btied\s+with\s+({REGION_LIST})", predicate) + if tie: + if re.search(r"\bor\b", tie[1]): + return set(), set(), True + regions.update(_names(tie[1])) + return regions, set(), False + + +class _Evidence: + """Keep independent observations so no conflicting row can be overwritten.""" + + def __init__(self): + self.groups = [] + self.denied = set() + self.counts = {} + self.ranks = {} + self.ambiguous = False + + def number(self, region, value, rank=False): + region = " ".join(region.split()) + if not REGIONS.fullmatch(region) or not re.fullmatch(NUMBER, value): + self.ambiguous = True + return + digits = value.replace(",", "") + # Counts in this dataset are small; avoid unbounded integer conversions. + if len(digits) > 15: + self.ambiguous = True + return + number = int(digits) + if rank and number < 1: + self.ambiguous = True + return + observations = self.ranks if rank else self.counts + if region in observations and observations[region] != number: + self.ambiguous = True + observations[region] = number + if rank and number > 1: + self.denied.add(region) + + def result(self): + first = {region for region, rank in self.ranks.items() if rank == 1} + if first: + self.groups.append(first) + if len(self.counts) > 1: + maximum = max(self.counts.values()) + leaders = {region for region, count in self.counts.items() if count == maximum} + # Every member of a claimed tie must agree with observed counts. + for group in self.groups: + if any(region in self.counts and region not in leaders for region in group): + self.ambiguous = True + if not self.groups or "africa" in self.counts: + self.groups.append(leaders) + common = self.ranks.keys() & self.counts.keys() + for left in common: + for right in common: + rank_delta = self.ranks[left] - self.ranks[right] + count_delta = self.counts[left] - self.counts[right] + if ((rank_delta == 0 and count_delta != 0) + or (rank_delta != 0 and rank_delta * count_delta >= 0)): + self.ambiguous = True + if any(group & self.denied for group in self.groups): + return False, "A stated winner is also denied first place." + if self.ambiguous: + return False, "Uncertain, unsupported or conflicting ranking/count evidence." + if not self.groups: + return False, "No supported, affirmative first-place answer was found." + if not all("africa" in group for group in self.groups): + return False, "The answer selects another region above Africa or has conflicting winners." + return True, "Africa is selected in first place; explicit ties are allowed." + + +def _count_list(clause, evidence): + """Consume an entire count list, never just valid-looking numeric prefixes. + + Return whether this clause has count-list syntax, including malformed lists. + Numeric prose is handled separately from explicit first-place assertions. + """ + body = clause.strip(" ()") + body = re.sub(r"^(?:and|but|however)\b\s*,?\s*", "", body) + if body.endswith("."): + body = body[:-1] + prefix = re.match(r"^([^:]+):\s*", body) + if (prefix and not REGIONS.search(prefix[1]) + and re.search(r"\b(?:counts?|breakdown|context|results)\b", prefix[1]) + and not re.search(r"\b(?:most|largest|highest|first|tied)\b", prefix[1])): + body = body[prefix.end():] + candidate = re.match(rf"^(?:{REGION}\s*(?::|\s+(?:(?:at|had|has)\s+)?[0-9-])|[a-z ]+\s*[:\-]\s*(?:[-0-9]|unknown|nan|n/a))", body) + if not candidate: + return False + item = re.compile(rf"({REGION})\s*(?::\s*|(?:at|had|has)\s+|\s+)({NUMBER})(?:\s+{ARTICLES})?") + observations = [] + while body: + match = item.match(body) + if not match: + evidence.ambiguous = True + return True + observations.append((match[1], match[2])) + body = body[match.end():] + if not body: + break + separator = re.match(r"(?:\s*,\s*(?:and\s+)?|\s+and\s+)", body) + if not separator or not body[separator.end():]: + evidence.ambiguous = True + return True + body = body[separator.end():] + for region, count in observations: + evidence.number(region, count) + return True + + +def _table_schema(cells): + roles = [] + for cell in cells: + if re.fullmatch(r"regions?", cell): + roles.append("region") + elif re.fullmatch(r"rank|ranking|position", cell): + roles.append("rank") + elif re.fullmatch(rf"(?:(?:world(?:[ -]category)?\s+)?(?:article\s+)?counts?|{ARTICLES}|number of {ARTICLES})", cell): + roles.append("count") + else: + roles.append(None) + if roles.count("region") == 1 and all(role is not None for role in roles) and len(set(roles)) == len(roles) and len(roles) > 1: + return roles + return None + + def validate(llm_output: str): - gt = "Africa" - if gt.lower() in llm_output.lower(): - return True, "Ground truth found in LLM output." - else: - reason = f"Ground truth '{gt}' not found in LLM output: {llm_output}" - return False, reason - - \ No newline at end of file + if not isinstance(llm_output, str): + return False, "The answer must be text." + text = _normalize(llm_output) + evidence = _Evidence() + section_in_scope = True + table_roles = None + table_active = False + table_in_scope = True + + for line in text.splitlines(): + line = re.sub(r"^\s*(?:[-+•]\s+|#+\s*)", "", line).strip() + if not line: + # Blank lines do not cancel an explicit year/category/metric heading. + continue + if "|" in line: + cells = [cell.strip() for cell in line.strip("|").split("|")] + if all(re.fullmatch(r":?-+:?", cell) for cell in cells): + continue + if any(re.fullmatch(r"regions?", cell) for cell in cells): + table_active = True + table_roles = _table_schema(cells) + table_in_scope = section_in_scope and _in_scope(line) + evidence.ambiguous |= _mixed_scope(line) + # Unknown metric columns cannot establish article rankings. + # A partially recognized ranking table is unsafe to ignore. + if table_in_scope and table_roles is None and any(re.search(r"\b(?:rank|articles|count)\b", cell) for cell in cells): + evidence.ambiguous = True + continue + if table_active: + if not table_in_scope or table_roles is None: + continue + if len(cells) != len(table_roles): + evidence.ambiguous = True + continue + row = dict(zip(table_roles, cells)) + for role in ("count", "rank"): + if role in row: + evidence.number(row["region"], row[role], rank=role == "rank") + continue + # A headerless table's column meaning cannot safely be inferred. + if section_in_scope: + evidence.ambiguous = True + continue + table_active = False + table_roles = None + heading = line.endswith(":") or re.search(r"\b(?:counts?|rankings?|population|height)$", line) + if not REGIONS.search(line) and heading: + evidence.ambiguous |= _mixed_scope(line) + section_in_scope = _heading_in_scope(line) + continue + # A new explicit answer or question-year assertion ends a background + # section; an empty line by itself does not. + if LABEL.match(line) or (REGIONS.search(line) and _years(line) == {"2015"}): + section_in_scope = True + if not section_in_scope: + continue + + rank = RANK_ROW.fullmatch(line) + if rank: + if not _in_scope(rank[2]): + continue + subject = SUBJECT.match(rank[2]) + if (not subject or UNCERTAIN.search(rank[2]) or NEGATION.search(rank[2]) + or re.search(r"\bor\b", rank[2]) or not _answer_tail(subject[2])): + evidence.ambiguous = True + continue + regions = _names(subject[1]) + tie = re.search(rf"\btied\s+with\s+({REGION_LIST})", subject[2]) + if tie: + regions.update(_names(tie[1])) + count = re.search(rf"\b({NUMBER})\s+{ARTICLES}\b", subject[2]) + for region in regions: + evidence.number(region, rank[1], rank=True) + if count: + evidence.number(region, count[1]) + continue + + if re.match(r"^[+-]?[0-9]+(?:\.[0-9]+)?[.)]\s+", line): + evidence.ambiguous = True + continue + + # Keep decimal points and coordinated region lists intact. + clauses = re.split( + rf"(?<=[.!?])\s+|;\s*|\b(?:but|whereas|while)\b(?=\s*{REGION}\b)|" + rf",\s*(?={REGION}\s+(?:had|has|is|was|ranked|published)\b)", line + ) + for clause in clauses: + if _in_scope(clause) and _count_list(clause, evidence): + continue + winners, losers, uncertain = _prose_claim(clause) + if winners: + evidence.groups.append(winners) + # A count attached to a supported first-place claim also + # constrains subsequent rankings and tables (including ties). + head = _without_comparison(clause) + count = re.search(rf"\b({NUMBER})\s+{ARTICLES}\b", head) + if count: + for region in winners: + evidence.number(region, count[1]) + if head != clause: + for item in re.finditer(rf"({REGION})\s+(?:at|with)\s+({NUMBER})", clause[len(head):]): + evidence.number(item[1], item[2]) + # Preserve explicit lower-place ties and positions as well as first + # place, so their accompanying counts cannot contradict each other. + if not uncertain and (winners or losers): + ordinal = re.search(r"\b(first|second|third|fourth|fifth|1st|2nd|3rd)\b", clause) + if ordinal and not NEGATION.search(clause): + positions = {"first": 1, "1st": 1, "second": 2, "2nd": 2, + "third": 3, "3rd": 3, "fourth": 4, "fifth": 5} + for region in winners or losers: + evidence.number(region, str(positions[ordinal[1]]), rank=True) + evidence.denied.update(losers) + evidence.ambiguous |= uncertain + + return evidence.result() diff --git a/tests/test_agnews_query4_validator.py b/tests/test_agnews_query4_validator.py new file mode 100644 index 0000000000..1a00d377f9 --- /dev/null +++ b/tests/test_agnews_query4_validator.py @@ -0,0 +1,367 @@ +"""Regression tests for answer-region validation, including contextual mentions.""" + +import importlib.util +from itertools import permutations, product +import unittest +from pathlib import Path + + +PATH = Path(__file__).resolve().parent.parent / "query_agnews/query4/validate.py" +SPEC = importlib.util.spec_from_file_location("agnews_query4_validator", PATH) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class TestAgnewsQuery4Validator(unittest.TestCase): + def test_accepts_correct_answer_surfaces(self): + answers = [ + "Africa", + " AFRICA. ", + "**Africa**", + '"Africa"', + "The answer is Africa.", + "Final answer: **Africa**.", + "The region is Africa.", + "Africa published the largest number of World articles in 2015.", + "In 2015, Africa published the largest number of World-category articles, with 337 articles.", + "The region with the largest number of World articles was Africa.", + "Africa had the most World articles, followed by North America and Europe.", + "Africa ranked first, ahead of North America.", + "Africa, with 337 World articles.", + "Africa (337 World-category articles).", + "1. Africa\n2. North America\n3. Europe", + "1) Africa (307 articles)\n2) North America (299 articles)", + "The answer is Africa, not North America.", + "Africa had the highest count; North America was the runner-up.", + "Africa had the highest count.\nAfrica: 307\nSouth America: 301\nNorth America: 299", + "| Region | World articles |\n|---|---|\n| North America | 299 |\n| Africa | 307 |\n| Asia | 295 |", + "Africa and North America tied for the largest number of World articles.", + "Africa ranked first, tied with North America.", + "1. Africa and North America (tied)\n3. Europe", + "1. North America\n1. Africa\n3. Europe", + "| Region | World articles |\n|---|---|\n| Africa | 317 |\n| North America | 317 |", + "In 2015, two regions tied for publishing the largest number of World-category articles: " + "North America and Africa, each with 317 World articles. " + "(For context: Europe had 315, South America had 315, and Asia had 304 World-category articles.)", + ] + for answer in answers: + with self.subTest(answer=answer): + self.assertTrue(MODULE.validate(answer)[0]) + + def test_rejects_contextual_africa_mentions(self): + answers = [ + # Real failure: the selected region is North America; Africa is third. + "In 2015, North America published the largest number of articles in the World category, " + "with 321 World articles (followed closely by South America at 320, Africa at 319, " + "Europe at 316, and Asia at 299).", + "North America", + "The answer is North America, not Africa.", + "Africa was the runner-up. North America had the most World articles.", + "Africa did not publish the largest number of World articles.", + "Africa isn't the winner.", + "Not Africa.", + "South Africa", + "South Africa had the most World articles.", + "The answer is Africa or North America.", + "Africa had the most articles. North America had the highest count.", + "Africa had the highest count, North America had the most World articles.", + "It is unclear whether Africa had the most World articles.", + "Perhaps Africa had the most World articles.", + "Africa, with 319 World articles, was third.", + "Africa had the second highest count.", + "Africa published the third largest number of World articles.", + "1. North America\n2. Africa\n3. Europe", + "Africa did not rank first. Final answer: Africa.", + "Africa: 319\nNorth America: 321\nSouth America: 320", + "Africa: 317\nNorth America: 317\nAsia: 318", + "North America and Asia tied for the largest number of World articles. Africa was third.", + "Africa and North America tied for the second highest count.", + "1. Africa or North America\n2. Europe", + "Africa: 307\nAfrica: 300\nNorth America: 301", + "I could not determine the answer.", + "", + ] + for answer in answers: + with self.subTest(answer=answer): + self.assertFalse(MODULE.validate(answer)[0]) + + def test_accepts_unnumbered_paraphrases_and_formats(self): + answers = [ + "In 2015, Africa published more World articles than any other region.", + "Africa published more articles than all other regions.", + "Africa came out on top.", + "Africa was number one.", + "Africa ranked 1st.", + "Africa finished first.", + "Africa is the winner.", + "Africa was jointly ranked first.", + "Africa ranked first, tied with North America.", + "- Africa: 321\n- North America: 319", + "• Africa: 321\n• North America: 319", + "Africa has the most World articles. North America has the highest population.", + "Africa ranked first in 2015. North America ranked first in 2014.", + "Africa had the most World articles. Asia had the most Sports articles.", + "In 2015, the region that published the largest number of articles in the World category " + "was **Africa**, with 337 World-category articles.", + "Africa: 2014\nNorth America: 2013", + "Africa: 2,014\nNorth America: 2,013", + "Africa has the most articles, North America was second.", + "Africa.\n| Region | Population |\n|---|---|\n| Africa | 100 |\n| Asia | 200 |", + "The most World articles were published by Africa.", + ] + for answer in answers: + with self.subTest(answer=answer): + self.assertTrue(MODULE.validate(answer)[0]) + + def test_rejects_misleading_relations_and_scope(self): + answers = [ + "Africa was discussed first, but North America actually published more articles.", + "Africa was the region we examined first.", + "Africa was first mentioned in the report.", + "Africa ranked first alphabetically.", + "Africa had fewer articles than North America. Final answer: Africa.", + "North America had more World articles than Africa. The answer is Africa.", + "The answer is Africa, Europe or Asia.", + "Did Africa publish the most World articles?", + "Could Africa have the highest count?", + "1. Africa (maybe)\n2. North America", + "1. Africa\n2. Africa", + "Africa isn’t the winner.", + "Africa wasn’t ranked first.", + "Africa didn’t publish the most World articles.", + "Final answer: Not Africa.", + "Africa published more articles than North America.", + "Africa ranked first in 2014.", + "Africa had the most Sports articles.", + "Africa had the highest population.", + "Africa had the highest count of countries.", + "2014 World article counts:\nAfrica: 321\nNorth America: 319", + "| Region | Population |\n|---|---|\n| Africa | 321 |\n| Asia | 319 |", + "Africa had the most articles. Africa: 319, North America: 321.", + "Africa: 319\nNorth America: 321\nFinal answer: Africa.", + "Africa or North America tied for first.", + "Africa was first in line.", + "Africa ranked first in height.", + "Africa is the winner, hypothetically.", + "The answer is Africa, or possibly some other region.", + "1. Africa (or North America)\n2. Europe", + "1. Africa by height\n2. North America", + "Height:\nAfrica: 321\nNorth America: 319", + "Maybe Africa: 321, North America: 319.", + "Africa: 321, North America: 319 (maybe).", + "Africa ranked first, but not for World articles.", + "2014 rankings:\nAfrica ranked first.", + "Population:\nAfrica", + "Perhaps these counts are correct:\nAfrica: 321\nNorth America: 319", + "Africa ranked first. However, North America published the most World articles.", + "Africa ranked first. It did not publish the most World articles.", + "Africa had the highest count. North America led in the World category.", + ] + for answer in answers: + with self.subTest(answer=answer): + self.assertFalse(MODULE.validate(answer)[0]) + + def test_region_substitution_changes_the_selected_winner(self): + templates = [ + "{region} published the most World articles in 2015.", + "In 2015, {region} published more World articles than any other region.", + "{region} came out on top.", + "The answer is **{region}**.", + "1. {region}\n2. {runner_up}", + "- {region}: 321\n- {runner_up}: 319", + ] + for region in ["Africa", "North America", "South America", "Europe", "Asia"]: + runner_up = "North America" if region == "Africa" else "Africa" + for template in templates: + answer = template.format(region=region, runner_up=runner_up) + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], region == "Africa") + + + def test_evidence_and_scope_regressions(self): + cases = [ + (False, '1. Africa (307 articles)\n2. North America (321 articles)'), + (False, 'Africa and North America tied for first.\nAfrica: 321\nNorth America: 319'), + (False, 'Africa and North America tied for first. North America was second.'), + (False, '1. Africa\n2. North America\n3. North America'), + (False, '1. Africa\n0. North America'), + (False, 'Africa: 3.5, North America: 3.4'), + (False, 'Africa: 1,23, North America: 1'), + (False, 'Africa: 321\nNorth America: -10\nFinal answer: Africa.'), + (False, 'Africa: 321\nNorth America: unknown\nFinal answer: Africa.'), + (False, 'Africa: 321\nAtlantis: 400\nNorth America: 300'), + (False, 'Africa: 321, North America: 319 (estimated)'), + (False, 'Population:\n\nAfrica: 321\nNorth America: 319'), + (False, '2014 World article counts:\n\nAfrica: 321\nNorth America: 319'), + (False, 'Africa. Actually, no.'), + (False, 'Africa. That is incorrect.'), + (False, 'Africa. It is not.'), + (False, 'Africa. North-America ranked first.'), + (False, 'Africa. NorthAmerica ranked first.'), + (False, 'Africa. Atlantis ranked first.'), + (False, 'For example: Africa ranked first.'), + (False, 'Africa / North America'), + (False, 'Africa, rather than North America, which actually ranked first.'), + (False, 'Africa ranked first, followed by North America in first place.'), + (False, 'Africa ranked first, ahead of North America if the estimate is correct.'), + (False, '1. Africa (tied with North America or Europe)'), + (False, '| Region | Height |\n|---|---|\n| Africa | 321 |\n| Asia | 319 |'), + (False, 'Africa.\n| Region | Rank | World articles |\n|---|---|---|\n| Africa | 2 | 319 |\n| North America | 1 | 321 |'), + (True, '| Region | Rank | World articles |\n|---|---|---|\n| Africa | 1 | 321 |\n| North America | 2 | 319 |'), + (True, '| World articles | Region |\n|---|---|\n| 321 | Africa |\n| 319 | North America |'), + (True, '| Rank | Region |\n|---|---|\n| 1 | Africa |\n| 2 | North America |'), + (True, 'Africa had the most World articles in 2015.\n2014 World article counts:\nAfrica: 10\nNorth America: 20'), + (True, 'Africa and North‑America tied for first.'), + (True, 'The winner is Africa.'), + (True, 'Final answer:\nAfrica'), + (True, 'Africa\xa0ranked\xa0first.'), + (False, '0 Africa\n1 North America'), + (False, 'The answer is Africa. Africa is not number one.'), + (False, '"Africa is the winner" is false.'), + (False, 'Africa ranked first. However, this is for Sports articles.'), + (False, 'Africa is n\u200bot the winner.'), + (False, 'Africa. No.'), + (False, 'Africa. The real leader is North America.'), + (False, 'Africa. North America tops the list.'), + (False, 'Africa. North America came in first.'), + (False, 'Africa. But the answer is North America.'), + (False, 'Africa. I retract that answer.'), + (False, 'Africa. This was just a guess.'), + (False, 'Africa\nAtlantis: unknown'), + (False, 'Africa\nAtlantis: -3'), + (False, 'Africa\nAtlantis - 400\nNorth America: 200'), + (False, 'Africa: 320\nNorth America - 400'), + (False, 'Africa\n1. Africa\n-1. North America'), + (False, 'Africa\n1. Africa\n2.5. North America'), + (False, 'Africa\n1. Africa\n2. North America (319 articles)\n3. Europe (320 articles)'), + (False, 'Africa ranked first with 310 articles, followed by North America with 320 articles.'), + (False, 'Africa ranked first. North America and Europe tied for second; North America: 319, Europe: 318.'), + (False, 'Africa. North America had 500 articles. Africa had 400 articles.'), + (False, 'Africa.\n| Region | World articles |\n|---|---|\n| Africa | 300 |\n| North America | 400 |'), + (False, 'Maybe:\nAfrica'), + (False, 'Africa. It might be wrong.'), + (False, 'Africa. However, the counts are estimates.'), + (False, 'Africa. Answer: Asia?'), + (False, 'Africa is first. Africa had exactly 321 articles; North America had exactly 400 articles.'), + (True, 'Africa ranked first. Europe: 200, Asia: 100.'), + (True, 'Africa: 321; North America: 319.'), + (True, 'Africa and North America tied for first. Africa: 321, North America: 321.'), + (True, 'Africa ranked first with 321 articles, followed by North America with 319 articles.'), + (True, 'Africa. North America was second. Europe was third.'), + (True, 'World article counts:\nAfrica: 321\nNorth America: 319'), + (True, 'Africa ranked first. The counts were computed from the dataset.'), + ] + for expected, answer in cases: + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], expected) + + def test_count_order_and_format_matrix(self): + regions = ["Africa", "North America", "South America", "Europe", "Asia"] + for left, right in permutations(regions, 2): + for left_count, right_count in product([319, 320, 321], repeat=2): + expected = ((left == "Africa" and left_count >= right_count) + or (right == "Africa" and right_count >= left_count)) + answers = [ + f"{left}: {left_count}, {right}: {right_count}.", + f"- {left}: {left_count}\n- {right}: {right_count}", + f"| Region | World articles |\n|---|---|\n| {left} | {left_count} |\n| {right} | {right_count} |", + f"| Count | Region |\n|---|---|\n| {left_count} | {left} |\n| {right_count} | {right} |", + ] + for answer in answers: + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], expected) + + def test_rank_count_consistency_matrix(self): + for other, other_rank, other_count, reverse in product( + ["North America", "South America", "Europe", "Asia"], [1, 2], [319, 320, 321], [False, True]): + # Africa is rank 1 with 320 articles. Ties require equal counts; + # strictly lower ranks require strictly smaller counts. + expected = ((other_rank == 1 and other_count == 320) + or (other_rank == 2 and other_count < 320)) + rows = [["Africa", "1", "320"], [other, str(other_rank), str(other_count)]] + if reverse: + rows.reverse() + for columns in permutations(range(3)): + headers = ["Region", "Rank", "World articles"] + table = ["| " + " | ".join(headers[i] for i in columns) + " |", "|---|---|---|"] + table.extend("| " + " | ".join(row[i] for i in columns) + " |" for row in rows) + answer = "\n".join(table) + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], expected) + + def test_malformed_count_matrix(self): + malformed = ["-1", "3.5", "1,23", "12,34,567", "3e2", "NaN", "unknown", + "321 (estimated)", "321?", "321 or 322", "321abc", "321%", "9" * 5000] + for value, bad_region, layout in product(malformed, ["Africa", "North America"], range(3)): + counts = {"Africa": "321", "North America": "319"} + counts[bad_region] = value + rows = [f"{region}: {count}" for region, count in counts.items()] + if layout == 0: + body = "\n".join(rows) + elif layout == 1: + body = ", ".join(rows) + else: + body = "| Region | World articles |\n|---|---|\n" + "\n".join( + f"| {region} | {count} |" for region, count in counts.items()) + answer = "Final answer: Africa.\n" + body + with self.subTest(value=value[:40], bad_region=bad_region, layout=layout): + self.assertFalse(MODULE.validate(answer)[0]) + + def test_competing_claim_and_normalization_matrix(self): + templates = ["{region} ranked first.", "The answer is {region}.", + "{region} came out on top.", "{region} tops the list.", + "The real leader is {region}.", "However, {region} is the winner."] + regions = ["North America", "South America", "Europe", "Asia", + "North-America", "SouthAmerica", "North‑America", "Atlantis"] + for template, region, separator, reverse in product(templates, regions, [" ", "\n", "; "], [False, True]): + claim = template.format(region=region) + answer = claim + separator + "Africa." if reverse else "Africa." + separator + claim + with self.subTest(answer=answer): + self.assertFalse(MODULE.validate(answer)[0]) + + def test_heading_scope_survives_whitespace_matrix(self): + for heading, blank, marker, prefix in product( + ["Population", "Height", "2014 World article counts", "Sports article counts"], + ["\n", "\n\n", "\n \n"], ["", "## "], ["", "Africa ranked first.\n"]): + answer = prefix + marker + heading + ":" + blank + "Africa: 321\nNorth America: 319" + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], bool(prefix)) + + def test_section_transitions_and_conflicting_identity(self): + cases = [ + (False, 'Africa. North America had the most World articles, not Sports articles.'), + (False, 'Africa. North America ranked first in 2015, not in 2014.'), + (False, 'Africa.\n2014 World article counts:\nAfrica: 300\nNorth America: 200\nFinal answer: North America.'), + (False, 'Africa.\nPopulation:\nAfrica: 300\nNorth America: 200\nNorth America ranked first in 2015.'), + (False, '2014 World article counts\nAfrica: 321\nNorth America: 319'), + (False, '## 2014 World article counts\nAfrica: 321\nNorth America: 319'), + (False, 'Africa.\n-1. Atlantis'), + (False, 'Africa.\n1.5. Atlantis'), + (False, 'Africa.\n| Region | World and Sports articles |\n|---|---|\n| North America | 400 |\n| Africa | 300 |'), + (False, 'Africa.\nWorld and Sports article counts:\nNorth America: 400\nAfrica: 300'), + (True, '2014 World article counts:\nAfrica: 300\nNorth America: 400\nFinal answer: Africa.'), + (True, 'Population:\nAfrica: 300\nNorth America: 400\nAfrica ranked first in 2015.'), + (True, '## World article counts\nAfrica: 321\nNorth America: 319'), + (False, 'Africa. North America ranked first and second.'), + (False, 'Africa. North America: 300\nNorth America: 301'), + (False, 'Africa.\n| Region | Rank |\n|---|---|\n| North America | 2 |\n| North America | 3 |'), + (True, 'Africa ranked first.\nNorth America: 319\nNorth America: 319'), + ] + for expected, answer in cases: + with self.subTest(answer=answer): + self.assertEqual(MODULE.validate(answer)[0], expected) + + def test_observed_submission_answers(self): + cases = [ + (True, 'In 2015, two regions tied for publishing the largest number of World-category articles: North America and Africa, each with 317 World articles. (For context: Europe had 315, South America had 315, and Asia had 304 World-category articles.)'), + (True, 'In 2015, Africa published the largest number of World-category articles, with 307 World articles. The runner-up was South America with 301 World articles.\n\nFull breakdown of World-category articles by region in 2015:\n- Africa: 307\n- South America: 301\n- North America: 299\n- Asia: 295\n- Europe: 295\n\n(All 6696 articles published in 2015 were classified into World, Sports, Business, or Science/Technology based on their title and description text.)'), + (False, 'In 2015, North America published the largest number of articles in the World category, with 321 World articles (followed closely by South America at 320, Africa at 319, Europe at 316, and Asia at 299).'), + (True, 'In 2015, Africa published the largest number of articles in the World category (320 World-category articles out of 1570 total World articles across all 6696 articles published in 2015). The full regional breakdown of World-category articles in 2015: Africa 320, South America 318, North America 316, Europe 310, Asia 306.'), + (True, 'In 2015, the region that published the largest number of articles in the World category was **Africa**, with 337 World-category articles.'), + ] + for run, (expected, answer) in enumerate(cases): + with self.subTest(run=run): + self.assertEqual(MODULE.validate(answer)[0], expected) + +if __name__ == "__main__": + unittest.main()