Hotfix/skip details

.github/workflows/pull-request.yml

@@ -34,4 +34,5 @@ jobs:
         AIRTABLE_BASE_ID: ${{ vars.AIRTABLE_BASE_ID }}
         WIKIDATA_BOT_PW: ${{ secrets.WIKIDATA_BOT_PW }}
         WIKIDATA_BOT_USERNAME: ${{ secrets.WIKIDATA_BOT_USERNAME }}
+        WIKIDATA_MAX_LISTINGS_TO_SEARCH: ${{ vars.WIKIDATA_MAX_LISTINGS_TO_SEARCH }}
         WIKIDATA_MAX_RESULTS_PER_SEARCH: ${{ vars.WIKIDATA_MAX_RESULTS_PER_SEARCH }}

.github/workflows/sync.yml

@@ -36,4 +36,5 @@ jobs:
         AIRTABLE_BASE_ID: ${{ vars.AIRTABLE_BASE_ID }}
         WIKIDATA_BOT_PW: ${{ secrets.WIKIDATA_BOT_PW }}
         WIKIDATA_BOT_USERNAME: ${{ secrets.WIKIDATA_BOT_USERNAME }}
+        WIKIDATA_MAX_LISTINGS_TO_SEARCH: ${{ vars.WIKIDATA_MAX_LISTINGS_TO_SEARCH }}
         WIKIDATA_MAX_RESULTS_PER_SEARCH: ${{ vars.WIKIDATA_MAX_RESULTS_PER_SEARCH }}

pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 readme = "README.md"
 requires-python = "<4.0,>=3.13"
 dependencies = [
-    "wikibaseintegrator @ git+https://github.com/LeMyst/WikibaseIntegrator.git@301067883ddce0bca31fdd54973c3d6c8cbbcb7d",
+    "wikibaseintegrator @ git+https://github.com/combinatorist/WikibaseIntegrator.git@feature/search_entities-exact-max_results",
     "pyairtable (>=3.1.1,<4.0.0)",
     "more_itertools"
 ]

src/config.py

@@ -13,6 +13,10 @@
     os.getenv("READ_CTFG_FROM_CACHE", "False").upper()
 )
 
+POST_DETAILS_TO_CTFG: bool = "TRUE".startswith(
+    os.getenv("POST_DETAILS_TO_CTFG", "False").upper()
+)
+
 
 @dataclass
 class airtable:
@@ -33,9 +37,12 @@ class airtable:
 
 wbi = WikibaseIntegrator()
 
-WIKIDATA_MAX_RESULTS_PER_SEARCH: int = int(
-    os.getenv("WIKIDATA_MAX_RESULTS_PER_SEARCH", 50)
-)
+def getEnvInt(name: str, default: int) -> int:
+    raw = os.getenv("WIKIDATA_MAX_RESULTS_PER_SEARCH")
+    return int(raw) if raw else default
+
+WIKIDATA_MAX_LISTINGS_TO_SEARCH: int = getEnvInt("WIKIDATA_MAX_LISTINGS_TO_SEARCH", 5)
 
+WIKIDATA_MAX_RESULTS_PER_SEARCH: int = getEnvInt("WIKIDATA_MAX_RESULTS_PER_SEARCH", 5)
 
 LANGUAGE_CODE = "en"

src/ctfg.py

@@ -158,7 +158,10 @@ def from_wiki_match(m: dict, keep_unknowns: bool = False):
         # pprint(claims)
 
         statements = [
-            WikidataStatement.from_wiki_statement(s) for p in claims.values() for s in p
+            WikidataStatement.from_wiki_statement(s)
+            for p in claims.values()
+            for s in p
+            if config.POST_DETAILS_TO_CTFG
         ]
         result = WikidataItem(**mappable, statements=statements)
         result.save()
@@ -271,17 +274,17 @@ def partition_matched(items: list[Listing]) -> tuple[list[Listing], list[Listing
 
 
 def upsert_matches(wiki_matches: dict[Listing, list[dict[str, Any]]]):
-    log("Updating CTFG with matching wikibase IDs...")
     with_wiki_items = {
         x: [WikidataItem.from_wiki_match(m) for m in matches]
         for x, matches in wiki_matches.items()
         if matches
     }
     wiki_items = list(set([x for y in with_wiki_items.values() for x in y]))
+    log(f"Updating CTFG with {len(wiki_items)} matching wikibase IDs...")
 
-    log("Example item")
-    # pprint(wiki_items[0].to_record())
-    # WikidataItem.recursive_save(wiki_items)
+    if wiki_items:
+        log("Example item")
+        pprint(wiki_items[0].to_record())
 
     for x, matches in with_wiki_items.items():
         x.wikidata_suggestions = matches

src/sync.py

@@ -11,7 +11,9 @@
 (unmatched_items, matched_items) = ctfg.partition_matched(items)
 
 wiki_matches = wiki.get_matches(
-    unmatched_items, max_attempts=5, max_results=config.WIKIDATA_MAX_RESULTS_PER_SEARCH
+    unmatched_items,
+    max_attempts=config.WIKIDATA_MAX_LISTINGS_TO_SEARCH,
+    max_results=config.WIKIDATA_MAX_RESULTS_PER_SEARCH,
 )
 # wiki_match_histogram = wiki.summarize_matches(wiki_matches)
 

src/wiki.py

@@ -18,7 +18,9 @@ def get_matches(
 ) -> dict[ctfg.Listing, list[dict[str, Any]]]:
     matchable_items = [x for x in items if x.name]
     attempting_items = (
-        sample(matchable_items, max_attempts) if max_attempts else matchable_items
+        sample(matchable_items, max_attempts)
+        if max_attempts and max_attempts < len(matchable_items)
+        else matchable_items
     )
     log(
         f"Searching for wikibase matches for {len(attempting_items)} {'random' if max_attempts else 'matchable'} items..."