diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 61d51ce..ee532de 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -34,4 +34,5 @@ jobs: AIRTABLE_BASE_ID: ${{ vars.AIRTABLE_BASE_ID }} WIKIDATA_BOT_PW: ${{ secrets.WIKIDATA_BOT_PW }} WIKIDATA_BOT_USERNAME: ${{ secrets.WIKIDATA_BOT_USERNAME }} + WIKIDATA_MAX_LISTINGS_TO_SEARCH: ${{ vars.WIKIDATA_MAX_LISTINGS_TO_SEARCH }} WIKIDATA_MAX_RESULTS_PER_SEARCH: ${{ vars.WIKIDATA_MAX_RESULTS_PER_SEARCH }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 6c32b06..9bf693f 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -36,4 +36,5 @@ jobs: AIRTABLE_BASE_ID: ${{ vars.AIRTABLE_BASE_ID }} WIKIDATA_BOT_PW: ${{ secrets.WIKIDATA_BOT_PW }} WIKIDATA_BOT_USERNAME: ${{ secrets.WIKIDATA_BOT_USERNAME }} + WIKIDATA_MAX_LISTINGS_TO_SEARCH: ${{ vars.WIKIDATA_MAX_LISTINGS_TO_SEARCH }} WIKIDATA_MAX_RESULTS_PER_SEARCH: ${{ vars.WIKIDATA_MAX_RESULTS_PER_SEARCH }} diff --git a/poetry.lock b/poetry.lock index 8fd2b82..1e3aa22 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1036,7 +1036,7 @@ files = [ [[package]] name = "wikibaseintegrator" -version = "0.12.14.dev0" +version = "0.12.15.dev0" description = "Python package for reading from and writing to a Wikibase instance" optional = false python-versions = "^3.9" @@ -1054,11 +1054,11 @@ ujson = "^5.10.0" [package.source] type = "git" -url = "https://github.com/LeMyst/WikibaseIntegrator.git" -reference = "301067883ddce0bca31fdd54973c3d6c8cbbcb7d" -resolved_reference = "301067883ddce0bca31fdd54973c3d6c8cbbcb7d" +url = "https://github.com/combinatorist/WikibaseIntegrator.git" +reference = "feature/search_entities-exact-max_results" +resolved_reference = "e6e48e9243b12854fa16e2c1c27590de320cf777" [metadata] lock-version = "2.1" python-versions = "<4.0,>=3.13" -content-hash = "a38a28e15ca1b872d79b3c7d1b698c4b0826fb79d185dd98c1a10473aa584b4e" +content-hash = "dd87a3ee33125d8face59db599ac939a15191468888423a725bd62c31f400312" diff --git a/pyproject.toml b/pyproject.toml index afd0a5a..1452c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ readme = "README.md" requires-python = "<4.0,>=3.13" dependencies = [ - "wikibaseintegrator @ git+https://github.com/LeMyst/WikibaseIntegrator.git@301067883ddce0bca31fdd54973c3d6c8cbbcb7d", + "wikibaseintegrator @ git+https://github.com/combinatorist/WikibaseIntegrator.git@feature/search_entities-exact-max_results", "pyairtable (>=3.1.1,<4.0.0)", "more_itertools" ] diff --git a/src/config.py b/src/config.py index 0b07d5c..f987c04 100644 --- a/src/config.py +++ b/src/config.py @@ -13,6 +13,10 @@ os.getenv("READ_CTFG_FROM_CACHE", "False").upper() ) +POST_DETAILS_TO_CTFG: bool = "TRUE".startswith( + os.getenv("POST_DETAILS_TO_CTFG", "False").upper() +) + @dataclass class airtable: @@ -33,9 +37,12 @@ class airtable: wbi = WikibaseIntegrator() -WIKIDATA_MAX_RESULTS_PER_SEARCH: int = int( - os.getenv("WIKIDATA_MAX_RESULTS_PER_SEARCH", 50) -) +def getEnvInt(name: str, default: int) -> int: + raw = os.getenv("WIKIDATA_MAX_RESULTS_PER_SEARCH") + return int(raw) if raw else default + +WIKIDATA_MAX_LISTINGS_TO_SEARCH: int = getEnvInt("WIKIDATA_MAX_LISTINGS_TO_SEARCH", 5) +WIKIDATA_MAX_RESULTS_PER_SEARCH: int = getEnvInt("WIKIDATA_MAX_RESULTS_PER_SEARCH", 5) LANGUAGE_CODE = "en" diff --git a/src/ctfg.py b/src/ctfg.py index 7eddf5f..47f610d 100644 --- a/src/ctfg.py +++ b/src/ctfg.py @@ -158,7 +158,10 @@ def from_wiki_match(m: dict, keep_unknowns: bool = False): # pprint(claims) statements = [ - WikidataStatement.from_wiki_statement(s) for p in claims.values() for s in p + WikidataStatement.from_wiki_statement(s) + for p in claims.values() + for s in p + if config.POST_DETAILS_TO_CTFG ] result = WikidataItem(**mappable, statements=statements) result.save() @@ -271,17 +274,17 @@ def partition_matched(items: list[Listing]) -> tuple[list[Listing], list[Listing def upsert_matches(wiki_matches: dict[Listing, list[dict[str, Any]]]): - log("Updating CTFG with matching wikibase IDs...") with_wiki_items = { x: [WikidataItem.from_wiki_match(m) for m in matches] for x, matches in wiki_matches.items() if matches } wiki_items = list(set([x for y in with_wiki_items.values() for x in y])) + log(f"Updating CTFG with {len(wiki_items)} matching wikibase IDs...") - log("Example item") - # pprint(wiki_items[0].to_record()) - # WikidataItem.recursive_save(wiki_items) + if wiki_items: + log("Example item") + pprint(wiki_items[0].to_record()) for x, matches in with_wiki_items.items(): x.wikidata_suggestions = matches diff --git a/src/sync.py b/src/sync.py index a1422b8..4f85ef3 100644 --- a/src/sync.py +++ b/src/sync.py @@ -11,7 +11,9 @@ (unmatched_items, matched_items) = ctfg.partition_matched(items) wiki_matches = wiki.get_matches( - unmatched_items, max_attempts=5, max_results=config.WIKIDATA_MAX_RESULTS_PER_SEARCH + unmatched_items, + max_attempts=config.WIKIDATA_MAX_LISTINGS_TO_SEARCH, + max_results=config.WIKIDATA_MAX_RESULTS_PER_SEARCH, ) # wiki_match_histogram = wiki.summarize_matches(wiki_matches) diff --git a/src/wiki.py b/src/wiki.py index 100f6e3..664412f 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -18,7 +18,9 @@ def get_matches( ) -> dict[ctfg.Listing, list[dict[str, Any]]]: matchable_items = [x for x in items if x.name] attempting_items = ( - sample(matchable_items, max_attempts) if max_attempts else matchable_items + sample(matchable_items, max_attempts) + if max_attempts and max_attempts < len(matchable_items) + else matchable_items ) log( f"Searching for wikibase matches for {len(attempting_items)} {'random' if max_attempts else 'matchable'} items..."