fix: page the Kaggle search by number, and require kaggle 2.x

Kaggle ignores page_size and never fills next_page_token, so the search
returned twenty rows whatever was asked and its cursor was always None:
the Hub could never load a second page. The page parameter does work, so
the cursor is now the page number as a string, the way the Zenodo source
already does it, and a page shorter than Kaggle's fixed size is read as
the last one. Entries are not trimmed to the requested limit, because the
next page starts where this one ended and a trimmed row is never served
again.

The credential and the source rely on the kaggle 2.x token API, which the
old ">=1.7.4.5" floor did not guarantee. With Python 3.10 gone the lock
already resolves 2.2.4 everywhere; the pin makes the requirement explicit.

Author: cristian-tamblay

DashAI/back/dataset_sources/kaggle_dataset_source.py

@@ -16,6 +16,11 @@
 
 log = logging.getLogger(__name__)
 
+# How many datasets Kaggle puts on one search page. It pages by number, ignores
+# ``page_size`` and never fills ``next_page_token``, so the cursor is the page
+# number, as in the Zenodo source, and a page shorter than this is the last one.
+_KAGGLE_PAGE_SIZE: Final[int] = 20
+
 
 def _import_kaggle():
     """Import the ``kaggle`` module, suppressing its import time auth noise.
@@ -143,10 +148,11 @@ def search(
         query : str
             Free text search string.
         limit : int, optional
-            Maximum number of results, by default 20.
+            Requested page size, by default 20. Passed to Kaggle, which today
+            serves ``_KAGGLE_PAGE_SIZE`` datasets a page whatever is asked.
         cursor : str or None, optional
-            Kaggle pagination token returned by the previous call.  ``None``
-            fetches the first page.
+            Page number returned by the previous call as ``next_cursor``.
+            ``None`` fetches the first page.
         **filters : Any
             Supported keys:
               sort_by (str): Kaggle dataset sort (e.g. ``"hottest"``).
@@ -162,13 +168,14 @@ def search(
         """
         kaggle = _import_kaggle()
         try:
+            page = int(cursor) if cursor else 1
             sort_by = filters.get("sort_by") or "hottest"
             tag_ids = filters.get("tags")
             if isinstance(tag_ids, list):
                 tag_ids = ",".join(tag_ids)
             params: dict[str, Any] = {
                 "search": query or None,
-                "page_token": cursor or None,
+                "page": page,
                 "page_size": limit,
                 "sort_by": sort_by,
             }
@@ -178,7 +185,12 @@ def search(
                     params[key] = value
             response = kaggle.api.dataset_list_with_response(**params)
             entries = [self._to_entry(item) for item in (response.datasets or [])]
-            next_cursor = response.next_page_token or None
+            # Kaggle answers with a full page or the tail of the results, never
+            # with a token, so a full page is the only sign of a next one. The
+            # entries are not trimmed to ``limit``: the next page starts where
+            # this one ended, so anything cut here would never be served again.
+            has_next = len(entries) >= min(limit, _KAGGLE_PAGE_SIZE)
+            next_cursor = str(page + 1) if has_next else None
             return SearchPage(entries=entries, next_cursor=next_cursor)
         except Exception:
             log.exception("Error searching Kaggle datasets")

pyproject.toml

@@ -78,7 +78,7 @@ dependencies = [
     "openml",
     "oslo.concurrency",
     "cryptography>=49.0.0",
-    "kaggle>=1.7.4.5",
+    "kaggle>=2",
     "grad-cam>=1.5.5",
     "dice-ml>=0.12",
     "lime>=0.2.0.1",

tests/back/dataset_sources/test_kaggle_dataset_source.py

@@ -62,15 +62,14 @@ def test_search_returns_dataset_entries():
             total_bytes=15347,
         )
     ]
-    response.next_page_token = ""
     api.dataset_list_with_response.return_value = response
 
     with fake_kaggle(api):
         source = _make_source()
         page = source.search("iris", limit=5)
 
     api.dataset_list_with_response.assert_called_once_with(
-        search="iris", page_token=None, page_size=5, sort_by="hottest"
+        search="iris", page=1, page_size=5, sort_by="hottest"
     )
     assert isinstance(page, SearchPage)
     assert len(page.entries) == 1
@@ -84,28 +83,59 @@ def test_search_returns_dataset_entries():
     assert page.next_cursor is None
 
 
-def test_search_passes_cursor_and_exposes_next_cursor():
+def test_search_reads_the_cursor_as_a_page_number_and_a_full_page_continues():
+    # Kaggle pages by number and never fills next_page_token, so the cursor is
+    # the page and a full page is the only sign that another one follows.
     api = MagicMock()
     response = MagicMock()
-    response.datasets = [_dataset("owner/repo")]
-    response.next_page_token = "tok123"
+    response.datasets = [_dataset(f"owner/repo{i}") for i in range(20)]
     api.dataset_list_with_response.return_value = response
 
     with fake_kaggle(api):
         source = _make_source()
-        page = source.search("q", limit=20, cursor="prevtok")
+        page = source.search("q", limit=20, cursor="2")
 
     api.dataset_list_with_response.assert_called_once_with(
-        search="q", page_token="prevtok", page_size=20, sort_by="hottest"
+        search="q", page=2, page_size=20, sort_by="hottest"
     )
-    assert page.next_cursor == "tok123"
+    assert len(page.entries) == 20
+    assert page.next_cursor == "3"
+
+
+def test_search_treats_a_short_page_as_the_last_one():
+    api = MagicMock()
+    response = MagicMock()
+    response.datasets = [_dataset(f"owner/repo{i}") for i in range(7)]
+    api.dataset_list_with_response.return_value = response
+
+    with fake_kaggle(api):
+        source = _make_source()
+        page = source.search("q", limit=20, cursor="3")
+
+    assert len(page.entries) == 7
+    assert page.next_cursor is None
+
+
+def test_search_does_not_trim_a_page_below_the_requested_limit():
+    # Kaggle ignores page_size and the next page starts where this one ended,
+    # so cutting the page down to ``limit`` would lose rows for good.
+    api = MagicMock()
+    response = MagicMock()
+    response.datasets = [_dataset(f"owner/repo{i}") for i in range(20)]
+    api.dataset_list_with_response.return_value = response
+
+    with fake_kaggle(api):
+        source = _make_source()
+        page = source.search("q", limit=3)
+
+    assert len(page.entries) == 20
+    assert page.next_cursor == "2"
 
 
 def test_search_uses_slug_as_name_when_title_missing():
     api = MagicMock()
     response = MagicMock()
     response.datasets = [_dataset("uciml/iris", title="")]
-    response.next_page_token = ""
     api.dataset_list_with_response.return_value = response
 
     with fake_kaggle(api):