Increase SERP download limit to 75k

webis-de · Mar 29, 2023 · 918b254 · 918b254
1 parent 6eb12de
commit 918b254
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/archive_query_log/download/warc.py b/archive_query_log/download/warc.py
@@ -377,7 +377,7 @@ async def download_service(
             archived_urls = self._deduplicate_urls(archived_urls, snippets)
             archived_urls = Random(0).sample(
                 archived_urls,
-                min(len(archived_urls), 50_000)
+                min(len(archived_urls), 75_000)
             )
 
         await self._download(archived_urls)