fix: Use batching for recipe addition to prevent 431 errors (request too large)

Author: aLEGEND21

scraper/add_recipes.py

@@ -11,6 +11,7 @@
 load_dotenv()
 
 POST_RECIPE_URL = os.getenv("POST_RECIPE_URL")
+BATCH_SIZE = 500
 
 
 def add_recipes_from_file(file_path: str) -> None:
@@ -22,14 +23,23 @@ def add_recipes_from_file(file_path: str) -> None:
     # Convert dict to list of recipes
     recipe_list = list(recipes.values())
 
-    # Send all recipes in a single batch request
-    resp = requests.post(POST_RECIPE_URL, json=recipe_list)
-    data = resp.json()
-
-    # Truncate the recipe IDs since there may be too many
-    if "recipe_ids" in data:
-        data["recipe_ids"] = data["recipe_ids"][:10]
-    print(json.dumps(data, indent=4))
+    # Send all recipes in batches
+    print(
+        f"Sending {len(recipe_list)} recipes to API using {len(recipe_list) // BATCH_SIZE + 1} batches"
+    )
+    for i in range(0, len(recipe_list), BATCH_SIZE):
+        current_batch = i // BATCH_SIZE + 1
+        batch = recipe_list[i : i + BATCH_SIZE]
+        resp = requests.post(POST_RECIPE_URL, json=batch)
+        data = resp.json()
+
+        # Truncate response to first 5 recipe IDs
+        if "recipe_ids" in data:
+            data["recipe_ids"] = data["recipe_ids"][:5]
+
+        print(f"Response from batch {current_batch}:")
+        print(json.dumps(data, indent=4))
+        print()
 
 
 if __name__ == "__main__":