WIP: Add backend validation to prevent adding failed crawls to collection

backend/btrixcloud/basecrawls.py

@@ -44,6 +44,7 @@
     UpdatedResponse,
     DeletedResponseQuota,
     CrawlSearchValuesResponse,
+    FAILED_STATES,
 )
 from .pagination import paginated_format, DEFAULT_PAGE_SIZE
 from .utils import dt_now, get_origin, date_to_str
@@ -601,6 +602,15 @@ async def bulk_presigned_files(
 
         return resources, pages_optimized
 
+    async def validate_all_crawls_successful(
+        self, crawl_ids: List[str], org: Organization
+    ):
+        """Validate that crawls in list exist and did not fail or else raise exception"""
+        for crawl_id in crawl_ids:
+            crawl = await self.get_base_crawl(crawl_id, org)
+            if crawl.state in FAILED_STATES:
+                raise HTTPException(status_code=400, detail="invalid_failed_crawl")
+
     async def add_to_collection(
         self, crawl_ids: List[str], collection_id: UUID, org: Organization
     ):

backend/btrixcloud/colls.py

@@ -121,17 +121,19 @@ async def init_index(self):
             [("oid", pymongo.ASCENDING), ("description", pymongo.ASCENDING)]
         )
 
-    async def add_collection(self, oid: UUID, coll_in: CollIn):
+    async def add_collection(self, org: Organization, coll_in: CollIn):
         """Add new collection"""
         crawl_ids = coll_in.crawlIds if coll_in.crawlIds else []
+        await self.crawl_ops.validate_all_crawls_successful(crawl_ids, org)
+
         coll_id = uuid4()
         created = dt_now()
 
         slug = coll_in.slug or slug_from_name(coll_in.name)
 
         coll = Collection(
             id=coll_id,
-            oid=oid,
+            oid=org.id,
             name=coll_in.name,
             slug=slug,
             description=coll_in.description,
@@ -144,7 +146,6 @@ async def add_collection(self, oid: UUID, coll_in: CollIn):
         )
         try:
             await self.collections.insert_one(coll.to_dict())
-            org = await self.orgs.get_org_by_id(oid)
             await self.clear_org_previous_slugs_matching_slug(slug, org)
 
             if crawl_ids:
@@ -229,7 +230,7 @@ async def add_crawls_to_collection(
         headers: Optional[dict] = None,
     ) -> CollOut:
         """Add crawls to collection"""
-        await self.crawl_ops.add_to_collection(crawl_ids, coll_id, org)
+        await self.crawl_ops.validate_all_crawls_successful(crawl_ids, org)
 
         modified = dt_now()
         result = await self.collections.find_one_and_update(
@@ -240,6 +241,8 @@ async def add_crawls_to_collection(
         if not result:
             raise HTTPException(status_code=404, detail="collection_not_found")
 
+        await self.crawl_ops.add_to_collection(crawl_ids, coll_id, org)
+
         await self.update_collection_counts_and_tags(coll_id)
         await self.update_collection_dates(coll_id, org.id)
 
@@ -1019,7 +1022,7 @@ def init_collections_api(
     async def add_collection(
         new_coll: CollIn, org: Organization = Depends(org_crawl_dep)
     ):
-        return await colls.add_collection(org.id, new_coll)
+        return await colls.add_collection(org, new_coll)
 
     @app.get(
         "/orgs/{oid}/collections",