Add project feature warning (10,000) and limit (30,000)

src/backend/app/projects/project_crud.py

@@ -587,6 +587,10 @@ async def generate_project_files(
         )
         for task in project.tasks
     ]
+    log.debug(
+        "Setting task feature counts in db for "
+        f"({len(project.tasks if project.tasks else 0)}) tasks",
+    )
     sql = """
         WITH task_update(id, feature_count) AS (
             VALUES {}
@@ -608,6 +612,7 @@ async def generate_project_files(
     async with db.cursor() as cur:
         await cur.execute(formatted_sql)
 
+    log.info("Finished generation of project additional files")
     return True
 
 

src/backend/app/projects/project_routes.py

@@ -965,8 +965,9 @@ async def generate_files(
     background_tasks: BackgroundTasks,
     xlsform_upload: Annotated[
         Optional[BytesIO], Depends(central_deps.read_optional_xlsform)
-    ],
-    additional_entities: Optional[list[str]] = None,
+    ] = None,
+    additional_entities: Annotated[Optional[list[str]], None] = None,
+    combined_features_count: Annotated[int, Form()] = 0,
 ):
     """Generate additional content to initialise the project.
 
@@ -985,6 +986,8 @@ async def generate_files(
             A file should be provided if user wants to upload a custom xls form.
         additional_entities (list[str]): If additional Entity lists need to be
             created (i.e. the project form references multiple geometries).
+        combined_features_count (int): Total count of features to be mapped, plus
+            additional dataset features, determined by frontend.
         db (Connection): The database connection.
         project_user_dict (ProjectUserDict): Project admin role.
         background_tasks (BackgroundTasks): FastAPI background tasks.
@@ -1050,39 +1053,53 @@ async def generate_files(
             },
         )
 
-    success = await project_crud.generate_project_files(
-        db,
-        project_id,
-    )
+    warning_message = None
 
-    if not success:
-        return JSONResponse(
-            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
-            content={
-                "message": (
-                    f"Failed project ({project_id}) creation. "
-                    "Please contact the server admin."
-                )
-            },
+    if combined_features_count > 10000:
+        # Return immediately and run in background if many features
+        background_tasks.add_task(
+            project_crud.generate_project_files,
+            db,
+            project_id,
+        )
+        warning_message = "There are lots of features to process. Please be patient 🙏"
+
+    else:
+        success = await project_crud.generate_project_files(
+            db,
+            project_id,
         )
 
+        if not success:
+            return JSONResponse(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                content={
+                    "message": (
+                        f"Failed project ({project_id}) creation. "
+                        "Please contact the server admin."
+                    )
+                },
+            )
+
     if project.custom_tms_url:
         basemap_in = project_schemas.BasemapGenerate(
             tile_source="custom", file_format="pmtiles", tms_url=project.custom_tms_url
         )
         org_id = project.organisation_id
         await generate_basemap(project_id, org_id, basemap_in, db, background_tasks)
 
-    return JSONResponse(
-        status_code=HTTPStatus.OK,
-        content={"message": "success"},
-    )
+    if warning_message:
+        return JSONResponse(
+            status_code=HTTPStatus.OK,
+            content={"message": warning_message},
+        )
+    return Response(status_code=HTTPStatus.OK)
 
 
 @router.post("/{project_id}/tiles-generate")
 async def generate_project_basemap(
     # NOTE we do not set the correct role on this endpoint yet
-    # FIXME once sub project creation implemented, this should be manager only
+    # FIXME once stub project creation implemented, this should be manager only
     project_user: Annotated[ProjectUserDict, Depends(mapper)],
     background_tasks: BackgroundTasks,
     db: Annotated[Connection, Depends(db_conn)],

src/frontend/src/api/CreateProjectService.ts

@@ -19,10 +19,11 @@ const CreateProjectService = (
   projectData: any,
   taskAreaGeojson: any,
   formUpload: any,
-  dataExtractFile: any,
+  dataExtractFile: File | null,
   isOsmExtract: boolean,
   additionalFeature: any,
   projectAdmins: number[],
+  combinedFeaturesCount: number,
 ) => {
   return async (dispatch: AppDispatch) => {
     dispatch(CreateProjectActions.CreateProjectLoading(true));
@@ -95,6 +96,7 @@ const CreateProjectService = (
             ? { ...projectData, additional_entities: [additionalFeature?.name?.split('.')?.[0]] }
             : projectData,
           formUpload,
+          combinedFeaturesCount,
         ),
       );
 
@@ -184,77 +186,56 @@ const UploadTaskAreasService = (url: string, filePayload: any) => {
   };
 };
 
-const GenerateProjectFilesService = (url: string, projectData: any, formUpload: any) => {
+const GenerateProjectFilesService = (url: string, projectData: any, formUpload: any, combinedFeaturesCount: number) => {
   return async (dispatch: AppDispatch) => {
     dispatch(CreateProjectActions.GenerateProjectLoading(true));
     dispatch(CommonActions.SetLoading(true));
 
-    const postUploadArea = async (url, projectData: any, formUpload) => {
-      let isAPISuccess = true;
-      try {
-        let response;
+    try {
+      const formData = new FormData();
 
-        const additional_entities =
-          projectData?.additional_entities?.length > 0
-            ? projectData.additional_entities.map((e: string) => e.replaceAll(' ', '_'))
-            : [];
-        const generateApiFormData = new FormData();
+      // Append additional_entities if they exist
+      const additionalEntities = projectData?.additional_entities?.map((e: string) => e.replaceAll(' ', '_')) ?? [];
+      if (additionalEntities.length > 0) {
+        formData.append('additional_entities', additionalEntities);
+      }
 
-        if (additional_entities?.length > 0) {
-          generateApiFormData.append('additional_entities', additional_entities);
-        }
+      // Append xlsform only if it's a custom form
+      if (projectData.form_ways === 'custom_form' && formUpload) {
+        formData.append('xlsform', formUpload);
+      }
 
-        if (projectData.form_ways === 'custom_form') {
-          // TODO move form upload to a separate service / endpoint?
-          generateApiFormData.append('xlsform', formUpload);
-          response = await axios.post(url, generateApiFormData, {
-            headers: {
-              'Content-Type': 'multipart/form-data',
-            },
-          });
-        } else {
-          if (additional_entities?.length > 0) {
-            response = await axios.post(url, generateApiFormData, {
-              headers: {
-                'Content-Type': 'multipart/form-data',
-              },
-            });
-          } else {
-            const payload = {
-              additional_entities: null,
-            };
-            response = await axios.post(url, payload, {
-              headers: {
-                'Content-Type': 'application/json',
-              },
-            });
-          }
-        }
+      // Add combined features count
+      formData.append('combined_features_count', combinedFeaturesCount.toString());
 
-        isAPISuccess = isStatusSuccess(response.status);
-        if (!isAPISuccess) {
-          throw new Error(`Request failed with status ${response.status}`);
-        }
+      const response = await axios.post(url, formData, {
+        headers: { 'Content-Type': 'multipart/form-data' },
+      });
 
-        dispatch(CreateProjectActions.GenerateProjectLoading(false));
-        dispatch(CommonActions.SetLoading(false));
-        // Trigger the watcher and redirect after success
-        dispatch(CreateProjectActions.GenerateProjectSuccess(true));
-      } catch (error: any) {
-        isAPISuccess = false;
-        dispatch(CommonActions.SetLoading(false));
-        dispatch(CreateProjectActions.GenerateProjectError(true));
-        dispatch(
-          CommonActions.SetSnackBar({
-            message: JSON.stringify(error?.response?.data?.detail),
-          }),
-        );
-        dispatch(CreateProjectActions.GenerateProjectLoading(false));
+      if (!isStatusSuccess(response.status)) {
+        throw new Error(`Request failed with status ${response.status}`);
+      }
+
+      // If warning provided, then inform user
+      const message = response.data?.message;
+      if (message) {
+        dispatch(CreateProjectActions.GenerateProjectWarning(message));
       }
-      return isAPISuccess;
-    };
 
-    return await postUploadArea(url, projectData, formUpload);
+      dispatch(CreateProjectActions.GenerateProjectSuccess(true));
+      return true; // ✅ Return success
+    } catch (error: any) {
+      dispatch(CreateProjectActions.GenerateProjectError(true));
+      dispatch(
+        CommonActions.SetSnackBar({
+          message: JSON.stringify(error?.response?.data?.detail),
+        }),
+      );
+      return false; // ❌ Return failure
+    } finally {
+      dispatch(CreateProjectActions.GenerateProjectLoading(false));
+      dispatch(CommonActions.SetLoading(false));
+    }
   };
 };
 
@@ -318,8 +299,8 @@ const GetDividedTaskFromGeojson = (url: string, projectData: Record<string, any>
         dividedTaskFormData.append('dimension_meters', projectData.dimension);
         const getGetDividedTaskFromGeojsonResponse = await axios.post(url, dividedTaskFormData);
         const resp: splittedGeojsonType = getGetDividedTaskFromGeojsonResponse.data;
-        dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'divide_on_square', value: true }));
-        dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'task_splitting_algorithm', value: false }));
+        dispatch(CreateProjectActions.SetIsTasksSplit({ key: 'divide_on_square', value: true }));
+        dispatch(CreateProjectActions.SetIsTasksSplit({ key: 'task_splitting_algorithm', value: false }));
         dispatch(CreateProjectActions.SetDividedTaskGeojson(resp));
         dispatch(CreateProjectActions.SetDividedTaskFromGeojsonLoading(false));
       } catch (error) {
@@ -393,8 +374,8 @@ const TaskSplittingPreviewService = (
           // TODO display error to user, perhaps there is not osm data here?
           return;
         }
-        dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'divide_on_square', value: false }));
-        dispatch(CreateProjectActions.SetIsTasksGenerated({ key: 'task_splitting_algorithm', value: true }));
+        dispatch(CreateProjectActions.SetIsTasksSplit({ key: 'divide_on_square', value: false }));
+        dispatch(CreateProjectActions.SetIsTasksSplit({ key: 'task_splitting_algorithm', value: true }));
         dispatch(CreateProjectActions.GetTaskSplittingPreview(resp));
       } catch (error) {
         dispatch(

src/frontend/src/components/createnewproject/DataExtract.tsx

@@ -33,13 +33,44 @@ const DataExtract = ({
   useDocumentTitle('Create Project: Map Data');
   const dispatch = useAppDispatch();
   const navigate = useNavigate();
+  const [disableNextButton, setDisableNextButton] = useState(true);
   const [extractWays, setExtractWays] = useState('');
   const projectDetails: any = useAppSelector((state) => state.createproject.projectDetails);
   const projectAoiGeojson = useAppSelector((state) => state.createproject.drawnGeojson);
   const dataExtractGeojson = useAppSelector((state) => state.createproject.dataExtractGeojson);
   const isFgbFetching = useAppSelector((state) => state.createproject.isFgbFetching);
   const additionalFeatureGeojson = useAppSelector((state) => state.createproject.additionalFeatureGeojson);
 
+  useEffect(() => {
+    const featureCount = dataExtractGeojson?.features?.length ?? 0;
+
+    if (featureCount > 10000) {
+      dispatch(
+        CommonActions.SetSnackBar({
+          message: `${featureCount} is a lot of features to map at once. Are you sure?`,
+          variant: 'warning',
+          duration: 10000,
+        }),
+      );
+    }
+
+    if (featureCount > 30000) {
+      setDisableNextButton(true);
+      dispatch(
+        CommonActions.SetSnackBar({
+          message: `${featureCount} is a lot of features! Please consider breaking this into smaller projects.`,
+          variant: 'error',
+          duration: 10000,
+        }),
+      );
+    } else {
+      // Enable/disable NEXT button based on conditions (if feature limit not exceeded)
+      const shouldDisableButton =
+        !dataExtractGeojson || (extractWays === 'osm_data_extract' && !dataExtractGeojson) || isFgbFetching;
+      setDisableNextButton(shouldDisableButton);
+    }
+  }, [dataExtractGeojson, additionalFeatureGeojson, extractWays, isFgbFetching]);
+
   const submission = () => {
     dispatch(CreateProjectActions.SetIndividualProjectDetailsData(formValues));
     dispatch(CommonActions.SetCurrentStepFormStep({ flag: flag, step: 5 }));
@@ -271,6 +302,12 @@ const DataExtract = ({
                   />
                 </>
               )}
+              <div className="fmtm-mt-5">
+                <p className="fmtm-text-gray-500">
+                  Total number of features:{' '}
+                  <span className="fmtm-font-bold">{dataExtractGeojson?.features?.length || 0}</span>
+                </p>
+              </div>
               {extractWays && (
                 <div className="fmtm-mt-4">
                   <div
@@ -331,15 +368,7 @@ const DataExtract = ({
               <Button variant="secondary-grey" onClick={() => toggleStep(3, '/upload-survey')}>
                 PREVIOUS
               </Button>
-              <Button
-                variant="primary-red"
-                type="submit"
-                disabled={
-                  !dataExtractGeojson || (extractWays === 'osm_data_extract' && !dataExtractGeojson) || isFgbFetching
-                    ? true
-                    : false
-                }
-              >
+              <Button variant="primary-red" type="submit" disabled={disableNextButton}>
                 NEXT
               </Button>
             </div>