Incorporate URL and Batch Materialized View Statuses into app logic

alembic/versions/2025_12_31_1106-759ce7d0772b_remove_url_status_attribute.py

@@ -0,0 +1,31 @@
+"""Remove URL Status attribute
+
+Revision ID: 759ce7d0772b
+Revises: 42933d84aa52
+Create Date: 2025-12-31 11:06:39.037486
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '759ce7d0772b'
+down_revision: Union[str, None] = '42933d84aa52'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.drop_column(
+        table_name="urls",
+        column_name="status"
+    )
+
+    op.execute("""DROP type url_status""")
+
+
+def downgrade() -> None:
+    pass

src/api/endpoints/annotate/_shared/queries/get_annotation_batch_info.py

@@ -4,7 +4,6 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src.api.endpoints.annotate.dtos.shared.batch import AnnotationBatchInfo
-from src.collectors.enums import URLStatus
 from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL
 from src.db.models.impl.url.core.sqlalchemy import URL
 from src.db.queries.base.builder import QueryBuilderBase
@@ -42,7 +41,6 @@ async def run(
         )
 
         common_where_clause = [
-            URL.status == URLStatus.OK.value,
             LinkBatchURL.batch_id == self.batch_id,
         ]
 

src/api/endpoints/annotate/_shared/queries/helper.py

@@ -5,7 +5,6 @@
 from sqlalchemy import Select, case, CTE, ColumnElement
 from sqlalchemy.orm import joinedload
 
-from src.collectors.enums import URLStatus
 from src.db.helpers.query import exists_url, not_exists_url
 from src.db.models.impl.flag.url_suspended.sqlalchemy import FlagURLSuspended
 from src.db.models.impl.url.core.enums import URLSource
@@ -33,7 +32,6 @@ def add_common_where_conditions(
     query: Select,
 ) -> Select:
     return query.where(
-        URL.status == URLStatus.OK.value,
         not_exists_url(
             FlagURLSuspended
         ),

src/api/endpoints/batch/dtos/get/summaries/counts.py

@@ -4,7 +4,6 @@
 class BatchSummaryURLCounts(BaseModel):
     total: int
     pending: int
-    duplicate: int
     not_relevant: int
     submitted: int
     errored: int

src/api/endpoints/batch/routes.py

@@ -10,7 +10,7 @@
 from src.api.endpoints.batch.urls.dto import GetURLsByBatchResponse
 from src.collectors.enums import CollectorType
 from src.core.core import AsyncCore
-from src.db.models.views.batch_url_status.enums import BatchURLStatusEnum
+from src.db.models.materialized_views.batch_url_status.enums import BatchURLStatusViewEnum
 from src.security.dtos.access_info import AccessInfo
 from src.security.manager import get_access_info
 
@@ -27,7 +27,7 @@ async def get_batch_status(
             description="Filter by collector type",
             default=None
         ),
-        status: BatchURLStatusEnum | None = Query(
+        status: BatchURLStatusViewEnum | None = Query(
             description="Filter by status",
             default=None
         ),

src/api/endpoints/collector/manual/query.py

@@ -3,7 +3,7 @@
 
 from src.api.endpoints.collector.dtos.manual_batch.post import ManualBatchInputDTO
 from src.api.endpoints.collector.dtos.manual_batch.response import ManualBatchResponseDTO
-from src.collectors.enums import CollectorType, URLStatus
+from src.collectors.enums import CollectorType
 from src.core.enums import BatchStatus
 from src.db.models.impl.batch.sqlalchemy import Batch
 from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL
@@ -53,7 +53,6 @@ async def run(self, session: AsyncSession) -> ManualBatchResponseDTO:
                 name=entry.name,
                 description=entry.description,
                 collector_metadata=entry.collector_metadata,
-                status=URLStatus.OK.value,
                 source=URLSource.MANUAL,
                 trailing_slash=url_and_scheme.url.endswith('/'),
             )

src/api/endpoints/metrics/batches/aggregated/query/core.py

@@ -1,29 +1,15 @@
-from sqlalchemy import case, select
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.sql.functions import coalesce, func
+from sqlalchemy.sql.functions import func
 
 from src.api.endpoints.metrics.batches.aggregated.dto import GetMetricsBatchesAggregatedResponseDTO, \
     GetMetricsBatchesAggregatedInnerResponseDTO
-from src.api.endpoints.metrics.batches.aggregated.query.all_urls.query import CountAllURLsByBatchStrategyQueryBuilder
-from src.api.endpoints.metrics.batches.aggregated.query.batch_status_.query import \
-    BatchStatusByBatchStrategyQueryBuilder
 from src.api.endpoints.metrics.batches.aggregated.query.requester_.requester import \
     GetBatchesAggregatedMetricsQueryRequester
-from src.api.endpoints.metrics.batches.aggregated.query.submitted_.query import \
-    CountSubmittedByBatchStrategyQueryBuilder
-from src.api.endpoints.metrics.batches.aggregated.query.url_error.query import URLErrorByBatchStrategyQueryBuilder
-from src.api.endpoints.metrics.batches.aggregated.query.validated_.query import \
-    ValidatedURLCountByBatchStrategyQueryBuilder
-from src.collectors.enums import URLStatus, CollectorType
+from src.collectors.enums import CollectorType
 from src.core.enums import BatchStatus
 from src.db.models.impl.batch.sqlalchemy import Batch
-from src.db.models.impl.flag.url_validated.enums import URLType
-from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
-from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL
-from src.db.models.impl.url.core.sqlalchemy import URL
-from src.db.models.impl.url.data_source.sqlalchemy import DSAppLinkDataSource
 from src.db.queries.base.builder import QueryBuilderBase
-from src.db.statement_composer import StatementComposer
 
 
 class GetBatchesAggregatedMetricsQueryBuilder(QueryBuilderBase):

src/api/endpoints/metrics/batches/aggregated/query/url_error/query.py

@@ -4,7 +4,6 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src.api.endpoints.metrics.batches.aggregated.query.models.strategy_count import CountByBatchStrategyResponse
-from src.collectors.enums import URLStatus
 from src.db.helpers.query import exists_url
 from src.db.helpers.session import session_helper as sh
 from src.db.models.impl.batch.sqlalchemy import Batch
@@ -28,7 +27,7 @@ async def run(self, session: AsyncSession) -> list[CountByBatchStrategyResponse]
             .where(
                 exists_url(URLTaskError)
             )
-            .group_by(Batch.strategy, URL.status)
+            .group_by(Batch.strategy)
         )
 
         mappings: Sequence[RowMapping] = await sh.mappings(session, query=query)

src/api/endpoints/metrics/batches/breakdown/query.py

@@ -1,4 +1,4 @@
-from sqlalchemy import select, case, Column
+from sqlalchemy import select, Column
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql.functions import coalesce
 
@@ -11,11 +11,9 @@
 from src.api.endpoints.metrics.batches.breakdown.templates.cte_ import BatchesBreakdownURLCTE
 from src.api.endpoints.metrics.batches.breakdown.total.cte_ import TOTAL_CTE
 from src.api.endpoints.metrics.batches.breakdown.validated.cte_ import VALIDATED_CTE
-from src.collectors.enums import URLStatus, CollectorType
+from src.collectors.enums import CollectorType
 from src.core.enums import BatchStatus
 from src.db.models.impl.batch.sqlalchemy import Batch
-from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL
-from src.db.models.impl.url.core.sqlalchemy import URL
 from src.db.queries.base.builder import QueryBuilderBase
 from src.db.statement_composer import StatementComposer
 

src/api/endpoints/metrics/dtos/get/urls/aggregated/core.py

@@ -4,7 +4,7 @@
 
 from src.core.enums import RecordType
 from src.db.models.impl.flag.url_validated.enums import URLType
-from src.db.models.views.url_status.enums import URLStatusViewEnum
+from src.db.models.materialized_views.url_status.enums import URLStatusViewEnum
 
 class GetMetricsURLValidatedOldestPendingURL(BaseModel):
     url_id: int

src/api/endpoints/metrics/urls/aggregated/query/core.py

@@ -11,7 +11,7 @@
 from src.core.enums import RecordType
 from src.db.helpers.session import session_helper as sh
 from src.db.models.impl.flag.url_validated.enums import URLType
-from src.db.models.views.url_status.enums import URLStatusViewEnum
+from src.db.models.materialized_views.url_status.enums import URLStatusViewEnum
 from src.db.queries.base.builder import QueryBuilderBase
 
 

src/api/endpoints/metrics/urls/aggregated/query/subqueries/oldest_pending_url.py

@@ -3,11 +3,10 @@
 
 from src.api.endpoints.metrics.dtos.get.urls.aggregated.core import GetMetricsURLValidatedOldestPendingURL
 from src.db.models.impl.url.core.sqlalchemy import URL
-from src.db.models.views.url_status.core import URLStatusMatView
-from src.db.models.views.url_status.enums import URLStatusViewEnum
+from src.db.models.materialized_views.url_status.sqlalchemy import URLStatusMaterializedView
+from src.db.models.materialized_views.url_status.enums import URLStatusViewEnum
 from src.db.queries.base.builder import QueryBuilderBase
 
-from src.db.helpers.session import session_helper as sh
 
 class GetOldestPendingURLQueryBuilder(QueryBuilderBase):
 
@@ -18,14 +17,14 @@ async def run(
 
         query = (
             select(
-                URLStatusMatView.url_id,
+                URLStatusMaterializedView.url_id,
                 URL.created_at
             )
             .join(
                 URL,
-                URLStatusMatView.url_id == URL.id
+                URLStatusMaterializedView.url_id == URL.id
             ).where(
-                URLStatusMatView.status.not_in(
+                URLStatusMaterializedView.status.not_in(
                     [
                         URLStatusViewEnum.SUBMITTED.value,
                         URLStatusViewEnum.ACCEPTED.value,

src/api/endpoints/metrics/urls/aggregated/query/subqueries/status.py

@@ -4,8 +4,8 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src.db.helpers.session import session_helper as sh
-from src.db.models.views.url_status.core import URLStatusMatView
-from src.db.models.views.url_status.enums import URLStatusViewEnum
+from src.db.models.materialized_views.url_status.sqlalchemy import URLStatusMaterializedView
+from src.db.models.materialized_views.url_status.enums import URLStatusViewEnum
 from src.db.queries.base.builder import QueryBuilderBase
 
 
@@ -18,13 +18,13 @@ async def run(
 
         query = (
             select(
-                URLStatusMatView.status,
+                URLStatusMaterializedView.status,
                 func.count(
-                    URLStatusMatView.url_id
+                    URLStatusMaterializedView.url_id
                 ).label("count")
             )
             .group_by(
-                URLStatusMatView.status
+                URLStatusMaterializedView.status
             )
         )
 

src/api/endpoints/metrics/urls/breakdown/query/core.py

@@ -3,12 +3,11 @@
 
 from src.api.endpoints.metrics.dtos.get.urls.breakdown.pending import GetMetricsURLsBreakdownPendingResponseInnerDTO, \
     GetMetricsURLsBreakdownPendingResponseDTO
-from src.collectors.enums import URLStatus
 from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser
-from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
-from src.db.models.impl.url.core.sqlalchemy import URL
 from src.db.models.impl.annotation.record_type.user.user import AnnotationRecordTypeUser
 from src.db.models.impl.annotation.url_type.user.sqlalchemy import AnnotationURLTypeUser
+from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
+from src.db.models.impl.url.core.sqlalchemy import URL
 from src.db.queries.base.builder import QueryBuilderBase
 
 
@@ -63,8 +62,7 @@ async def run(self, session: AsyncSession) -> GetMetricsURLsBreakdownPendingResp
                 FlagURLValidated.url_id == URL.id
             )
             .where(
-                FlagURLValidated.url_id.is_(None),
-                URL.status == URLStatus.OK
+                FlagURLValidated.url_id.is_(None)
             )
             .group_by(month)
             .order_by(month.asc())

src/api/endpoints/review/reject/query.py

@@ -4,7 +4,6 @@
 from starlette.status import HTTP_400_BAD_REQUEST
 
 from src.api.endpoints.review.enums import RejectionReason
-from src.collectors.enums import URLStatus
 from src.db.models.impl.flag.url_validated.enums import URLType
 from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
 from src.db.models.impl.url.core.sqlalchemy import URL

src/api/endpoints/submit/data_source/models/response/duplicate.py

@@ -1,11 +1,11 @@
 from pydantic import BaseModel
 
-from src.collectors.enums import URLStatus
 from src.db.models.impl.flag.url_validated.enums import URLType
+from src.db.models.materialized_views.url_status.enums import URLStatusViewEnum
 
 
 class SubmitDataSourceURLDuplicateSubmissionResponse(BaseModel):
     message: str
     url_id: int
     url_type: URLType | None
-    url_status: URLStatus
+    url_status: URLStatusViewEnum

src/api/endpoints/submit/data_source/queries/core.py

@@ -1,12 +1,9 @@
 import uuid
-from typing import Any
 
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse
 from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest
-from src.collectors.enums import URLStatus
 from src.core.enums import BatchStatus
 from src.db.models.impl.annotation.agency.anon.sqlalchemy import AnnotationAgencyAnon
 from src.db.models.impl.annotation.location.anon.sqlalchemy import AnnotationLocationAnon
@@ -44,7 +41,6 @@ async def run(
             trailing_slash=full_url.has_trailing_slash,
             name=self.request.name,
             description=self.request.description,
-            status=URLStatus.OK,
             source=URLSource.MANUAL,
         )
 

src/api/endpoints/submit/data_source/queries/duplicate.py

@@ -8,6 +8,7 @@
     SubmitDataSourceURLDuplicateSubmissionResponse
 from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated
 from src.db.models.impl.url.core.sqlalchemy import URL
+from src.db.models.materialized_views.url_status.sqlalchemy import URLStatusMaterializedView
 from src.db.queries.base.builder import QueryBuilderBase
 
 
@@ -29,13 +30,17 @@ async def run(self, session: AsyncSession) -> None:
         query = (
             select(
                 URL.id,
-                URL.status,
+                URLStatusMaterializedView.status,
                 FlagURLValidated.type
             )
             .outerjoin(
                 FlagURLValidated,
                 FlagURLValidated.url_id == URL.id
             )
+            .outerjoin(
+                URLStatusMaterializedView,
+                URLStatusMaterializedView.url_id == URL.id
+            )
             .where(
                 URL.url == self.url
             )
@@ -48,7 +53,7 @@ async def run(self, session: AsyncSession) -> None:
         model = SubmitDataSourceURLDuplicateSubmissionResponse(
             message="Duplicate URL found",
             url_id=mapping[URL.id],
-            url_status=mapping[URL.status],
+            url_status=mapping[URLStatusMaterializedView.status],
             url_type=mapping[FlagURLValidated.type]
         )
         raise HTTPException(

src/api/endpoints/submit/url/queries/core.py

@@ -7,16 +7,15 @@
 from src.api.endpoints.submit.url.queries.convert import convert_invalid_url_to_url_response, \
     convert_duplicate_urls_to_url_response
 from src.api.endpoints.submit.url.queries.dedupe import DeduplicateURLQueryBuilder
-from src.collectors.enums import URLStatus
 from src.db.models.impl.annotation.agency.user.sqlalchemy import AnnotationAgencyUser
 from src.db.models.impl.annotation.location.user.sqlalchemy import AnnotationLocationUser
 from src.db.models.impl.annotation.name.suggestion.enums import NameSuggestionSource
 from src.db.models.impl.annotation.name.suggestion.sqlalchemy import AnnotationNameSuggestion
 from src.db.models.impl.annotation.name.user.sqlalchemy import AnnotationNameUserEndorsement
+from src.db.models.impl.annotation.record_type.user.user import AnnotationRecordTypeUser
 from src.db.models.impl.link.user_suggestion_not_found.users_submitted_url.sqlalchemy import LinkUserSubmittedURL
 from src.db.models.impl.url.core.enums import URLSource
 from src.db.models.impl.url.core.sqlalchemy import URL
-from src.db.models.impl.annotation.record_type.user.user import AnnotationRecordTypeUser
 from src.db.queries.base.builder import QueryBuilderBase
 from src.util.models.url_and_scheme import URLAndScheme
 from src.util.url import clean_url, get_url_and_scheme, is_valid_url
@@ -61,7 +60,6 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse:
             url=url_and_scheme.url,
             scheme=url_and_scheme.scheme,
             source=URLSource.MANUAL,
-            status=URLStatus.OK,
             description=self.request.description,
             trailing_slash=url_and_scheme.url.endswith('/'),
         )