Dev to Main December 2025

DESIGN.md

@@ -49,12 +49,13 @@ Previously, SQLAlchemy CTEs and Subqueries were created and referenced via `.c.[
 Instead, CTEs and Subqueries should be wrapped in classes positioned in separate `cte.py` files or `cte` subdirectories, where individual columns can be referenced via type-hinted properties that internally reference the `CTE` or `Subquery` attributes via `.c` access. These can then be imported into relevant files.
 
 Example:
+
 ```python
 from typing import final
 
 from sqlalchemy import select, func
 
-from db.models.implementations import LinkAgencyDataSource
+from db.models.implementations.links.agency__data_source import LinkAgencyDataSource
 
 
 @final
@@ -65,11 +66,11 @@ class AgencyIdsCTE:
             func.unnest(LinkAgencyDataSource.agency_id).label("agency_ids"),
             LinkAgencyDataSource.data_source_id,
         ).cte(name="agency_ids")
-        
+
     @property
     def agency_ids(self) -> list[int]:
         return self.query.c.agency_ids
-    
+
     @property
     def data_source_id(self) -> int:
         return self.query.c.data_source_id

alembic/versions/2025_10_09_1646-8c3153d94dfb_remove_recent_searches_table_constraint.py

@@ -0,0 +1,30 @@
+"""Remove recent searches table constraint
+
+Revision ID: 8c3153d94dfb
+Revises: 2f8bd4749166
+Create Date: 2025-10-09 16:46:49.858892
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "8c3153d94dfb"
+down_revision: Union[str, None] = "2f8bd4749166"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "recent_searches",
+        column_name="location_id",
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    pass

alembic/versions/2025_10_12_0857-e51211c51b29_remove_approval_status_for_data_sources_.py

@@ -0,0 +1,126 @@
+"""Remove approval status for data sources and agencies
+
+Revision ID: e51211c51b29
+Revises: 8c3153d94dfb
+Create Date: 2025-10-12 08:57:00.734088
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "e51211c51b29"
+down_revision: Union[str, None] = "8c3153d94dfb"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _delete_agencies_and_data_sources_without_approval():
+    op.execute("""DELETE FROM data_sources WHERE approval_status != 'approved'""")
+    op.execute("""DELETE FROM agencies WHERE approval_status != 'approved'""")
+
+
+def upgrade() -> None:
+    _delete_agencies_and_data_sources_without_approval()
+    _drop_views()
+    _drop_columns()
+    _rebuild_views()
+    op.execute("drop trigger update_approval_status_updated_at on data_sources;")
+    op.execute("drop function update_approval_status_updated_at;")
+
+
+def _rebuild_views():
+    op.execute("""
+    create materialized view distinct_source_urls as
+    SELECT DISTINCT rtrim(ltrim(ltrim(ltrim(data_sources.source_url::text, 'https://'::text), 'http://'::text),
+                                'www.'::text), '/'::text) AS base_url,
+                    data_sources.source_url               AS original_url
+    FROM data_sources
+    WHERE data_sources.source_url IS NOT NULL;
+    """)
+    op.execute("""
+    create materialized view typeahead_agencies as
+    SELECT
+        a.id,
+        a.name,
+        a.jurisdiction_type,
+        l.state_iso,
+        l.locality_name AS municipality,
+        l.county_name
+    FROM
+        agencies a
+            LEFT JOIN link_agencies_locations lal ON lal.agency_id = a.id
+            LEFT JOIN locations_expanded l ON lal.location_id = l.id
+    """)
+    op.execute("""
+    create view data_sources_expanded
+            (name, description, source_url, agency_supplied, supplying_entity, agency_originated, agency_aggregation,
+             coverage_start, coverage_end, updated_at, detail_level, data_portal_type, update_method, readme_url,
+             originating_entity, retention_schedule, id, scraper_url, created_at, submission_notes, 
+             agency_described_not_in_database, data_portal_type_other,
+             data_source_request, broken_source_url_as_of, access_notes, url_status, record_type_id,
+             record_type_name, access_types, tags, record_formats)
+    as
+    SELECT
+        ds.name,
+        ds.description,
+        ds.source_url,
+        ds.agency_supplied,
+        ds.supplying_entity,
+        ds.agency_originated,
+        ds.agency_aggregation,
+        ds.coverage_start,
+        ds.coverage_end,
+        ds.updated_at,
+        ds.detail_level,
+        ds.data_portal_type,
+        ds.update_method,
+        ds.readme_url,
+        ds.originating_entity,
+        ds.retention_schedule,
+        ds.id,
+        ds.scraper_url,
+        ds.created_at,
+        ds.submission_notes,
+        ds.agency_described_not_in_database,
+        ds.data_portal_type_other,
+        ds.data_source_request,
+        ds.broken_source_url_as_of,
+        ds.access_notes,
+        ds.url_status,
+        ds.record_type_id,
+        rt.name AS record_type_name,
+        ds.access_types,
+        ds.tags,
+        ds.record_formats
+    FROM
+        data_sources ds
+        LEFT JOIN record_types rt
+                  ON ds.record_type_id = rt.id
+
+    """)
+
+
+def _drop_views():
+    op.execute("""drop view data_sources_expanded""")
+    op.execute("""drop materialized view distinct_source_urls""")
+    op.execute("""drop materialized view typeahead_agencies""")
+
+
+def _drop_columns():
+    op.drop_column(table_name="data_sources", column_name="approval_status")
+    op.drop_column(table_name="data_sources", column_name="last_approval_editor")
+    op.drop_column(table_name="data_sources", column_name="submitter_contact_info")
+    op.drop_column(table_name="data_sources", column_name="approval_status_updated_at")
+    op.drop_column(table_name="agencies", column_name="approval_status")
+    op.drop_column(table_name="agencies", column_name="last_approval_editor")
+    op.drop_column(table_name="agencies", column_name="creator_user_id")
+    op.drop_column(table_name="agencies", column_name="rejection_reason")
+    op.drop_column(table_name="agencies", column_name="submitter_contact")
+
+
+def downgrade() -> None:
+    pass

alembic/versions/2025_10_21_0955-a41df84338bb_remove_unneeded_agencies_columns.py

@@ -0,0 +1,60 @@
+"""Remove unneeded agencies columns
+
+Revision ID: a41df84338bb
+Revises: e51211c51b29
+Create Date: 2025-10-21 09:55:20.961243
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "a41df84338bb"
+down_revision: Union[str, None] = "e51211c51b29"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+AGENCIES_TABLE_NAME: str = "agencies"
+
+
+def upgrade() -> None:
+    op.alter_column(
+        table_name=AGENCIES_TABLE_NAME,
+        column_name="agency_created",
+        new_column_name="created_at",
+    )
+    op.drop_column(AGENCIES_TABLE_NAME, "multi_agency")
+    op.drop_column(AGENCIES_TABLE_NAME, "airtable_agency_last_modified")
+    op.drop_column(AGENCIES_TABLE_NAME, "airtable_uid")
+    op.execute("""
+    DROP TRIGGER IF EXISTS set_agency_updated_at ON public.agencies
+    """)
+    op.execute("""
+    DROP FUNCTION IF EXISTS update_airtable_agency_last_modified_column
+    """)
+
+    op.execute("""
+    CREATE OR REPLACE FUNCTION update_agency_updated_at_column()
+
+    RETURNS TRIGGER language plpgsql AS $$
+    BEGIN
+        NEW.updated_at = NOW();
+        RETURN NEW;
+    END;
+    $$
+    """)
+
+    op.execute("""
+    CREATE TRIGGER set_agency_updated_at
+    BEFORE UPDATE ON public.agencies
+    FOR EACH ROW
+    WHEN (OLD.* IS DISTINCT FROM NEW.*)
+    EXECUTE FUNCTION update_agency_updated_at_column()
+    """)
+
+
+def downgrade() -> None:
+    pass

alembic/versions/2025_10_21_1133-a1a8d636f4dd_clean_up_data_sources_table_structure.py

@@ -0,0 +1,97 @@
+"""Clean up data sources table structure
+
+Revision ID: a1a8d636f4dd
+Revises: a41df84338bb
+Create Date: 2025-10-21 11:33:46.002274
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "a1a8d636f4dd"
+down_revision: Union[str, None] = "a41df84338bb"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+DATA_SOURCES_TABLE_NAME: str = "data_sources"
+
+
+def upgrade() -> None:
+    op.execute("""
+    drop view data_sources_expanded
+    """)
+
+    op.alter_column(
+        table_name=DATA_SOURCES_TABLE_NAME, column_name="record_type_id", nullable=True
+    )
+    # Drop columns
+    op.drop_column(
+        table_name=DATA_SOURCES_TABLE_NAME,
+        column_name="airtable_uid",
+    )
+    op.drop_column(
+        table_name=DATA_SOURCES_TABLE_NAME,
+        column_name="tags",
+    )
+    op.drop_column(
+        table_name=DATA_SOURCES_TABLE_NAME, column_name="broken_source_url_as_of"
+    )
+    op.drop_column(
+        table_name=DATA_SOURCES_TABLE_NAME,
+        column_name="record_download_option_provided",
+    )
+    op.drop_column(
+        table_name=DATA_SOURCES_TABLE_NAME, column_name="data_source_request"
+    )
+    op.drop_column(table_name=DATA_SOURCES_TABLE_NAME, column_name="submission_notes")
+
+    op.execute("""
+    create view data_sources_expanded
+            (name, description, source_url, agency_supplied, supplying_entity, agency_originated, agency_aggregation,
+             coverage_start, coverage_end, updated_at, detail_level, data_portal_type, update_method, readme_url,
+             originating_entity, retention_schedule, id, scraper_url, created_at, 
+             agency_described_not_in_database, data_portal_type_other,
+             access_notes, url_status, record_type_id, record_type_name, access_types, record_formats)
+    as
+    SELECT
+        ds.name,
+        ds.description,
+        ds.source_url,
+        ds.agency_supplied,
+        ds.supplying_entity,
+        ds.agency_originated,
+        ds.agency_aggregation,
+        ds.coverage_start,
+        ds.coverage_end,
+        ds.updated_at,
+        ds.detail_level,
+        ds.data_portal_type,
+        ds.update_method,
+        ds.readme_url,
+        ds.originating_entity,
+        ds.retention_schedule,
+        ds.id,
+        ds.scraper_url,
+        ds.created_at,
+        ds.agency_described_not_in_database,
+        ds.data_portal_type_other,
+        ds.access_notes,
+        ds.url_status,
+        ds.record_type_id,
+        rt.name AS record_type_name,
+        ds.access_types,
+        ds.record_formats
+    FROM
+        data_sources ds
+        LEFT JOIN record_types rt
+                  ON ds.record_type_id = rt.id
+
+    """)
+
+
+def downgrade() -> None:
+    pass