feat(app): allow retrieving URLs for annotation without html info

collector_db/AsyncDatabaseClient.py

@@ -129,7 +129,6 @@ async def get_next_url_for_user_annotation(
                 URL,
             )
             .where(URL.outcome == URLStatus.PENDING.value)
-            .where(exists(select(URLHTMLContent).where(URLHTMLContent.url_id == URL.id)))
             # URL must not have metadata annotation by this user
             .where(
                 not_(

tests/conftest.py

@@ -1,11 +1,13 @@
 import pytest
 from alembic import command
 from alembic.config import Config
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect, MetaData
+from sqlalchemy.orm import scoped_session, sessionmaker
 
 from collector_db.DatabaseClient import DatabaseClient
 from collector_db.helper_functions import get_postgres_connection_string
 from collector_db.models import Base
+from tests.helpers.AlembicRunner import AlembicRunner
 from tests.helpers.DBDataCreator import DBDataCreator
 
 
@@ -19,7 +21,22 @@
         "sqlalchemy.url",
         get_postgres_connection_string()
     )
-    command.upgrade(alembic_cfg, "head")
+    live_connection = engine.connect()
+    runner = AlembicRunner(
+        alembic_config=alembic_cfg,
+        inspector=inspect(live_connection),
+        metadata=MetaData(),
+        connection=live_connection,
+        session=scoped_session(sessionmaker(bind=live_connection)),
+    )
+    try:
+        runner.upgrade("head")
+    except Exception as e:
+        runner.reset_schema()
+        runner.stamp("base")
+        runner.upgrade("head")
+
+    live_connection.close()
     engine.dispose()
     yield
 

tests/helpers/complex_test_data_functions.py

@@ -36,11 +36,12 @@ class AnnotateAgencySetupInfo(BaseModel):
 async def setup_for_annotate_agency(
         db_data_creator: DBDataCreator,
         url_count: int,
-        suggestion_type: SuggestionType = SuggestionType.UNKNOWN
+        suggestion_type: SuggestionType = SuggestionType.UNKNOWN,
+        with_html_content: bool = True
 ):
     buci: BatchURLCreationInfo = await db_data_creator.batch_and_urls(
         url_count=url_count,
-        with_html_content=True
+        with_html_content=with_html_content
     )
     await db_data_creator.auto_suggestions(
         url_ids=buci.url_ids,

tests/test_alembic/conftest.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import scoped_session, sessionmaker
 
 from collector_db.helper_functions import get_postgres_connection_string
-from tests.test_alembic.AlembicRunner import AlembicRunner
+from tests.helpers.AlembicRunner import AlembicRunner
 
 
 @pytest.fixture()

tests/test_alembic/helpers.py

@@ -3,7 +3,7 @@
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from tests.test_alembic.AlembicRunner import AlembicRunner      
+from tests.helpers.AlembicRunner import AlembicRunner
 
 
 def get_enum_values(enum_name: str, session: Session) -> list[str]:

tests/test_automated/integration/api/test_annotate.py

@@ -28,8 +28,12 @@
 def check_html_info_not_empty(
     html_info: ResponseHTMLInfo
 ):
-    assert html_info.description != ""
-    assert html_info.title != ""
+    assert not html_info_empty(html_info)
+
+def html_info_empty(
+    html_info: ResponseHTMLInfo
+) -> bool:
+    return html_info.description == "" and html_info.title == ""
 
 @pytest.mark.asyncio
 async def test_annotate_relevancy(api_test_helper):
@@ -123,6 +127,36 @@
             assert results[0].relevant is True
 
 
+@pytest.mark.asyncio
+async def test_annotate_relevancy_no_html(api_test_helper):
+    ath = api_test_helper
+
+    batch_id = ath.db_data_creator.batch()
+
+    # Create 2 URLs with outcome `pending`
+    iui: InsertURLsInfo = ath.db_data_creator.urls(batch_id=batch_id, url_count=2)
+
+    url_1 = iui.url_mappings[0]
+    url_2 = iui.url_mappings[1]
+
+    # Add `Relevancy` attribute with value `True` to 1st URL
+    await ath.db_data_creator.auto_relevant_suggestions(
+        url_id=url_1.url_id,
+        relevant=True
+    )
+
+    # Add 'Relevancy' attribute with value `False` to 2nd URL
+    await ath.db_data_creator.auto_relevant_suggestions(
+        url_id=url_2.url_id,
+        relevant=False
+    )
+
+    # Call `GET` `/annotate/relevance` and receive next URL
+    request_info_1: GetNextRelevanceAnnotationResponseOuterInfo = api_test_helper.request_validator.get_next_relevance_annotation()
+    inner_info_1 = request_info_1.next_annotation
+
+    check_url_mappings_match(inner_info_1.url_info, url_1)
+    assert html_info_empty(inner_info_1.html_info)
 
 @pytest.mark.asyncio
 async def test_annotate_record_type(api_test_helper):
@@ -213,6 +247,36 @@
         if result.url_id == inner_info_1.url_info.url_id:
             assert result.record_type == RecordType.BOOKING_REPORTS.value
 
+@pytest.mark.asyncio
+async def test_annotate_record_type_no_html_info(api_test_helper):
+    ath = api_test_helper
+
+    batch_id = ath.db_data_creator.batch()
+
+    # Create 2 URLs with outcome `pending`
+    iui: InsertURLsInfo = ath.db_data_creator.urls(batch_id=batch_id, url_count=2)
+
+    url_1 = iui.url_mappings[0]
+    url_2 = iui.url_mappings[1]
+
+    # Add record type attribute with value `Accident Reports` to 1st URL
+    await ath.db_data_creator.auto_record_type_suggestions(
+        url_id=url_1.url_id,
+        record_type=RecordType.ACCIDENT_REPORTS
+    )
+
+    # Add 'Record Type' attribute with value `Dispatch Recordings` to 2nd URL
+    await ath.db_data_creator.auto_record_type_suggestions(
+        url_id=url_2.url_id,
+        record_type=RecordType.DISPATCH_RECORDINGS
+    )
+
+    # Call `GET` `/annotate/record-type` and receive next URL
+    request_info_1: GetNextRecordTypeAnnotationResponseOuterInfo = api_test_helper.request_validator.get_next_record_type_annotation()
+    inner_info_1 = request_info_1.next_annotation
+
+    check_url_mappings_match(inner_info_1.url_info, url_1)
+    assert html_info_empty(inner_info_1.html_info)
 
 @pytest.mark.asyncio
 async def test_annotate_agency_multiple_auto_suggestions(api_test_helper):
@@ -256,6 +320,36 @@
         assert agency_suggestion.locality is not None
 
 
+@pytest.mark.asyncio
+async def test_annotate_agency_multiple_auto_suggestions_no_html(api_test_helper):
+    """
+    Test Scenario: Multiple Auto Suggestions
+    A URL has multiple Agency Auto Suggestion and has not been annotated by the User
+    The user should receive all of the auto suggestions with full detail
+    """
+    ath = api_test_helper
+    buci: BatchURLCreationInfo = await ath.db_data_creator.batch_and_urls(
+        url_count=1,
+        with_html_content=False
+    )
+    await ath.db_data_creator.auto_suggestions(
+        url_ids=buci.url_ids,
+        num_suggestions=2,
+        suggestion_type=SuggestionType.AUTO_SUGGESTION
+    )
+
+    # User requests next annotation
+    response = await ath.request_validator.get_next_agency_annotation()
+
+    assert response.next_annotation
+    next_annotation = response.next_annotation
+    # Check that url_id matches the one we inserted
+    assert next_annotation.url_id == buci.url_ids[0]
+
+    # Check that html data is not present
+    assert next_annotation.html_info.description == ""
+    assert next_annotation.html_info.title == ""
+
 @pytest.mark.asyncio
 async def test_annotate_agency_single_unknown_auto_suggestion(api_test_helper):
     """