ff-171 simplifying rules creation (#306)

changes/unreleased.md

@@ -1,3 +1,11 @@
 
 - ff-174 — Custom API entry points for OpenAI and Gemini clients.
 - ff-175 — Export feeds to OPML file.
+- ff-171 — Simplified rules creation:
+  - The rules creation form was moved to the tags filter.
+  - Rules now allow specifying excluded tags (will not apply if the news has any of them).
+  - Tags under the news caption now behave as tags in the tags filter.
+  - Button `[more]` on the tags line under the news caption now opens the whole news.
+  - Openned news items now always show all tags.
+  - Only a single news item now can be opened.
+  - Performance of the News page improved 2-3 times.

ffun/ffun/api/entities.py

@@ -1,6 +1,5 @@
 import datetime
 import enum
-import uuid
 from decimal import Decimal
 from typing import Any, Iterable
 
@@ -10,7 +9,7 @@
 from ffun.api import front_events
 from ffun.core import api
 from ffun.core.entities import BaseEntity
-from ffun.domain.entities import AbsoluteUrl, EntryId, FeedId, FeedUrl, UnknownUrl, UserId
+from ffun.domain.entities import AbsoluteUrl, EntryId, FeedId, FeedUrl, RuleId, UnknownUrl, UserId
 from ffun.feeds import entities as f_entities
 from ffun.feeds_collections import entities as fc_entities
 from ffun.feeds_links import entities as fl_entities
@@ -103,8 +102,9 @@ def from_internal(  # noqa: CFQ002
 
 
 class Rule(BaseEntity):
-    id: uuid.UUID
-    tags: list[str]
+    id: RuleId
+    requiredTags: list[str]
+    excludedTags: list[str]
     score: int
     createdAt: datetime.datetime
     updatedAt: datetime.datetime
@@ -113,7 +113,8 @@ class Rule(BaseEntity):
     def from_internal(cls, rule: s_entities.Rule, tags_mapping: dict[int, str]) -> "Rule":
         return cls(
             id=rule.id,
-            tags=[tags_mapping[tag_id] for tag_id in rule.tags],
+            requiredTags=[tags_mapping[tag_id] for tag_id in rule.required_tags],
+            excludedTags=[tags_mapping[tag_id] for tag_id in rule.excluded_tags],
             score=rule.score,
             createdAt=rule.created_at,
             updatedAt=rule.updated_at,
@@ -319,7 +320,8 @@ class GetEntriesByIdsResponse(api.APISuccess):
 
 
 class CreateOrUpdateRuleRequest(api.APIRequest):
-    tags: list[str]
+    requiredTags: list[str]
+    excludedTags: list[str]
     score: int
 
 
@@ -328,16 +330,17 @@ class CreateOrUpdateRuleResponse(api.APISuccess):
 
 
 class DeleteRuleRequest(api.APIRequest):
-    id: uuid.UUID
+    id: RuleId
 
 
 class DeleteRuleResponse(api.APISuccess):
     pass
 
 
 class UpdateRuleRequest(api.APIRequest):
-    id: uuid.UUID
-    tags: list[str]
+    id: RuleId
+    requiredTags: list[str]
+    excludedTags: list[str]
     score: int
 
 

ffun/ffun/api/http_handlers.py

@@ -148,9 +148,15 @@ async def api_get_entries_by_ids(
 async def api_create_or_update_rule(
     request: entities.CreateOrUpdateRuleRequest, user: User
 ) -> entities.CreateOrUpdateRuleResponse:
-    tags_ids = await o_domain.get_ids_by_uids(request.tags)
-
-    await s_domain.create_or_update_rule(user_id=user.id, tags=set(tags_ids.values()), score=request.score)
+    required_tags_ids = await o_domain.get_ids_by_uids(request.requiredTags)
+    excluded_tags_ids = await o_domain.get_ids_by_uids(request.excludedTags)
+
+    await s_domain.create_or_update_rule(
+        user_id=user.id,
+        score=request.score,
+        required_tags=required_tags_ids.values(),
+        excluded_tags=excluded_tags_ids.values(),
+    )
 
     return entities.CreateOrUpdateRuleResponse()
 
@@ -164,9 +170,16 @@ async def api_delete_rule(request: entities.DeleteRuleRequest, user: User) -> en
 
 @router.post("/api/update-rule")
 async def api_update_rule(request: entities.UpdateRuleRequest, user: User) -> entities.UpdateRuleResponse:
-    tags_ids = await o_domain.get_ids_by_uids(request.tags)
-
-    await s_domain.update_rule(user_id=user.id, rule_id=request.id, score=request.score, tags=tags_ids.values())
+    required_tags_ids = await o_domain.get_ids_by_uids(request.requiredTags)
+    excluded_tags_ids = await o_domain.get_ids_by_uids(request.excludedTags)
+
+    await s_domain.update_rule(
+        user_id=user.id,
+        rule_id=request.id,
+        score=request.score,
+        required_tags=required_tags_ids.values(),
+        excluded_tags=excluded_tags_ids.values(),
+    )
 
     return entities.UpdateRuleResponse()
 
@@ -175,7 +188,8 @@ async def _prepare_rules(rules: Iterable[s_entities.Rule]) -> list[entities.Rule
     all_tags = set()
 
     for rule in rules:
-        all_tags.update(rule.tags)
+        all_tags.update(rule.required_tags)
+        all_tags.update(rule.excluded_tags)
 
     tags_mapping = await o_domain.get_tags_by_ids(all_tags)
 

ffun/ffun/cli/application.py

@@ -2,6 +2,7 @@
 
 from ffun.cli.commands import cleaner  # noqa: F401
 from ffun.cli.commands import experiments  # noqa: F401
+from ffun.cli.commands import fixtures  # noqa: F401
 from ffun.cli.commands import metrics  # noqa: F401
 from ffun.cli.commands import processors_quality  # noqa: F401
 from ffun.cli.commands import profile  # noqa: F401
@@ -19,6 +20,7 @@
 app.add_typer(metrics.cli_app, name="metrics")
 app.add_typer(profile.cli_app, name="profile")
 app.add_typer(experiments.cli_app, name="experiments")
+app.add_typer(fixtures.cli_app, name="fixtures")
 
 
 if __name__ == "__main__":

ffun/ffun/cli/commands/fixtures.py

@@ -0,0 +1,142 @@
+"""
+Development utilities for feeds.
+
+This code is intended to be used by developers to simplify our work, therefore:
+
+- It MUST NOT be used in production code.
+- It MUST NOT be used in tests.
+- no tests are required for this code.
+"""
+
+import asyncio
+import uuid
+
+import typer
+
+from ffun.application.application import with_app
+from ffun.auth.settings import settings as a_settings
+from ffun.core import logging, utils
+from ffun.domain.domain import new_entry_id, new_feed_id
+from ffun.domain.entities import UnknownUrl, UserId
+from ffun.domain.urls import adjust_classic_relative_url, str_to_feed_url, url_to_source_uid
+from ffun.feeds.domain import get_feeds, get_source_ids, save_feed
+from ffun.feeds.entities import Feed, FeedState
+from ffun.feeds_links.domain import add_link
+from ffun.library.domain import catalog_entries, get_entries_by_ids
+from ffun.library.entities import Entry
+from ffun.ontology.domain import apply_tags_to_entry
+from ffun.ontology.entities import ProcessorTag
+from ffun.users import domain as u_domain
+from ffun.users import entities as u_entities
+
+logger = logging.get_module_logger()
+
+cli_app = typer.Typer()
+
+
+async def fake_feed() -> Feed:
+
+    _id = uuid.uuid4().hex
+
+    url = str_to_feed_url(f"https://{_id}.com")
+
+    source_uid = url_to_source_uid(url)
+
+    source_ids = await get_source_ids([source_uid])
+
+    timestamp = utils.now()
+
+    feed = Feed(
+        id=new_feed_id(),
+        source_id=source_ids[source_uid],
+        url=url,
+        state=FeedState.loaded,
+        last_error=None,
+        load_attempted_at=timestamp,
+        loaded_at=timestamp,
+        title=f"Title {_id}",
+        description=f"Description {_id}",
+    )
+
+    feed_id = await save_feed(feed)
+
+    feeds = await get_feeds([feed_id])
+
+    return feeds[0]
+
+
+async def fake_entry(feed: Feed) -> Entry:
+    _id = uuid.uuid4().hex
+
+    timestamp = utils.now()
+
+    url = adjust_classic_relative_url(UnknownUrl(f"enrty-{_id}"), feed.url)
+
+    assert url is not None
+
+    entry = Entry(
+        id=new_entry_id(),
+        source_id=feed.source_id,
+        title=f"Title {_id}",
+        body=f"Body {_id}",
+        external_id=uuid.uuid4().hex,
+        external_url=url,
+        external_tags=set(),
+        published_at=timestamp,
+        cataloged_at=timestamp,
+    )
+
+    await catalog_entries(feed.id, [entry])
+
+    entries = await get_entries_by_ids([entry.id])
+
+    assert entry is not None
+
+    returned_entry = entries[entry.id]
+
+    assert returned_entry is not None
+
+    return returned_entry
+
+
+async def run_fill_db(feeds_number: int, entries_per_feed: int, tags_per_entry: int) -> None:
+    async with with_app():
+        external_user_id = a_settings.single_user.external_id
+
+        internal_user_id = await u_domain.get_or_create_user_id(u_entities.Service.single, external_user_id)
+
+        for _ in range(feeds_number):
+            feed = await fake_feed()
+
+            await add_link(internal_user_id, feed.id)
+
+            entries = [await fake_entry(feed) for _ in range(entries_per_feed)]
+
+            for i, entry in enumerate(entries, start=1):
+
+                tags = []
+
+                for j, _ in enumerate(range(tags_per_entry), start=1):
+                    raw_uid = f"some-long-tag-name-{j}-{i % j}"
+                    tags.append(ProcessorTag(raw_uid=raw_uid))
+
+                await apply_tags_to_entry(entry.id, 100500, tags)
+
+
+@cli_app.command()
+def fill_db(feeds_number: int = 10, entries_per_feed: int = 100, tags_per_entry: int = 25) -> None:
+    asyncio.run(run_fill_db(feeds_number, entries_per_feed, tags_per_entry))
+
+
+async def run_supertokens_user_to_dev(intenal_user_id: UserId) -> None:
+    async with with_app():
+        external_user_id = a_settings.single_user.external_id
+
+        to_user_id = await u_domain.get_or_create_user_id(u_entities.Service.single, external_user_id)
+
+        await u_domain.tech_move_user(from_user_id=intenal_user_id, to_user_id=to_user_id)
+
+
+@cli_app.command()
+def supertokens_user_to_dev(intenal_user_id: str) -> None:
+    asyncio.run(run_supertokens_user_to_dev(UserId(uuid.UUID(intenal_user_id))))

ffun/ffun/domain/domain.py

@@ -1,6 +1,6 @@
 import uuid
 
-from ffun.domain.entities import CollectionId, EntryId, FeedId, SourceId, UserId
+from ffun.domain.entities import CollectionId, EntryId, FeedId, RuleId, SourceId, UserId
 
 
 def new_user_id() -> UserId:
@@ -21,3 +21,7 @@ def new_collection_id() -> CollectionId:
 
 def new_source_id() -> SourceId:
     return SourceId(uuid.uuid4())
+
+
+def new_rule_id() -> RuleId:
+    return RuleId(uuid.uuid4())

ffun/ffun/domain/entities.py

@@ -6,6 +6,7 @@
 FeedId = NewType("FeedId", uuid.UUID)
 CollectionId = NewType("CollectionId", uuid.UUID)
 SourceId = NewType("SourceId", uuid.UUID)
+RuleId = NewType("RuleId", uuid.UUID)
 
 # URL types for better normalization control in code
 # conversion schemas:

ffun/ffun/library/migrations/20240928_01_fIeAw-feeds-to-entries-many-to-many.py

@@ -83,43 +83,26 @@
 """
 
 
-# TODO: remove prints after migration applied to prod
 def apply_step(conn: Connection[dict[str, Any]]) -> None:
     cursor = conn.cursor(row_factory=dict_row)
 
-    print("Creating l_feeds_to_entries table")  # noqa
-
     cursor.execute(sql_create_feeds_to_entries_table)
 
-    print("Creating l_orphaned_entries table")  # noqa
-
     cursor.execute(sql_create_orphaned_entries_table)
 
-    print("Filling l_feeds_to_entries table")  # noqa
-
     cursor.execute(sql_fill_from_entries_table)
 
-    print("Filling l_orphaned_entries table")  # noqa
-
     cursor.execute(sql_fill_orphaned_entries)
 
     cursor.execute("CREATE INDEX l_feeds_to_entries_entry_id_idx ON l_feeds_to_entries (entry_id)")
 
-    print("Filling duplicated entries")  # noqa
-
     cursor.execute(sql_fill_duplicated_entries)
 
-    print("Removing duplicated entries")  # noqa
-
     cursor.execute(sql_remove_duplicated_entries)
 
-    print("Creating unique index on l_entries")  # noqa
-
     # external_id goes first to optimize queries
     cursor.execute("CREATE UNIQUE INDEX l_entries_source_id_external_id_idx ON l_entries (external_id, source_id)")
 
-    print("Removing feed_id column from l_entries")  # noqa
-
     cursor.execute("ALTER TABLE l_entries DROP COLUMN feed_id")
 
     cursor.execute(
@@ -129,8 +112,6 @@ def apply_step(conn: Connection[dict[str, Any]]) -> None:
 
     cursor.execute("CREATE INDEX l_entries_created_at_idx ON l_entries (created_at ASC)")
 
-    print("Completed")  # noqa
-
 
 def rollback_step(conn: Connection[dict[str, Any]]) -> None:
     cursor = conn.cursor()

ffun/ffun/ontology/tests/fixtures.py

@@ -25,3 +25,25 @@ def three_tags_ids(three_tags_by_uids: dict[str, int]) -> tuple[int, int, int]:
 @pytest.fixture
 def three_processor_tags(three_tags_by_ids: dict[int, str]) -> tuple[ProcessorTag, ProcessorTag, ProcessorTag]:
     return tuple(ProcessorTag(raw_uid=tag) for tag in three_tags_by_ids.values())  # type: ignore
+
+
+@pytest_asyncio.fixture
+async def five_tags_by_uids() -> dict[str, int]:
+    return await get_ids_by_uids([uuid.uuid4().hex for _ in range(5)])
+
+
+@pytest.fixture
+def five_tags_by_ids(five_tags_by_uids: dict[str, int]) -> dict[int, str]:
+    return {v: k for k, v in five_tags_by_uids.items()}
+
+
+@pytest.fixture
+def five_tags_ids(five_tags_by_uids: dict[str, int]) -> tuple[int, int, int, int, int]:
+    return tuple(five_tags_by_uids.values())  # type: ignore
+
+
+@pytest.fixture
+def five_processor_tags(
+    five_tags_by_ids: dict[int, str]
+) -> tuple[ProcessorTag, ProcessorTag, ProcessorTag, ProcessorTag, ProcessorTag]:
+    return tuple(ProcessorTag(raw_uid=tag) for tag in five_tags_by_ids.values())  # type: ignore