Add Notion import

env.d/development/common

@@ -66,3 +66,6 @@ COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/
 DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token
 Y_PROVIDER_API_BASE_URL=http://y-provider-development:4444/api/
 Y_PROVIDER_API_KEY=yprovider-api-key
+
+# Frontend
+FRONTEND_URL=http://localhost:3000

src/backend/core/api/viewsets.py

@@ -18,6 +18,7 @@
 from django.db.models.expressions import RawSQL
 from django.db.models.functions import Left, Length
 from django.http import Http404, StreamingHttpResponse
+from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils.functional import cached_property
 from django.utils.text import capfirst, slugify
@@ -37,9 +38,18 @@
 from core import authentication, choices, enums, models
 from core.services.ai_services import AIService
 from core.services.collaboration_services import CollaborationService
+from core.services.converter_services import YdocConverter
+from core.services.notion_import import (
+    ImportedDocument,
+    build_notion_session,
+    fetch_all_pages,
+    import_page,
+    link_child_page_to_parent,
+)
 from core.tasks.mail import send_ask_for_access_mail
 from core.utils import extract_attachments, filter_descendants
 
+from ..notion_schemas.notion_page import NotionPage
 from . import permissions, serializers, utils
 from .filters import DocumentFilter, ListDocumentFilter
 
@@ -2072,3 +2082,174 @@ def _load_theme_customization(self):
             )
 
         return theme_customization
+
+
+@drf.decorators.api_view()
+def notion_import_redirect(request):
+    query = urlencode(
+        {
+            "client_id": settings.NOTION_CLIENT_ID,
+            "response_type": "code",
+            "owner": "user",
+            "redirect_uri": settings.NOTION_REDIRECT_URI,
+        }
+    )
+    return redirect("https://api.notion.com/v1/oauth/authorize?" + query)
+
+
+@drf.decorators.api_view()
+def notion_import_callback(request):
+    code = request.GET.get("code")
+    resp = requests.post(
+        "https://api.notion.com/v1/oauth/token",
+        auth=requests.auth.HTTPBasicAuth(
+            settings.NOTION_CLIENT_ID, settings.NOTION_CLIENT_SECRET
+        ),
+        headers={"Accept": "application/json"},
+        data={
+            "grant_type": "authorization_code",
+            "code": code,
+            "redirect_uri": settings.NOTION_REDIRECT_URI,
+        },
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    request.session["notion_token"] = data["access_token"]
+    return redirect(f"{settings.FRONTEND_URL}/import-notion/")
+
+
+def _import_notion_doc_content(imported_doc, obj, user):
+    for att in imported_doc.attachments:
+        extra_args = {
+            "Metadata": {
+                "owner": str(user.id),
+                "status": enums.DocumentAttachmentStatus.READY,  # TODO
+            },
+        }
+        file_id = uuid.uuid4()
+        key = f"{obj.key_base}/{enums.ATTACHMENTS_FOLDER:s}/{file_id!s}.raw"
+        with requests.get(att.file.file["url"], stream=True) as resp:
+            default_storage.connection.meta.client.upload_fileobj(
+                resp.raw, default_storage.bucket_name, key
+            )
+        obj.attachments.append(key)
+        att.block["props"]["url"] = (
+            f"{settings.MEDIA_BASE_URL}{settings.MEDIA_URL}{key}"
+        )
+
+    obj.content = YdocConverter().convert_blocks(imported_doc.blocks)
+    obj.save()
+
+
+def _import_notion_child_page(imported_doc, parent_doc, user, imported_ids):
+    obj = parent_doc.add_child(
+        creator=user,
+        title=imported_doc.page.get_title() or "Child page",
+    )
+
+    models.DocumentAccess.objects.create(
+        document=obj,
+        user=user,
+        role=models.RoleChoices.OWNER,
+    )
+
+    _import_notion_doc_content(imported_doc, obj, user)
+
+    imported_ids.append(imported_doc.page.id)
+
+    for child in imported_doc.children:
+        _import_notion_child_page(child, obj, user, imported_ids)
+
+
+def _import_notion_root_page(imported_doc, user) -> list[str]:
+    obj = models.Document.add_root(
+        depth=1,
+        creator=user,
+        title=imported_doc.page.get_title() or "Imported Notion page",
+        link_reach=models.LinkReachChoices.RESTRICTED,
+    )
+
+    models.DocumentAccess.objects.create(
+        document=obj,
+        user=user,
+        role=models.RoleChoices.OWNER,
+    )
+
+    imported_ids = [imported_doc.page.id]
+
+    _import_notion_doc_content(imported_doc, obj, user)
+
+    for child in imported_doc.children:
+        _import_notion_child_page(child, obj, user, imported_ids)
+
+    return imported_ids
+
+
+def _generate_notion_progress(
+    all_pages: list[NotionPage], page_statuses: dict[str, str]
+) -> str:
+    raw = json.dumps(
+        [
+            {
+                "title": page.get_title(),
+                "status": page_statuses[page.id],
+            }
+            for page in all_pages
+        ]
+    )
+    return f"data: {raw}\n\n"
+
+
+def _notion_import_event_stream(request):
+    session = build_notion_session(request.session["notion_token"])
+    all_pages = fetch_all_pages(session)
+
+    page_statuses = {}
+    for page in all_pages:
+        page_statuses[page.id] = "pending"
+
+    yield _generate_notion_progress(all_pages, page_statuses)
+
+    docs_by_page_id: dict[str, ImportedDocument] = {}
+    child_page_blocs_ids_to_parent_page_ids: dict[str, str] = {}
+
+    for page in all_pages:
+        docs_by_page_id[page.id] = import_page(
+            session, page, child_page_blocs_ids_to_parent_page_ids
+        )
+        page_statuses[page.id] = "fetched"
+        yield _generate_notion_progress(all_pages, page_statuses)
+
+    for page in all_pages:
+        link_child_page_to_parent(
+            page, docs_by_page_id, child_page_blocs_ids_to_parent_page_ids
+        )
+
+    root_docs = [doc for doc in docs_by_page_id.values() if doc.page.is_root()]
+
+    for root_doc in root_docs:
+        imported_ids = _import_notion_root_page(root_doc, request.user)
+        for imported_id in imported_ids:
+            page_statuses[imported_id] = "imported"
+
+        yield _generate_notion_progress(all_pages, page_statuses)
+
+
+class IgnoreClientContentNegotiation(drf.negotiation.BaseContentNegotiation):
+    def select_parser(self, request, parsers):
+        return parsers[0]
+
+    def select_renderer(self, request, renderers, format_suffix):
+        return (renderers[0], renderers[0].media_type)
+
+
+class NotionImportRunView(drf.views.APIView):
+    content_negotiation_class = IgnoreClientContentNegotiation
+
+    def get(self, request, format=None):
+        if "notion_token" not in request.session:
+            raise drf.exceptions.PermissionDenied()
+
+        return StreamingHttpResponse(
+            _notion_import_event_stream(request), content_type="text/event-stream"
+        )