✨Collaboration long polling fallback

.github/workflows/docker-hub.yml

@@ -6,6 +6,7 @@ on:
   push:
     branches:
       - 'main'
+      - 'feature/collab-long-polling'
     tags:
       - 'v*'
   pull_request:

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to
 
 ## [Unreleased]
 
+## Added
+
+- ✨Collaboration long polling fallback #517
+
 ## Changed
 
 - 🛂(frontend) Restore version visibility #629
@@ -18,6 +22,7 @@ and this project adheres to
 
 - ♻️(frontend) improve table pdf rendering
 
+
 ## [2.2.0] - 2025-02-10
 
 ## Added

docker/files/etc/nginx/conf.d/00-collaboration-proxy-cache.conf

@@ -0,0 +1 @@
+proxy_cache_path /tmp/auth_cache levels=1:2 keys_zone=auth_cache:10m inactive=60s max_size=100m;

docker/files/etc/nginx/conf.d/default.conf

@@ -1,11 +1,18 @@
-
 server {
     listen 8083;
     server_name localhost;
     charset utf-8;
 
     # Proxy auth for collaboration server
     location /collaboration/ws/ {
+        if ($request_method = OPTIONS) {
+            add_header 'Access-Control-Allow-Origin' 'http://localhost:3000';
+            add_header 'Access-Control-Allow-Credentials' 'true';
+            add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+            add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type';
+            return 204;
+        }
+
         # Collaboration Auth request configuration
         auth_request /collaboration-auth;
         auth_request_set $authHeader $upstream_http_authorization;
@@ -34,6 +41,10 @@ server {
     }
 
     location /collaboration-auth {
+        proxy_cache auth_cache;
+        proxy_cache_key "$http_authorization-$arg_room";
+        proxy_cache_valid 200 30s;
+
         proxy_pass http://app-dev:8000/api/v1.0/documents/collaboration-auth/;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -43,10 +54,11 @@ server {
         # Prevent the body from being passed
         proxy_pass_request_body off;
         proxy_set_header Content-Length "";
+        proxy_set_header Accept "application/json";
         proxy_set_header X-Original-Method $request_method;
     }
 
-    location  /collaboration/api/ {
+    location /collaboration/api/ {
         # Collaboration server
         proxy_pass http://y-provider:4444;
         proxy_set_header Host $host;
@@ -76,7 +88,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Original-URL $request_uri;
-        
+
         # Prevent the body from being passed
         proxy_pass_request_body off;
         proxy_set_header Content-Length "";

docs/collaboration-architecture.md

@@ -0,0 +1,84 @@
+# Architecture Overview
+
+This architecture showcases different ways for clients to interact with a **Hocus Pocus Server** (a [Y.js](https://github.com/yjs/yjs) provider) through either WebSockets, HTTP fallbacks, or Server-Sent Events (SSE) when WebSockets are not available.
+
+**Main Components**:
+
+- **Client**: The front-end application or user agent.
+- **Nginx**: A reverse proxy handling incoming requests, forwarding them to the appropriate services, and managing SSL/TLS termination if needed.
+- **Auth Sub Request (Django)**: Handles authentication/authorization, ensuring requests have valid credentials or permissions.
+- **Hocus Pocus Server**: The core collaborative editing server (powered by [Y.js](https://github.com/yjs/yjs) libraries) that manages document state and synchronization.
+- **Express**: Fallback server to handle push or pull requests when WebSocket connections fail.
+- **SSE**: A mechanism (Server-Sent Events) for real-time updates when WebSockets are unavailable.
+
+## Mermaid Diagram
+
+```mermaid
+flowchart TD
+    title1[WebSocket Success]-->Client1(Client)<--->|WebSocket Success|WS1(Websocket) --> Nginx1(Ngnix) <--> Auth1("Auth Sub Request (Django)") --->|With the good right|YServer1("Hocus Pocus Server")
+    YServer1 --> WS1
+    YServer1 <--> clients(Dispatch to clients)
+
+    title2[WebSocket Fails - Push data]-->Client2(Client)---|WebSocket fails|HTTP2(HTTP) --> Nginx2(Ngnix) <--> Auth2("Auth Sub Request (Django)")--->|With the good right|Express2(Express) --> YServer2("Hocus Pocus Server") --> clients(Dispatch to clients)
+
+    title3[WebSocket Fails - Pull data]-->Client3(Client)<--->|WebSocket fails|SSE(SSE) --> Nginx3(Ngnix) <--> Auth3("Auth Sub Request (Django)") --->|With the good right|Express3(Express) --> YServer3("Listen Hocus Pocus Server")
+    YServer3("Listen Hocus Pocus Server") --> SSE
+    YServer3("Listen Hocus Pocus Server") <--> clients(Data from clients)
+```
+
+---
+
+## Detailed Flows
+
+### 1. WebSocket Success
+1. **Client** attempts a WebSocket connection.
+2. **Nginx** proxies the WebSocket connection through the **Auth Sub Request (Django)** for authentication.
+3. Once authenticated, traffic is routed to the **Hocus Pocus Server**.
+4. The server can broadcast data to all clients connected through WebSockets.  
+   - Note: The path `YServer1 --> WS1` indicates the two-way real-time communication between the server and client(s).
+
+### 2. WebSocket Fails — Push Data (HTTP)
+If WebSocket connections fail, clients can **push** data via HTTP:
+1. **Client** detects WebSocket failure and falls back to sending data over **HTTP**.
+2. **Nginx** handles HTTP requests and authenticates them via the **Auth Sub Request (Django)**.
+3. After successful authentication, the requests go to an **Express** server.
+4. The **Express** server relays changes to the **Hocus Pocus Server**.
+5. The **Hocus Pocus Server** dispatches updated content to connected clients.  
+
+### 3. WebSocket Fails — Pull Data (SSE)
+For continuously receiving data when WebSockets fail, the client can **pull** data using SSE:
+1. **Client** sets up an **SSE** connection.
+2. **Nginx** proxies the SSE stream request through the **Auth Sub Request (Django)** for authentication.
+3. Once authenticated, the **Express** server listens to the **Hocus Pocus Server** for changes.
+4. The server then sends updates back to the **Client** through SSE in near real-time.
+
+---
+
+## Component Responsibilities
+
+| **Component**               | **Responsibility**                                                                     |
+|-----------------------------|-----------------------------------------------------------------------------------------|
+| **Client**                  | Initiates connections (WebSocket/HTTP/SSE), displays and interacts with data           |
+| **Nginx**                   | Acts as a reverse proxy, routes traffic, handles SSL, and passes auth sub requests     |
+| **Auth Sub Request (Django)** | Validates requests, ensuring correct permissions and tokens                           |
+| **WebSocket**               | Real-time two-way communication channel                                                |
+| **HTTP**                    | Fallback method for sending updates when WebSockets are not available                  |
+| **Express**                 | Fallback server for handling requests (push/pull of data)                              |
+| **SSE**                     | Mechanism for real-time one-way updates from server to client                          |
+| **Hocus Pocus Server**      | Core Y.js server for collaboration, managing document states and synchronization       |
+
+---
+
+## Why This Setup?
+
+- **Reliability:** Ensures that when a user’s browser or network environment does not support WebSockets, there are fallback mechanisms (HTTP for push updates and SSE for server-initiated updates).
+- **Scalability:** Nginx can efficiently proxy requests and scale horizontally, while the authentication step is centralized in Django.
+- **Security:** The Auth Sub Request in Django enforces proper permissions before data is relayed to the collaboration server.
+- **Real-time Collaboration:** The Hocus Pocus Server provides low-latency updates, essential for collaborative editing, supported by [Y.js](https://github.com/yjs/yjs).
+
+---
+
+### Contributing
+If you have any suggestions or improvements, feel free to open an issue or submit a pull request.
+
+**Thank you for exploring this architecture!** If you have any questions or need more detailed explanations, please let us know.

src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts

@@ -0,0 +1,97 @@
+import { expect, test } from '@playwright/test';
+
+import { createDoc, verifyDocName } from './common';
+
+test.beforeEach(async ({ page }) => {
+  await page.goto('/');
+});
+
+test.describe('Doc Collaboration', () => {
+  /**
+   * We check:
+   *  - connection to the collaborative server
+   *  - signal of the backend to the collaborative server (connection should close)
+   *  - reconnection to the collaborative server
+   */
+  test('checks the connection with collaborative server', async ({
+    page,
+    browserName,
+  }) => {
+    let webSocketPromise = page.waitForEvent('websocket', (webSocket) => {
+      return webSocket
+        .url()
+        .includes('ws://localhost:8083/collaboration/ws/?room=');
+    });
+
+    const [title] = await createDoc(page, 'doc-editor', browserName, 1);
+    await verifyDocName(page, title);
+
+    let webSocket = await webSocketPromise;
+    expect(webSocket.url()).toContain(
+      'ws://localhost:8083/collaboration/ws/?room=',
+    );
+
+    // Is connected
+    let framesentPromise = webSocket.waitForEvent('framesent');
+
+    await page.locator('.ProseMirror.bn-editor').click();
+    await page.locator('.ProseMirror.bn-editor').fill('Hello World');
+
+    let framesent = await framesentPromise;
+    expect(framesent.payload).not.toBeNull();
+
+    await page.getByRole('button', { name: 'Share' }).click();
+
+    const selectVisibility = page.getByLabel('Visibility', { exact: true });
+
+    // When the visibility is changed, the ws should closed the connection (backend signal)
+    const wsClosePromise = webSocket.waitForEvent('close');
+
+    await selectVisibility.click();
+    await page
+      .getByRole('button', {
+        name: 'Connected',
+      })
+      .click();
+
+    // Assert that the doc reconnects to the ws
+    const wsClose = await wsClosePromise;
+    expect(wsClose.isClosed()).toBeTruthy();
+
+    // Checkt the ws is connected again
+    webSocketPromise = page.waitForEvent('websocket', (webSocket) => {
+      return webSocket
+        .url()
+        .includes('ws://localhost:8083/collaboration/ws/?room=');
+    });
+
+    webSocket = await webSocketPromise;
+    framesentPromise = webSocket.waitForEvent('framesent');
+    framesent = await framesentPromise;
+    expect(framesent.payload).not.toBeNull();
+  });
+
+  test('checks the connection switch to polling after websocket failure', async ({
+    page,
+    browserName,
+  }) => {
+    const responsePromise = page.waitForResponse(
+      (response) =>
+        response.url().includes('/poll/') && response.status() === 200,
+    );
+
+    await page.routeWebSocket(
+      'ws://localhost:8083/collaboration/ws/**',
+      async (ws) => {
+        await ws.close();
+      },
+    );
+
+    await page.reload();
+
+    await createDoc(page, 'doc-polling', browserName, 1);
+
+    const response = await responsePromise;
+    expect(response.ok()).toBeTruthy();
+  });
+});

src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts

@@ -88,70 +88,6 @@ test.describe('Doc Editor', () => {
     ).toBeVisible();
   });
 
-  /**
-   * We check:
-   *  - connection to the collaborative server
-   *  - signal of the backend to the collaborative server (connection should close)
-   *  - reconnection to the collaborative server
-   */
-  test('checks the connection with collaborative server', async ({
-    page,
-    browserName,
-  }) => {
-    let webSocketPromise = page.waitForEvent('websocket', (webSocket) => {
-      return webSocket
-        .url()
-        .includes('ws://localhost:8083/collaboration/ws/?room=');
-    });
-
-    const randomDoc = await createDoc(page, 'doc-editor', browserName, 1);
-    await verifyDocName(page, randomDoc[0]);
-
-    let webSocket = await webSocketPromise;
-    expect(webSocket.url()).toContain(
-      'ws://localhost:8083/collaboration/ws/?room=',
-    );
-
-    // Is connected
-    let framesentPromise = webSocket.waitForEvent('framesent');
-
-    await page.locator('.ProseMirror.bn-editor').click();
-    await page.locator('.ProseMirror.bn-editor').fill('Hello World');
-
-    let framesent = await framesentPromise;
-    expect(framesent.payload).not.toBeNull();
-
-    await page.getByRole('button', { name: 'Share' }).click();
-
-    const selectVisibility = page.getByLabel('Visibility', { exact: true });
-
-    // When the visibility is changed, the ws should closed the connection (backend signal)
-    const wsClosePromise = webSocket.waitForEvent('close');
-
-    await selectVisibility.click();
-    await page
-      .getByRole('button', {
-        name: 'Connected',
-      })
-      .click();
-
-    // Assert that the doc reconnects to the ws
-    const wsClose = await wsClosePromise;
-    expect(wsClose.isClosed()).toBeTruthy();
-
-    // Checkt the ws is connected again
-    webSocketPromise = page.waitForEvent('websocket', (webSocket) => {
-      return webSocket
-        .url()
-        .includes('ws://localhost:8083/collaboration/ws/?room=');
-    });
-
-    webSocket = await webSocketPromise;
-    framesentPromise = webSocket.waitForEvent('framesent');
-    framesent = await framesentPromise;
-    expect(framesent.payload).not.toBeNull();
-  });
-
   test('markdown button converts from markdown to the editor syntax json', async ({
     page,
     browserName,

src/frontend/apps/impress/package.json

@@ -62,6 +62,7 @@
     "@types/node": "*",
     "@types/react": "18.3.12",
     "@types/react-dom": "*",
+    "@types/ws": "8.5.13",
     "cross-env": "7.0.3",
     "dotenv": "16.4.7",
     "eslint-config-impress": "*",

src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx

@@ -2,12 +2,10 @@ import { useRouter } from 'next/router';
 import { useCallback, useEffect, useRef, useState } from 'react';
 import * as Y from 'yjs';
 
-import { useUpdateDoc } from '@/features/docs/doc-management/';
+import { toBase64, useUpdateDoc } from '@/features/docs/doc-management/';
 import { KEY_LIST_DOC_VERSIONS } from '@/features/docs/doc-versioning';
 import { isFirefox } from '@/utils/userAgent';
 
-import { toBase64 } from '../utils';
-
 const useSaveDoc = (docId: string, doc: Y.Doc, canSave: boolean) => {
   const { mutate: updateDoc } = useUpdateDoc({
     listInvalideQueries: [KEY_LIST_DOC_VERSIONS],

src/frontend/apps/impress/src/features/docs/doc-editor/utils.ts

@@ -22,6 +22,3 @@ function hslToHex(h: number, s: number, l: number) {
   };
   return `#${f(0)}${f(8)}${f(4)}`;
 }
-
-export const toBase64 = (str: Uint8Array) =>
-  Buffer.from(str).toString('base64');