From 2a3fa1acdb342c583189b7cfb62572440a3c31d4 Mon Sep 17 00:00:00 2001 From: Yifeng Wang Date: Wed, 22 May 2024 14:52:57 +0800 Subject: [PATCH 1/5] feat: add basic ystream setup --- packages/server/package.json | 2 + packages/server/src/ystream-test.ts | 38 +++++ packages/server/src/ystream/utils.ts | 39 +++++ pnpm-lock.yaml | 213 +++++++++++++++++++++++++++ tests/common/demo.ts | 6 +- 5 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 packages/server/src/ystream-test.ts create mode 100644 packages/server/src/ystream/utils.ts diff --git a/packages/server/package.json b/packages/server/package.json index 058fcd8..3dd5e82 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -4,11 +4,13 @@ "repository": "toeverything/notesuite", "type": "module", "scripts": { + "start:ystream": "NODE_OPTIONS=--import=./register.js node src/ystream-test.ts", "start": "NODE_OPTIONS=--import=./register.js node src/index.ts", "clean": "rm -rf ./.db*" }, "dependencies": { "@notesuite/common": "workspace:*", + "@y/stream": "https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz", "body-parser": "^1.20.2", "cors": "^2.8.5", "express": "^4.19.2", diff --git a/packages/server/src/ystream-test.ts b/packages/server/src/ystream-test.ts new file mode 100644 index 0000000..efb7564 --- /dev/null +++ b/packages/server/src/ystream-test.ts @@ -0,0 +1,38 @@ +import * as Ystream from '@y/stream'; +import * as wscomm from '@y/stream/comms/ws'; +import * as authentication from '@y/stream/api/authentication'; +import { createWSServer } from '@y/stream/comms/ws-server'; +import { + collectionDef, + testServerIdentity, + testUser, +} from './ystream/utils.js'; + +const dbname = './.db-ystream/test'; + +const server = await createWSServer({ + port: 9000, + dbname, + identity: testServerIdentity, +}); + +const comm = new wscomm.WebSocketComm('ws://localhost:9000', [collectionDef]); +// await Ystream.remove(dbname); +const ystream = await Ystream.open(dbname, { + comms: [comm], +}); +await authentication.registerUser(ystream, testServerIdentity.user, { + isTrusted: true, +}); +await authentication.setUserIdentity( + ystream, + testUser.user, + await testUser.user.publicKey, + testUser.privateKey +); + +console.log(ystream.getCollection); +// const collection = ystream.getCollection(collectionDef.collection); +// const ydoc1 = collection.getYdoc('ydoc'); +// await ydoc1.whenLoaded; +// console.log(ydoc1); diff --git a/packages/server/src/ystream/utils.ts b/packages/server/src/ystream/utils.ts new file mode 100644 index 0000000..d57d52e --- /dev/null +++ b/packages/server/src/ystream/utils.ts @@ -0,0 +1,39 @@ +import * as dbtypes from '@y/stream/api/dbtypes'; +import * as json from 'lib0/json'; +import * as buffer from 'lib0/buffer'; +import * as decoding from 'lib0/decoding'; +import * as ecdsa from 'lib0/crypto/ecdsa'; + +const testUserRaw = { + privateKey: + '{"key_ops":["sign"],"ext":true,"kty":"EC","x":"pAUmLYc-UFmPIt7leafPTbhxQyygcaW7__nPcUNCuu0wH27yS9P_pWFP1GwcsoAN","y":"u3109KjrPGsNUn2k5Whn2uHLAckQPdLNqtM4GpBEpUJwlvVDvk71-lS3YOEYJ_Sq","crv":"P-384","d":"OHnRw5an9hlSqSKg966lFRvB7dow669pVSn7sFZUi7UQh_Y9Xc95SQ6pEWsofsYD"}', + user: 'AMgBeyJrZXlfb3BzIjpbInZlcmlmeSJdLCJleHQiOnRydWUsImt0eSI6IkVDIiwieCI6InBBVW1MWWMtVUZtUEl0N2xlYWZQVGJoeFF5eWdjYVc3X19uUGNVTkN1dTB3SDI3eVM5UF9wV0ZQMUd3Y3NvQU4iLCJ5IjoidTMxMDlLanJQR3NOVW4yazVXaG4ydUhMQWNrUVBkTE5xdE00R3BCRXBVSndsdlZEdms3MS1sUzNZT0VZSl9TcSIsImNydiI6IlAtMzg0In0=', +}; + +const testServerIdentityRaw = { + privateKey: + '{"key_ops":["sign"],"ext":true,"kty":"EC","x":"CYwMakpn0onaNeCa-wqLn4Fzsris_UY4Z5gRQUA9xQOoh94YG9OHhItr6rovaYpZ","y":"74Ulju86IUMJZsYsSjxSjusLjj9U6rozZwbK9Xaqj3MgIWtnjNyjL1D-NzOP3FJ7","crv":"P-384","d":"-yKNOty9EshGL0yAOQ2q6c_b_PNCpeEK9FVPoB0wc9EUyt9BR4DZuqrC9t_DgNaF"}', + user: 'AMgBeyJrZXlfb3BzIjpbInZlcmlmeSJdLCJleHQiOnRydWUsImt0eSI6IkVDIiwieCI6IkNZd01ha3BuMG9uYU5lQ2Etd3FMbjRGenNyaXNfVVk0WjVnUlFVQTl4UU9vaDk0WUc5T0hoSXRyNnJvdmFZcFoiLCJ5IjoiNzRVbGp1ODZJVU1KWnNZc1NqeFNqdXNMamo5VTZyb3pad2JLOVhhcWozTWdJV3Ruak55akwxRC1Oek9QM0ZKNyIsImNydiI6IlAtMzg0In0=', +}; + +export const testUser = { + privateKey: await ecdsa.importKeyJwk(json.parse(testUserRaw.privateKey)), + user: dbtypes.UserIdentity.decode( + decoding.createDecoder(buffer.fromBase64(testUserRaw.user)) + ), +}; + +export const testServerIdentity = { + privateKey: await ecdsa.importKeyJwk( + json.parse(testServerIdentityRaw.privateKey) + ), + user: dbtypes.UserIdentity.decode( + decoding.createDecoder(buffer.fromBase64(testServerIdentityRaw.user)) + ), +}; + +export const owner = testUser.user.hash; +export const collectionDef = { + owner: buffer.toBase64(owner), + collection: 'c1', +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e148d1..9652daa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -124,6 +124,9 @@ importers: '@notesuite/common': specifier: workspace:* version: link:../common + '@y/stream': + specifier: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz + version: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz body-parser: specifier: ^1.20.2 version: 1.20.2 @@ -623,10 +626,70 @@ packages: '@lit/reactive-element@2.0.4': resolution: {integrity: sha512-GFn91inaUa2oHLak8awSIigYz0cU0Payr1rcFsrkf5OJ5eSPxElyZfKh0f2p9FsTiZWXQdWGJeXZICEfXXYSXQ==} + '@lmdb/lmdb-darwin-arm64@2.9.4': + resolution: {integrity: sha512-38XmbES/wVcvMXdwcM5QzL0cSaZu3VwE7mCd0I89eliHQTMQblgWXsr2HQoP9v0JnH6jVt7+E/TkeGNLmp4wzA==} + cpu: [arm64] + os: [darwin] + + '@lmdb/lmdb-darwin-x64@2.9.4': + resolution: {integrity: sha512-JkPrV8rEu88FzMcuouZeU2b/NuVC3KwQxKo5vKhNycBtsCn7KCWHalxL4sdTiHQ4xtgMca3mmeDAdxgqQqnDig==} + cpu: [x64] + os: [darwin] + + '@lmdb/lmdb-linux-arm64@2.9.4': + resolution: {integrity: sha512-aIzmw0g4Wdd/w2rDppGfo1JEl4xWpg6HPbf5ZeuWXCjFms8oc8cazm6oBEAimiZEgYYBFPDPdM644xJcwuJbxQ==} + cpu: [arm64] + os: [linux] + + '@lmdb/lmdb-linux-arm@2.9.4': + resolution: {integrity: sha512-b3JZL5pLuvcGEbcsThUQPFlQdBFaBxImrlNbFUeJmzLwpdgGRi0RSQdZZ2PuIoZvpRj0tfNlhXQwXiYMz+9iTw==} + cpu: [arm] + os: [linux] + + '@lmdb/lmdb-linux-x64@2.9.4': + resolution: {integrity: sha512-Yj6Nb+/j+ZZ65oH/UCE0UfUu/6TO5wWLIeE2izGCpsCxcozZVbzwhzrCs0FUXf6lXv46DJteONosWH9o1XjzqQ==} + cpu: [x64] + os: [linux] + + '@lmdb/lmdb-win32-x64@2.9.4': + resolution: {integrity: sha512-0L6Tyun47/kQb+FzTDIumrfZgU6oEos0RgekKa/3YC7nsUY+ZASZHikzGgEZpMQHSz5YeR+DDUtOMSwqodWHDg==} + cpu: [x64] + os: [win32] + '@msgpack/msgpack@3.0.0-beta2': resolution: {integrity: sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==} engines: {node: '>= 14'} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2': + resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2': + resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2': + resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2': + resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2': + resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2': + resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==} + cpu: [x64] + os: [win32] + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1076,6 +1139,12 @@ packages: resolution: {integrity: sha512-FNoYzHawTMk/6KMQoEG5O4PuioX19UbwdQKF44yw0nLfOypfQdjtfZzo/UIJWAJ23sNIFbD1Ug9lbaDGMwbqQA==} engines: {node: '>=8'} + '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz': + resolution: {tarball: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz} + version: 0.0.1 + engines: {node: '>=20'} + hasBin: true + abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -1942,6 +2011,10 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isodb@0.1.12: + resolution: {integrity: sha512-JJHYkvs9HNGotOUMq5yqDEL7VPQ1LJ+gxqsPG/ksmszwPb6SSvPQKca1Sa51/XDwnH0ROHkgchPwy2GnCgymXQ==} + engines: {node: '>=14'} + isomorphic.js@0.2.5: resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==} @@ -2068,6 +2141,10 @@ packages: lit@3.1.2: resolution: {integrity: sha512-VZx5iAyMtX7CV4K8iTLdCkMaYZ7ipjJZ0JcSdJ0zIdGxxyurjIn7yuuSxNBD7QmjvcNJwr0JS4cAdAtsy7gZ6w==} + lmdb@2.9.4: + resolution: {integrity: sha512-Kri5TSKgpLk5q1VO7vYCcqAMyXTxmis6Et+6UARkU7ygvg3ZxUX2oEu/UwBkBskaS1d73effiBrTiHYyDppcBg==} + hasBin: true + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -2303,6 +2380,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.2: + resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} + hasBin: true + + msgpackr@1.10.2: + resolution: {integrity: sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==} + muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} @@ -2331,6 +2415,9 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} + node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2344,6 +2431,14 @@ packages: encoding: optional: true + node-gyp-build-optional-packages@5.0.7: + resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==} + hasBin: true + + node-gyp-build-optional-packages@5.1.1: + resolution: {integrity: sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw==} + hasBin: true + node-gyp-build@4.1.1: resolution: {integrity: sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==} hasBin: true @@ -2380,6 +2475,9 @@ packages: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + ordered-binary@1.5.1: + resolution: {integrity: sha512-5VyHfHY3cd0iza71JepYG50My+YUbrFtGoUz2ooEydPyPM7Aai/JW098juLr+RG6+rDJuzNNTsEQu2DZa1A41A==} + p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} @@ -2856,6 +2954,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + uWebSockets.js@https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/1977b5039938ad863d42fc4958d48c17e5a1fa06: + resolution: {tarball: https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/1977b5039938ad863d42fc4958d48c17e5a1fa06} + version: 20.43.0 + undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} @@ -2995,6 +3097,9 @@ packages: wasm-imagemagick@1.2.8: resolution: {integrity: sha512-V7u80n7g+iAoV7sYgQKGSdG59J6/aSMGO0DDK0zxKnwOGjmVXyjP0yU4tX4cMrfC0t/Wk3I8TX7cmdbFQOYHpg==} + weak-lru-cache@1.2.2: + resolution: {integrity: sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==} + web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} @@ -3593,8 +3698,44 @@ snapshots: dependencies: '@lit-labs/ssr-dom-shim': 1.2.0 + '@lmdb/lmdb-darwin-arm64@2.9.4': + optional: true + + '@lmdb/lmdb-darwin-x64@2.9.4': + optional: true + + '@lmdb/lmdb-linux-arm64@2.9.4': + optional: true + + '@lmdb/lmdb-linux-arm@2.9.4': + optional: true + + '@lmdb/lmdb-linux-x64@2.9.4': + optional: true + + '@lmdb/lmdb-win32-x64@2.9.4': + optional: true + '@msgpack/msgpack@3.0.0-beta2': {} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2': + optional: true + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -4066,6 +4207,19 @@ snapshots: dependencies: tslib: 2.6.2 + '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz': + dependencies: + isodb: 0.1.12 + lib0: 0.2.93 + yjs: 13.6.15 + optionalDependencies: + chokidar: 3.6.0 + uws: uWebSockets.js@https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/1977b5039938ad863d42fc4958d48c17e5a1fa06 + ws: 8.16.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -5048,6 +5202,12 @@ snapshots: isexe@2.0.0: {} + isodb@0.1.12: + dependencies: + lib0: 0.2.93 + optionalDependencies: + lmdb: 2.9.4 + isomorphic.js@0.2.5: {} isstream@0.1.2: {} @@ -5204,6 +5364,22 @@ snapshots: lit-element: 4.0.4 lit-html: 3.1.2 + lmdb@2.9.4: + dependencies: + msgpackr: 1.10.2 + node-addon-api: 6.1.0 + node-gyp-build-optional-packages: 5.1.1 + ordered-binary: 1.5.1 + weak-lru-cache: 1.2.2 + optionalDependencies: + '@lmdb/lmdb-darwin-arm64': 2.9.4 + '@lmdb/lmdb-darwin-x64': 2.9.4 + '@lmdb/lmdb-linux-arm': 2.9.4 + '@lmdb/lmdb-linux-arm64': 2.9.4 + '@lmdb/lmdb-linux-x64': 2.9.4 + '@lmdb/lmdb-win32-x64': 2.9.4 + optional: true + locate-path@6.0.0: dependencies: p-locate: 5.0.0 @@ -5583,6 +5759,23 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.2: + dependencies: + node-gyp-build-optional-packages: 5.0.7 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 + optional: true + + msgpackr@1.10.2: + optionalDependencies: + msgpackr-extract: 3.0.2 + optional: true + muggle-string@0.4.1: {} naive-ui@2.38.1(vue@3.4.21(typescript@5.4.3)): @@ -5618,12 +5811,23 @@ snapshots: negotiator@0.6.3: {} + node-addon-api@6.1.0: + optional: true + node-domexception@1.0.0: {} node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-gyp-build-optional-packages@5.0.7: + optional: true + + node-gyp-build-optional-packages@5.1.1: + dependencies: + detect-libc: 2.0.3 + optional: true + node-gyp-build@4.1.1: {} normalize-path@3.0.0: {} @@ -5672,6 +5876,9 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + ordered-binary@1.5.1: + optional: true + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 @@ -6198,6 +6405,9 @@ snapshots: typescript@5.4.3: {} + uWebSockets.js@https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/1977b5039938ad863d42fc4958d48c17e5a1fa06: + optional: true + undici-types@5.26.5: {} unified@11.0.4: @@ -6344,6 +6554,9 @@ snapshots: p-map: 2.1.0 stacktrace-js: 2.0.2 + weak-lru-cache@1.2.2: + optional: true + web-namespaces@2.0.1: {} web-streams-polyfill@3.3.3: {} diff --git a/tests/common/demo.ts b/tests/common/demo.ts index 2e2c302..73cf2c2 100644 --- a/tests/common/demo.ts +++ b/tests/common/demo.ts @@ -9,12 +9,12 @@ test('demo', async ({ page }) => { backendPort: 3000, }); await page.goto(agent.web.baseUrl); - // await page.pause(); + await page.pause(); await agent.web.createWorkspace(page, 'hello'); await agent.web.createDoc(page, 'First Doc'); await agent.web.createDoc(page, 'Second Doc'); await agent.web.createDoc(page, 'Third Doc'); - // await page.pause(); - // await agent.stop(); + await page.pause(); + await agent.stop(); }); From b17911b613b0c38f88704d5216b4a15118229012 Mon Sep 17 00:00:00 2001 From: Yifeng Wang Date: Thu, 23 May 2024 10:43:03 +0800 Subject: [PATCH 2/5] chore: update upstream --- packages/server/package.json | 2 +- packages/server/src/ystream-test.ts | 4 ++-- pnpm-lock.yaml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 3dd5e82..8158a10 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@notesuite/common": "workspace:*", - "@y/stream": "https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz", + "@y/stream": "https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz", "body-parser": "^1.20.2", "cors": "^2.8.5", "express": "^4.19.2", diff --git a/packages/server/src/ystream-test.ts b/packages/server/src/ystream-test.ts index efb7564..9517210 100644 --- a/packages/server/src/ystream-test.ts +++ b/packages/server/src/ystream-test.ts @@ -1,7 +1,7 @@ import * as Ystream from '@y/stream'; -import * as wscomm from '@y/stream/comms/ws'; +import * as wscomm from '@y/stream/comms/websocket'; import * as authentication from '@y/stream/api/authentication'; -import { createWSServer } from '@y/stream/comms/ws-server'; +import { createWSServer } from '@y/stream/comms/websocket-server'; import { collectionDef, testServerIdentity, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9652daa..1245568 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -125,8 +125,8 @@ importers: specifier: workspace:* version: link:../common '@y/stream': - specifier: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz - version: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz + specifier: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz + version: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz body-parser: specifier: ^1.20.2 version: 1.20.2 @@ -1139,9 +1139,9 @@ packages: resolution: {integrity: sha512-FNoYzHawTMk/6KMQoEG5O4PuioX19UbwdQKF44yw0nLfOypfQdjtfZzo/UIJWAJ23sNIFbD1Ug9lbaDGMwbqQA==} engines: {node: '>=8'} - '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz': - resolution: {tarball: https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz} - version: 0.0.1 + '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz': + resolution: {tarball: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz} + version: 0.0.0 engines: {node: '>=20'} hasBin: true @@ -4207,7 +4207,7 @@ snapshots: dependencies: tslib: 2.6.2 - '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.1-240522/y-stream-0.0.1-2.tgz': + '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz': dependencies: isodb: 0.1.12 lib0: 0.2.93 From 5d61e718960fd1739a6f4adbbdb6f112e2d06978 Mon Sep 17 00:00:00 2001 From: Yifeng Wang Date: Tue, 28 May 2024 00:17:13 +0800 Subject: [PATCH 3/5] chore: bump ystream --- packages/server/package.json | 2 +- pnpm-lock.yaml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 8158a10..6e0f7b2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@notesuite/common": "workspace:*", - "@y/stream": "https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz", + "@y/stream": "^0.0.3", "body-parser": "^1.20.2", "cors": "^2.8.5", "express": "^4.19.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1245568..cec9df6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -125,8 +125,8 @@ importers: specifier: workspace:* version: link:../common '@y/stream': - specifier: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz - version: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz + specifier: ^0.0.3 + version: 0.0.3 body-parser: specifier: ^1.20.2 version: 1.20.2 @@ -1139,9 +1139,8 @@ packages: resolution: {integrity: sha512-FNoYzHawTMk/6KMQoEG5O4PuioX19UbwdQKF44yw0nLfOypfQdjtfZzo/UIJWAJ23sNIFbD1Ug9lbaDGMwbqQA==} engines: {node: '>=8'} - '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz': - resolution: {tarball: https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz} - version: 0.0.0 + '@y/stream@0.0.3': + resolution: {integrity: sha512-mfQfrJI/ZEcfwKTKYfKc9CfWmG+ovZ2mvSPn59wc3817qefGD8QKfMOZQf0lbhPAW3F04hu4zCuFVUJrVUcklA==} engines: {node: '>=20'} hasBin: true @@ -4207,7 +4206,7 @@ snapshots: dependencies: tslib: 2.6.2 - '@y/stream@https://github.com/doodlewind/ystream/releases/download/0.0.0-240523/y-stream-0.0.0.tgz': + '@y/stream@0.0.3': dependencies: isodb: 0.1.12 lib0: 0.2.93 From 91756b077f31b30551709fb599957899e4c24aba Mon Sep 17 00:00:00 2001 From: Yifeng Wang Date: Tue, 28 May 2024 00:20:57 +0800 Subject: [PATCH 4/5] refactor: bump api --- packages/server/src/ystream-test.ts | 10 +++++----- packages/server/src/ystream/utils.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/ystream-test.ts b/packages/server/src/ystream-test.ts index 9517210..ccc87c0 100644 --- a/packages/server/src/ystream-test.ts +++ b/packages/server/src/ystream-test.ts @@ -16,7 +16,7 @@ const server = await createWSServer({ identity: testServerIdentity, }); -const comm = new wscomm.WebSocketComm('ws://localhost:9000', [collectionDef]); +const comm = new wscomm.WebSocketComm('ws://localhost:9000', collectionDef); // await Ystream.remove(dbname); const ystream = await Ystream.open(dbname, { comms: [comm], @@ -31,8 +31,8 @@ await authentication.setUserIdentity( testUser.privateKey ); -console.log(ystream.getCollection); -// const collection = ystream.getCollection(collectionDef.collection); -// const ydoc1 = collection.getYdoc('ydoc'); -// await ydoc1.whenLoaded; +const { owner, name } = collectionDef; +const collection = ystream.getCollection(owner, name); +const ydoc1 = collection.getYdoc('ydoc'); +await ydoc1.whenLoaded; // console.log(ydoc1); diff --git a/packages/server/src/ystream/utils.ts b/packages/server/src/ystream/utils.ts index d57d52e..da4c02d 100644 --- a/packages/server/src/ystream/utils.ts +++ b/packages/server/src/ystream/utils.ts @@ -35,5 +35,5 @@ export const testServerIdentity = { export const owner = testUser.user.hash; export const collectionDef = { owner: buffer.toBase64(owner), - collection: 'c1', + name: 'c1', }; From 59fb83f47c91517c4f388d021289f89e622fa5dc Mon Sep 17 00:00:00 2001 From: Yifeng Wang Date: Tue, 28 May 2024 09:56:40 +0800 Subject: [PATCH 5/5] feat: switch persistence --- .vscode/launch.template.json | 2 +- packages/server/package.json | 4 +- packages/server/src/api.ts | 2 +- .../server/src/third-party/y-websocket.js | 46 +++---------------- packages/server/src/utils.ts | 4 +- .../{ystream-test.ts => ystream/adaptor.ts} | 22 +++++---- .../server/src/ystream/{utils.ts => auth.ts} | 0 pnpm-lock.yaml | 40 ++++++++++++---- 8 files changed, 54 insertions(+), 66 deletions(-) rename packages/server/src/{ystream-test.ts => ystream/adaptor.ts} (61%) rename packages/server/src/ystream/{utils.ts => auth.ts} (100%) diff --git a/.vscode/launch.template.json b/.vscode/launch.template.json index 546f785..3b1b4e6 100644 --- a/.vscode/launch.template.json +++ b/.vscode/launch.template.json @@ -8,7 +8,7 @@ "skipFiles": ["/**"], "env": { "NODE_OPTIONS": "--import=./register.js", - "INSTANCE_NAME": "", + "INSTANCE_NAME": "demo", "PORT": "3000" }, "program": "${workspaceFolder}/packages/server/src/index.ts", diff --git a/packages/server/package.json b/packages/server/package.json index 6e0f7b2..3fb225b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -4,9 +4,8 @@ "repository": "toeverything/notesuite", "type": "module", "scripts": { - "start:ystream": "NODE_OPTIONS=--import=./register.js node src/ystream-test.ts", "start": "NODE_OPTIONS=--import=./register.js node src/index.ts", - "clean": "rm -rf ./.db*" + "clean": "rm -rf ./.db* db*.json" }, "dependencies": { "@notesuite/common": "workspace:*", @@ -21,7 +20,6 @@ "ts-node": "^10.9.2", "webdav-server": "^2.6.2", "ws": "^8.16.0", - "y-leveldb": "^0.1.2", "y-protocols": "^1.0.6", "y-websocket": "^2.0.2", "yjs": "^13.5.6" diff --git a/packages/server/src/api.ts b/packages/server/src/api.ts index 5a9d409..0522c82 100644 --- a/packages/server/src/api.ts +++ b/packages/server/src/api.ts @@ -1,6 +1,6 @@ import * as Y from 'yjs'; // @ts-ignore -import { getYDoc } from './third-party/y-websocket.js'; +import { getYDoc } from './ystream/adaptor.js'; import type { AppContext } from './utils.js'; // import { exportSnapshot } from '@notesuite/common/dist/editor.js'; diff --git a/packages/server/src/third-party/y-websocket.js b/packages/server/src/third-party/y-websocket.js index 084f709..c70d8f7 100644 --- a/packages/server/src/third-party/y-websocket.js +++ b/packages/server/src/third-party/y-websocket.js @@ -7,6 +7,7 @@ import * as encoding from 'lib0/encoding' import * as decoding from 'lib0/decoding' import * as map from 'lib0/map' import * as lodash from 'lodash' +import { getYDoc as getYstreamDoc } from '../ystream/adaptor.js' const { debounce } = lodash @@ -25,44 +26,8 @@ const wsReadyStateClosed = 3 // eslint-disable-line // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' -const persistenceDir = `./.db-${process.env.INSTANCE_NAME || 'default'}` -/** - * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} - */ -let persistence = null -if (typeof persistenceDir === 'string') { - console.info('Persisting documents to "' + persistenceDir + '"') - // @ts-ignore - const { LeveldbPersistence } = await import('y-leveldb') - const ldb = new LeveldbPersistence(persistenceDir) - persistence = { - provider: ldb, - bindState: async (docName, ydoc) => { - const persistedYdoc = await ldb.getYDoc(docName) - const newUpdates = Y.encodeStateAsUpdate(ydoc) - ldb.storeUpdate(docName, newUpdates) - Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc)) - ydoc.on('update', update => { - ldb.storeUpdate(docName, update) - }) - }, - writeState: async (_docName, _ydoc) => {} - } -} - -/** - * @param {{bindState: function(string,WSSharedDoc):void, - * writeState:function(string,WSSharedDoc):Promise,provider:any}|null} persistence_ - */ -export const setPersistence = persistence_ => { - persistence = persistence_ -} -/** - * @return {null|{bindState: function(string,WSSharedDoc):void, - * writeState:function(string,WSSharedDoc):Promise}|null} used persistence layer - */ -export const getPersistence = () => persistence +let persistence = null /** * @type {Map} @@ -167,10 +132,11 @@ export const getYDoc = async (docname, gc = true) => { if (!doc) { doc = new WSSharedDoc(docname) doc.gc = gc - if (persistence !== null) { - await persistence.bindState(docname, doc) - } docs.set(docname, doc) + + const peerDoc = await getYstreamDoc(docname) + peerDoc.on('update', update => Y.applyUpdate(doc, update)) + doc.on('update', update => Y.applyUpdate(peerDoc, update)) } return doc; } diff --git a/packages/server/src/utils.ts b/packages/server/src/utils.ts index ae395f3..aeba67d 100644 --- a/packages/server/src/utils.ts +++ b/packages/server/src/utils.ts @@ -6,11 +6,11 @@ import { JSONFilePreset } from 'lowdb/node'; import type http from 'http'; import { setupWSConnection, - getYDoc, docs as serverDocs, // @ts-ignore } from './third-party/y-websocket.js'; import * as Y from 'yjs'; +import { getYDoc } from './ystream/adaptor.js'; const instanceName = process.env.INSTANCE_NAME || 'default'; const docs = serverDocs as Map; @@ -44,7 +44,7 @@ async function initDB() { activeWorkspaceId?: string; workspaces: { id: string; rootId: string; name: string }[]; } = { workspaces: [] }; - const db = await JSONFilePreset(`./.db-${instanceName}/db.json`, defaultData); + const db = await JSONFilePreset(`./db-${instanceName}.json`, defaultData); await db.read(); await db.write(); return db; diff --git a/packages/server/src/ystream-test.ts b/packages/server/src/ystream/adaptor.ts similarity index 61% rename from packages/server/src/ystream-test.ts rename to packages/server/src/ystream/adaptor.ts index ccc87c0..6caa175 100644 --- a/packages/server/src/ystream-test.ts +++ b/packages/server/src/ystream/adaptor.ts @@ -6,20 +6,20 @@ import { collectionDef, testServerIdentity, testUser, -} from './ystream/utils.js'; +} from './auth.js'; -const dbname = './.db-ystream/test'; +const dbname = `./.db-ystream-${process.env.INSTANCE_NAME}`; -const server = await createWSServer({ +const remoteServer = await createWSServer({ port: 9000, dbname, identity: testServerIdentity, }); -const comm = new wscomm.WebSocketComm('ws://localhost:9000', collectionDef); +// const comm = new wscomm.WebSocketComm('ws://localhost:9000', collectionDef); // await Ystream.remove(dbname); const ystream = await Ystream.open(dbname, { - comms: [comm], + // comms: [comm], }); await authentication.registerUser(ystream, testServerIdentity.user, { isTrusted: true, @@ -32,7 +32,11 @@ await authentication.setUserIdentity( ); const { owner, name } = collectionDef; -const collection = ystream.getCollection(owner, name); -const ydoc1 = collection.getYdoc('ydoc'); -await ydoc1.whenLoaded; -// console.log(ydoc1); +const collection = ystream.getCollection(owner, name) as Ystream.Collection; + + +export async function getYDoc(id: string) { + const ydoc = collection.getYdoc(id); + await ydoc.whenLoaded; + return ydoc; +} diff --git a/packages/server/src/ystream/utils.ts b/packages/server/src/ystream/auth.ts similarity index 100% rename from packages/server/src/ystream/utils.ts rename to packages/server/src/ystream/auth.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cec9df6..1f4688e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -157,9 +157,6 @@ importers: ws: specifier: ^8.16.0 version: 8.16.0 - y-leveldb: - specifier: ^0.1.2 - version: 0.1.2(yjs@13.6.15) y-protocols: specifier: ^1.0.6 version: 1.0.6(yjs@13.6.15) @@ -4230,6 +4227,7 @@ snapshots: level-concat-iterator: 2.0.1 level-supports: 1.0.1 xtend: 4.0.2 + optional: true abstract-leveldown@6.3.0: dependencies: @@ -4238,6 +4236,7 @@ snapshots: level-concat-iterator: 2.0.1 level-supports: 1.0.1 xtend: 4.0.2 + optional: true accepts@1.3.8: dependencies: @@ -4369,6 +4368,7 @@ snapshots: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + optional: true buffer@6.0.3: dependencies: @@ -4566,6 +4566,7 @@ snapshots: dependencies: abstract-leveldown: 6.2.3 inherits: 2.0.4 + optional: true define-data-property@1.1.4: dependencies: @@ -4627,6 +4628,7 @@ snapshots: inherits: 2.0.4 level-codec: 9.0.2 level-errors: 2.0.1 + optional: true enhanced-resolve@5.16.0: dependencies: @@ -4638,6 +4640,7 @@ snapshots: errno@0.1.8: dependencies: prr: 1.0.1 + optional: true error-stack-parser@2.1.4: dependencies: @@ -5151,7 +5154,8 @@ snapshots: immediate@3.0.6: {} - immediate@3.3.0: {} + immediate@3.3.0: + optional: true import-fresh@3.3.0: dependencies: @@ -5269,18 +5273,22 @@ snapshots: level-codec@9.0.2: dependencies: buffer: 5.7.1 + optional: true - level-concat-iterator@2.0.1: {} + level-concat-iterator@2.0.1: + optional: true level-errors@2.0.1: dependencies: errno: 0.1.8 + optional: true level-iterator-stream@4.0.2: dependencies: inherits: 2.0.4 readable-stream: 3.6.2 xtend: 4.0.2 + optional: true level-js@5.0.2: dependencies: @@ -5288,27 +5296,32 @@ snapshots: buffer: 5.7.1 inherits: 2.0.4 ltgt: 2.2.1 + optional: true level-packager@5.1.1: dependencies: encoding-down: 6.3.0 levelup: 4.4.0 + optional: true level-supports@1.0.1: dependencies: xtend: 4.0.2 + optional: true level@6.0.1: dependencies: level-js: 5.0.2 level-packager: 5.1.1 leveldown: 5.6.0 + optional: true leveldown@5.6.0: dependencies: abstract-leveldown: 6.2.3 napi-macros: 2.0.0 node-gyp-build: 4.1.1 + optional: true levelup@4.4.0: dependencies: @@ -5317,6 +5330,7 @@ snapshots: level-iterator-stream: 4.0.2 level-supports: 1.0.1 xtend: 4.0.2 + optional: true levn@0.4.1: dependencies: @@ -5414,7 +5428,8 @@ snapshots: dependencies: yallist: 4.0.0 - ltgt@2.2.1: {} + ltgt@2.2.1: + optional: true lz-string@1.5.0: {} @@ -5804,7 +5819,8 @@ snapshots: nanoid@5.0.6: {} - napi-macros@2.0.0: {} + napi-macros@2.0.0: + optional: true natural-compare@1.4.0: {} @@ -5827,7 +5843,8 @@ snapshots: detect-libc: 2.0.3 optional: true - node-gyp-build@4.1.1: {} + node-gyp-build@4.1.1: + optional: true normalize-path@3.0.0: {} @@ -5963,7 +5980,8 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - prr@1.0.1: {} + prr@1.0.1: + optional: true psl@1.9.0: {} @@ -6615,13 +6633,15 @@ snapshots: xstate@4.38.3: {} - xtend@4.0.2: {} + xtend@4.0.2: + optional: true y-leveldb@0.1.2(yjs@13.6.15): dependencies: level: 6.0.1 lib0: 0.2.93 yjs: 13.6.15 + optional: true y-protocols@1.0.6(yjs@13.6.15): dependencies: