Orion 5.0.0 (#365)

* - Curator schema and Curator view permissions
- Support for setting channel YppStatus with a tier
- `isYtSyncEnabled` field + related mutations
- Removal of redundant `excludeChannel` / `excludeVideo` mutations (`excludeContent` should be used instead)

* - Relevance service
- Updated channelWeight / videoRelevance formula
- Separated index creation from migrations
- Moved views creation script to `/src`

* - Reduce the usage of NextEntityId to mappings only
	 - Remove NextEntityId from offchainState migration

* Formatting

* Fix linter errors

* Restore accidently removed `save` in `createAccount`, fix notifications test

* Fix notifications tests (part 2)

* Bump version to 5.0.0 and adjust migrations

* Fix notifications tests (part 3)

* Fix notifications tests (part 4)

* Final improvements

* Initial CHANGELOG

* Fix: SumTo constraint

* Fix: TypeOrm subscribers config

* Fix rate query, adjust percentiles

* Fix: Prevent index creation from breaking import

Author: Leszek Wiesner

.env

@@ -1,5 +1,4 @@
 # LOCAL DEV ENVIRONMENT
-
 ORION_ENV=development
 DEV_DISABLE_SAME_SITE=true
 
@@ -17,6 +16,9 @@ PROCESSOR_PROMETHEUS_PORT=3337
 GQL_PORT=4350
 # Auth api port
 AUTH_API_PORT=4074
+# RabbitMQ
+RABBITMQ_PORT=5672
+RABBITMQ_URL=amqp://orion_rabbitmq
 
 # Archive gateway url
 ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://localhost:8888/graphql}
@@ -33,17 +35,10 @@ KILL_SWITCH_ON=false
 VIDEO_VIEW_PER_USER_TIME_LIMIT=10
 # Operator API secret
 OPERATOR_SECRET=this-is-not-so-secret-change-it
-# every 50 views video relevance score will be recalculated
-VIDEO_RELEVANCE_VIEWS_TICK=50
-# [
-#   newness (negative number of days since created) weight,
-#   views weight,
-#   comments weight,
-#   rections weights,
-#   [joystream creation weight, YT creation weight],
-#   Default channel weight/bias
-# ]
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3], 1]"
+# every 10 views video relevance score will be recalculated
+VIDEO_RELEVANCE_VIEWS_TICK=10
+# every time a channel is followed / unfollowed, its weight will be recalculated 
+CHANNEL_WEIGHT_FOLLOWS_TICK=1
 COMMENT_TIP_TIERS='{"SILVER": 100, "GOLD": 500, "DIAMOND": 1000}'
 MAX_CACHED_ENTITIES=5000
 APP_PRIVATE_KEY=this-is-not-so-secret-change-it

CHANGELOG.md

@@ -1,3 +1,168 @@
+# 5.0.0
+
+## Changes
+
+### Channel weights and video relevance calculation
+
+- **Automatic channel weights:** The new _Relevance Service_ supports automatic calculation of channel weights based on factors such as channel's _cumulative revenue_, _YPP tier_, _number of followers_, _CRT liquidity_ and _CRT volume_ (the weight of which can be configured).
+- **Improved video relevance formula:** Factors such as number of _views_ / _comments_ / _reactions_ are now represented as `0 - 1` range rates, which simplifies assigning adequate weights.
+- **Fixed video relevance calculation bugs** (such as https://github.com/Joystream/orion/issues/361)
+- **More flexible configuration** (updates interval or number of scored videos per channel can now be configured via the GraphQL API)
+
+### Improved permissions system
+
+- **New view/read permission levels: `VIEW_CURATOR_SCHEMA`, `VIEW_ADMIN_SCHEMA`** . Thanks to the introduction of a new `curator` database schema, the view permissions to access hidden content (hidden / censored channels and videos) can be separated from view permissions to access more sensitive data (Orion account emails, session data, configuration etc.) and other root-level permissions.
+- Changing YPP status of a channel now requires `SET_CHANNEL_YPP_STATUS` permission.
+- Changing other app-scoped configuration settings which don't have a separate permission category now requires `SET_APP_CONFIGS` permission. 
+- Changing the new relevance service weights and configuration requires `SET_RELEVANCE_WEIGHTS` and `SET_RELEVANCE_CONFIG` permissions respectively.
+
+### Support for setting channel YPP tiers and sync status
+
+- `Channel.yppStatus` now includes YPP tier
+- `Channel` now has a new `isYtSyncEnabled` field
+- Those values can be set via the new `setChannelYppStatus` and `setChannelYoutubeSyncEnabled` mutations respectively (given sufficient permissions)
+
+### Fixed _Offchain state_ migrations
+
+- Fixed https://github.com/Joystream/orion/issues/360
+
+### Performance impovements (indexes, faster sync etc.)
+
+- **New indexes:** New PostgreSQL indexes have been added to support faster filtering / ordering of channels and videos in Atlas and [Content admin dashboard](https://github.com/Joystream/gleev/issues/72).
+- **Post-sync index creation:** The creation of some custom PostgreSQL indexes has been delayed to post-sync phase (ie. the time when processor catches up to chain head) to allow faster from-scratch syncing of new Orion instances and faster migrations.
+
+
+## Upgrade instructions
+
+TBD.
+
+## Affected components:
+- **(M)** `assets/patches/@subsquid+typeorm-config+2.0.2.patch` (support for custom TypeORM subscribers)
+- Processor
+    - **(M)** `processor.ts`
+        - support for new `RelevanceService`
+        - support for post-sync index generation (via `model/indexes.ts`)
+        - improved logging
+    - **(A)** `mappings/subscribers/TransactionCommitSubscriber.ts`
+    - **(M)** `mappings/utils.ts` (added `relevanceQueuePublisher` instance)
+    - Event handlers:
+        - **(M)** `Content.ChannelCreated`
+        - **(M)** `Content.VideoCreated`
+        - Events triggering re-calculation of channel / video relevance:
+            - **(M)** `Content.ChannelRewardUpdated`
+            - **(M)** `Content.ChannelRewardClaimedAndWithdrawn`
+            - **(M)** `Content.EnglishAuctionSettled`
+            - **(M)** `Content.BidMadeCompletingAuction`
+            - **(M)** `Content.OpenAuctionBidAccepted`
+            - **(M)** `Content.OfferAccepted`
+            - **(M)** `Content.NftBought`
+            - **(M)** `Members.MemberRemarked`
+                - `ReactVideo` (meta-action)
+                - `CreateComment` (meta-action)
+                - `DeleteComment` (meta-action)
+                - `MakeChannelPayment` (meta-action)
+- Migrations:
+    - **(!) Re-generated all data migrations**
+    - **(!) Custom indexes are no longer part of migrations, moved to `model/indexes.ts`**
+    - **(M)** `db/migrations/1000000000000-Admin.js`
+    - **(D)** `db/migrations/2200000000000-Indexes.js`
+    - **(R)** `db/viewDefinitions.js` => `model/views.ts`
+    - **(M)** `db/generateViewsMigration.js`
+- Schema / Models:
+    - Entities moved from `admin` to `curator` schema 
+        - **(M)** `OwnedNft`
+        - **(M)** `Auction`
+        - **(M)** `Bid`
+        - **(M)** `Channel`
+        - **(M)** `BannedMember`
+        - **(M)** `Event`
+        - **(M)** `NftHistoryEntry`
+        - **(M)** `NftActivity`
+        - **(M)** `UserInteractionCount`
+        - **(M)** `VideoViewEvent`
+        - **(M)** `Report`
+        - **(M)** `NftFeaturingRequest`
+        - **(M)** `ChannelFollow`
+        - **(M)** `StorageDataObject`
+        - **(M)** `MarketplaceToken`
+        - **(M)** `CommentReaction`
+        - **(M)** `Comment`
+        - **(M)** `VideoCategory`
+        - **(M)** `Video`
+        - **(M)** `VideoFeaturedInCategory`
+        - **(M)** `VideoHero`
+        - **(M)** `VideoMediaMetadata`
+        - **(M)** `VideoMediaEncoding`
+        - **(M)** `License`
+        - **(M)** `VideoSubtitle`
+        - **(M)** `VideoReaction`
+    - Entities with removed `@index` decorators (index defs moved to `model/indexes.ts`):
+        - **(M)** `Channel` (`createdAt`, `language`)
+        - **(M)** `Video` (`createdAt`, `orionLanguage`, `videoRelevance`)
+    - Auth server:
+        - **(M)** `auth-server/handlers/createAccount.ts` (dependency on `NextEntityId` removed)
+    - Entities removed:
+        - **(D)** `ChannelVerification`
+        - **(D)** `ChannelSuspension`
+        - **(D)** `Exclusion`
+    - **(M)** `OperatorPermission`
+    - **(M)** `Channel`
+        - added `isYtSyncEnabled`
+        - extended `yppStatus`
+    - **(M)** `ChannelYppStatus`
+    - **(A)** `model/views.ts` (PostgreSQL view creation utils)
+    - **(A)** `model/indexes.ts` (PostgreSQL index creation utils)
+- Auth server:
+    - **(M)** `openapi.yml` spec
+    - **(M)** `anonymousAuth` route
+- GraphQL server:
+    - **(M)** `server-extension/check.ts` (more granular view permissions, `curator` / `admin` schema)
+    - **(M)** `server-extension/utils.ts`
+    - **(A)** `server-extension/subscribers/TransactionCommitSubscriber.ts`
+    - Fixed permissions:
+        - **(M)** `setAppAssetStorage`
+        - **(M)** `setAppNameAlt`
+        - setNewNotificationAssetRoot
+        - setMaxAttemptsOnMailDelivery
+    - **(A)** `setChannelYppStatus`
+    - **(M)** `followChannel`
+    - **(M)** `addVideoView`
+    - **(D)** `suspendChannels` (replaced by `setChannelYppStatus`)
+    - **(D)** `verifyChannel` (replaced by `setChannelYppStatus`)
+    - **(D)** `excludeChannel` (duplicate of `excludeContent`)
+    - **(D)** `excludeVideo` (duplicate of `excludeContent`)
+    - **(D)** `setVideoWeights`
+    - **(D)** `setNewNotificationCenterPath` (duplicate of `setMaxAttemptsOnMailDelivery`)
+    - **(D)** `setChannelsWeights`
+    - **(M)** `processCommentsCensorshipStatusUpdate`
+    - **(A)** `setRelevanceWeights` mutation
+    - **(A)** `setRelevanceServiceConfig` mutation
+- **(A) NEW RELEVANCE SERVICE (`relevance-service`)**
+- **(D)** `utils/VideoRelevanceManager.ts`
+- Config:
+    - **Defaults now can be specified directly in `src/utils/config.ts`** (but can still be overriden by `env` variables and/or db config)
+    - **(A)** `RELEVANCE_SERVICE_CONFIG`
+    - **(A)** `CHANNEL_WEIGHT_FOLLOWS_TICK`
+- Environment:
+    - **(A)** `RABBITMQ_PORT` (required)
+    - **(A)** `RABBITMQ_URL` (required)
+    - **(A)** `CHANNEL_WEIGHT_FOLLOWS_TICK` (required)
+    - **(M)** `VIDEO_RELEVANCE_VIEWS_TICK` (`50` => `10`)
+    - **(D)** `RELEVANCE_WEIGHTS`
+- Utils:
+    - **(M)** `utils/OrionVideoLanguageManager.ts` (`admin` => `curator` schema)
+    - **(M)** `utils/customMigrations/setOrionLanguageProvider.ts` (`admin` => `curator` schema)
+    - **(D)** `utils/nextEntityId.ts`
+    - **(M)** `notification/helpers.ts` (dependency on `NextEntityId` removed)
+    - **(M)** `utils/offchainState.ts`
+    - **(M)** `utils/overlay.ts` (prevent deadlocks)
+- **(M)** `docker-compose.yml`
+    - Added RabbitMQ queue
+    - Added relevance service
+- **(M)** `Makefile`
+    - Turned off `SQD_DEBUG` for mappings in `process`
+    - Added `relevance-service`
+
 # 4.5.0
 ## Affected components:
 - Auth server:

Makefile

@@ -1,5 +1,5 @@
 process: migrate
-	@SQD_DEBUG=sqd:processor:mapping node -r dotenv-expand/config lib/processor.js
+	@node -r dotenv-expand/config lib/processor.js
 
 install:
 	@rm -rf node_modules # clean up node_modules to avoid issues with patch-package
@@ -18,6 +18,9 @@ serve:
 serve-auth-api:
 	@npm run auth-server-start
 
+relevance-service:
+	@npm run relevance-service-start
+
 migrate:
 	@npx squid-typeorm-migration apply
 

assets/patches/@subsquid+typeorm-config+2.0.2.patch

@@ -1,8 +1,8 @@
 diff --git a/node_modules/@subsquid/typeorm-config/lib/config.js b/node_modules/@subsquid/typeorm-config/lib/config.js
-index 046611f..28fd4e2 100644
+index 046611f..04a0ad2 100644
 --- a/node_modules/@subsquid/typeorm-config/lib/config.js
 +++ b/node_modules/@subsquid/typeorm-config/lib/config.js
-@@ -28,22 +28,22 @@ const path = __importStar(require("path"));
+@@ -28,22 +28,26 @@ const path = __importStar(require("path"));
  const process = __importStar(require("process"));
  const connectionOptions_1 = require("./connectionOptions");
  const namingStrategy_1 = require("./namingStrategy");
@@ -13,6 +13,9 @@ index 046611f..28fd4e2 100644
 -    let model = resolveModel(path.join(dir, 'lib/model'));
 +    let model = resolveModel(path.join(dir, "lib/model"));
      let migrationsDir = path.join(dir, exports.MIGRATIONS_DIR);
++    let subscribers = process.env.TYPEORM_SUBSCRIBERS_DIR
++        ? [`${path.join(dir, process.env.TYPEORM_SUBSCRIBERS_DIR)}/*.js`]
++        : undefined
      return {
 -        type: 'postgres',
 +        type: "postgres",
@@ -21,6 +24,7 @@ index 046611f..28fd4e2 100644
 -        migrations: [migrationsDir + '/*.js'],
 -        ...(0, connectionOptions_1.createConnectionOptions)()
 +        migrations: [migrationsDir + `/${options?.file || "*.js"}`],
++        subscribers,
 +        ...(0, connectionOptions_1.createConnectionOptions)(),
      };
  }

db/generateViewsMigration.js

@@ -8,7 +8,7 @@ const generateViewsMigration = (versionNumber) => {
   const className = `Views${versionNumber}`
   const fileName = `${versionNumber}-Views.js`
   const fileContent = `
-const { getViewDefinitions } = require('../viewDefinitions')
+const { createViews } = require('../../lib/model/views')
 
 module.exports = class ${className} {
   name = '${className}'
@@ -21,31 +21,11 @@ module.exports = class ${className} {
       id SERIAL PRIMARY KEY,
       height INT
     );\`)
-    const viewDefinitions = getViewDefinitions(db);
-    for (const [tableName, viewConditions] of Object.entries(viewDefinitions)) {
-      if (Array.isArray(viewConditions)) {
-        await db.query(\`
-          DROP VIEW IF EXISTS "\${tableName}" CASCADE
-        \`)
-        await db.query(\`
-          CREATE OR REPLACE VIEW "\${tableName}" AS
-            SELECT *
-            FROM "admin"."\${tableName}" AS "this"
-            WHERE \${viewConditions.map(cond => \`(\${cond})\`).join(' AND ')}
-        \`);
-      } else {
-        await db.query(\`
-          CREATE OR REPLACE VIEW "\${tableName}" AS (\${viewConditions})
-        \`);
-      }
-    }
-  }  
+    await createViews(db);
+  }
 
   async down(db) {
-    const viewDefinitions = this.getViewDefinitions(db)
-    for (const viewName of Object.keys(viewDefinitions)) {
-      await db.query(\`DROP VIEW "\${viewName}"\`)
-    }
+    await dropViews(db);
   }
 }
 `

db/migrations/1000000000000-Admin.js

@@ -2,15 +2,18 @@ module.exports = class Admin1000000000000 {
   name = 'Admin1000000000000'
 
   async up(db) {
-    // Create a new "admin" schema through which the "hidden" entities can be accessed
+    // Create "admin" and "curator" schemas, through which some of the "hidden" entities can be accessed
     await db.query(`CREATE SCHEMA "admin"`)
-    // Create admin user with "admin" schema in default "search_path"
+    await db.query(`CREATE SCHEMA "curator"`)
+    // Create admin user with "admin" and "curator" schemas in default "search_path"
     await db.query(
       `CREATE USER "${process.env.DB_ADMIN_USER}" WITH PASSWORD '${process.env.DB_ADMIN_PASS}'`
     )
     await db.query(`GRANT pg_read_all_data TO "${process.env.DB_ADMIN_USER}"`)
     await db.query(`GRANT pg_write_all_data TO "${process.env.DB_ADMIN_USER}"`)
-    await db.query(`ALTER USER "${process.env.DB_ADMIN_USER}" SET search_path TO admin,public`)
+    await db.query(
+      `ALTER USER "${process.env.DB_ADMIN_USER}" SET search_path TO admin,curator,public`
+    )
   }
 
   async down(db) {