diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 182f83b09..1ac381051 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,8 +4,8 @@ about: Tell us about something that went wrong title: '' labels: bug assignees: '' - --- + Thanks for reporting a bug! **Describe the bug** @@ -13,6 +13,7 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' diff --git a/.github/ISSUE_TEMPLATE/documentation-issue.md b/.github/ISSUE_TEMPLATE/documentation-issue.md index 659e89678..3b69e4719 100644 --- a/.github/ISSUE_TEMPLATE/documentation-issue.md +++ b/.github/ISSUE_TEMPLATE/documentation-issue.md @@ -4,19 +4,22 @@ about: Report an issue found in the Documentation title: '' labels: bug, documentation assignees: '' - --- ### Where is the Issue? + Please link to the file(s) and line number(s) that has the issue(s) ### What is wrong with it? + Spelling/typo out of date, doesn't work, broken link, something missing, etc. ### Optional: Suggested change ... + ### [PM/Eng] Acceptance Criteria + - [ ] list of clear directives - [ ] that would enable someone - [ ] to begin work diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 5d903d258..f0ad7da23 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,17 +4,20 @@ about: Request additional Documentation, updates, features title: '' labels: enhancement assignees: '' - --- + Thanks for requesting an improvement! ### Context -Why is this important? + +Why is this important? ### Outline or additional Details + Description of the feature/update/docs needed ### [PM/Eng]: Acceptance criteria + - [ ] list of clear directives - [ ] that would enable someone - [ ] to begin work diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 329da0cff..d1624f8d2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,10 +12,10 @@ A short description of what you have done to implement/fix the above mentioned f ### Change summary -* A detailed list of bulleted -* changes that go into detail about -* the specifics of the changes -* to the codebase +- A detailed list of bulleted +- changes that go into detail about +- the specifics of the changes +- to the codebase ### Steps to Verify diff --git a/.spellcheckerrc.json b/.spellcheckerrc.json index 90a4cac2c..86e706c1e 100644 --- a/.spellcheckerrc.json +++ b/.spellcheckerrc.json @@ -7,7 +7,5 @@ "!CLAUDE.md", "**/.github/**/*.md" ], - "dictionaries": [ - ".spellcheckerdict.txt" - ] + "dictionaries": [".spellcheckerdict.txt"] } diff --git a/CLAUDE.md b/CLAUDE.md index c20239b9c..cb3340166 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,15 +9,18 @@ This is the **Frequency Developer Gateway** - a NestJS monorepo containing micro ### Architecture **Monorepo Structure:** + - **Apps** (9 services): Independent microservices with API endpoints and background workers - **Libs** (16 libraries): Shared functionality and utilities **Core Services:** + - **Account** (`account-api` + `account-worker`): User account management, authentication via Sign In With Frequency - **Content Publishing** (`content-publishing-api` + `content-publishing-worker`): Content creation, file upload, IPFS storage, blockchain publishing - **Content Watcher** (`content-watcher`): Blockchain monitoring for content changes **Key Libraries:** + - **blockchain**: Polkadot API integration, Frequency chain interactions - **queue**: BullMQ job processing with Redis - **cache**: Redis caching layer @@ -27,6 +30,7 @@ This is the **Frequency Developer Gateway** - a NestJS monorepo containing micro ## Development Commands ### Building + ```bash # Build all services npm run build @@ -39,6 +43,7 @@ npm run build:libs ``` ### Running Services + ```bash # Start individual services in development mode npm run start:account-api:dev @@ -51,6 +56,7 @@ npm run start:content-publishing-worker:dev ``` ### Testing + ```bash # Run all tests npm test @@ -75,6 +81,7 @@ jest path/to/test.spec.ts ``` ### Linting and Formatting + ```bash # Lint all code npm run lint @@ -95,6 +102,7 @@ npm run spellcheck ``` ### OpenAPI Documentation Generation + ```bash # Generate all OpenAPI specs npm run generate:openapi @@ -108,6 +116,7 @@ npm run generate:swagger-ui ``` ### Load Testing + ```bash # Run k6 load tests npm run test:k6:account @@ -120,7 +129,9 @@ SCENARIO=light k6 run apps/content-publishing-api/k6-test/batch-announcement-loa ## Architecture Patterns ### Import Path Aliases + Uses `#` prefix for internal module resolution: + - `#account-api/*` → `apps/account-api/src/*` - `#content-publishing-lib/*` → `libs/content-publishing-lib/src/*` - `#blockchain` → `libs/blockchain/src` @@ -129,33 +140,39 @@ Uses `#` prefix for internal module resolution: - `#validation` → `libs/types/src/validation` ### Queue Architecture + Each service uses BullMQ with Redis for background job processing: **Content Publishing Queues:** + - `REQUEST_QUEUE_NAME`: Incoming announcement requests - `ASSET_QUEUE_NAME`: File upload processing - `BATCH_QUEUE_NAME`: Batching announcements for efficiency - `PUBLISH_QUEUE_NAME`: Publishing to blockchain **Queue Processing Flow:** + 1. API receives request → enqueues job 2. Worker processes job → transforms data 3. Publishes to blockchain via batch transactions 4. Monitors transaction status ### Service Communication + - **API Layer**: REST endpoints with OpenAPI/Swagger documentation - **Worker Layer**: Background job processors using BullMQ - **Blockchain Layer**: Polkadot API integration for Frequency chain - **Storage Layer**: IPFS for content, Redis for caching, file system for temporary storage ### Configuration Management + - Environment-specific `.env` files for each service - Template files in `env-files/` directory - Joi validation schemas for configuration - Service-specific config modules in each app ### Database/Storage Architecture + - **No traditional database** - leverages blockchain as source of truth - **Redis**: Caching, job queues, temporary data storage - **IPFS**: Persistent content storage @@ -164,32 +181,38 @@ Each service uses BullMQ with Redis for background job processing: ## Testing Architecture ### Test Structure + - **Unit tests**: `.spec.ts` files alongside source code - **E2E tests**: `.e2e-spec.ts` files in each app's test directory - **Load tests**: k6 scripts in each app's `k6-test/` directory - **Mocks**: Shared mocks in `__mocks__/` directory ### Module Path Mapping + Jest configuration maps `#` aliases to actual paths, matching TypeScript configuration. ## Key Development Practices ### Error Handling + - Use NestJS HTTP exceptions (`BadRequestException`, `InternalServerErrorException`) - Blockchain errors are wrapped and handled gracefully - Queue job failures use exponential backoff retry strategies ### Logging + - Uses Pino logger with structured JSON output - Set `PRETTY=true` for human-readable development logs - Log levels: `error`, `warn`, `info`, `debug`, `trace` ### Capacity Management + - Services check Frequency blockchain capacity before operations - Automatic pausing/resuming of queues based on capacity - Graceful handling of capacity exhaustion ### Transaction Monitoring + - All blockchain transactions are monitored for finality - Status tracking via Redis with transaction hash keys - Automatic cleanup of completed/failed transactions @@ -197,6 +220,7 @@ Jest configuration maps `#` aliases to actual paths, matching TypeScript configu ## File Organization Patterns ### Apps Structure + ``` apps/[service-name]/ ├── src/ @@ -211,6 +235,7 @@ apps/[service-name]/ ``` ### Libs Structure + ``` libs/[lib-name]/ ├── src/ diff --git a/README.md b/README.md index 4f285b73d..2195a9700 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ bridging the gap between Web2 and Web3 development. - [🔍 Architecture Map](#-arch-maps) - [🔍 Gateway Microservices](#gateway-microservices) - [💻 Getting Started](#getting-started) - - [🚀 Quick Start Guide](#quick-start-guide) - - [💻 Getting Started with Microservices](#microservices-start-guide) + - [🚀 Quick Start Guide](#quick-start-guide) + - [💻 Getting Started with Microservices](#microservices-start-guide) - [🛫 Deployment](#deployment) - [📝 Logging](#logging) - [📊 Metrics](#metrics) @@ -125,7 +125,7 @@ Gateway consists of four independent microservices, each designed to handle spec Frequency blockchain. Below is a detailed overview of each service: | Service | Description | API Documentation | README | -|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|---------------------------------------------------------| +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------- | | Account Service | Manages user accounts and authentication on the Frequency blockchain using [Sign In With Frequency](https://github.com/ProjectLibertyLabs/siwf). It handles tasks such as account creation and key management. | [API Docs](https://projectlibertylabs.github.io/gateway/account) | [README](./developer-docs/account/README.md) | | Content Publishing Service | Facilitates the creation and publication of content on the Frequency blockchain. It manages tasks such as posting messages, attachments, replies, and reactions. | [API Docs](https://projectlibertylabs.github.io/gateway/content-publishing/) | [README](./developer-docs/content-publishing/README.md) | | Content Watcher Service | Monitors and retrieves content updates from the Frequency blockchain. It allows applications to efficiently track new content as it's published. | [API Docs](https://projectlibertylabs.github.io/gateway/content-watcher/) | [README](./developer-docs/content-watcher/README.md) | @@ -274,8 +274,7 @@ cd apps/content-publishing-api/k6-test SCENARIO=heavy k6 run batch-announcement-load.k6.js ``` -Detailed information, configuration options, and best practices, are included in our * -*[k6 testing documentation](./docs/k6/k6-documentation.md)**. +Detailed information, configuration options, and best practices, are included in our \* \*[k6 testing documentation](./docs/k6/k6-documentation.md)\*\*. diff --git a/apps/account-api/src/controllers/v1/ics.controller.v1.ts b/apps/account-api/src/controllers/v1/ics.controller.v1.ts index fcde0fa00..2970150dc 100644 --- a/apps/account-api/src/controllers/v1/ics.controller.v1.ts +++ b/apps/account-api/src/controllers/v1/ics.controller.v1.ts @@ -1,15 +1,7 @@ import { EnqueueService } from '#account-lib/services/enqueue-request.service'; import { AddNewPublicKeyAgreementRequestDto, IcsPublishAllRequestDto, UpsertPagePayloadDto } from '#types/dtos/account'; import { AccountIdDto } from '#types/dtos/common'; -import { - Body, - Controller, - HttpCode, - HttpException, - HttpStatus, - Param, - Post, -} from '@nestjs/common'; +import { Body, Controller, HttpCode, HttpException, HttpStatus, Param, Post } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { ApiPromise } from '@polkadot/api'; import { SubmittableExtrinsic } from '@polkadot/api/types'; @@ -17,9 +9,7 @@ import { ISubmittableResult } from '@polkadot/types/types'; import { HexString } from '@polkadot/util/types'; import { InjectPinoLogger, PinoLogger } from 'nestjs-pino'; import { BlockchainRpcQueryService } from '#blockchain/blockchain-rpc-query.service'; -import { - chainSignature, -} from '#utils/common/signature.util'; +import { chainSignature } from '#utils/common/signature.util'; @Controller({ version: '1', path: 'ics' }) @ApiTags('v1/ics') @@ -51,12 +41,12 @@ export class IcsControllerV1 { payload: AddNewPublicKeyAgreementRequestDto, api: ApiPromise, ): SubmittableExtrinsic<'promise', ISubmittableResult> { - const encodedPayload = this.blockchainService.createItemizedSignaturePayloadV2Type(payload.payload); - return api.tx.statefulStorage.applyItemActionsWithSignatureV2( - accountId, - chainSignature({ algo: 'Sr25519', encodedValue: payload.proof}), // TODO: determine signature algo - encodedPayload, - ); + const encodedPayload = this.blockchainService.createItemizedSignaturePayloadV2Type(payload.payload); + return api.tx.statefulStorage.applyItemActionsWithSignatureV2( + accountId, + chainSignature({ algo: 'Sr25519', encodedValue: payload.proof }), // TODO: determine signature algo + encodedPayload, + ); } buildUpsertPageExtrinsic( diff --git a/apps/account-api/src/services/siwfV2.service.ts b/apps/account-api/src/services/siwfV2.service.ts index 7426e748c..692f009c8 100644 --- a/apps/account-api/src/services/siwfV2.service.ts +++ b/apps/account-api/src/services/siwfV2.service.ts @@ -99,7 +99,7 @@ export class SiwfV2Service { return null; } - const api: ApiPromise = await this.blockchainService.getApi() as ApiPromise; + const api: ApiPromise = (await this.blockchainService.getApi()) as ApiPromise; const { pallet, extrinsic: extrinsicName } = payload.endpoint; switch (`${pallet}.${extrinsicName}`) { diff --git a/apps/content-publishing-api/k6-test/batch-announcement-load.k6.js b/apps/content-publishing-api/k6-test/batch-announcement-load.k6.js index e5b344660..f1d95c6ea 100644 --- a/apps/content-publishing-api/k6-test/batch-announcement-load.k6.js +++ b/apps/content-publishing-api/k6-test/batch-announcement-load.k6.js @@ -90,8 +90,8 @@ export default function () { const params = { headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; const request = http.post(url, JSON.stringify(body), params); @@ -113,8 +113,8 @@ export default function () { const params = { headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; const request = http.post(url, JSON.stringify(body), params); @@ -205,8 +205,8 @@ export default function () { const params = { headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; const request = http.post(url, JSON.stringify(body), params); @@ -221,10 +221,13 @@ export default function () { const avgResponseTime = totalResponseTime / batchCount; - check({ successCount, avgResponseTime }, { - 'sequential batches - success rate > 80%': (r) => (r.successCount / batchCount) > 0.8, - 'sequential batches - avg response time < 8s': (r) => r.avgResponseTime < 8000, - }); + check( + { successCount, avgResponseTime }, + { + 'sequential batches - success rate > 80%': (r) => r.successCount / batchCount > 0.8, + 'sequential batches - avg response time < 8s': (r) => r.avgResponseTime < 8000, + }, + ); sleep(randomIntBetween(3, 6)); }); @@ -251,10 +254,13 @@ export default function () { const avgResponseTime = totalResponseTime / uploadCount; - check({ successCount, avgResponseTime }, { - 'v3 sequential uploads - success rate > 80%': (r) => (r.successCount / uploadCount) > 0.8, - 'v3 sequential uploads - avg response time < 20s': (r) => r.avgResponseTime < 20000, - }); + check( + { successCount, avgResponseTime }, + { + 'v3 sequential uploads - success rate > 80%': (r) => r.successCount / uploadCount > 0.8, + 'v3 sequential uploads - avg response time < 20s': (r) => r.avgResponseTime < 20000, + }, + ); sleep(randomIntBetween(5, 10)); }); @@ -266,8 +272,8 @@ export default function () { const params = { headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; const request = http.post(url, JSON.stringify(maxBatchData), params); @@ -305,15 +311,15 @@ export default function () { const params = { headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; const request = http.post(url, JSON.stringify(body), params); check(request, { 'different batch sizes - status is 202': (r) => r.status === 202, - 'different batch sizes - response time reasonable': (r) => r.timings.duration < (selectedSize * 2000), // 2s per file max + 'different batch sizes - response time reasonable': (r) => r.timings.duration < selectedSize * 2000, // 2s per file max }); sleep(randomIntBetween(3, 8)); diff --git a/apps/content-publishing-api/k6-test/batch-announcement-stress.k6.js b/apps/content-publishing-api/k6-test/batch-announcement-stress.k6.js index d8197a3e3..3b27b3441 100644 --- a/apps/content-publishing-api/k6-test/batch-announcement-stress.k6.js +++ b/apps/content-publishing-api/k6-test/batch-announcement-stress.k6.js @@ -2,12 +2,12 @@ import http from 'k6/http'; import { check, group, sleep } from 'k6'; import { randomIntBetween } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; import { Trend, Counter } from 'k6/metrics'; -import { - createRealisticBatchData, - createMultipartBatchData, +import { + createRealisticBatchData, + createMultipartBatchData, BATCH_SCENARIOS, createErrorScenarios, - BATCH_CONSTANTS + BATCH_CONSTANTS, } from './helpers.js'; const BASE_URL = 'http://localhost:3010'; @@ -26,7 +26,7 @@ export const options = { ], gracefulRampDown: '30s', }, - + // Sustained load phase sustained_load: { executor: 'constant-vus', @@ -34,7 +34,7 @@ export const options = { duration: '2m', startTime: '2m', }, - + // Spike testing spike_test: { executor: 'ramping-arrival-rate', @@ -49,7 +49,7 @@ export const options = { ], startTime: '5m', }, - + // Burst testing burst_test: { executor: 'constant-arrival-rate', @@ -61,21 +61,21 @@ export const options = { startTime: '7m', }, }, - + thresholds: { // Overall performance thresholds checks: ['rate>=0.90'], http_req_duration: ['avg<15000', 'p(95)<30000', 'p(99)<60000'], http_req_failed: ['rate<0.10'], http_reqs: ['rate>=50'], - + // Error rate thresholds by scenario 'http_req_failed{scenario:ramp_up}': ['rate<0.05'], 'http_req_failed{scenario:sustained_load}': ['rate<0.08'], 'http_req_failed{scenario:spike_test}': ['rate<0.15'], 'http_req_failed{scenario:burst_test}': ['rate<0.20'], }, - + noConnectionReuse: true, // Add connection settings to handle EOF errors // Increase timeouts to handle file uploads @@ -86,31 +86,28 @@ const successfulRequests = new Counter('successful_requests'); const failedRequests = new Counter('failed_requests'); const responseTime = new Trend('response_time'); - - export default function () { const scenario = __ENV.SCENARIO || 'stress'; - + // Test v2 batch announcement with different scenarios group('v2/batchAnnouncement - Stress Testing', () => { const scenarios = ['small', 'medium', 'large', 'mixed']; const selectedScenario = scenarios[randomIntBetween(0, scenarios.length - 1)]; - + const url = `${BASE_URL}/v2/content/batchAnnouncement`; - const batchData = createRealisticBatchData( - BATCH_SCENARIOS[selectedScenario].fileCount, - { fileSize: BATCH_SCENARIOS[selectedScenario].fileSize } - ); - - const params = { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json' - } + const batchData = createRealisticBatchData(BATCH_SCENARIOS[selectedScenario].fileCount, { + fileSize: BATCH_SCENARIOS[selectedScenario].fileSize, + }); + + const params = { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, }; - + const request = http.post(url, JSON.stringify(batchData), params); - + // Track metrics if (request.status === 202) { successfulRequests.add(1); @@ -118,13 +115,13 @@ export default function () { failedRequests.add(1); } responseTime.add(request.timings.duration); - + check(request, { 'v2 stress - status is 202': (r) => r.status === 202, 'v2 stress - response time < 20s': (r) => r.timings.duration < 20000, 'v2 stress - has response body': (r) => r.body.length > 0, }); - + sleep(randomIntBetween(2, 8)); }); @@ -132,14 +129,12 @@ export default function () { group('v3/batchAnnouncement - Multipart Stress', () => { const fileCount = randomIntBetween(1, 3); // Keep small to avoid overwhelming const formData = createMultipartBatchData(fileCount, { fileSize: 'sm' }); - + const url = `${BASE_URL}/v3/content/batchAnnouncement`; - + // No need to set Content-Type header - k6 will set it automatically for multipart const request = http.post(url, formData); - - // Track metrics if (request.status === 202) { successfulRequests.add(1); @@ -147,7 +142,7 @@ export default function () { failedRequests.add(1); } responseTime.add(request.timings.duration); - + check(request, { 'v3 multipart stress - status is 202': (r) => r.status === 202, 'v3 multipart stress - response time < 40s': (r) => r.timings.duration < 40000, @@ -160,7 +155,7 @@ export default function () { } }, }); - + sleep(randomIntBetween(10, 20)); // Longer sleep for file uploads }); @@ -170,23 +165,20 @@ export default function () { const responses = []; let totalDuration = 0; let successCount = 0; - + for (let i = 0; i < concurrentCount; i++) { const url = `${BASE_URL}/v2/content/batchAnnouncement`; - const batchData = createRealisticBatchData( - randomIntBetween(1, 3), - { fileSize: 'sm' } - ); - const params = { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json' - } + const batchData = createRealisticBatchData(randomIntBetween(1, 3), { fileSize: 'sm' }); + const params = { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, }; - + const response = http.post(url, JSON.stringify(batchData), params); responses.push(response); - + // Track metrics for concurrent requests if (response.status === 202) { successfulRequests.add(1); @@ -197,7 +189,7 @@ export default function () { responseTime.add(response.timings.duration); totalDuration += response.timings.duration; } - + check(responses, { 'concurrent stress - all successful': (r) => successCount === concurrentCount, 'concurrent stress - avg response time < 25s': (r) => { @@ -205,7 +197,7 @@ export default function () { return avgTime < 25000; }, }); - + sleep(randomIntBetween(3, 8)); }); @@ -214,23 +206,23 @@ export default function () { const errorScenarios = createErrorScenarios(); const scenarioNames = Object.keys(errorScenarios); const selectedScenario = scenarioNames[randomIntBetween(0, scenarioNames.length - 1)]; - + const url = `${BASE_URL}/v2/content/batchAnnouncement`; const body = errorScenarios[selectedScenario]; - const params = { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json' - } + const params = { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, }; - + const request = http.post(url, JSON.stringify(body), params); - + check(request, { 'error stress - appropriate error status': (r) => r.status >= 400 && r.status < 500, 'error stress - response time < 5s': (r) => r.timings.duration < 5000, }); - + sleep(randomIntBetween(1, 3)); }); @@ -238,20 +230,20 @@ export default function () { group('Large Batch Processing - Stress', () => { const url = `${BASE_URL}/v2/content/batchAnnouncement`; const batchData = createRealisticBatchData(10, { fileSize: 'lg' }); - const params = { - headers: { + const params = { + headers: { 'Content-Type': 'application/json', - Accept: 'application/json' - } + Accept: 'application/json', + }, }; - + const request = http.post(url, JSON.stringify(batchData), params); - + check(request, { 'large batch stress - status is 202': (r) => r.status === 202, 'large batch stress - response time < 60s': (r) => r.timings.duration < 60000, }); - + sleep(randomIntBetween(10, 20)); }); @@ -259,38 +251,35 @@ export default function () { group('Large File Uploads - Stress', () => { const url = `${BASE_URL}/v3/content/batchAnnouncement`; const formData = createMultipartBatchData(1, { fileSize: 'md' }); // Use medium size to avoid overwhelming - + const request = http.post(url, formData); - + check(request, { 'large file upload stress - status is 202': (r) => r.status === 202, 'large file upload stress - response time < 60s': (r) => r.timings.duration < 60000, }); - + sleep(randomIntBetween(15, 25)); // Longer sleep for large file uploads }); // Test maximum batch limits group('Maximum Batch Limits - Stress', () => { const url = `${BASE_URL}/v2/content/batchAnnouncement`; - const maxBatchData = createRealisticBatchData( - BATCH_CONSTANTS.MAX_FILES_PER_BATCH, - { fileSize: 'sm' } - ); - const params = { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json' - } + const maxBatchData = createRealisticBatchData(BATCH_CONSTANTS.MAX_FILES_PER_BATCH, { fileSize: 'sm' }); + const params = { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, }; - + const request = http.post(url, JSON.stringify(maxBatchData), params); - + check(request, { 'max batch stress - status is 202': (r) => r.status === 202, 'max batch stress - response time < 30s': (r) => r.timings.duration < 30000, }); - + sleep(randomIntBetween(5, 15)); }); } @@ -298,34 +287,34 @@ export default function () { // Setup function for stress test export function setup() { console.log('Starting batch announcement stress test...'); - + // Health check const healthCheck = http.get(`${BASE_URL}/healthz`); if (healthCheck.status !== 200) { throw new Error('Service is not healthy for stress testing'); } - + // Warm up the service console.log('Warming up the service...'); for (let i = 0; i < 10; i++) { const warmupData = createRealisticBatchData(1, { fileSize: 'sm' }); const url = `${BASE_URL}/v2/content/batchAnnouncement`; - const params = { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json' - } + const params = { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, }; - + const request = http.post(url, JSON.stringify(warmupData), params); if (request.status !== 202) { console.warn(`Warmup request ${i} failed with status ${request.status}`); } } - - return { + + return { startTime: new Date().toISOString(), - testType: 'stress' + testType: 'stress', }; } @@ -334,7 +323,7 @@ export function teardown(data) { console.log('Completing batch announcement stress test...'); console.log(`Test started at: ${data.startTime}`); console.log(`Test completed at: ${new Date().toISOString()}`); - + // Note: Custom metrics are handled in handleSummary console.log('Test completed. Check the summary for detailed metrics.'); } @@ -343,24 +332,24 @@ export function teardown(data) { export function handleSummary(data) { const testStartTime = new Date().toISOString(); const testEndTime = new Date().toISOString(); - + console.log('\n' + '='.repeat(80)); console.log('BATCH ANNOUNCEMENT STRESS TEST SUMMARY'); console.log('='.repeat(80)); - + console.log(`Test Type: batch-announcement-stress`); console.log(`Timestamp: ${new Date().toISOString()}`); console.log(`Test started at: ${testStartTime}`); console.log(`Test completed at: ${testEndTime}`); console.log(''); - + // Get metrics from data const successfulCount = data.metrics.successful_requests?.values?.count || 0; const failedCount = data.metrics.failed_requests?.values?.count || 0; const avgResponseTime = data.metrics.response_time?.values?.avg || 0; const totalRequests = successfulCount + failedCount; - const successRate = totalRequests > 0 ? (successfulCount / totalRequests * 100).toFixed(2) : '0.00'; - + const successRate = totalRequests > 0 ? ((successfulCount / totalRequests) * 100).toFixed(2) : '0.00'; + // Basic summary console.log('BASIC METRICS'); console.log('-'.repeat(40)); @@ -370,7 +359,7 @@ export function handleSummary(data) { console.log(`Success rate: ${successRate}%`); console.log(`Average response time: ${avgResponseTime.toFixed(2)}ms`); console.log(''); - + // Detailed HTTP metrics console.log('HTTP METRICS'); console.log('-'.repeat(40)); @@ -384,31 +373,33 @@ export function handleSummary(data) { console.log(` Min: ${duration.min?.toFixed(2)}ms`); console.log(` Max: ${duration.max?.toFixed(2)}ms`); } - + if (data.metrics.http_reqs) { const reqs = data.metrics.http_reqs.values; console.log(`HTTP Requests:`); console.log(` Rate: ${reqs.rate?.toFixed(2)} req/s`); console.log(` Count: ${reqs.count}`); } - + if (data.metrics.http_req_failed) { const failed = data.metrics.http_req_failed.values; console.log(`HTTP Request Failures:`); console.log(` Rate: ${failed.rate?.toFixed(2)} failed/s`); - console.log(` Percentage: ${(failed.rate / (data.metrics.http_reqs?.values?.rate || 1) * 100).toFixed(2)}%`); + console.log(` Percentage: ${((failed.rate / (data.metrics.http_reqs?.values?.rate || 1)) * 100).toFixed(2)}%`); } console.log(''); - - if ((!data.thresholds?.passed || data.thresholds.passed.length === 0) && - (!data.thresholds?.failed || data.thresholds.failed.length === 0)) { + + if ( + (!data.thresholds?.passed || data.thresholds.passed.length === 0) && + (!data.thresholds?.failed || data.thresholds.failed.length === 0) + ) { console.log('No thresholds defined'); } console.log(''); console.log('='.repeat(80)); console.log('SUMMARY COMPLETE'); console.log('='.repeat(80) + '\n'); - + // Return empty object to prevent file generation return {}; } diff --git a/apps/content-publishing-api/k6-test/helpers.js b/apps/content-publishing-api/k6-test/helpers.js index 2a8cee817..16c89f19b 100644 --- a/apps/content-publishing-api/k6-test/helpers.js +++ b/apps/content-publishing-api/k6-test/helpers.js @@ -112,11 +112,11 @@ export const BATCH_CONSTANTS = { SUPPORTED_SCHEMA_IDS: [12, 13], // Valid schema IDs SUPPORTED_FILE_TYPES: ['parquet', 'jpg', 'png'], // TODO: Add all mime types relevantIDs EXPECTED_RESPONSE_TIMES: { - singleFile: 5000, // 5s + singleFile: 5000, // 5s multipleFiles: 10000, // 10s - largeFiles: 30000, // 30s - concurrent: 15000 // 15s - } + largeFiles: 30000, // 30s + concurrent: 15000, // 15s + }, }; // Generate test data for different batch scenarios @@ -125,21 +125,21 @@ export const BATCH_SCENARIOS = { small: { fileCount: 1, fileSize: 'sm', - description: 'Single small file batch' + description: 'Single small file batch', }, // Medium batches - normal usage medium: { fileCount: 3, fileSize: 'sm', - description: 'Multiple small files' + description: 'Multiple small files', }, // Large batches - stress testing large: { fileCount: 10, fileSize: 'md', - description: 'Multiple medium files' + description: 'Multiple medium files', }, // Mixed batches - realistic usage @@ -147,26 +147,24 @@ export const BATCH_SCENARIOS = { fileCount: 5, fileSize: 'sm', description: 'Mixed file sizes and schemas', - mixedSizes: true + mixedSizes: true, }, // Edge case - maximum files maxFiles: { fileCount: 20, fileSize: 'sm', - description: 'Maximum number of files' - } + description: 'Maximum number of files', + }, }; - - // Create realistic batch data with different schemas export const createRealisticBatchData = (fileCount = 1, options = {}) => { const { useRealUploads = false, // Changed back to false to avoid overwhelming the service fileSize = 'sm', schemaIds = BATCH_CONSTANTS.SUPPORTED_SCHEMA_IDS, - baseUrl = 'http://localhost:3010' + baseUrl = 'http://localhost:3010', } = options; const batchFiles = []; @@ -210,7 +208,7 @@ export const createMultipartBatchData = (fileCount = 1, options = {}) => { const { fileSize = 'sm', schemaIds = BATCH_CONSTANTS.SUPPORTED_SCHEMA_IDS, - baseUrl = 'http://localhost:3010' + baseUrl = 'http://localhost:3010', } = options; // Create multipart form data with actual files @@ -234,39 +232,45 @@ export const createErrorScenarios = () => { return { // Invalid schema ID invalidSchema: { - batchFiles: [{ - cid: generateValidCid(), - schemaId: 99999 - }] + batchFiles: [ + { + cid: generateValidCid(), + schemaId: 99999, + }, + ], }, // Invalid CID format invalidCid: { - batchFiles: [{ - cid: 'invalid-cid-format', - schemaId: 12 - }] + batchFiles: [ + { + cid: 'invalid-cid-format', + schemaId: 12, + }, + ], }, // Empty batch emptyBatch: { - batchFiles: [] + batchFiles: [], }, // Missing required fields missingFields: { - batchFiles: [{ - cid: generateValidCid(), - // Missing schemaId - }] + batchFiles: [ + { + cid: generateValidCid(), + // Missing schemaId + }, + ], }, // Too many files tooManyFiles: { batchFiles: Array.from({ length: 25 }, (_, i) => ({ cid: generateValidCid(), - schemaId: 12 - })) - } + schemaId: 12, + })), + }, }; }; diff --git a/apps/content-publishing-api/src/api.service.ts b/apps/content-publishing-api/src/api.service.ts index 31adc4f43..d879a8fc6 100644 --- a/apps/content-publishing-api/src/api.service.ts +++ b/apps/content-publishing-api/src/api.service.ts @@ -19,7 +19,8 @@ import { isImage, isParquet, DSNP_VALID_IMAGE_MIME_TYPES_REGEX, - DSNP_VALID_MIME_TYPES_REGEX, VALID_BATCH_MIME_TYPES_REGEX, + DSNP_VALID_MIME_TYPES_REGEX, + VALID_BATCH_MIME_TYPES_REGEX, } from '#validation'; import { IRequestJob, @@ -127,10 +128,13 @@ export class ApiService { async validateAssetsAndFetchMetadata( content: AssetIncludedRequestDto, ): Promise { - const checkingList: { allowedMimeTypesRegex: RegExp, referenceId: string }[] = []; + const checkingList: { allowedMimeTypesRegex: RegExp; referenceId: string }[] = []; if (content.profile) { content.profile.icon?.forEach((reference) => - checkingList.push({ allowedMimeTypesRegex: DSNP_VALID_IMAGE_MIME_TYPES_REGEX, referenceId: reference.referenceId }), + checkingList.push({ + allowedMimeTypesRegex: DSNP_VALID_IMAGE_MIME_TYPES_REGEX, + referenceId: reference.referenceId, + }), ); } else if (content.content) { content.content.assets?.forEach((asset) => @@ -142,7 +146,9 @@ export class ApiService { ), ); } else if (content.batchFiles) { - content.batchFiles.forEach((batchFile) => checkingList.push({ allowedMimeTypesRegex: VALID_BATCH_MIME_TYPES_REGEX, referenceId: batchFile.cid })); + content.batchFiles.forEach((batchFile) => + checkingList.push({ allowedMimeTypesRegex: VALID_BATCH_MIME_TYPES_REGEX, referenceId: batchFile.cid }), + ); } const redisResults = await Promise.all( @@ -161,7 +167,9 @@ export class ApiService { // checks that the MIME type is allowed for this type of asset if (!checkingList[index].allowedMimeTypesRegex.test(metadata.mimeType)) { - errors.push(`Uploaded asset referenceId ${checkingList[index].referenceId} has invalid MIME type ${metadata.mimeType}!`); + errors.push( + `Uploaded asset referenceId ${checkingList[index].referenceId} has invalid MIME type ${metadata.mimeType}!`, + ); } } }); @@ -267,12 +275,17 @@ export class ApiService { }; } - public async uploadStreamedAsset(stream: Readable, filename: string, mimetype: string, allowedMimeTypes = VALID_UPLOAD_MIME_TYPES_REGEX): Promise { + public async uploadStreamedAsset( + stream: Readable, + filename: string, + mimetype: string, + allowedMimeTypes = VALID_UPLOAD_MIME_TYPES_REGEX, + ): Promise { this.logger.debug(`Processing file: ${filename} (${mimetype})`); if (!allowedMimeTypes.test(mimetype)) { this.logger.warn(`Skipping file: ${filename} due to unsupported file type (${mimetype}).`); - stream.resume(); // Make sure we consume the entire file stream so the rest of the request can be processed + stream.resume(); // Make sure we consume the entire file stream so the rest of the request can be processed return { error: `Unsupported file type (${mimetype})` }; } diff --git a/apps/content-publishing-api/src/controllers/v2/asset.controller.v2.ts b/apps/content-publishing-api/src/controllers/v2/asset.controller.v2.ts index c9e495b47..ea1a03601 100644 --- a/apps/content-publishing-api/src/controllers/v2/asset.controller.v2.ts +++ b/apps/content-publishing-api/src/controllers/v2/asset.controller.v2.ts @@ -45,7 +45,9 @@ export class AssetControllerV2 { busboy.on('file', (_fieldname, fileStream, fileinfo) => { fileIndex += 1; - this.logger.trace(`on file event: ${fileinfo.filename} (${fileinfo.mimeType}) is truncated: ${fileStream.truncated}`); + this.logger.trace( + `on file event: ${fileinfo.filename} (${fileinfo.mimeType}) is truncated: ${fileStream.truncated}`, + ); fileStream.on('end', () => { this.logger.trace(`Finished writing ${fileinfo.filename}`); }); diff --git a/apps/content-publishing-api/src/controllers/v3/content.controller.v3.ts b/apps/content-publishing-api/src/controllers/v3/content.controller.v3.ts index 2c5e75dd4..478d011c8 100644 --- a/apps/content-publishing-api/src/controllers/v3/content.controller.v3.ts +++ b/apps/content-publishing-api/src/controllers/v3/content.controller.v3.ts @@ -109,7 +109,12 @@ export class ContentControllerV3 { // Upload file to IPFS fileProcessingPromises.push( - this.apiService.uploadStreamedAsset(fileStream, fileinfo.filename, fileinfo.mimeType, VALID_BATCH_MIME_TYPES_REGEX), + this.apiService.uploadStreamedAsset( + fileStream, + fileinfo.filename, + fileinfo.mimeType, + VALID_BATCH_MIME_TYPES_REGEX, + ), ); }); diff --git a/apps/content-publishing-api/test/mockRequestData.ts b/apps/content-publishing-api/test/mockRequestData.ts index 1d5f06ba2..20ee9f88f 100644 --- a/apps/content-publishing-api/test/mockRequestData.ts +++ b/apps/content-publishing-api/test/mockRequestData.ts @@ -2,7 +2,9 @@ import { IAssetMetadata, IBroadcast, ILocation, - INoteActivity, IProfile, IProfileActivity, + INoteActivity, + IProfile, + IProfileActivity, IReply, ITag, ITombstone, @@ -87,7 +89,7 @@ export const validReplyNoUploadedAssets: IReply = { export const validUpdateNoUploadedAssets: IUpdate = { targetContentHash: 'bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna', targetAnnouncementType: ModifiableAnnouncementType.BROADCAST, - content: validContentWithHrefAsset + content: validContentWithHrefAsset, }; export const validReaction = { @@ -152,7 +154,7 @@ export function generateUpdate(assets: string[] = []): IUpdate { export function generateProfile(assets: string[] = []): IProfile { const payload: IProfile = { profile: structuredClone(validProfileNoUploadedAssets), - } + }; if (assets.length > 0) { payload.profile.icon = assets.map((referenceId) => ({ referenceId })); } diff --git a/developer-docs/account/ENVIRONMENT.md b/developer-docs/account/ENVIRONMENT.md index 244a166fa..1237a11c3 100644 --- a/developer-docs/account/ENVIRONMENT.md +++ b/developer-docs/account/ENVIRONMENT.md @@ -4,7 +4,7 @@ This application recognizes the following environment variables: | Name | Description | Range/Type | Required? | Default | -|-------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------:|:------------------------------------:|:-----------------------------------------------:| +| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------: | :----------------------------------: | :---------------------------------------------: | | `API_BODY_JSON_LIMIT` | Api json body size limit in string (some examples: 100kb or 5mb or etc) | string | | 1mb | | `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 | | `API_TIMEOUT_MS` | Overall API timeout limit in milliseconds. This is the maximum time allowed for any API request to complete. Any `HTTP_RESPONSE_TIMEOUT_MS` value must be less than this value | > 0 | | 30000 | @@ -18,12 +18,12 @@ This application recognizes the following environment variables: | `HEALTH_CHECK_SUCCESS_THRESHOLD` | Minimum number of consecutive successful calls to the provider webhook `/health` endpoint before it is marked up again | > 0 | | 10 | | `HTTP_RESPONSE_TIMEOUT_MS` | Timeout in milliseconds to wait for a response as part of a request to an HTTP endpoint. Must be less than `API_TIMEOUT_MS` | > 0 and < API_TIMEOUT_MS | | 3000 | | `LOG_LEVEL` | Verbosity level for logging | `trace` \| `debug` \| `info` \| `warn` \| `error` \| `fatal` | N | `info` | -| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | +| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | | `PROVIDER_ACCESS_TOKEN` | An optional bearer token authentication to the provider webhook | string | | | | `PROVIDER_ACCOUNT_SEED_PHRASE` | Seed phrase or URI or Ethereum private key that is used for provider MSA control key | string | Y | | | `PROVIDER_ID` | Provider MSA Id | integer | Y | | | `RATE_LIMIT_TTL` | Rate limiting time window in milliseconds. Requests are limited within this time window | > 0 | | 60000 | -| `RATE_LIMIT_MAX_REQUESTS` | Maximum number of requests allowed per time window (TTL) | > 0 | | 100 | +| `RATE_LIMIT_MAX_REQUESTS` | Maximum number of requests allowed per time window (TTL) | > 0 | | 100 | | `RATE_LIMIT_SKIP_SUCCESS` | Whether to skip counting successful requests (2xx status codes) towards the rate limit | boolean | | false | | `RATE_LIMIT_SKIP_FAILED` | Whether to skip counting failed requests (4xx/5xx status codes) towards the rate limit | boolean | | false | | `RATE_LIMIT_BLOCK_ON_EXCEEDED` | Whether to block requests when rate limit is exceeded (true) or just log them (false) | boolean | | true | diff --git a/developer-docs/account/ICS.md b/developer-docs/account/ICS.md index 27f2a8040..683c176d5 100644 --- a/developer-docs/account/ICS.md +++ b/developer-docs/account/ICS.md @@ -1,7 +1,6 @@ # Changes to support ICS -The payload body for submitting the ICS AddPublicKey/AddContextGroupPRIDEntry/AddContextGroupMetadata payloads together is defined as `IcsPublishAllRequestDto`. - +The payload body for submitting the ICS AddPublicKey/AddContextGroupPRIDEntry/AddContextGroupMetadata payloads together is defined as `IcsPublishAllRequestDto`. ## ICS Add Public Key Payload @@ -18,15 +17,16 @@ The schema is **`Itemized` and `SignatureRequired`**. The body DTO is `AddNewPublicKeyAgreementRequestDto`, which is already defined. ## Add `ContextGroup` Metadata + We will use existing extrinsic `upsert_page_with_signature_v2` to store `ContextGroup` Metadata on a chain. The schema -is * -*`Paginated`, `SignatureRequired`.** +is \* \*`Paginated`, `SignatureRequired`.\*\* The body DTO is a new one, `UpsertPagePayloadDto` ```typescript import { IsMsaId } from './is-msa-id.decorator'; -import { IsSchemaId } from './is-schema-id.decorator';`` +import { IsSchemaId } from './is-schema-id.decorator'; +``; export class UpsertPagePayloadDto extends ItemizedSignaturePayloadDto { @IsPageId() @@ -36,7 +36,7 @@ export class UpsertPagePayloadDto extends ItemizedSignaturePayloadDto { ## ICS is a part of the `account-api` microservice -For MVI, so that we need to stand up only a single service, ICS endpoints will be part of the `account-api` microservice. +For MVI, so that we need to stand up only a single service, ICS endpoints will be part of the `account-api` microservice. Attachments are not supported as of this writing and not included in this proposal/update @@ -50,11 +50,11 @@ This endpoint requires a body of type `IcsPublishAllRequestDto`, which contains addContextGroupPRIDEntryPayload: ItemizedSignaturePayloadDto; addContetGroupMetadataPayload: UpsertPagePayloadDto; } - + @Post(':accountId/addIcsPublicKey') @HttpCode(HttpStatus.ACCEPTED) - async publishAll( - @Param() { accountId }: AccountIdDto, + async publishAll( + @Param() { accountId }: AccountIdDto, @Body() payload: IcsPublishAllRequestDto): Promise {} ``` @@ -68,7 +68,9 @@ async enqueueIcsRequest( ) {} ``` + ## `account-worker`: Processing ICS jobs + There is a new request processor service for ICS jobs, `icsRequest.publisher.service.ts` -There is a new queue, `icsPublishQueue`, for adding ICS keys and PRIDs. These will call `payWithCapacityBatchAll`, batching all ICS calls. \ No newline at end of file +There is a new queue, `icsPublishQueue`, for adding ICS keys and PRIDs. These will call `payWithCapacityBatchAll`, batching all ICS calls. diff --git a/developer-docs/account/README.md b/developer-docs/account/README.md index 612839049..94ed3de77 100644 --- a/developer-docs/account/README.md +++ b/developer-docs/account/README.md @@ -207,13 +207,13 @@ preferred debugger. Each queue lists the [DTOs](https://en.wikipedia.org/wiki/Data_transfer_object) used with it. -* transactionPublish - the sole queue for all Account operations, with +- transactionPublish - the sole queue for all Account operations, with related [DTOs](https://en.wikipedia.org/wiki/Data_transfer_object). - * Managing handles: `CreateHandleRequest`, `ChangeHandleRequest` - * Managing control keys: `AddKeyRequestDto` `PublicKeyAgreementDto` - * Handling SIWF sign up requests: `PublishSIWFSignupRequestDto` - * Retiring an MSA. `PublishRetireMsaRequestDto` - * Revoking a delegation to the Provider operating this gateway: `PublishRevokeDelegationRequestDto` + - Managing handles: `CreateHandleRequest`, `ChangeHandleRequest` + - Managing control keys: `AddKeyRequestDto` `PublicKeyAgreementDto` + - Handling SIWF sign up requests: `PublishSIWFSignupRequestDto` + - Retiring an MSA. `PublishRetireMsaRequestDto` + - Revoking a delegation to the Provider operating this gateway: `PublishRevokeDelegationRequestDto` ### Built With diff --git a/developer-docs/content-publishing/CONTENT_PUBLISHING_V3.md b/developer-docs/content-publishing/CONTENT_PUBLISHING_V3.md index 3b2b04ac5..d24a0effd 100644 --- a/developer-docs/content-publishing/CONTENT_PUBLISHING_V3.md +++ b/developer-docs/content-publishing/CONTENT_PUBLISHING_V3.md @@ -102,4 +102,4 @@ sequenceDiagram PublishSvc -->> PQueue: completed deactivate PublishSvc Note right of Chain: Files are now published:
- Content stored in IPFS
- CID announced on Frequency chain
- Transaction monitored for finality -``` \ No newline at end of file +``` diff --git a/developer-docs/content-publishing/ENVIRONMENT.md b/developer-docs/content-publishing/ENVIRONMENT.md index 7242d7132..622d86618 100644 --- a/developer-docs/content-publishing/ENVIRONMENT.md +++ b/developer-docs/content-publishing/ENVIRONMENT.md @@ -3,34 +3,34 @@ ℹ️ Feel free to adjust your environment variables to taste. This application recognizes the following environment variables: -| Name | Description | Range/Type | Required? | Default | -|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------:|:------------------------------------:|:--------------------------:| -| `API_BODY_JSON_LIMIT` | Api json body size limit in string (some examples: 100kb or 5mb or etc) | string | | 1mb | -| `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 | -| `API_TIMEOUT_MS` | Overall API timeout limit in milliseconds. This is the maximum time allowed for any API request to complete. Any `HTTP_RESPONSE_TIMEOUT_MS` value must be less than this value | > 0 | | 30000 | -| `ASSET_EXPIRATION_INTERVAL_SECONDS` | Number of seconds to keep completed asset entries in the cache before expiring them | > 0 | Y | | -| `ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS` | Base delay in seconds used for exponential backoff while waiting for uploaded assets to be verified available before publishing a content notice | >= 0 | Y | | -| `BATCH_INTERVAL_SECONDS` | Number of seconds between publishing batches. This is so that the service waits a reasonable amount of time for additional content to publish before submitting a batch--it represents a trade-off between maximum batch fullness and minimal wait time for content | > 0 | Y | | -| `BATCH_MAX_COUNT` | Maximum number of items that can be submitted in a single batch | > 0 | Y | | -| `CACHE_KEY_PREFIX` | Prefix to use for Redis cache keys | string | Y | | -| `CAPACITY_LIMIT` | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity) | JSON [(example)](https://github.com/ProjectLibertyLabs/gateway/blob/main/env-files/content-publishing.template.env) | Y | | -| `FILE_UPLOAD_COUNT_LIMIT` | Max number of files to be able to upload at the same time via one upload call | > 0 | Y | | -| `FILE_UPLOAD_MAX_SIZE_IN_BYTES` | Maximum size (in bytes) allowed for each uploaded file. This limit applies to individual files in both single and batch uploads. | > 0 | Y | | | -| `FREQUENCY_API_WS_URL` | Blockchain API Websocket URL | ws(s): URL | Y | | -| `FREQUENCY_TIMEOUT_SECS` | Frequency chain connection timeout limit; app will terminate if disconnected longer | integer | | 10 | -| `HTTP_RESPONSE_TIMEOUT_MS` | Timeout in milliseconds to wait for a response as part of a request to an HTTP endpoint. Must be less than `API_TIMEOUT_MS` | > 0 and < API_TIMEOUT_MS | | 3000 | -| `IPFS_BASIC_AUTH_SECRET` | If using Infura, put auth token here, or leave blank for Kubo RPC | string | | blank | -| `IPFS_BASIC_AUTH_USER` | If using Infura, put Project ID here, or leave blank for Kubo RPC | string | | blank | -| `IPFS_ENDPOINT` | URL to IPFS endpoint | URL | Y | | -| `IPFS_GATEWAY_URL` | IPFS gateway URL '[CID]' is a token that will be replaced with an actual content ID | URL template | Y | | -| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | -| `PROVIDER_ACCOUNT_SEED_PHRASE` | Seed phrase or URI or Ethereum private key that is used for provider MSA control key | string | Y | | -| `PROVIDER_ID` | Provider MSA Id | integer | Y | | -| `RATE_LIMIT_TTL` | Rate limiting time window in milliseconds. Requests are limited within this time window | > 0 | | 60000 | -| `RATE_LIMIT_MAX_REQUESTS` | Maximum number of requests allowed per time window (TTL) | > 0 | | 100 | -| `RATE_LIMIT_SKIP_SUCCESS` | Whether to skip counting successful requests (2xx status codes) towards the rate limit | boolean | | false | -| `RATE_LIMIT_SKIP_FAILED` | Whether to skip counting failed requests (4xx/5xx status codes) towards the rate limit | boolean | | false | -| `RATE_LIMIT_BLOCK_ON_EXCEEDED` | Whether to block requests when rate limit is exceeded (true) or just log them (false) | boolean | | true | +| Name | Description | Range/Type | Required? | Default | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-----------------------------------------------------------------------------------------------------------------: | :----------------------------------: | :-------------------------: | --- | +| `API_BODY_JSON_LIMIT` | Api json body size limit in string (some examples: 100kb or 5mb or etc) | string | | 1mb | +| `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 | +| `API_TIMEOUT_MS` | Overall API timeout limit in milliseconds. This is the maximum time allowed for any API request to complete. Any `HTTP_RESPONSE_TIMEOUT_MS` value must be less than this value | > 0 | | 30000 | +| `ASSET_EXPIRATION_INTERVAL_SECONDS` | Number of seconds to keep completed asset entries in the cache before expiring them | > 0 | Y | | +| `ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS` | Base delay in seconds used for exponential backoff while waiting for uploaded assets to be verified available before publishing a content notice | >= 0 | Y | | +| `BATCH_INTERVAL_SECONDS` | Number of seconds between publishing batches. This is so that the service waits a reasonable amount of time for additional content to publish before submitting a batch--it represents a trade-off between maximum batch fullness and minimal wait time for content | > 0 | Y | | +| `BATCH_MAX_COUNT` | Maximum number of items that can be submitted in a single batch | > 0 | Y | | +| `CACHE_KEY_PREFIX` | Prefix to use for Redis cache keys | string | Y | | +| `CAPACITY_LIMIT` | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity) | JSON [(example)](https://github.com/ProjectLibertyLabs/gateway/blob/main/env-files/content-publishing.template.env) | Y | | +| `FILE_UPLOAD_COUNT_LIMIT` | Max number of files to be able to upload at the same time via one upload call | > 0 | Y | | +| `FILE_UPLOAD_MAX_SIZE_IN_BYTES` | Maximum size (in bytes) allowed for each uploaded file. This limit applies to individual files in both single and batch uploads. | > 0 | Y | | | +| `FREQUENCY_API_WS_URL` | Blockchain API Websocket URL | ws(s): URL | Y | | +| `FREQUENCY_TIMEOUT_SECS` | Frequency chain connection timeout limit; app will terminate if disconnected longer | integer | | 10 | +| `HTTP_RESPONSE_TIMEOUT_MS` | Timeout in milliseconds to wait for a response as part of a request to an HTTP endpoint. Must be less than `API_TIMEOUT_MS` | > 0 and < API_TIMEOUT_MS | | 3000 | +| `IPFS_BASIC_AUTH_SECRET` | If using Infura, put auth token here, or leave blank for Kubo RPC | string | | blank | +| `IPFS_BASIC_AUTH_USER` | If using Infura, put Project ID here, or leave blank for Kubo RPC | string | | blank | +| `IPFS_ENDPOINT` | URL to IPFS endpoint | URL | Y | | +| `IPFS_GATEWAY_URL` | IPFS gateway URL '[CID]' is a token that will be replaced with an actual content ID | URL template | Y | | +| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | +| `PROVIDER_ACCOUNT_SEED_PHRASE` | Seed phrase or URI or Ethereum private key that is used for provider MSA control key | string | Y | | +| `PROVIDER_ID` | Provider MSA Id | integer | Y | | +| `RATE_LIMIT_TTL` | Rate limiting time window in milliseconds. Requests are limited within this time window | > 0 | | 60000 | +| `RATE_LIMIT_MAX_REQUESTS` | Maximum number of requests allowed per time window (TTL) | > 0 | | 100 | +| `RATE_LIMIT_SKIP_SUCCESS` | Whether to skip counting successful requests (2xx status codes) towards the rate limit | boolean | | false | +| `RATE_LIMIT_SKIP_FAILED` | Whether to skip counting failed requests (4xx/5xx status codes) towards the rate limit | boolean | | false | +| `RATE_LIMIT_BLOCK_ON_EXCEEDED` | Whether to block requests when rate limit is exceeded (true) or just log them (false) | boolean | | true | | `RATE_LIMIT_KEY_PREFIX` | Redis key prefix for rate limiting storage. Used to namespace rate limiting data in Redis | string | | content-publishing:throttle | -| `REDIS_OPTIONS` | Additional Redis options.
See [ioredis configuration](https://ioredis.readthedocs.io/en/latest/API/#new-redisport-host-options) | JSON string | Y
(either this or REDIS_URL) | '{"commandTimeout":10000}' | -| `REDIS_URL` | Connection URL for Redis | URL | Y
(either this or REDIS_OPTIONS) | | +| `REDIS_OPTIONS` | Additional Redis options.
See [ioredis configuration](https://ioredis.readthedocs.io/en/latest/API/#new-redisport-host-options) | JSON string | Y
(either this or REDIS_URL) | '{"commandTimeout":10000}' | +| `REDIS_URL` | Connection URL for Redis | URL | Y
(either this or REDIS_OPTIONS) | | diff --git a/developer-docs/content-publishing/README.md b/developer-docs/content-publishing/README.md index d3483e800..d443eaa67 100644 --- a/developer-docs/content-publishing/README.md +++ b/developer-docs/content-publishing/README.md @@ -94,6 +94,7 @@ Ensure you have the following installed: ``` 5. Set up with account data: + ```bash make setup-account ``` @@ -180,27 +181,27 @@ Each queue lists the [DTOs](https://en.wikipedia.org/wiki/Data_transfer_object) These queues contain individual DSNP message data. `DsnpAnnouncementProcessor` enqueues these jobs, and `RequestProcessorService` processes the jobs. -* broadcastQueue: `BroadcastDto` -* replyQueue: `ReplyDto` -* reactionQueue: `ReactionDto` -* updateQueue: `UpdateDto` -* tombstoneQueue: `TombstoneDto` -* profileQueue: `ProfileDto` +- broadcastQueue: `BroadcastDto` +- replyQueue: `ReplyDto` +- reactionQueue: `ReactionDto` +- updateQueue: `UpdateDto` +- tombstoneQueue: `TombstoneDto` +- profileQueue: `ProfileDto` ### Other Queues These jobs are enqueued within Content Publisher's `ApiService`, triggered by an API call, **except for _statusQueue_** -* **assetQueue** contains metadata about stored assets. , and `AssetProcessor` processes the jobs. No DTO is used. -* **batchQueue** holds batch file metadata for announcing batch files on Frequency. `BatchProcessor` processes the jobs. +- **assetQueue** contains metadata about stored assets. , and `AssetProcessor` processes the jobs. No DTO is used. +- **batchQueue** holds batch file metadata for announcing batch files on Frequency. `BatchProcessor` processes the jobs. DTOs: `BatchFileDto`. -* **publishQueue** stores on-chain content associated with an MSA, for the purpose of publishing the content on +- **publishQueue** stores on-chain content associated with an MSA, for the purpose of publishing the content on Frequency. `PublishingService` processes the jobs. DTOs: `OnChainContentDto` -* **requestQueue**: a generic queue available for any announcement type, including DSNP. `RequestProcessorService` +- **requestQueue**: a generic queue available for any announcement type, including DSNP. `RequestProcessorService` processes the jobs. DTOs: `RequestTypeDto` -* **statusQueue**: stores jobs that need to run periodically, and items that need their status checked: No DTO used. +- **statusQueue**: stores jobs that need to run periodically, and items that need their status checked: No DTO used. ### Built With @@ -214,7 +215,7 @@ These jobs are enqueued within Content Publisher's `ApiService`, triggered by an ## Sequence Diagrams -* [Content Publishing V3 Batch File Upload](CONTENT_PUBLISHING_V3.md) +- [Content Publishing V3 Batch File Upload](CONTENT_PUBLISHING_V3.md) ## 🤝 Contributing diff --git a/developer-docs/content-watcher/ENVIRONMENT.md b/developer-docs/content-watcher/ENVIRONMENT.md index db989fab6..6be8dcd9b 100644 --- a/developer-docs/content-watcher/ENVIRONMENT.md +++ b/developer-docs/content-watcher/ENVIRONMENT.md @@ -4,7 +4,7 @@ This application recognizes the following environment variables: | Name | Description | Range/Type | Required? | Default | -|------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------:|:------------------------------------:|:--------------------------:| +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------: | :----------------------------------: | :------------------------: | | `API_BODY_JSON_LIMIT` | Api json body size limit in string (some examples: 100kb or 5mb or etc) | string | | 1mb | | `API_PORT` | HTTP port that the application listens on | 1025 - 65535 | | 3000 | | `API_TIMEOUT_MS` | Overall API timeout limit in milliseconds. This is the maximum time allowed for any API request to complete. Any `HTTP_RESPONSE_TIMEOUT_MS` value must be less than this value | > 0 | | 30000 | @@ -18,14 +18,14 @@ This application recognizes the following environment variables: | `IPFS_ENDPOINT` | URL to IPFS endpoint | URL | Y | | | `IPFS_GATEWAY_URL` | IPFS gateway URL '[CID]' is a token that will be replaced with an actual content ID | URL template | Y | | | `LOG_LEVEL` | Verbosity level for logging | `trace` \| `debug` \| `info` \| `warn` \| `error` \| `fatal` | N | `info` | -| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | +| `PRETTY` | Whether logs should be pretty-printed on one line or multiple lines, or plain JSON | `true` \| `false` \| `compact` | N | `false` | | `QUEUE_HIGH_WATER` | Max number of jobs allowed on the queue before blockchain scan will be paused to allow queue to drain | >= 100 | | 1000 | | `RATE_LIMIT_TTL` | Rate limiting time window in milliseconds. Requests are limited within this time window | > 0 | | 60000 | | `RATE_LIMIT_MAX_REQUESTS` | Maximum number of requests allowed per time window (TTL) | > 0 | | 100 | | `RATE_LIMIT_SKIP_SUCCESS` | Whether to skip counting successful requests (2xx status codes) towards the rate limit | boolean | | false | | `RATE_LIMIT_SKIP_FAILED` | Whether to skip counting failed requests (4xx/5xx status codes) towards the rate limit | boolean | | false | -| `RATE_LIMIT_BLOCK_ON_EXCEEDED` | Whether to block requests when rate limit is exceeded (true) or just log them (false) | boolean | | true | -| `RATE_LIMIT_KEY_PREFIX` | Redis key prefix for rate limiting storage. Used to namespace rate limiting data in Redis | string | | content-watcher:throttle | +| `RATE_LIMIT_BLOCK_ON_EXCEEDED` | Whether to block requests when rate limit is exceeded (true) or just log them (false) | boolean | | true | +| `RATE_LIMIT_KEY_PREFIX` | Redis key prefix for rate limiting storage. Used to namespace rate limiting data in Redis | string | | content-watcher:throttle | | `REDIS_OPTIONS` | Additional Redis options.
See [ioredis configuration](https://ioredis.readthedocs.io/en/latest/API/#new-redisport-host-options) | JSON string | Y
(either this or REDIS_URL) | '{"commandTimeout":10000}' | | `REDIS_URL` | Connection URL for Redis | URL | Y
(either this or REDIS_OPTIONS) | | | `STARTING_BLOCK` | Block number from which the service will start scanning the chain | > 0 | | 1 | diff --git a/libs/blockchain/src/blockchain-rpc-query.service.ts b/libs/blockchain/src/blockchain-rpc-query.service.ts index a619fba1d..0a19e328e 100644 --- a/libs/blockchain/src/blockchain-rpc-query.service.ts +++ b/libs/blockchain/src/blockchain-rpc-query.service.ts @@ -272,7 +272,9 @@ export class BlockchainRpcQueryService extends PolkadotApiService { @RpcCall('rpc.msa.checkDelegations') public async checkCurrentDelegation(msaId: AnyNumber, schemaId: AnyNumber, providerId: AnyNumber): Promise { const header = await this.api.rpc.chain.getHeader(); - const delegation = (await this.api.rpc.msa.checkDelegations([msaId], providerId, header.number.toNumber(), schemaId)) + const delegation = ( + await this.api.rpc.msa.checkDelegations([msaId], providerId, header.number.toNumber(), schemaId) + ) .toArray() .find(([delegatorId, _]) => delegatorId.toString() === (typeof msaId === 'string' ? msaId : msaId.toString())); diff --git a/libs/blockchain/src/blockchain.service.ts b/libs/blockchain/src/blockchain.service.ts index 0d12dde6e..89ca72689 100644 --- a/libs/blockchain/src/blockchain.service.ts +++ b/libs/blockchain/src/blockchain.service.ts @@ -102,10 +102,7 @@ export class BlockchainService extends BlockchainRpcQueryService implements OnAp if (this.connected) { const latestFinalizedBlock = await this.getFinalizedHead(); const latestFinalizedHeader = await this.getHeaderByHash(latestFinalizedBlock); - await this.defaultRedis.set( - 'latestFinalizedHeader', - JSON.stringify(latestFinalizedHeader), - ); + await this.defaultRedis.set('latestFinalizedHeader', JSON.stringify(latestFinalizedHeader)); } } diff --git a/libs/blockchain/src/decorators/rpc-call.decorator.ts b/libs/blockchain/src/decorators/rpc-call.decorator.ts index 71b9cc87f..cf2b6ae87 100644 --- a/libs/blockchain/src/decorators/rpc-call.decorator.ts +++ b/libs/blockchain/src/decorators/rpc-call.decorator.ts @@ -3,9 +3,9 @@ import { PinoLogger } from 'nestjs-pino'; /** * Decorator that wraps RPC method calls with error handling and logging. * Automatically logs the RPC method name, arguments, and error details when errors occur. - * + * * @param rpcMethodName - The name of the RPC method being called (e.g., 'rpc.chain.getBlockHash') - * + * * @example * ```typescript * @RpcCall('rpc.chain.getBlockHash') @@ -56,4 +56,3 @@ export function RpcCall(rpcMethodName: string) { return descriptor; }; } - diff --git a/libs/consumer/src/base-chain-publisher.service.ts b/libs/consumer/src/base-chain-publisher.service.ts index 835839def..8510a0291 100644 --- a/libs/consumer/src/base-chain-publisher.service.ts +++ b/libs/consumer/src/base-chain-publisher.service.ts @@ -183,9 +183,7 @@ export abstract class BaseChainPublisherService } // Get the failed jobs and check if they failed due to capacity const failedJobs = await this.queue.getFailed(); - const capacityFailedJobs = failedJobs.filter((job) => - /inability to pay some fees/i.test(job.failedReason), - ); + const capacityFailedJobs = failedJobs.filter((job) => /inability to pay some fees/i.test(job.failedReason)); // Retry the failed jobs await Promise.all( capacityFailedJobs.map(async (job) => { diff --git a/libs/logger/src/logLevel-common-config.ts b/libs/logger/src/logLevel-common-config.ts index e3ff3b79a..1e2ada998 100644 --- a/libs/logger/src/logLevel-common-config.ts +++ b/libs/logger/src/logLevel-common-config.ts @@ -19,7 +19,7 @@ function getPrettyOptions() { ignore: 'hostname,context,levelStr', messageFormat: `[{context}] {msg}`, singleLine: process.env?.PRETTY === 'compact', - } + }; } export function getPinoTransport() { @@ -67,7 +67,8 @@ export function getPinoHttpOptions() { paths: ['ip', '*.ip', 'ipAddress'], }, transport: getPinoTransport(), - stream: process.env.NODE_ENV === 'test' ? require('pino-pretty')({ ...getPrettyOptions(), sync: true }) : undefined, + stream: + process.env.NODE_ENV === 'test' ? require('pino-pretty')({ ...getPrettyOptions(), sync: true }) : undefined, }, }; } diff --git a/libs/types/src/constants/queue.constants.ts b/libs/types/src/constants/queue.constants.ts index 57f7b45e5..d1c496ceb 100644 --- a/libs/types/src/constants/queue.constants.ts +++ b/libs/types/src/constants/queue.constants.ts @@ -222,7 +222,7 @@ export namespace ContentPublishingQueues { backoff: { type: 'exponential', delay: 6000, - } + }, }, }, { diff --git a/libs/types/src/dtos/account/ics.request.dto.ts b/libs/types/src/dtos/account/ics.request.dto.ts index de39398f2..119ea7d4c 100644 --- a/libs/types/src/dtos/account/ics.request.dto.ts +++ b/libs/types/src/dtos/account/ics.request.dto.ts @@ -1,14 +1,12 @@ -import { - AddNewPublicKeyAgreementRequestDto, -} from '#types/dtos/account/graphs.request.dto'; +import { AddNewPublicKeyAgreementRequestDto } from '#types/dtos/account/graphs.request.dto'; import { IsSchemaId } from '#utils/decorators/is-schema-id.decorator'; import { HexString } from '@polkadot/util/types'; import { IsNotEmpty, ValidateNested } from 'class-validator'; import { Type } from 'class-transformer'; -import { IsHexValue } from "#utils/decorators"; -import { IsSignature } from "#utils/decorators/is-signature.decorator"; -import { IsIntValue } from "#utils/decorators/is-int-value.decorator"; -import { IsAccountIdOrAddress } from "#utils/decorators/is-account-id-address.decorator"; +import { IsHexValue } from '#utils/decorators'; +import { IsSignature } from '#utils/decorators/is-signature.decorator'; +import { IsIntValue } from '#utils/decorators/is-int-value.decorator'; +import { IsAccountIdOrAddress } from '#utils/decorators/is-account-id-address.decorator'; export class UpsertedPageDto { /** diff --git a/testlib/index.ts b/testlib/index.ts index 29f318bf7..69f60203e 100644 --- a/testlib/index.ts +++ b/testlib/index.ts @@ -1,5 +1,5 @@ export * from './configProviders.mock.spec'; export * from './keys.spec'; export * from './polkadot-api.mock.spec'; -export * from './redis-provider.mock.spec'; +export * from './redis-provider.mock.spec'; export * from './utils.config-tests';