From 18f612114b8dee6482ddcb075b78dcd7f25eed18 Mon Sep 17 00:00:00 2001 From: Promise Date: Tue, 28 Apr 2026 05:27:08 +0100 Subject: [PATCH 1/2] feat: enhance CI workflow, add CORS handling, and update documentation - Added Stellar CLI installation and WASM optimization steps to CI workflow. - Improved CORS middleware to default to localhost in development. - Added tests for CORS functionality. - Updated architecture and development documentation for clarity and completeness. --- .github/workflows/ci.yml | 27 ++++++ backend/.env.example | 2 + backend/src/app.ts | 23 +++-- backend/src/controllers/sse.controller.ts | 2 +- backend/tests/cors.test.ts | 15 ++++ docs/ARCHITECTURE.md | 8 ++ docs/DEVELOPMENT.md | 95 ++++++++++++++++++++ package-lock.json | 105 +++++++++++++++++++++- 8 files changed, 267 insertions(+), 10 deletions(-) create mode 100644 backend/tests/cors.test.ts create mode 100644 docs/DEVELOPMENT.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdcec16f..70d1ed87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,3 +116,30 @@ jobs: - name: Run Contract Tests run: cargo test working-directory: contracts + + - name: Install Stellar CLI + run: | + curl -fsSL https://github.com/stellar/stellar-cli/raw/main/install.sh | sh -s -- --install-deps + shell: bash + + - name: Optimize WASM files + run: | + set -euo pipefail + WASMS=$(find contracts/target -type f -name "*.wasm" -print) + if [ -z "$WASMS" ]; then + echo "No wasm files found" + exit 1 + fi + for w in $WASMS; do + out="${w%%.wasm}.optimized.wasm" + echo "Optimizing $w -> $out" + stellar contract optimize --wasm "$w" --wasm-out "$out" + done + shell: bash + + - name: Upload optimized WASM artifacts + uses: actions/upload-artifact@v4 + with: + name: optimized-wasm + path: | + contracts/target/**/**/*.optimized.wasm diff --git a/backend/.env.example b/backend/.env.example index 1d388c2d..9d90cc90 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -5,6 +5,8 @@ DATABASE_URL="postgresql://user:password@localhost:5432/flowfi?schema=public" PORT=3001 NODE_ENV=development CORS_ALLOWED_ORIGINS="https://app.flowfi.xyz,https://flowfi.xyz" +# Comma-separated list of allowed origins for CORS. In development, if unset, +# defaults to http://localhost:3000 # Stellar Network (Testnet/Mainnet) STELLAR_NETWORK=testnet diff --git a/backend/src/app.ts b/backend/src/app.ts index 2d96cf32..17c83057 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -9,11 +9,17 @@ import v1Routes from './routes/v1/index.js'; const app = express(); const isProduction = process.env.NODE_ENV === 'production'; -const allowedOrigins = (process.env.CORS_ALLOWED_ORIGINS ?? '') +const rawCors = process.env.CORS_ALLOWED_ORIGINS ?? ''; +const allowedOrigins = rawCors .split(',') .map((origin) => origin.trim()) .filter(Boolean); +// Default in development to only localhost:3000 (frontend dev server) +if (!process.env.CORS_ALLOWED_ORIGINS && !isProduction) { + allowedOrigins.push('http://localhost:3000'); +} + // Apply global rate limiter first app.use(globalRateLimiter); @@ -35,11 +41,6 @@ app.use((req: Request, res: Response, next: NextFunction) => { app.use(cors({ origin(origin, callback) { - if (!isProduction) { - callback(null, true); - return; - } - // Allow non-browser clients (no Origin header) if (!origin) { callback(null, true); @@ -51,10 +52,20 @@ app.use(cors({ return; } + // Not allowed callback(new Error('CORS origin not allowed')); }, credentials: true, })); + +// Convert CORS errors into 403 responses so callers get a clear status code +app.use((err: any, req: Request, res: Response, next: NextFunction) => { + if (err && err.message === 'CORS origin not allowed') { + res.status(403).json({ error: 'CORS origin not allowed' }); + return; + } + next(err); +}); app.use(express.json()); // Sandbox mode detection (before versioning) diff --git a/backend/src/controllers/sse.controller.ts b/backend/src/controllers/sse.controller.ts index 438ca518..24356bab 100644 --- a/backend/src/controllers/sse.controller.ts +++ b/backend/src/controllers/sse.controller.ts @@ -9,7 +9,7 @@ const subscribeSchema = z.object({ all: z.boolean().optional().default(false), }); -export const subscribe = (req: Request, res: Response) => { +export const subscribe = async (req: Request, res: Response) => { if (sseService.isShuttingDown()) { return res.status(503).json({ message: 'Server is shutting down, please reconnect shortly.' }); } diff --git a/backend/tests/cors.test.ts b/backend/tests/cors.test.ts new file mode 100644 index 00000000..53ae7f52 --- /dev/null +++ b/backend/tests/cors.test.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from 'vitest'; +import request from 'supertest'; +import app from '../src/app.js'; + +describe('CORS middleware', () => { + it('returns 403 for non-whitelisted origin', async () => { + const response = await request(app) + .get('/') + .set('Origin', 'https://evil.example') + .set('Accept', 'text/plain'); + + expect(response.status).toBe(403); + expect(response.body.error).toBe('CORS origin not allowed'); + }); +}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7b078da5..cf2ea751 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,6 +2,14 @@ This document provides a high-level overview of how FlowFi's components interact and how the system processes on-chain events. +```mermaid +flowchart LR + Contract[Stream Contract (Soroban WASM)] --> Indexer[Soroban Event Indexer]\n Indexer --> DB[(Postgres DB)] + DB --> API[Backend API (Express + SSE)] + API --> UI[Frontend (Next.js)] + UI --> API +``` + ## System Components FlowFi consists of three main components: diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 00000000..e32ad044 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,95 @@ +# Development Guide + +## Prerequisites +- Rust toolchain (stable) +- Node.js 20 +- npm +- Docker & Docker Compose (for Postgres) +- `stellar` CLI (install from https://github.com/stellar/stellar-cli) +- PostgreSQL (or use Docker) + +## Local setup (quick) + +1. Start Postgres (docker-compose): + +```bash +docker compose up -d +``` + +2. Backend + +```bash +cd backend +npm ci +# Set env vars (see .env.example) +cp .env.example .env +# Edit .env as needed (DATABASE_URL, STELLAR_NETWORK, etc.) +npm run dev +``` + +3. Frontend + +```bash +cd frontend +npm ci +npm run dev +# open http://localhost:3000 +``` + +4. Contracts (build and test) + +```bash +cd contracts +# Run unit tests +cargo test +# Build WASM +cargo build --target wasm32-unknown-unknown --release +``` + +5. Deploy contract to testnet (optional) + +Install `stellar` CLI then run (example): + +```bash +./scripts/deploy.sh --network testnet --source-account YOUR_KEY_OR_IDENTITY +``` + +This will build, optimize, deploy the WASM and save `deploy/deployment-info.json` with results. + +## Running the full stack locally +- Start Postgres: `docker compose up -d` +- Run backend: `cd backend && npm run dev` +- Run frontend: `cd frontend && npm run dev` +- Build contracts (if developing contracts): `cd contracts && cargo build --target wasm32-unknown-unknown --release` + +## Running tests +- Backend tests (Vitest): + +```bash +cd backend +npm ci +npx vitest +``` + +- Contract tests (cargo): + +```bash +cd contracts +cargo test +``` + +## How to run against testnet vs local sandbox +- Use `.env` to set `SANDBOX_MODE_ENABLED=true` to use local sandbox settings. +- Set `STELLAR_NETWORK=testnet` and `STELLAR_HORIZON_URL` to talk to testnet. +- Use `stellar container start` (stellar CLI) to start a local Soroban sandbox environment. + +## Troubleshooting +- If the backend cannot connect to Postgres, verify `DATABASE_URL` and that Docker is running. +- If contracts fail to build, ensure `wasm32-unknown-unknown` target is installed: `rustup target add wasm32-unknown-unknown`. +- If `stellar` CLI commands fail in CI, ensure the CLI is installed in the CI environment or use the GitHub Action `stellar/stellar-cli@vX`. + +## Links +- Architecture overview: [ARCHITECTURE.md](ARCHITECTURE.md) +- Contracts: `contracts/stream_contract` +- Backend: `backend/` +- Frontend: `frontend/` diff --git a/package-lock.json b/package-lock.json index c7b19666..c694f212 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "dotenv": "^17.3.1", "express": "^5.2.1", "express-rate-limit": "^8.2.1", + "ioredis": "^5.3.2", "pg": "^8.18.0", "stellar-sdk": "^13.3.0", "swagger-jsdoc": "^6.2.8", @@ -81,6 +82,7 @@ "version": "25.3.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -284,6 +286,7 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -500,6 +503,7 @@ "version": "7.29.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -751,7 +755,8 @@ "node_modules/@electric-sql/pglite": { "version": "0.3.15", "dev": true, - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/@electric-sql/pglite-socket": { "version": "0.0.20", @@ -1435,6 +1440,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", + "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", + "license": "MIT" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "dev": true, @@ -2019,6 +2030,7 @@ "version": "20.19.33", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -2052,6 +2064,7 @@ "version": "19.2.14", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -2158,6 +2171,7 @@ "version": "8.56.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -2409,6 +2423,7 @@ "version": "8.16.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2900,6 +2915,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3201,6 +3217,15 @@ "version": "0.0.1", "license": "MIT" }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "5.0.3", "license": "MIT", @@ -3386,7 +3411,8 @@ }, "node_modules/csstype": { "version": "3.2.3", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/damerau-levenshtein": { "version": "1.0.8", @@ -3524,7 +3550,6 @@ }, "node_modules/denque": { "version": "2.1.0", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=0.10" @@ -4268,6 +4293,7 @@ "version": "9.39.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -4437,6 +4463,7 @@ "version": "2.32.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -4689,6 +4716,7 @@ "node_modules/express": { "version": "5.2.1", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -5368,6 +5396,7 @@ "version": "4.11.4", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -5502,6 +5531,30 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.5.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-address": { "version": "10.0.1", "license": "MIT", @@ -6202,10 +6255,22 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, "node_modules/lodash.get": { "version": "4.4.2", "license": "MIT" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isequal": { "version": "4.5.0", "license": "MIT" @@ -7035,6 +7100,7 @@ "node_modules/pg": { "version": "8.18.0", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.11.0", "pg-pool": "^3.11.0", @@ -7238,6 +7304,7 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@prisma/config": "7.4.1", "@prisma/dev": "0.20.0", @@ -7404,6 +7471,7 @@ "node_modules/react": { "version": "19.2.4", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -7411,6 +7479,7 @@ "node_modules/react-dom": { "version": "19.2.4", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -7461,6 +7530,27 @@ "node": ">=8.10.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "dev": true, @@ -8123,6 +8213,12 @@ "dev": true, "license": "MIT" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "license": "MIT", @@ -8562,6 +8658,7 @@ "version": "4.0.3", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -8835,6 +8932,7 @@ "version": "5.9.3", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -9439,6 +9537,7 @@ "node_modules/zod": { "version": "4.3.6", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } From 16dcd0574f45f04f5baa9a5215a6f5081ad5be73 Mon Sep 17 00:00:00 2001 From: Promise Date: Tue, 28 Apr 2026 05:36:39 +0100 Subject: [PATCH 2/2] feat: update Redis client import and enhance connection logic; add triggerPoll method to SorobanEventWorker --- backend/src/lib/redis.ts | 7 ++++--- backend/src/routes/v1/index.ts | 1 - backend/src/services/sorobanService.ts | 4 +++- backend/src/workers/soroban-event-worker.ts | 5 +++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/src/lib/redis.ts b/backend/src/lib/redis.ts index 524f5f92..db93ac73 100644 --- a/backend/src/lib/redis.ts +++ b/backend/src/lib/redis.ts @@ -1,4 +1,5 @@ -import Redis from 'ioredis'; +import type { Redis } from 'ioredis'; +import RedisClass from 'ioredis'; import logger from '../logger.js'; const REDIS_URL = process.env.REDIS_URL; @@ -20,9 +21,9 @@ export function isRedisAvailable(): boolean { } function makeClient(url: string): Redis { - return new Redis(url, { + return new (RedisClass as any)(url, { maxRetriesPerRequest: 3, - retryStrategy: (times) => (times > 3 ? null : Math.min(times * 200, 2000)), + retryStrategy: (times: number) => (times > 3 ? null : Math.min(times * 200, 2000)), enableOfflineQueue: false, lazyConnect: true, }); diff --git a/backend/src/routes/v1/index.ts b/backend/src/routes/v1/index.ts index 37b34eae..7bd36572 100644 --- a/backend/src/routes/v1/index.ts +++ b/backend/src/routes/v1/index.ts @@ -4,7 +4,6 @@ import eventsRoutes from './events.routes.js'; import userRoutes from './user.routes.js'; import authRoutes from './auth.routes.js'; import adminRoutes from './admin.routes.js'; -import adminRoutes from '../adminRoutes.js'; const router = Router(); diff --git a/backend/src/services/sorobanService.ts b/backend/src/services/sorobanService.ts index ed7b9117..b8bdb9ab 100644 --- a/backend/src/services/sorobanService.ts +++ b/backend/src/services/sorobanService.ts @@ -68,6 +68,8 @@ export async function getStreamFromChain(streamId: number): Promise { + await this.poll(); + } + // ─── Internal ────────────────────────────────────────────────────────────── private scheduleNext(): void {