Deploy randomness service

.github/workflows/deploy-randomness.yml

@@ -0,0 +1,91 @@
+name: Deploy Randomness
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  ORGANIZATION: happychaindevs
+  PM2_PROCESS_NAME: randomness
+  RANDOMNESS_DB_PATH: /home/randomness/randomness.sqlite
+  TXM_DB_PATH: /home/randomness/txm.sqlite
+
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.2.0
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Build code
+        run: |
+          make randomness.build
+
+      - name: Copy files to server
+        uses: appleboy/scp-action@v0.1.7
+        with:
+          host: ${{ secrets.SERVER_HOST }}
+          username: ${{ secrets.SERVER_USERNAME }}
+          key: ${{ secrets.SERVER_KEY }}
+          port: ${{ secrets.SERVER_PORT }}
+          source: "apps/randomness/build/*,node_modules/better-sqlite3,node_modules/bindings,node_modules/file-uri-to-path"
+          target: /home/deployer/randomness
+          strip_components: 1   
+          rm: true
+
+      - name: Deploy randomness to server
+        uses: appleboy/ssh-action@v1.1.0
+        with:
+          host: ${{ secrets.SERVER_HOST }}
+          username: ${{ secrets.SERVER_USERNAME }}
+          key: ${{ secrets.SERVER_KEY }}
+          port: ${{ secrets.SERVER_PORT }}
+          # We copy better-sqlite3, bindings, and file-uri-to-path separately because the bundled file does not include
+          # better-sqlite3 and its dependencies
+          script: |
+            chmod -R o+rX /home/deployer/randomness
+            mv /home/deployer/randomness /tmp
+            sudo -u randomness bash -c '
+              rm -rf /home/randomness/build
+              rm -rf /home/randomness/node_modules
+              cp -r /tmp/randomness/randomness/build /home/randomness/build
+
+              mkdir -p /home/randomness/node_modules
+              cp -r /tmp/randomness/better-sqlite3 /home/randomness/node_modules/better-sqlite3
+              cp -r /tmp/randomness/bindings /home/randomness/node_modules/bindings
+              cp -r /tmp/randomness/file-uri-to-path /home/randomness/node_modules/file-uri-to-path
+
+              cd /home/randomness
+
+              rm -f .env
+
+              cat > .env <<-EOF
+            export PRIVATE_KEY=${{ secrets.RANDOMNESS_PRIVATE_KEY }}
+            export TXM_DB_PATH=${{ env.TXM_DB_PATH }}
+            export RANDOMNESS_DB_PATH=${{ env.RANDOMNESS_DB_PATH }}
+            export ${{vars.RANDOMNESS_ENV}}
+            EOF
+
+              npm rebuild
+
+              pm2 delete ${{ env.PM2_PROCESS_NAME }}
+
+              source .env && node ./build/migrate.es.js
+
+              source .env && pm2 start ./build/index.es.js \
+                    --name ${{ env.PM2_PROCESS_NAME }}
+              pm2 save
+            '
+            rm -rf /tmp/randomness

Makefile

@@ -339,6 +339,11 @@ contracts.build:
 	cd contracts && make build
 .PHONY: contracts.build
 
+randomness.build: setup.ts shared.build
+	cd packages/txm && make build
+	cd apps/randomness && make build
+.PHONY: randomness.build
+
 # ==================================================================================================
 # DOCS
 

apps/randomness/Makefile

@@ -18,8 +18,8 @@ start: ## Starts the randomness service
 .PHONY: start
 
 migrate: ## Runs pending migrations
-	tsx --env-file=.env src/migrate.ts
-.PHONY: migrate
+	node --env-file=.env dist/migrate.es.js
+PHONY: migrate
 
 link-anvil:
 	rm src/ABI/random.ts

apps/randomness/migrations/Migration20241210123000.ts → apps/randomness/src/db/migrations/Migration20241210123000.ts

@@ -1,5 +1,5 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../src/db/types"
+import type { Database } from "../types"
 
 export async function up(db: Kysely<Database>) {
     await db.schema
@@ -12,3 +12,5 @@ export async function up(db: Kysely<Database>) {
         .addColumn("status", "text", (col) => col.notNull())
         .execute()
 }
+
+export const migration20241210123000 = { up }

apps/randomness/migrations/Migration20241219104500.ts → apps/randomness/src/db/migrations/Migration20241219104500.ts

@@ -1,5 +1,5 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../src/db/types"
+import type { Database } from "../types"
 
 export async function up(db: Kysely<Database>) {
     await db.schema
@@ -10,3 +10,5 @@ export async function up(db: Kysely<Database>) {
         .addColumn("transactionIntentId", "text")
         .execute()
 }
+
+export const migration20241219104500 = { up }

apps/randomness/src/db/migrations/index.ts

@@ -0,0 +1,7 @@
+import { migration20241210123000 } from "./Migration20241210123000"
+import { migration20241219104500 } from "./Migration20241219104500"
+
+export const migrations = {
+    "20241210123000": migration20241210123000,
+    "20241219104500": migration20241219104500,
+}

apps/randomness/src/index.ts

@@ -27,7 +27,9 @@ class RandomnessService {
             rpc: {
                 url: env.RPC_URL,
                 allowDebug: true,
+                pollingInterval: 250,
             },
+            maxPriorityFeePerGas: 10n,
         })
         this.transactionFactory = new TransactionFactory(this.txm, env.RANDOM_CONTRACT_ADDRESS, env.PRECOMMIT_DELAY)
         this.drandService = new DrandService(this.drandRepository, this.transactionFactory)

apps/randomness/src/migrate.ts

@@ -1,19 +1,20 @@
-import { promises as fs } from "node:fs"
-import path from "node:path"
-import { fileURLToPath } from "node:url"
-import { FileMigrationProvider, Migrator } from "kysely"
+import { type Migration, type MigrationProvider, Migrator } from "kysely"
 import { db } from "./db/driver"
+import { migrations } from "./db/migrations"
+import "@happy.tech/txm/migrate"
 
-const dirname = path.dirname(fileURLToPath(import.meta.url))
+class ObjectMigrationProvider implements MigrationProvider {
+    constructor(private migrations: Record<string, Migration>) {}
+
+    async getMigrations(): Promise<Record<string, Migration>> {
+        return this.migrations
+    }
+}
 
 async function migrateToLatest() {
     const migrator = new Migrator({
         db,
-        provider: new FileMigrationProvider({
-            fs,
-            path,
-            migrationFolder: path.join(dirname, "../migrations"),
-        }),
+        provider: new ObjectMigrationProvider(migrations),
     })
 
     const { error, results } = await migrator.migrateToLatest()

apps/randomness/tsconfig.json

@@ -1,4 +1,4 @@
 {
     "extends": ["../../support/configs/tsconfig.base.json"],
-    "include": ["*.ts", "src", "migrations"]
+    "include": ["*.ts", "src"]
 }

packages/txm/build.config.ts

@@ -1,6 +1,7 @@
 import { defineConfig } from "@happy.tech/happybuild"
 
 export default defineConfig({
+    exports: [".", "./migrate"],
     bunConfig: {
         minify: false,
         target: "node",

packages/txm/lib/NonceManager.ts

@@ -41,6 +41,7 @@ export class NonceManager {
 
         const blockchainNonce = await this.txmgr.viemClient.getTransactionCount({
             address: address,
+            blockTag: "pending",
         })
 
         const highestDbNonce = this.txmgr.transactionRepository.getHighestNonce()

packages/txm/migrations/Migration20241111163800.ts → packages/txm/lib/db/migrations/Migration20241111163800.ts

@@ -1,7 +1,7 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../lib/db/types"
+import type { Database } from "../types"
 
-export async function up(db: Kysely<Database>) {
+async function up(db: Kysely<Database>) {
     await db.schema
         .createTable("transaction")
         .addColumn("intentId", "text", (col) => col.notNull())
@@ -16,3 +16,5 @@ export async function up(db: Kysely<Database>) {
         .addColumn("metadata", "json")
         .execute()
 }
+
+export const migration20241111163800 = { up }

packages/txm/migrations/Migration20241111223000.ts → packages/txm/lib/db/migrations/Migration20241111223000.ts

@@ -1,13 +1,15 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../lib/db/types"
+import type { Database } from "../types"
 
 /*
     SQLite does not have native time types. The SQL interface allows arbitrary type names including "DATE" and "DATETIME",
     but this is invalid in this API, and results in "NUMERIC" affinity instead of "INTEGER" affinity, 
     which is the one we want here
 */
-export async function up(db: Kysely<Database>) {
+async function up(db: Kysely<Database>) {
     await db.schema.alterTable("transaction").addColumn("createdAt", "integer").execute()
 
     await db.schema.alterTable("transaction").addColumn("updatedAt", "integer").execute()
 }
+
+export const migration20241111223000 = { up }

packages/txm/migrations/Migration20241205104400.ts → packages/txm/lib/db/migrations/Migration20241205104400.ts

@@ -1,9 +1,11 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../lib/db/types"
+import type { Database } from "../types"
 
-export async function up(db: Kysely<Database>) {
+async function up(db: Kysely<Database>) {
     await db.schema
         .alterTable("transaction")
         .addColumn("from", "text", (col) => col.notNull())
         .execute()
 }
+
+export const migration20241205104400 = { up }

packages/txm/migrations/Migration20250121110600.ts → packages/txm/lib/db/migrations/Migration20250121110600.ts

@@ -1,6 +1,8 @@
 import type { Kysely } from "kysely"
-import type { Database } from "../lib/db/types"
+import type { Database } from "../types"
 
 export async function up(db: Kysely<Database>) {
     await db.schema.alterTable("transaction").addColumn("collectionBlock", "integer").execute()
 }
+
+export const migration20250121110600 = { up }

packages/txm/lib/db/migrations/index.ts

@@ -0,0 +1,11 @@
+import { migration20241111163800 } from "./Migration20241111163800"
+import { migration20241111223000 } from "./Migration20241111223000"
+import { migration20241205104400 } from "./Migration20241205104400"
+import { migration20250121110600 } from "./Migration20250121110600"
+
+export const migrations = {
+    "20241111163800": migration20241111163800,
+    "20241111223000": migration20241111223000,
+    "20241205104400": migration20241205104400,
+    "20250121110600": migration20250121110600,
+}

packages/txm/lib/migrate.ts

@@ -1,19 +1,19 @@
-import { promises as fs } from "node:fs"
-import path from "node:path"
-import { fileURLToPath } from "node:url"
-import { FileMigrationProvider, Migrator } from "kysely"
+import { type Migration, type MigrationProvider, Migrator } from "kysely"
 import { db } from "./db/driver"
+import { migrations } from "./db/migrations"
 
-const dirname = path.dirname(fileURLToPath(import.meta.url))
+class ObjectMigrationProvider implements MigrationProvider {
+    constructor(private migrations: Record<string, Migration>) {}
 
-async function migrateToLatest() {
+    async getMigrations(): Promise<Record<string, Migration>> {
+        return this.migrations
+    }
+}
+
+export async function migrateToLatest() {
     const migrator = new Migrator({
         db,
-        provider: new FileMigrationProvider({
-            fs,
-            path,
-            migrationFolder: path.join(dirname, "../migrations"),
-        }),
+        provider: new ObjectMigrationProvider(migrations),
     })
 
     const { error, results } = await migrator.migrateToLatest()

packages/txm/package.json

@@ -10,6 +10,10 @@
         ".": {
             "default": "./dist/index.es.js",
             "types": "./dist/index.es.d.ts"
+        },
+        "./migrate": {
+            "default": "./dist/migrate.es.js",
+            "types": "./dist/migrate.es.d.ts"
         }
     },
     "dependencies": {

packages/txm/tsconfig.json

@@ -9,5 +9,5 @@
             "ts-node": ["./node_modules/ts-node/dist/bin.js"]
         }
     },
-    "include": ["lib", "*.ts", "package.json", "migrations"]
+    "include": ["lib", "*.ts", "package.json"]
 }