From e94c3e539b144234b657d2a572e69a4e27147305 Mon Sep 17 00:00:00 2001 From: Anas Lecaillon Date: Tue, 2 Dec 2025 16:13:39 +0100 Subject: [PATCH] Added human review capabilities --- src/db/schema.ts | 4 +- src/migrations/0011_chilly_exiles.sql | 20 + src/migrations/meta/0011_snapshot.json | 944 +++++++++++++++++++++++++ src/migrations/meta/_journal.json | 7 + src/resources/publication.ts | 143 +++- src/runner/config.ts | 1 + src/server/experiments.ts | 6 +- src/srchd.ts | 45 ++ src/tools/publications.ts | 14 +- 9 files changed, 1141 insertions(+), 43 deletions(-) create mode 100644 src/migrations/0011_chilly_exiles.sql create mode 100644 src/migrations/meta/0011_snapshot.json diff --git a/src/db/schema.ts b/src/db/schema.ts index f8217d3c0..fc0099d2f 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -228,9 +228,7 @@ export const reviews = sqliteTable( publication: integer("publication") .notNull() .references(() => publications.id), - author: integer("author") - .notNull() - .references(() => agents.id), + author: integer("author").references(() => agents.id), // null when requested by the system until submitted grade: text("grade", { diff --git a/src/migrations/0011_chilly_exiles.sql b/src/migrations/0011_chilly_exiles.sql new file mode 100644 index 000000000..493e4a8f2 --- /dev/null +++ b/src/migrations/0011_chilly_exiles.sql @@ -0,0 +1,20 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_reviews` ( + `id` integer PRIMARY KEY NOT NULL, + `created` integer NOT NULL, + `updated` integer NOT NULL, + `experiment` integer NOT NULL, + `publication` integer NOT NULL, + `author` integer, + `grade` text, + `content` text, + FOREIGN KEY (`experiment`) REFERENCES `experiments`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`publication`) REFERENCES `publications`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`author`) REFERENCES `agents`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +INSERT INTO `__new_reviews`("id", "created", "updated", "experiment", "publication", "author", "grade", "content") SELECT "id", "created", "updated", "experiment", "publication", "author", "grade", "content" FROM `reviews`;--> statement-breakpoint +DROP TABLE `reviews`;--> statement-breakpoint +ALTER TABLE `__new_reviews` RENAME TO `reviews`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `reviews_author_publication_unique` ON `reviews` (`author`,`publication`); \ No newline at end of file diff --git a/src/migrations/meta/0011_snapshot.json b/src/migrations/meta/0011_snapshot.json new file mode 100644 index 000000000..e34b673d9 --- /dev/null +++ b/src/migrations/meta/0011_snapshot.json @@ -0,0 +1,944 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "e91cb032-b16c-41e9-b8c0-ce5c7aaab992", + "prevId": "b1993905-2f97-4674-9267-0971eab844df", + "tables": { + "agents": { + "name": "agents", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tools": { + "name": "tools", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "agents_name_experiment_unique": { + "name": "agents_name_experiment_unique", + "columns": [ + "name", + "experiment" + ], + "isUnique": true + } + }, + "foreignKeys": { + "agents_experiment_experiments_id_fk": { + "name": "agents_experiment_experiments_id_fk", + "tableFrom": "agents", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "citations": { + "name": "citations", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from": { + "name": "from", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to": { + "name": "to", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "citations_from_to_experiment_unique": { + "name": "citations_from_to_experiment_unique", + "columns": [ + "from", + "to", + "experiment" + ], + "isUnique": true + } + }, + "foreignKeys": { + "citations_experiment_experiments_id_fk": { + "name": "citations_experiment_experiments_id_fk", + "tableFrom": "citations", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "citations_from_publications_id_fk": { + "name": "citations_from_publications_id_fk", + "tableFrom": "citations", + "tableTo": "publications", + "columnsFrom": [ + "from" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "citations_to_publications_id_fk": { + "name": "citations_to_publications_id_fk", + "tableFrom": "citations", + "tableTo": "publications", + "columnsFrom": [ + "to" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "evolutions": { + "name": "evolutions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "evolutions_idx_experiment_agent_created": { + "name": "evolutions_idx_experiment_agent_created", + "columns": [ + "experiment", + "agent", + "created" + ], + "isUnique": false + } + }, + "foreignKeys": { + "evolutions_experiment_experiments_id_fk": { + "name": "evolutions_experiment_experiments_id_fk", + "tableFrom": "evolutions", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "evolutions_agent_agents_id_fk": { + "name": "evolutions_agent_agents_id_fk", + "tableFrom": "evolutions", + "tableTo": "agents", + "columnsFrom": [ + "agent" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "experiments": { + "name": "experiments", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "problem": { + "name": "problem", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "experiments_name_unique": { + "name": "experiments_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "messages_experiment_agent_position_unique": { + "name": "messages_experiment_agent_position_unique", + "columns": [ + "experiment", + "agent", + "position" + ], + "isUnique": true + } + }, + "foreignKeys": { + "messages_experiment_experiments_id_fk": { + "name": "messages_experiment_experiments_id_fk", + "tableFrom": "messages", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "messages_agent_agents_id_fk": { + "name": "messages_agent_agents_id_fk", + "tableFrom": "messages", + "tableTo": "agents", + "columnsFrom": [ + "agent" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "publications": { + "name": "publications", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "abstract": { + "name": "abstract", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reference": { + "name": "reference", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "publications_experiment_reference_unique": { + "name": "publications_experiment_reference_unique", + "columns": [ + "experiment", + "reference" + ], + "isUnique": true + } + }, + "foreignKeys": { + "publications_experiment_experiments_id_fk": { + "name": "publications_experiment_experiments_id_fk", + "tableFrom": "publications", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "publications_author_agents_id_fk": { + "name": "publications_author_agents_id_fk", + "tableFrom": "publications", + "tableTo": "agents", + "columnsFrom": [ + "author" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "reviews": { + "name": "reviews", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "publication": { + "name": "publication", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "grade": { + "name": "grade", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "reviews_author_publication_unique": { + "name": "reviews_author_publication_unique", + "columns": [ + "author", + "publication" + ], + "isUnique": true + } + }, + "foreignKeys": { + "reviews_experiment_experiments_id_fk": { + "name": "reviews_experiment_experiments_id_fk", + "tableFrom": "reviews", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "reviews_publication_publications_id_fk": { + "name": "reviews_publication_publications_id_fk", + "tableFrom": "reviews", + "tableTo": "publications", + "columnsFrom": [ + "publication" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "reviews_author_agents_id_fk": { + "name": "reviews_author_agents_id_fk", + "tableFrom": "reviews", + "tableTo": "agents", + "columnsFrom": [ + "author" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "solutions": { + "name": "solutions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "publication": { + "name": "publication", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "solutions_idx_experiment_agent_created": { + "name": "solutions_idx_experiment_agent_created", + "columns": [ + "experiment", + "agent", + "created" + ], + "isUnique": false + } + }, + "foreignKeys": { + "solutions_experiment_experiments_id_fk": { + "name": "solutions_experiment_experiments_id_fk", + "tableFrom": "solutions", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "solutions_publication_publications_id_fk": { + "name": "solutions_publication_publications_id_fk", + "tableFrom": "solutions", + "tableTo": "publications", + "columnsFrom": [ + "publication" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "solutions_agent_agents_id_fk": { + "name": "solutions_agent_agents_id_fk", + "tableFrom": "solutions", + "tableTo": "agents", + "columnsFrom": [ + "agent" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "token_usages": { + "name": "token_usages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created": { + "name": "created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated": { + "name": "updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "experiment": { + "name": "experiment", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input": { + "name": "input", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cached": { + "name": "cached", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "token_usages_idx_experiment_agent": { + "name": "token_usages_idx_experiment_agent", + "columns": [ + "experiment", + "agent" + ], + "isUnique": false + } + }, + "foreignKeys": { + "token_usages_experiment_experiments_id_fk": { + "name": "token_usages_experiment_experiments_id_fk", + "tableFrom": "token_usages", + "tableTo": "experiments", + "columnsFrom": [ + "experiment" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "token_usages_agent_agents_id_fk": { + "name": "token_usages_agent_agents_id_fk", + "tableFrom": "token_usages", + "tableTo": "agents", + "columnsFrom": [ + "agent" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "token_usages_message_messages_id_fk": { + "name": "token_usages_message_messages_id_fk", + "tableFrom": "token_usages", + "tableTo": "messages", + "columnsFrom": [ + "message" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/migrations/meta/_journal.json b/src/migrations/meta/_journal.json index 11a0dbbae..ab2747983 100644 --- a/src/migrations/meta/_journal.json +++ b/src/migrations/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1762451040363, "tag": "0010_premium_krista_starr", "breakpoints": true + }, + { + "idx": 11, + "version": "6", + "when": 1764685712729, + "tag": "0011_chilly_exiles", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/resources/publication.ts b/src/resources/publication.ts index b8d0749c2..b1f8b1541 100644 --- a/src/resources/publication.ts +++ b/src/resources/publication.ts @@ -22,8 +22,14 @@ import { DEFAULT_TOOLS } from "@app/tools/constants"; export type Publication = InferSelectModel; export type Review = Omit, "author"> & { + author: Agent | "human"; +}; +export type AgentReview = Omit & { author: Agent; }; +export function isAgentReview(r: Review): r is AgentReview { + return r.author !== "human"; +} export type Citation = InferInsertModel; export class PublicationResource { @@ -83,6 +89,12 @@ export class PublicationResource { this.reviews = await concurrentExecutor( reviewsResults, async (review) => { + if (!review.author) { + return { + ...review, + author: "human", + }; + } const reviewAgent = await AgentResource.findById( this.experiment, review.author, @@ -338,13 +350,16 @@ export class PublicationResource { return ok(await new PublicationResource(created, experiment).finalize()); } - async maybePublishOrReject(): Promise< - "SUBMITTED" | "PUBLISHED" | "REJECTED" - > { + async maybePublishOrReject( + humanReview: boolean, + ): Promise<"SUBMITTED" | "PUBLISHED" | "REJECTED"> { const grades = removeNulls(this.reviews.map((r) => r.grade ?? null)); // If we are mising reviews return early - if (grades.length < this.reviews.length) { + if ( + grades.length < this.reviews.length || + (humanReview && !this.reviews.some((r) => r.author === "human")) + ) { return "SUBMITTED"; } @@ -456,45 +471,107 @@ export class PublicationResource { } async submitReview( - reviewer: AgentResource, + reviewer: AgentResource | "human", data: Omit< InferInsertModel, "id" | "created" | "updated" | "experiment" | "publication" | "author" >, ): Promise> { - const idx = this.reviews.findIndex( - (r) => r.author?.id === reviewer.toJSON().id, - ); - if (idx === -1) { - return err( - "resource_creation_error", - "Review submitted does not match any review request.", + if (reviewer !== "human") { + const agentReviews: AgentReview[] = this.reviews.filter(isAgentReview); + const idx = agentReviews.findIndex( + (r) => r.author.id === reviewer.toJSON().id, ); - } + if (idx === -1) { + return err( + "resource_creation_error", + "Review submitted does not match any review request.", + ); + } - const [updated] = await db - .update(reviews) - .set({ - grade: data.grade, - content: data.content, - updated: new Date(), - }) - .where( - and( - eq(reviews.experiment, this.experiment.toJSON().id), - eq(reviews.publication, this.data.id), - eq(reviews.author, reviewer.toJSON().id), - ), - ) - .returning(); + const [updated] = await db + .update(reviews) + .set({ + grade: data.grade, + content: data.content, + updated: new Date(), + }) + .where( + and( + eq(reviews.experiment, this.experiment.toJSON().id), + eq(reviews.publication, this.data.id), + eq(reviews.author, reviewer.toJSON().id), + ), + ) + .returning(); - if (!updated) { - return err("not_found_error", "Review not found"); - } + if (!updated) { + return err("not_found_error", "Review not found"); + } + + this.reviews[idx] = { ...updated, author: reviewer.toJSON() }; + return ok(this.reviews[idx]); + } else { + // For humans, check if a review already exists and update it, or insert new + const existingHumanReviewIdx = this.reviews.findIndex( + (r) => r.author === "human", + ); - this.reviews[idx] = { ...updated, author: reviewer.toJSON() }; + let humanReview; + + if (existingHumanReviewIdx !== -1) { + // Update existing human review + const [updated] = await db + .update(reviews) + .set({ + grade: data.grade, + content: data.content, + updated: new Date(), + }) + .where( + and( + eq(reviews.experiment, this.experiment.toJSON().id), + eq(reviews.publication, this.data.id), + isNull(reviews.author), + ), + ) + .returning(); + + if (!updated) { + return err("not_found_error", "Human review not found"); + } - return ok(this.reviews[idx]); + humanReview = updated; + this.reviews[existingHumanReviewIdx] = { + ...updated, + author: "human", + }; + } else { + // Insert new human review + const [inserted] = await db + .insert(reviews) + .values({ + experiment: this.experiment.toJSON().id, + publication: this.data.id, + author: null, // in db kept as null for human reviews + ...data, + }) + .returning(); + + humanReview = inserted; + this.reviews.push({ ...inserted, author: "human" }); + } + + // If publication is already published and human rejects it, unpublish it + if ( + this.data.status === "PUBLISHED" && + (data.grade === "REJECT" || data.grade === "STRONG_REJECT") + ) { + await this.reject(); + } + + return ok({ ...humanReview, author: "human" }); + } } toJSON() { diff --git a/src/runner/config.ts b/src/runner/config.ts index da40b89ec..ee171ce6c 100644 --- a/src/runner/config.ts +++ b/src/runner/config.ts @@ -1,3 +1,4 @@ export type RunConfig = { reviewers: number; + humanReview?: boolean; }; diff --git a/src/server/experiments.ts b/src/server/experiments.ts index 5ee91ab45..6b9d2c4c4 100644 --- a/src/server/experiments.ts +++ b/src/server/experiments.ts @@ -1,7 +1,7 @@ import { ExperimentResource } from "@app/resources/experiment"; import { AgentResource } from "@app/resources/agent"; import { MessageResource } from "@app/resources/messages"; -import { PublicationResource } from "@app/resources/publication"; +import { isAgentReview, PublicationResource } from "@app/resources/publication"; import { SolutionResource } from "@app/resources/solutions"; import { TokenUsageResource } from "@app/resources/token_usage"; import { @@ -548,7 +548,7 @@ export const publicationDetail = async (c: Input) => { .map( (review) => `
-

Review by ${sanitizeText(review.author.name || "Unknown")}

+

Review by ${sanitizeText(isAgentReview(review) ? review.author.name : "Human")}

${review.grade ? `) => { console.error( @@ -403,6 +404,10 @@ agentCmd .command("run ") .description("Run an agent") .requiredOption("-e, --experiment ", "Experiment name") + .option( + "-h, --human", + "Run with human reviewer (human review mandatory for publishing)", + ) .option( "-r, --reviewers ", "Number of required reviewers for each publication", @@ -451,6 +456,7 @@ agentCmd agents.map((a) => Runner.builder(experiment, a, { reviewers, + humanReview: options.human, }), ), ); @@ -660,4 +666,43 @@ program }); }); +program + .command("review ") + .description("Add a human review to a publication") + .requiredOption("-e, --experiment ", "Experiment") + .requiredOption("-c, --content ", "Review Content") + .requiredOption("-g, --grade ", "Review Content") + .action(async (reference, options) => { + const experimentRes = await experimentAndAgents({ + experiment: options.experiment, + }); + if (experimentRes.isErr()) { + return exitWithError(experimentRes); + } + const [experiment] = experimentRes.value; + const publication = await PublicationResource.findByReference( + experiment, + reference, + ); + + if (!publication) { + return exitWithError( + err( + "invalid_parameters_error", + `Reference ${reference} does not exist`, + ), + ); + } + + const publicationRes = await publication.submitReview("human", { + content: options.content, + grade: options.grade, + }); + if (publicationRes.isErr()) { + return exitWithError(publicationRes); + } else { + console.log(`Successfully reviewed ${reference}`); + } + }); + program.parse(); diff --git a/src/tools/publications.ts b/src/tools/publications.ts index 771995a9e..507e507a9 100644 --- a/src/tools/publications.ts +++ b/src/tools/publications.ts @@ -2,7 +2,11 @@ import { z } from "zod"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { AgentResource } from "@app/resources/agent"; import { errorToCallToolResult } from "@app/lib/mcp"; -import { PublicationResource, Review } from "@app/resources/publication"; +import { + isAgentReview, + PublicationResource, + Review, +} from "@app/resources/publication"; import { ExperimentResource } from "@app/resources/experiment"; import { err } from "@app/lib/error"; import { PUBLICATIONS_SERVER_NAME as SERVER_NAME } from "@app/tools/constants"; @@ -12,7 +16,7 @@ const SERVER_VERSION = "0.1.0"; export const reviewHeader = (review: Review) => { return `\ -reviewer=${review.author.name} +reviewer=${isAgentReview(review) ? review.author.name : "human"} grade=${review.grade ?? "PENDING"}`; }; @@ -231,7 +235,9 @@ ${r.content}`; return errorToCallToolResult(reviews); } if (reviewers.length === 0) { - await publication.value.maybePublishOrReject(); + await publication.value.maybePublishOrReject( + config.humanReview ?? false, + ); } const res = publication.value.toJSON(); @@ -333,7 +339,7 @@ ${r.content}`; return errorToCallToolResult(review); } - await publication.maybePublishOrReject(); + await publication.maybePublishOrReject(config.humanReview ?? false); return { isError: false,