feat(orion-scheduler): support multi-VM keyed by server_ws domain (#2146)

* feat(orion-scheduler): support multi-VM keyed by server_ws domain

Allow concurrent runners for different domains with same-domain
idempotency/409/replace, and wire mono client + OC Start Runner
through the updated scheduler APIs and docs.

* fix eslint

Author: benjamin-747

Cargo.lock

@@ -5,6 +5,9 @@ use utoipa::ToSchema;
 pub struct StartRunnerRequest {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub target: Option<String>,
+    /// Force recreate when a Running VM already exists for this mono's domain.
+    #[serde(default)]
+    pub replace: bool,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub image_path: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -23,13 +26,17 @@ pub struct StartRunnerRequest {
 pub struct StartRunnerResponse {
     pub vm_id: String,
     pub phase: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub domain: Option<String>,
 }
 
 #[derive(Serialize, Deserialize, ToSchema, Debug, Clone)]
 pub struct RunnerStatusResponse {
     pub vm_id: String,
     pub phase: String,
     #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub domain: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub vm_ip: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub log_file: Option<String>,

ceres/src/model/orion_runner.rs

@@ -13,4 +13,5 @@ common = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 anyhow = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 tracing = { workspace = true }

clients/orion-scheduler-client/Cargo.toml

@@ -59,7 +59,8 @@ impl OrionSchedulerHttpClient {
         let res = self.auth_headers(req).send().await?;
         let status = res.status();
         let body: StartRunnerSchedulerResponse = res.json().await?;
-        if status.is_success() || status.as_u16() == 202 {
+        // 200 OK (idempotent), 202 Accepted (provisioning), 409 Conflict
+        if status.is_success() || status.as_u16() == 202 || status.as_u16() == 409 {
             Ok(body)
         } else {
             Err(anyhow::anyhow!(
@@ -70,12 +71,56 @@ impl OrionSchedulerHttpClient {
         }
     }
 
+    pub async fn get_vm_status(&self, vm_id: &str) -> anyhow::Result<SchedulerStatusResponse> {
+        let url = format!("{}/vms/{}", self.base_url, vm_id);
+        let req = self.client.get(&url).timeout(Duration::from_secs(30));
+        let res = self.auth_headers(req).send().await?;
+        let status = res.status();
+        if status.as_u16() == 404 {
+            return Ok(SchedulerStatusResponse {
+                status: "no_vm".to_string(),
+                phase: Some("no_vm".to_string()),
+                vm_id: Some(vm_id.to_string()),
+                domain: None,
+                vm_ip: None,
+                uptime_secs: None,
+                log_file: None,
+                error: Some("VM not found".to_string()),
+            });
+        }
+        if status.is_success() {
+            Ok(res.json().await?)
+        } else {
+            Err(anyhow::anyhow!(
+                "Scheduler get_vm_status failed: {}",
+                status
+            ))
+        }
+    }
+
     pub async fn get_status(&self) -> anyhow::Result<SchedulerStatusResponse> {
         let url = format!("{}/status", self.base_url);
         let req = self.client.get(&url).timeout(Duration::from_secs(30));
         let res = self.auth_headers(req).send().await?;
         if res.status().is_success() {
-            Ok(res.json().await?)
+            // List form — not used by mono GET by id path anymore.
+            let v: serde_json::Value = res.json().await?;
+            if let Some(vms) = v.get("vms").and_then(|x| x.as_array()) {
+                if let Some(first) = vms.first() {
+                    return Ok(serde_json::from_value(first.clone())?);
+                }
+                return Ok(SchedulerStatusResponse {
+                    status: "no_vm".to_string(),
+                    phase: Some("no_vm".to_string()),
+                    vm_id: None,
+                    domain: None,
+                    vm_ip: None,
+                    uptime_secs: None,
+                    log_file: None,
+                    error: None,
+                });
+            }
+            Ok(serde_json::from_value(v)?)
         } else {
             Err(anyhow::anyhow!(
                 "Scheduler get_status failed: {}",

clients/orion-scheduler-client/src/http_client.rs

@@ -1,4 +1,4 @@
-//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`).
+//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`, `/vms/{id}`).
 
 mod http_client;
 
@@ -11,6 +11,8 @@ use serde::{Deserialize, Serialize};
 pub struct StartRunnerPayload {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub target: Option<String>,
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub replace: bool,
     pub server_ws: String,
     pub scorpio_base_url: String,
     pub scorpio_lfs_url: String,
@@ -28,24 +30,30 @@ pub struct StartRunnerPayload {
     pub image_memory_mb: Option<u32>,
 }
 
-/// Response from scheduler `POST /webhook` (async 202 or sync 200).
+/// Response from scheduler `POST /webhook` (async 202, sync 200, conflict 409).
 #[derive(Debug, Clone, Deserialize)]
 pub struct StartRunnerSchedulerResponse {
     pub status: String,
     pub vm_id: Option<String>,
+    #[serde(default)]
+    pub domain: Option<String>,
+    #[serde(default)]
+    pub phase: Option<String>,
     pub error: Option<String>,
     #[serde(default)]
     pub orion_log_file: Option<String>,
 }
 
-/// Response from scheduler `GET /status`.
+/// Response from scheduler `GET /vms/{id}` or filtered `/status`.
 #[derive(Debug, Clone, Deserialize)]
 pub struct SchedulerStatusResponse {
     pub status: String,
     #[serde(default)]
     pub phase: Option<String>,
     pub vm_id: Option<String>,
     #[serde(default)]
+    pub domain: Option<String>,
+    #[serde(default)]
     pub vm_ip: Option<String>,
     #[serde(default)]
     pub uptime_secs: Option<u64>,
@@ -79,6 +87,11 @@ impl OrionSchedulerClient {
         self.http.start_runner(payload).await
     }
 
+    pub async fn get_vm_status(&self, vm_id: &str) -> anyhow::Result<SchedulerStatusResponse> {
+        self.http.get_vm_status(vm_id).await
+    }
+
+    /// Backward-compatible list/status endpoint.
     pub async fn get_status(&self) -> anyhow::Result<SchedulerStatusResponse> {
         self.http.get_status().await
     }

clients/orion-scheduler-client/src/lib.rs

@@ -140,6 +140,7 @@ async fn start_runner(
 
     let payload = StartRunnerPayload {
         target: req.target,
+        replace: req.replace,
         server_ws: env.server_ws,
         scorpio_base_url: env.scorpio_base_url,
         scorpio_lfs_url: env.scorpio_lfs_url,
@@ -158,6 +159,17 @@ async fn start_runner(
         )
     })?;
 
+    if sched_resp.status == "conflict" {
+        return Err(ApiError::with_status(
+            StatusCode::CONFLICT,
+            anyhow!(
+                "Runner already provisioning for domain {:?}: {}",
+                sched_resp.domain,
+                sched_resp.error.unwrap_or_else(|| "conflict".to_string())
+            ),
+        ));
+    }
+
     let vm_id = sched_resp.vm_id.ok_or_else(|| {
         ApiError::with_status(
             StatusCode::BAD_GATEWAY,
@@ -170,17 +182,20 @@ async fn start_runner(
         )
     })?;
 
-    let phase = if sched_resp.status == "provisioning" {
-        "provisioning".to_string()
-    } else if sched_resp.status == "ok" {
-        "running".to_string()
-    } else {
-        sched_resp.status
-    };
+    let phase = sched_resp.phase.unwrap_or_else(|| {
+        if sched_resp.status == "provisioning" {
+            "provisioning".to_string()
+        } else if sched_resp.status == "ok" {
+            "running".to_string()
+        } else {
+            sched_resp.status
+        }
+    });
 
     Ok(Json(CommonResult::success(Some(StartRunnerResponse {
         vm_id,
         phase,
+        domain: sched_resp.domain,
     }))))
 }
 
@@ -209,25 +224,27 @@ async fn get_runner_status(
     ensure_admin(&state, &user).await?;
     let client = scheduler_client(&state)?;
 
-    let sched = client.get_status().await.map_err(|e| {
+    let sched = client.get_vm_status(&id).await.map_err(|e| {
         ApiError::with_status(
             StatusCode::BAD_GATEWAY,
             anyhow!("Scheduler request failed: {}", e),
         )
     })?;
 
-    if sched.vm_id.as_deref() != Some(id.as_str()) {
-        return Err(ApiError::not_found(anyhow!("Runner VM '{}' not found", id)));
-    }
-
     let phase = sched
         .phase
+        .clone()
         .or_else(|| Some(sched.status.clone()))
         .unwrap_or_else(|| "unknown".to_string());
 
+    if phase == "no_vm" || sched.vm_id.as_deref() != Some(id.as_str()) {
+        return Err(ApiError::not_found(anyhow!("Runner VM '{}' not found", id)));
+    }
+
     Ok(Json(CommonResult::success(Some(RunnerStatusResponse {
         vm_id: id,
         phase,
+        domain: sched.domain,
         vm_ip: sched.vm_ip,
         log_file: sched.log_file,
         error: sched.error,

mono/src/api/router/orion_runner_router.rs

@@ -1,23 +1,27 @@
 import { useMutation } from '@tanstack/react-query'
 import { toast } from 'react-hot-toast'
 
-import type { StartRunnerResponse } from '@gitmono/types/generated'
+import type { StartRunnerRequest, StartRunnerResponse } from '@gitmono/types/generated'
 
 import { legacyApiClient } from '@/utils/queryClient'
 
 export function usePostStartRunner() {
   const mutation = legacyApiClient.v1.postApiOrionRunners()
 
-  return useMutation<StartRunnerResponse, Error, void>({
-    mutationFn: async () => {
-      const result = await mutation.request({})
+  return useMutation<StartRunnerResponse, Error, StartRunnerRequest | void>({
+    mutationFn: async (body) => {
+      const result = await mutation.request(body ?? {})
       if (!result.req_result || !result.data) {
         throw new Error(result.err_message || 'Failed to start runner')
       }
       return result.data
     },
-    onSuccess: () => {
-      toast.success('Runner provisioning started')
+    onSuccess: (data) => {
+      if (data.phase === 'running') {
+        toast.success(data.domain ? `Runner already running (${data.domain})` : 'Runner already running')
+      } else {
+        toast.success(data.domain ? `Runner provisioning started (${data.domain})` : 'Runner provisioning started')
+      }
     },
     onError: (error) => {
       toast.error(error?.message || 'Failed to start runner')

moon/apps/web/hooks/OrionClient/usePostStartRunner.ts

@@ -29,6 +29,7 @@ const OrionClientPage: PageWithLayout<any> = () => {
   const [currentPage, setCurrentPage] = React.useState<number>(1)
   const [activeVmId, setActiveVmId] = React.useState<string | null>(null)
   const [activePhase, setActivePhase] = React.useState<string | null>(null)
+  const [activeDomain, setActiveDomain] = React.useState<string | null>(null)
 
   const perPage = 8
 
@@ -90,6 +91,9 @@ const OrionClientPage: PageWithLayout<any> = () => {
   React.useEffect(() => {
     if (!runnerStatus) return
     setActivePhase(runnerStatus.phase)
+    if (runnerStatus.domain) {
+      setActiveDomain(runnerStatus.domain)
+    }
   }, [runnerStatus])
 
   React.useEffect(() => {
@@ -98,14 +102,21 @@ const OrionClientPage: PageWithLayout<any> = () => {
     }
   }, [runnerStatus?.phase, handleRefresh])
 
-  const handleStartRunner = React.useCallback(() => {
-    startRunner(undefined, {
-      onSuccess: (data) => {
-        setActiveVmId(data.vm_id)
-        setActivePhase(data.phase)
-      }
-    })
-  }, [startRunner])
+  const handleStartRunner = React.useCallback(
+    (replace = false) => {
+      startRunner(
+        { replace },
+        {
+          onSuccess: (data) => {
+            setActiveVmId(data.vm_id)
+            setActivePhase(data.phase)
+            setActiveDomain(data.domain ?? null)
+          }
+        }
+      )
+    },
+    [startRunner]
+  )
 
   React.useEffect(() => {
     mutate(requestPayload, {
@@ -159,7 +170,7 @@ const OrionClientPage: PageWithLayout<any> = () => {
               {isAdmin ? (
                 <Button
                   variant='primary'
-                  onClick={handleStartRunner}
+                  onClick={() => handleStartRunner(false)}
                   disabled={isStartingRunner || activePhase === 'provisioning'}
                 >
                   {isStartingRunner ? 'Starting…' : 'Start Runner'}
@@ -182,6 +193,11 @@ const OrionClientPage: PageWithLayout<any> = () => {
                 Runner {activeVmId}
               </UIText>
               <div className='mt-1 flex flex-col gap-1'>
+                {(runnerStatus?.domain ?? activeDomain) ? (
+                  <UIText size='text-sm' color='text-muted'>
+                    Domain: {runnerStatus?.domain ?? activeDomain}
+                  </UIText>
+                ) : null}
                 <UIText size='text-sm'>
                   Phase:{' '}
                   <span className='font-medium capitalize'>{runnerStatus?.phase ?? activePhase ?? 'unknown'}</span>
@@ -197,7 +213,7 @@ const OrionClientPage: PageWithLayout<any> = () => {
                   </UIText>
                 ) : null}
                 {runnerStatus?.phase === 'failed' ? (
-                  <Button variant='primary' size='sm' className='mt-1 w-fit' onClick={handleStartRunner}>
+                  <Button variant='primary' size='sm' className='mt-1 w-fit' onClick={() => handleStartRunner(true)}>
                     Retry
                   </Button>
                 ) : null}