feat: add a force flag to enable DPU reprovisioning during states oth…

crates/admin-cli/src/dpu/reprovision/args.rs

@@ -62,6 +62,9 @@ Reprovision and update DPU firmware, recording a maintenance message:
     $ nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 \
     --update-firmware --update-message \"scheduled firmware refresh\"
 
+Force reprovisioning to recover a DPU stuck in ingestion:
+    $ nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 --force
+
 ")]
 pub(crate) struct DpuReprovisionSet {
     #[clap(
@@ -80,6 +83,13 @@ pub(crate) struct DpuReprovisionSet {
         help = "If set, a HostUpdateInProgress health alert will be applied to the host"
     )]
     pub(super) update_message: Option<String>,
+
+    #[clap(
+        long,
+        action,
+        help = "Force reprovisioning regardless of the Machine state to recover DPUs stuck in ingestion. Restarts the ingestion state machine and skips the HostUpdateInProgress precondition. Has no effect on assigned Machines."
+    )]
+    force: bool,
 }
 
 impl From<&DpuReprovisionSet> for DpuReprovisioningRequest {
@@ -90,6 +100,7 @@ impl From<&DpuReprovisionSet> for DpuReprovisioningRequest {
             mode: Mode::Set as i32,
             initiator: UpdateInitiator::AdminCli as i32,
             update_firmware: args.update_firmware,
+            force: args.force,
         }
     }
 }
@@ -125,6 +136,7 @@ impl From<&DpuReprovisionClear> for DpuReprovisioningRequest {
             mode: Mode::Clear as i32,
             initiator: UpdateInitiator::AdminCli as i32,
             update_firmware: args.update_firmware,
+            force: false,
         }
     }
 }
@@ -161,6 +173,7 @@ impl From<&DpuReprovisionRestart> for DpuReprovisioningRequest {
             mode: Mode::Restart as i32,
             initiator: UpdateInitiator::AdminCli as i32,
             update_firmware: args.update_firmware,
+            force: false,
         }
     }
 }

crates/api-core/src/handlers/dpu.rs

@@ -1307,27 +1307,37 @@ pub(crate) async fn trigger_dpu_reprovisioning(
         id: machine_id.to_string(),
     })?;
 
+    // A forced reprovision recovers DPUs stuck in ingestion (non-Ready,
+    // non-Assigned states). Such a host is already non-allocatable, so the
+    // HostUpdateInProgress precondition is meaningless and skipped, and a
+    // forced request may re-kick even if a previous request is in progress.
+    let force = req.force;
+
     // Start reprovisioning only if the host has an HostUpdateInProgress health alert
-    let update_alert = snapshot
-        .aggregate_health
-        .alerts
-        .iter()
-        .find(|a| a.id == *HOST_UPDATE_HEALTH_PROBE_ID);
-    if !update_alert.is_some_and(|alert| {
-        alert
-            .classifications
-            .contains(&health_report::HealthAlertClassification::prevent_allocations())
-    }) {
-        return Err(CarbideError::InvalidArgument(format!(
-            "machine {machine_id} must have a 'HostUpdateInProgress' health alert with the 'PreventAllocations' classification before reprovisioning. set this precondition with: `machine health-override add --template host-update <id>`",
-        )).into());
+    if !force {
+        let update_alert = snapshot
+            .aggregate_health
+            .alerts
+            .iter()
+            .find(|a| a.id == *HOST_UPDATE_HEALTH_PROBE_ID);
+        if !update_alert.is_some_and(|alert| {
+            alert
+                .classifications
+                .contains(&health_report::HealthAlertClassification::prevent_allocations())
+        }) {
+            return Err(CarbideError::InvalidArgument(format!(
+                "machine {machine_id} must have a 'HostUpdateInProgress' health alert with the 'PreventAllocations' classification before reprovisioning. set this precondition with: `machine health-override add --template host-update <id>`",
+            )).into());
+        }
     }
 
-    if snapshot.dpu_snapshots.iter().any(|ms| {
-        ms.reprovision_requested
-            .as_ref()
-            .is_some_and(|x| x.started_at.is_some())
-    }) {
+    if !force
+        && snapshot.dpu_snapshots.iter().any(|ms| {
+            ms.reprovision_requested
+                .as_ref()
+                .is_some_and(|x| x.started_at.is_some())
+        })
+    {
         match req.mode() {
             Mode::Restart => {}
             _ => {
@@ -1348,6 +1358,7 @@ pub(crate) async fn trigger_dpu_reprovisioning(
                     &mut txn,
                     initiator,
                     req.update_firmware,
+                    force,
                 )
                 .await?;
             } else {
@@ -1357,6 +1368,7 @@ pub(crate) async fn trigger_dpu_reprovisioning(
                         &mut txn,
                         initiator,
                         req.update_firmware,
+                        force,
                     )
                     .await?;
                 }
@@ -1411,6 +1423,22 @@ pub(crate) async fn trigger_dpu_reprovisioning(
 
     txn.commit().await?;
 
+    // A stuck-in-ingestion host may have backed off with a long state-controller
+    // wait, so a forced request explicitly wakes the host to be re-evaluated
+    // promptly.
+    if force
+        && let Err(err) = api
+            .machine_state_handler_enqueuer
+            .enqueue_object(&snapshot.host_snapshot.id)
+            .await
+    {
+        tracing::warn!(
+            host_machine_id = %snapshot.host_snapshot.id,
+            error = %err,
+            "failed to wake host state handler after forced DPU reprovisioning request",
+        );
+    }
+
     Ok(Response::new(()))
 }
 

crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs

@@ -146,6 +146,7 @@ impl TestMachine {
                     mode: mode as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
                     update_firmware,
+                    force: false,
                 },
             ))
             .await

crates/api-core/src/tests/dpu_reprovisioning.rs

@@ -27,9 +27,10 @@ use common::api_fixtures::{
 use libredfish::{EnabledDisabled, SystemPowerControl};
 use model::instance::status::tenant::TenantState;
 use model::machine::{
-    DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState,
-    Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState,
-    ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState,
+    DpuDiscoveringState, DpuDiscoveringStates, DpuInitState, DpuInitStates, FailureCause,
+    FailureDetails, FailureSource, InstallDpuOsState, InstanceState, Machine,
+    MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, ReprovisionState,
+    SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState,
 };
 use model::test_support::HardwareInfoTemplate;
 use rpc::forge::MachineArchitecture;
@@ -273,9 +274,15 @@ async fn prepare_dpu_reprovision_host_boot_check(
     )
     .await
     .unwrap();
-    db::machine::trigger_dpu_reprovisioning_request(&dpu_machine.id, &mut txn, "AdminCli", true)
-        .await
-        .unwrap();
+    db::machine::trigger_dpu_reprovisioning_request(
+        &dpu_machine.id,
+        &mut txn,
+        "AdminCli",
+        true,
+        false,
+    )
+    .await
+    .unwrap();
     txn.commit().await.unwrap();
 
     dpu_machine
@@ -591,7 +598,8 @@ async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgP
                     machine_id: mh.dpu().id.into(),
                     mode: rpc::forge::dpu_reprovisioning_request::Mode::Set as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
-                    update_firmware: true
+                    update_firmware: true,
+                    force: false,
                 },
             ))
             .await
@@ -612,7 +620,8 @@ async fn test_dpu_for_reprovisioning_fail_if_state_is_not_ready(pool: sqlx::PgPo
                     machine_id: dpu_machine_id.into(),
                     mode: rpc::forge::dpu_reprovisioning_request::Mode::Set as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
-                    update_firmware: true
+                    update_firmware: true,
+                    force: false,
                 },
             ))
             .await
@@ -1155,7 +1164,8 @@ async fn test_dpu_for_set_but_clear_failed(pool: sqlx::PgPool) {
                     machine_id: mh.dpu().id.into(),
                     mode: rpc::forge::dpu_reprovisioning_request::Mode::Clear as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
-                    update_firmware: true
+                    update_firmware: true,
+                    force: false,
                 },
             ))
             .await
@@ -1492,6 +1502,7 @@ async fn test_restart_dpu_reprov(pool: sqlx::PgPool) {
                     mode: Mode::Restart as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
                     update_firmware: false,
+                    force: false,
                 },
             ))
             .await
@@ -1570,9 +1581,15 @@ async fn test_restart_dpu_reprov_unassigned_host_boot_failure(pool: sqlx::PgPool
 
     let failed_at = Utc::now();
     let mut txn = env.pool.begin().await.unwrap();
-    db::machine::trigger_dpu_reprovisioning_request(&dpu_machine.id, &mut txn, "AdminCli", true)
-        .await
-        .unwrap();
+    db::machine::trigger_dpu_reprovisioning_request(
+        &dpu_machine.id,
+        &mut txn,
+        "AdminCli",
+        true,
+        false,
+    )
+    .await
+    .unwrap();
     db::machine::update_dpu_reprovision_explicit_start_time(&dpu_machine.id, failed_at, &mut txn)
         .await
         .unwrap();
@@ -2110,6 +2127,7 @@ async fn test_instance_reprov_restart_failed_impl(pool: sqlx::PgPool) {
                     mode: Mode::Restart as i32,
                     initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
                     update_firmware: false,
+                    force: false,
                 },
             ))
             .await
@@ -2226,6 +2244,7 @@ async fn test_dpu_for_reprovisioning_cannot_restart_if_not_started(pool: sqlx::P
                 mode: rpc::forge::dpu_reprovisioning_request::Mode::Restart as i32,
                 initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
                 update_firmware: true,
+                force: false,
             },
         ))
         .await
@@ -2271,3 +2290,99 @@ impl TestManagedHost {
             .unwrap();
     }
 }
+
+// A DPU stuck in ingestion has no HostUpdateInProgress health alert, so a
+// non-forced request is rejected (see
+// test_dpu_for_reprovisioning_fail_if_maintenance_not_set). A forced request
+// must instead succeed and persist the force flag for the controller to act on.
+#[crate::sqlx_test]
+async fn test_dpu_force_reprovisioning_bypasses_precondition(pool: sqlx::PgPool) {
+    let env = create_test_env(pool).await;
+    let mh = common::api_fixtures::create_managed_host(&env).await;
+
+    env.api
+        .trigger_dpu_reprovisioning(tonic::Request::new(
+            ::rpc::forge::DpuReprovisioningRequest {
+                dpu_id: None,
+                machine_id: mh.dpu().id.into(),
+                mode: Mode::Set as i32,
+                initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32,
+                update_firmware: false,
+                force: true,
+            },
+        ))
+        .await
+        .expect("forced reprovisioning should bypass the health-alert precondition");
+
+    let mut txn = env.pool.begin().await.unwrap();
+    let dpu = mh.dpu().db_machine(&mut txn).await;
+    let req = dpu
+        .reprovision_requested
+        .expect("forced reprovisioning should persist a request");
+    assert!(req.force, "persisted request should carry the force flag");
+}
+
+// A forced request on a host still in an ingestion substate restarts the
+// ingestion state machine from discovery and clears the one-shot force flag.
+#[crate::sqlx_test]
+async fn test_force_reprovisioning_restarts_ingestion(pool: sqlx::PgPool) {
+    let env = create_test_env(pool).await;
+
+    for ingestion_substate in [
+        DpuInitState::WaitingForPlatformConfiguration,
+        DpuInitState::WaitingForNetworkConfig,
+    ] {
+        let mh = common::api_fixtures::create_managed_host(&env).await;
+
+        let mut txn = env.pool.begin().await.unwrap();
+        db::machine::update_state(
+            &mut txn,
+            &mh.id,
+            &ManagedHostState::DPUInit {
+                dpu_states: DpuInitStates {
+                    states: mh
+                        .dpu_ids
+                        .iter()
+                        .map(|id| (*id, ingestion_substate.clone()))
+                        .collect(),
+                },
+            },
+        )
+        .await
+        .unwrap();
+        db::machine::trigger_dpu_reprovisioning_request(
+            &mh.dpu().id,
+            &mut txn,
+            "AdminCli",
+            false,
+            true,
+        )
+        .await
+        .unwrap();
+        txn.commit().await.unwrap();
+
+        let dpu = mh.dpu().next_iteration_machine(&env).await;
+
+        let expected = ManagedHostState::DpuDiscoveringState {
+            dpu_states: DpuDiscoveringStates {
+                states: mh
+                    .dpu_ids
+                    .iter()
+                    .map(|id| (*id, DpuDiscoveringState::Initializing))
+                    .collect(),
+            },
+        };
+        assert_eq!(
+            dpu.current_state(),
+            &expected,
+            "forced reprovisioning from {ingestion_substate:?} should restart ingestion from discovery",
+        );
+
+        let mut txn = env.pool.begin().await.unwrap();
+        let dpu = mh.dpu().db_machine(&mut txn).await;
+        assert!(
+            dpu.reprovision_requested.is_none(),
+            "forced reprovisioning should clear the one-shot request",
+        );
+    }
+}

crates/api-core/src/tests/machine_update_manager.rs

@@ -363,7 +363,7 @@ async fn test_get_updating_machines(pool: sqlx::PgPool) -> Result<(), Box<dyn st
     )
     .await?;
 
-    db::machine::trigger_dpu_reprovisioning_request(&host_machine_id1, &mut txn, "test", true)
+    db::machine::trigger_dpu_reprovisioning_request(&host_machine_id1, &mut txn, "test", true, false)
         .await?;
     txn.commit().await.unwrap();
 

crates/api-db/src/dpu_machine_update.rs

@@ -57,6 +57,7 @@ pub async fn trigger_reprovisioning_for_managed_host(
             started_at: None,
             user_approval_received: false,
             restart_reprovision_requested_at: reprovision_time,
+            force: false,
         };
 
         let query = r#"UPDATE machines SET reprovisioning_requested=$1 WHERE controller_state = '{"state": "ready"}' AND id=$2 RETURNING id"#;

crates/api-db/src/machine.rs

@@ -1732,6 +1732,7 @@ pub async fn trigger_dpu_reprovisioning_request(
     txn: &mut PgConnection,
     initiator: &str,
     update_firmware: bool,
+    force: bool,
 ) -> Result<(), DatabaseError> {
     let reprovision_time = chrono::Utc::now();
     let req = ReprovisionRequest {
@@ -1741,6 +1742,7 @@ pub async fn trigger_dpu_reprovisioning_request(
         started_at: None,
         user_approval_received: false,
         restart_reprovision_requested_at: reprovision_time,
+        force,
     };
 
     let query = "UPDATE machines SET reprovisioning_requested=$2 WHERE id=$1 RETURNING id";