diff --git a/crates/admin-cli/src/dpu/reprovision/args.rs b/crates/admin-cli/src/dpu/reprovision/args.rs index 75a4d744d3..a59c53567b 100644 --- a/crates/admin-cli/src/dpu/reprovision/args.rs +++ b/crates/admin-cli/src/dpu/reprovision/args.rs @@ -62,6 +62,9 @@ Reprovision and update DPU firmware, recording a maintenance message: $ nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 \ --update-firmware --update-message \"scheduled firmware refresh\" +Force reprovisioning to recover a DPU stuck in ingestion: + $ nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 --force + ")] pub(crate) struct DpuReprovisionSet { #[clap( @@ -80,6 +83,13 @@ pub(crate) struct DpuReprovisionSet { help = "If set, a HostUpdateInProgress health alert will be applied to the host" )] pub(super) update_message: Option, + + #[clap( + long, + action, + help = "Force reprovisioning regardless of the Machine state to recover DPUs stuck in ingestion. Restarts the ingestion state machine and skips the HostUpdateInProgress precondition. Has no effect on assigned Machines." + )] + force: bool, } impl From<&DpuReprovisionSet> for DpuReprovisioningRequest { @@ -90,6 +100,7 @@ impl From<&DpuReprovisionSet> for DpuReprovisioningRequest { mode: Mode::Set as i32, initiator: UpdateInitiator::AdminCli as i32, update_firmware: args.update_firmware, + force: args.force, } } } @@ -125,6 +136,7 @@ impl From<&DpuReprovisionClear> for DpuReprovisioningRequest { mode: Mode::Clear as i32, initiator: UpdateInitiator::AdminCli as i32, update_firmware: args.update_firmware, + force: false, } } } @@ -161,6 +173,7 @@ impl From<&DpuReprovisionRestart> for DpuReprovisioningRequest { mode: Mode::Restart as i32, initiator: UpdateInitiator::AdminCli as i32, update_firmware: args.update_firmware, + force: false, } } } diff --git a/crates/api-core/src/handlers/dpu.rs b/crates/api-core/src/handlers/dpu.rs index 22b9f3673c..e9c94ced46 100644 --- a/crates/api-core/src/handlers/dpu.rs +++ b/crates/api-core/src/handlers/dpu.rs @@ -1307,27 +1307,37 @@ pub(crate) async fn trigger_dpu_reprovisioning( id: machine_id.to_string(), })?; + // A forced reprovision recovers DPUs stuck in ingestion (non-Ready, + // non-Assigned states). Such a host is already non-allocatable, so the + // HostUpdateInProgress precondition is meaningless and skipped, and a + // forced request may re-kick even if a previous request is in progress. + let force = req.force; + // Start reprovisioning only if the host has an HostUpdateInProgress health alert - let update_alert = snapshot - .aggregate_health - .alerts - .iter() - .find(|a| a.id == *HOST_UPDATE_HEALTH_PROBE_ID); - if !update_alert.is_some_and(|alert| { - alert - .classifications - .contains(&health_report::HealthAlertClassification::prevent_allocations()) - }) { - return Err(CarbideError::InvalidArgument(format!( - "machine {machine_id} must have a 'HostUpdateInProgress' health alert with the 'PreventAllocations' classification before reprovisioning. set this precondition with: `machine health-override add --template host-update `", - )).into()); + if !force { + let update_alert = snapshot + .aggregate_health + .alerts + .iter() + .find(|a| a.id == *HOST_UPDATE_HEALTH_PROBE_ID); + if !update_alert.is_some_and(|alert| { + alert + .classifications + .contains(&health_report::HealthAlertClassification::prevent_allocations()) + }) { + return Err(CarbideError::InvalidArgument(format!( + "machine {machine_id} must have a 'HostUpdateInProgress' health alert with the 'PreventAllocations' classification before reprovisioning. set this precondition with: `machine health-override add --template host-update `", + )).into()); + } } - if snapshot.dpu_snapshots.iter().any(|ms| { - ms.reprovision_requested - .as_ref() - .is_some_and(|x| x.started_at.is_some()) - }) { + if !force + && snapshot.dpu_snapshots.iter().any(|ms| { + ms.reprovision_requested + .as_ref() + .is_some_and(|x| x.started_at.is_some()) + }) + { match req.mode() { Mode::Restart => {} _ => { @@ -1348,6 +1358,7 @@ pub(crate) async fn trigger_dpu_reprovisioning( &mut txn, initiator, req.update_firmware, + force, ) .await?; } else { @@ -1357,6 +1368,7 @@ pub(crate) async fn trigger_dpu_reprovisioning( &mut txn, initiator, req.update_firmware, + force, ) .await?; } @@ -1411,6 +1423,22 @@ pub(crate) async fn trigger_dpu_reprovisioning( txn.commit().await?; + // A stuck-in-ingestion host may have backed off with a long state-controller + // wait, so a forced request explicitly wakes the host to be re-evaluated + // promptly. + if force + && let Err(err) = api + .machine_state_handler_enqueuer + .enqueue_object(&snapshot.host_snapshot.id) + .await + { + tracing::warn!( + host_machine_id = %snapshot.host_snapshot.id, + error = %err, + "failed to wake host state handler after forced DPU reprovisioning request", + ); + } + Ok(Response::new(())) } diff --git a/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs b/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs index 30a042c349..eb302f2a0c 100644 --- a/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/test_machine/mod.rs @@ -146,6 +146,7 @@ impl TestMachine { mode: mode as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, update_firmware, + force: false, }, )) .await diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 9e7ebcd3bc..6a8ed381bf 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -27,9 +27,10 @@ use common::api_fixtures::{ use libredfish::{EnabledDisabled, SystemPowerControl}; use model::instance::status::tenant::TenantState; use model::machine::{ - DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState, - Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, - ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, + DpuDiscoveringState, DpuDiscoveringStates, DpuInitState, DpuInitStates, FailureCause, + FailureDetails, FailureSource, InstallDpuOsState, InstanceState, Machine, + MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, ReprovisionState, + SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, }; use model::test_support::HardwareInfoTemplate; use rpc::forge::MachineArchitecture; @@ -273,9 +274,15 @@ async fn prepare_dpu_reprovision_host_boot_check( ) .await .unwrap(); - db::machine::trigger_dpu_reprovisioning_request(&dpu_machine.id, &mut txn, "AdminCli", true) - .await - .unwrap(); + db::machine::trigger_dpu_reprovisioning_request( + &dpu_machine.id, + &mut txn, + "AdminCli", + true, + false, + ) + .await + .unwrap(); txn.commit().await.unwrap(); dpu_machine @@ -591,7 +598,8 @@ async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgP machine_id: mh.dpu().id.into(), mode: rpc::forge::dpu_reprovisioning_request::Mode::Set as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, - update_firmware: true + update_firmware: true, + force: false, }, )) .await @@ -612,7 +620,8 @@ async fn test_dpu_for_reprovisioning_fail_if_state_is_not_ready(pool: sqlx::PgPo machine_id: dpu_machine_id.into(), mode: rpc::forge::dpu_reprovisioning_request::Mode::Set as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, - update_firmware: true + update_firmware: true, + force: false, }, )) .await @@ -1155,7 +1164,8 @@ async fn test_dpu_for_set_but_clear_failed(pool: sqlx::PgPool) { machine_id: mh.dpu().id.into(), mode: rpc::forge::dpu_reprovisioning_request::Mode::Clear as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, - update_firmware: true + update_firmware: true, + force: false, }, )) .await @@ -1492,6 +1502,7 @@ async fn test_restart_dpu_reprov(pool: sqlx::PgPool) { mode: Mode::Restart as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, update_firmware: false, + force: false, }, )) .await @@ -1570,9 +1581,15 @@ async fn test_restart_dpu_reprov_unassigned_host_boot_failure(pool: sqlx::PgPool let failed_at = Utc::now(); let mut txn = env.pool.begin().await.unwrap(); - db::machine::trigger_dpu_reprovisioning_request(&dpu_machine.id, &mut txn, "AdminCli", true) - .await - .unwrap(); + db::machine::trigger_dpu_reprovisioning_request( + &dpu_machine.id, + &mut txn, + "AdminCli", + true, + false, + ) + .await + .unwrap(); db::machine::update_dpu_reprovision_explicit_start_time(&dpu_machine.id, failed_at, &mut txn) .await .unwrap(); @@ -2110,6 +2127,7 @@ async fn test_instance_reprov_restart_failed_impl(pool: sqlx::PgPool) { mode: Mode::Restart as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, update_firmware: false, + force: false, }, )) .await @@ -2226,6 +2244,7 @@ async fn test_dpu_for_reprovisioning_cannot_restart_if_not_started(pool: sqlx::P mode: rpc::forge::dpu_reprovisioning_request::Mode::Restart as i32, initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, update_firmware: true, + force: false, }, )) .await @@ -2271,3 +2290,99 @@ impl TestManagedHost { .unwrap(); } } + +// A DPU stuck in ingestion has no HostUpdateInProgress health alert, so a +// non-forced request is rejected (see +// test_dpu_for_reprovisioning_fail_if_maintenance_not_set). A forced request +// must instead succeed and persist the force flag for the controller to act on. +#[crate::sqlx_test] +async fn test_dpu_force_reprovisioning_bypasses_precondition(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let mh = common::api_fixtures::create_managed_host(&env).await; + + env.api + .trigger_dpu_reprovisioning(tonic::Request::new( + ::rpc::forge::DpuReprovisioningRequest { + dpu_id: None, + machine_id: mh.dpu().id.into(), + mode: Mode::Set as i32, + initiator: ::rpc::forge::UpdateInitiator::AdminCli as i32, + update_firmware: false, + force: true, + }, + )) + .await + .expect("forced reprovisioning should bypass the health-alert precondition"); + + let mut txn = env.pool.begin().await.unwrap(); + let dpu = mh.dpu().db_machine(&mut txn).await; + let req = dpu + .reprovision_requested + .expect("forced reprovisioning should persist a request"); + assert!(req.force, "persisted request should carry the force flag"); +} + +// A forced request on a host still in an ingestion substate restarts the +// ingestion state machine from discovery and clears the one-shot force flag. +#[crate::sqlx_test] +async fn test_force_reprovisioning_restarts_ingestion(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + for ingestion_substate in [ + DpuInitState::WaitingForPlatformConfiguration, + DpuInitState::WaitingForNetworkConfig, + ] { + let mh = common::api_fixtures::create_managed_host(&env).await; + + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state( + &mut txn, + &mh.id, + &ManagedHostState::DPUInit { + dpu_states: DpuInitStates { + states: mh + .dpu_ids + .iter() + .map(|id| (*id, ingestion_substate.clone())) + .collect(), + }, + }, + ) + .await + .unwrap(); + db::machine::trigger_dpu_reprovisioning_request( + &mh.dpu().id, + &mut txn, + "AdminCli", + false, + true, + ) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let dpu = mh.dpu().next_iteration_machine(&env).await; + + let expected = ManagedHostState::DpuDiscoveringState { + dpu_states: DpuDiscoveringStates { + states: mh + .dpu_ids + .iter() + .map(|id| (*id, DpuDiscoveringState::Initializing)) + .collect(), + }, + }; + assert_eq!( + dpu.current_state(), + &expected, + "forced reprovisioning from {ingestion_substate:?} should restart ingestion from discovery", + ); + + let mut txn = env.pool.begin().await.unwrap(); + let dpu = mh.dpu().db_machine(&mut txn).await; + assert!( + dpu.reprovision_requested.is_none(), + "forced reprovisioning should clear the one-shot request", + ); + } +} diff --git a/crates/api-core/src/tests/machine_update_manager.rs b/crates/api-core/src/tests/machine_update_manager.rs index e43c37196b..3fef7349ff 100644 --- a/crates/api-core/src/tests/machine_update_manager.rs +++ b/crates/api-core/src/tests/machine_update_manager.rs @@ -363,7 +363,7 @@ async fn test_get_updating_machines(pool: sqlx::PgPool) -> Result<(), Box Result<(), DatabaseError> { let reprovision_time = chrono::Utc::now(); let req = ReprovisionRequest { @@ -1741,6 +1742,7 @@ pub async fn trigger_dpu_reprovisioning_request( started_at: None, user_approval_received: false, restart_reprovision_requested_at: reprovision_time, + force, }; let query = "UPDATE machines SET reprovisioning_requested=$2 WHERE id=$1 RETURNING id"; diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 4940e16ec7..80e8f23c05 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -2497,6 +2497,11 @@ pub struct ReprovisionRequest { pub user_approval_received: bool, #[serde(default)] pub restart_reprovision_requested_at: DateTime, + /// When set, reprovisioning is forced regardless of the managed host state. + /// Used to recover DPUs stuck in ingestion (non-Ready, non-Assigned states) + /// by restarting the ingestion state machine. + #[serde(default)] + pub force: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -3605,6 +3610,29 @@ mod tests { ); } + #[test] + fn reprovision_request_force_defaults_false_for_legacy_blobs() { + // A request persisted before the force flag existed omits it and must + // load as a non-forced request. + let legacy = r#"{ + "requested_at": "2026-01-01T00:00:00Z", + "initiator": "AdminCli", + "update_firmware": false + }"#; + let req: ReprovisionRequest = + serde_json::from_str(legacy).expect("legacy request blob should deserialize"); + assert!(!req.force, "a missing force flag must default to false"); + + // A forced request round-trips through JSON. + let forced = ReprovisionRequest { + force: true, + ..req + }; + let json = serde_json::to_string(&forced).unwrap(); + let round_tripped: ReprovisionRequest = serde_json::from_str(&json).unwrap(); + assert!(round_tripped.force, "force flag must survive serialization"); + } + #[test] fn ready_boot_config_defaults_survive_persisted_state_loading() { scenarios!( @@ -3805,6 +3833,7 @@ mod tests { started_at: None, user_approval_received: false, restart_reprovision_requested_at: DateTime::::UNIX_EPOCH, + force: false, }); dpu }) diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 431a76ac03..2fa21d61bb 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -75,7 +75,8 @@ use model::machine::nvlink::nvlink_config_synced; use model::machine::{ AttestationMode, BomValidating, BomValidatingContext, CleanupContext, CleanupState, CreateBossVolumeContext, CreateBossVolumeState, DecommissioningState, DpuDiscoveringState, - DpuInitNextStateResolver, DpuInitState, FactoryResetBmcState, FailureCause, FailureDetails, + DpuDiscoveringStates, DpuInitNextStateResolver, DpuInitState, FactoryResetBmcState, + FailureCause, FailureDetails, FailureSource, HostPlatformConfigurationState, HostReprovisionState, InitialResetPhase, InstallDpuOsState, InstanceNextStateResolver, InstanceState, LockdownInfo, LockdownState, MAX_FIRMWARE_UPGRADE_RETRIES, Machine, MachineLastRebootRequested, @@ -747,6 +748,54 @@ impl MachineStateHandler { return Ok(outcome); } + // A forced reprovision request recovers DPUs stuck in ingestion by + // restarting the ingestion state machine from the beginning. Ready and + // Assigned hosts already have working reprovision paths (handled below + // and in their state handlers), so the forced restart only applies to + // the remaining non-Ready, non-Assigned states. The request is a + // one-shot: it is cleared here so it is not re-applied on the next tick. + if force_reprovision_requested(&mh_snapshot.dpu_snapshots) + && !matches!( + mh_state, + ManagedHostState::Ready | ManagedHostState::Assigned { .. } + ) + { + tracing::warn!( + host_machine_id = %host_machine_id, + current_state = %mh_state, + "Force reprovisioning requested; restarting DPU ingestion", + ); + + for dpu_snapshot in &mh_snapshot.dpu_snapshots { + if dpu_snapshot + .reprovision_requested + .as_ref() + .is_some_and(|r| r.force) + { + handler_restart_dpu( + dpu_snapshot, + ctx, + mh_snapshot.host_snapshot.config.dpf.used_for_ingestion, + ) + .await?; + } + } + + let mut txn = ctx.services.db_pool.begin().await?; + Self::clear_dpu_reprovision(mh_snapshot, &mut txn).await?; + + let states = mh_snapshot + .dpu_snapshots + .iter() + .map(|dpu| (dpu.id, DpuDiscoveringState::Initializing)) + .collect(); + let next_state = ManagedHostState::DpuDiscoveringState { + dpu_states: DpuDiscoveringStates { states }, + }; + + return Ok(StateHandlerOutcome::transition(next_state).with_txn(txn)); + } + if dpu_reprovisioning_needed(&mh_snapshot.dpu_snapshots) { // Reprovision is started and user requested for restart of reprovision. let restart_reprov = can_restart_reprovision( @@ -2324,6 +2373,15 @@ fn dpu_reprovisioning_needed(dpu_snapshots: &[Machine]) -> bool { .any(|x| x.reprovision_requested.is_some()) } +/// This function checks if a forced reprovisioning is requested for any DPU. +/// A forced request recovers DPUs stuck in ingestion by restarting the +/// ingestion state machine regardless of the current managed host state. +fn force_reprovision_requested(dpu_snapshots: &[Machine]) -> bool { + dpu_snapshots + .iter() + .any(|x| x.reprovision_requested.as_ref().is_some_and(|r| r.force)) +} + async fn handle_restart_verification( mh_snapshot: &ManagedHostStateSnapshot, ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, diff --git a/crates/rpc/proto/forge.proto b/crates/rpc/proto/forge.proto index 4aa70e80cd..a3027f9f04 100644 --- a/crates/rpc/proto/forge.proto +++ b/crates/rpc/proto/forge.proto @@ -6253,6 +6253,11 @@ message DpuReprovisioningRequest { bool update_firmware = 4; common.MachineId machine_id = 5; + + // Force reprovisioning regardless of the managed host state. Intended to + // recover DPUs stuck in ingestion (non-Ready, non-Assigned states) by + // restarting the ingestion state machine. Ignored for Assigned hosts. + bool force = 6; } message DpuReprovisioningListRequest { diff --git a/docs/dpu-management/dpu-lifecycle-management.md b/docs/dpu-management/dpu-lifecycle-management.md index b0d87fb723..8bb13ad83e 100644 --- a/docs/dpu-management/dpu-lifecycle-management.md +++ b/docs/dpu-management/dpu-lifecycle-management.md @@ -322,6 +322,31 @@ curl -X PATCH "${BASE_URL}/v2/org/${ORG}/nico/machine/${MACHINE_ID}/dpu/reprovis Use `Set` mode in the PATCH operation to start reprovisioning and `Clear` to remove a pending request. `Restart` mode accepts a host ID only, and restarts DPUs that already have a reprovisioning request. If an Instance is attached to the Machine, also pass `"acknowledgeAttachedInstance": true`. The REST `updateFirmware` field is accepted for compatibility, but firmware is always verified and updated during reprovisioning. +### Recovering a DPU Stuck in Ingestion + +The manual and automatic flows above act on hosts that have reached `Ready` (or a tenant-allocated `Assigned` host, which follows the approval flow). A DPU that never finished ingestion — one still in a `DPUInitializing` substate such as `WaitingForPlatformConfiguration`, `WaitingForNetworkConfig`, or `WaitingForPlatformPowercycle` — is not `Ready`, so an ordinary reprovisioning request does not move it. + +To recover such a DPU, set `--force` (CLI) or `"force": true` (REST). A forced request restarts the ingestion state machine from the beginning: NICo reinstalls the NICo BFB and drives the DPU back through discovery, network configuration, and health. + +```bash +nico-admin-cli -a dpu reprovision set \ + --id \ + --force +``` + +```bash +curl -X PATCH "${BASE_URL}/v2/org/${ORG}/nico/machine/${MACHINE_ID}/dpu/reprovision" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"mode":"Set","force":true}' +``` + +A forced request behaves differently from a standard one in three ways: + +- It does **not** require the `HostUpdateInProgress` health alert precondition, so no health report needs to be created first. +- It is a one-shot kick: NICo clears the force flag as soon as it restarts ingestion, so ingestion is not repeatedly reset on later ticks. +- It has **no effect on tenant-allocated (`Assigned`) hosts**, which continue to use the standard reprovisioning approval flow. + ### Monitoring Reprovisioning Progress ```bash diff --git a/docs/manuals/nico-admin-cli/commands/dpu/dpu-reprovision-set.md b/docs/manuals/nico-admin-cli/commands/dpu/dpu-reprovision-set.md index b858ac7f57..280ca2f1ae 100644 --- a/docs/manuals/nico-admin-cli/commands/dpu/dpu-reprovision-set.md +++ b/docs/manuals/nico-admin-cli/commands/dpu/dpu-reprovision-set.md @@ -9,7 +9,7 @@ nico-admin-cli-dpu-reprovision-set - Set the DPU in reprovisioning mode. ## SYNOPSIS **nico-admin-cli dpu reprovision set** \<**-i**\|**--id**\> -\[**-u**\|**--update-firmware**\] \[**--update-message**\] +\[**-u**\|**--update-firmware**\] \[**--update-message**\] \[**--force**\] \[**--extended**\] \[**--sort-by**\] \[**-h**\|**--help**\] ## DESCRIPTION @@ -26,6 +26,11 @@ all DPUs should be reprovisioned. **--update-message** *\* If set, a HostUpdateInProgress health alert will be applied to the host +**--force** +Force reprovisioning regardless of the Machine state to recover DPUs +stuck in ingestion. Restarts the ingestion state machine and skips the +HostUpdateInProgress precondition. Has no effect on assigned Machines. + **--extended** Extended result output. @@ -52,6 +57,7 @@ Print help (see a summary with -h) nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 nico-admin-cli dpu reprovision set --id abcdef01-2345-6789-abcd-ef0123456789 nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 --update-firmware --update-message "scheduled firmware refresh" +nico-admin-cli dpu reprovision set --id 12345678-1234-5678-90ab-cdef01234567 --force ``` --- diff --git a/rest-api/api/pkg/api/model/dpureprovision.go b/rest-api/api/pkg/api/model/dpureprovision.go index f884010d36..3fcd725c6e 100644 --- a/rest-api/api/pkg/api/model/dpureprovision.go +++ b/rest-api/api/pkg/api/model/dpureprovision.go @@ -45,6 +45,8 @@ type APIMachineDpuReprovisionRequest struct { UpdateFirmware bool `json:"updateFirmware"` // AcknowledgeAttachedInstance indicates the caller is aware that an Instance is currently attached to the Machine. AcknowledgeAttachedInstance *bool `json:"acknowledgeAttachedInstance"` + // Force restarts reprovisioning regardless of the Machine state, to recover DPUs stuck in ingestion. It has no effect on assigned Machines. + Force bool `json:"force"` } // Validate validates the APIMachineDpuReprovisionRequest @@ -64,5 +66,6 @@ func (amdrr *APIMachineDpuReprovisionRequest) ToProto(machineID string) *corev1. // TODO: Add end user initiator in Core gRPC API Initiator: corev1.UpdateInitiator_AdminCli, UpdateFirmware: amdrr.UpdateFirmware, + Force: amdrr.Force, } } diff --git a/rest-api/api/pkg/api/model/dpureprovision_test.go b/rest-api/api/pkg/api/model/dpureprovision_test.go index 681357587a..06adc23f81 100644 --- a/rest-api/api/pkg/api/model/dpureprovision_test.go +++ b/rest-api/api/pkg/api/model/dpureprovision_test.go @@ -21,6 +21,11 @@ func TestAPIMachineDpuReprovisionRequestValidateAndToProto(t *testing.T) { assert.Equal(t, corev1.DpuReprovisioningRequest_Restart, protoReq.GetMode()) assert.Equal(t, corev1.UpdateInitiator_AdminCli, protoReq.GetInitiator()) assert.True(t, protoReq.GetUpdateFirmware()) + assert.False(t, protoReq.GetForce(), "force must default to false when unset") + + forced := APIMachineDpuReprovisionRequest{Mode: MachineDpuReprovisionModeSet, Force: true} + require.NoError(t, forced.Validate()) + assert.True(t, forced.ToProto("machine-2").GetForce(), "force must map through to the proto request") assert.Error(t, (&APIMachineDpuReprovisionRequest{}).Validate()) assert.Error(t, (&APIMachineDpuReprovisionRequest{Mode: MachineDpuReprovisionMode("restart")}).Validate()) diff --git a/rest-api/docs/index.html b/rest-api/docs/index.html index e53fbd9cf8..eb818c7cee 100644 --- a/rest-api/docs/index.html +++ b/rest-api/docs/index.html @@ -12281,6 +12281,11 @@

Typical API Call Flow for Tenant

<code>HostUpdateInProgress</code> and whose classifications include <code>PreventAllocations</code>. When an Instance is attached to the Machine, <code>acknowledgeAttachedInstance</code> must be <code>true</code>.</p> +<p>To recover DPUs stuck in ingestion (before the Machine reaches a ready +state), set <code>force</code> to <code>true</code>. A forced request restarts the ingestion +state machine and does not require the <code>HostUpdateInProgress</code> health +report precondition. <code>force</code> has no effect on assigned Machines, which +retain the standard reprovisioning approval flow.</p> <p>For Infrastructure Providers: Org must have an Infrastructure Provider entity and own the Site that the Machine belongs to. User must have authorization role with <code>PROVIDER_ADMIN</code> suffix.</p> <p>For Tenants: Org must have a Tenant with <code>TargetedInstanceCreation</code> capability enabled and Tenant Account with Machine&#39;s Provider. User must have authorization role with <code>TENANT_ADMIN</code> suffix.</p> " class="sc-iJSMbW sc-cBEgGa fiNpIH ewCFMV">

Trigger DPU reprovisioning for a Machine. Before calling this operation, @@ -12288,6 +12293,11 @@

Typical API Call Flow for Tenant

HostUpdateInProgress and whose classifications include PreventAllocations. When an Instance is attached to the Machine, acknowledgeAttachedInstance must be true.

+

To recover DPUs stuck in ingestion (before the Machine reaches a ready +state), set force to true. A forced request restarts the ingestion +state machine and does not require the HostUpdateInProgress health +report precondition. force has no effect on assigned Machines, which +retain the standard reprovisioning approval flow.

For Infrastructure Providers: Org must have an Infrastructure Provider entity and own the Site that the Machine belongs to. User must have authorization role with PROVIDER_ADMIN suffix.

For Tenants: Org must have a Tenant with TargetedInstanceCreation capability enabled and Tenant Account with Machine's Provider. User must have authorization role with TENANT_ADMIN suffix.

Authorizations:
JWTBearerToken
path Parameters
org
required
string
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Use Set to start reprovisioning, Clear to remove a pending request, or Restart to restart DPUs that already have a request. Restart accepts a host Machine ID only.

updateFirmware
boolean
Deprecated
Default: false

Deprecated compatibility field. Firmware is always verified and updated during reprovisioning.

-
acknowledgeAttachedInstance
boolean
acknowledgeAttachedInstance
boolean

Acknowledges that an Instance is currently attached to the Machine and this action may disrupt Tenant workload on the Instance.

+
force
boolean
Default: false

Forces reprovisioning regardless of the Machine state to recover DPUs stuck in ingestion, restarting the ingestion state machine. When set, the HostUpdateInProgress health report precondition is not required. It has no effect on assigned Machines, which retain the standard reprovisioning approval flow.

Responses