feat (postStart) : Allow debugging poststart failures with sleep by trapping errors

Add an optional debug mechanism for postStart lifecycle hooks. When enabled via the
`controller.devfile.io/debug-start: "true"` annotation, any failure in a postStart command results in the container sleeping for 3600 seconds, allowing developers time to inspect the container state.

- Added `enableDebugStart` parameter to poststart methods.
- Injects `trap ... sleep` into postStart scripts when debug mode is enabled.
- Includes support for both timeout-wrapped (`postStartTimeout`) and non-timeout lifecycle scripts.

This feature improves debuggability of DevWorkspaces where postStart hooks fail and would otherwise cause container crash/restarts.

Signed-off-by: Rohan Kumar <[email protected]>

Author: rohanKanojia

controllers/workspace/devworkspace_controller.go

@@ -329,6 +329,7 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		workspace.Config.Workspace.ImagePullPolicy,
 		workspace.Config.Workspace.DefaultContainerResources,
 		workspace.Config.Workspace.PostStartTimeout,
+		workspace.Annotations[constants.DevWorkspaceDebugStartAnnotation] == "true",
 	)
 	if err != nil {
 		return r.failWorkspace(workspace, fmt.Sprintf("Error processing devfile: %s", err), metrics.ReasonBadRequest, reqLogger, &reconcileStatus), nil

pkg/library/container/container.go

@@ -45,7 +45,7 @@ import (
 // rewritten as Volumes are added to PodAdditions, in order to support e.g. using one PVC to hold all volumes
 //
 // Note: Requires DevWorkspace to be flattened (i.e. the DevWorkspace contains no Parent or Components of type Plugin)
-func GetKubeContainersFromDevfile(workspace *dw.DevWorkspaceTemplateSpec, securityContext *corev1.SecurityContext, pullPolicy string, defaultResources *corev1.ResourceRequirements, postStartTimeout string) (*v1alpha1.PodAdditions, error) {
+func GetKubeContainersFromDevfile(workspace *dw.DevWorkspaceTemplateSpec, securityContext *corev1.SecurityContext, pullPolicy string, defaultResources *corev1.ResourceRequirements, postStartTimeout string, enableDebugStart bool) (*v1alpha1.PodAdditions, error) {
 	if !flatten.DevWorkspaceIsFlattened(workspace, nil) {
 		return nil, fmt.Errorf("devfile is not flattened")
 	}
@@ -77,7 +77,7 @@ func GetKubeContainersFromDevfile(workspace *dw.DevWorkspaceTemplateSpec, securi
 		podAdditions.Containers = append(podAdditions.Containers, *k8sContainer)
 	}
 
-	if err := lifecycle.AddPostStartLifecycleHooks(workspace, podAdditions.Containers, postStartTimeout); err != nil {
+	if err := lifecycle.AddPostStartLifecycleHooks(workspace, podAdditions.Containers, postStartTimeout, enableDebugStart); err != nil {
 		return nil, err
 	}
 

pkg/library/container/container_test.go

@@ -87,7 +87,7 @@ func TestGetKubeContainersFromDevfile(t *testing.T) {
 		t.Run(tt.Name, func(t *testing.T) {
 			// sanity check that file is read correctly.
 			assert.True(t, len(tt.Input.Components) > 0, "Input defines no components")
-			gotPodAdditions, err := GetKubeContainersFromDevfile(tt.Input, nil, testImagePullPolicy, defaultResources, "")
+			gotPodAdditions, err := GetKubeContainersFromDevfile(tt.Input, nil, testImagePullPolicy, defaultResources, "", false)
 			if tt.Output.ErrRegexp != nil && assert.Error(t, err) {
 				assert.Regexp(t, *tt.Output.ErrRegexp, err.Error(), "Error message should match")
 			} else {

pkg/library/lifecycle/poststart.go

@@ -41,7 +41,7 @@ const (
 `
 )
 
-func AddPostStartLifecycleHooks(wksp *dw.DevWorkspaceTemplateSpec, containers []corev1.Container, postStartTimeout string) error {
+func AddPostStartLifecycleHooks(wksp *dw.DevWorkspaceTemplateSpec, containers []corev1.Container, postStartTimeout string, enableDebugStart bool) error {
 	if wksp.Events == nil || len(wksp.Events.PostStart) == 0 {
 		return nil
 	}
@@ -69,7 +69,7 @@ func AddPostStartLifecycleHooks(wksp *dw.DevWorkspaceTemplateSpec, containers []
 			return fmt.Errorf("failed to process postStart event %s: %w", commands[0].Id, err)
 		}
 
-		postStartHandler, err := processCommandsForPostStart(commands, postStartTimeout)
+		postStartHandler, err := processCommandsForPostStart(commands, postStartTimeout, enableDebugStart)
 		if err != nil {
 			return fmt.Errorf("failed to process postStart event %s: %w", commands[0].Id, err)
 		}
@@ -85,10 +85,10 @@ func AddPostStartLifecycleHooks(wksp *dw.DevWorkspaceTemplateSpec, containers []
 
 // processCommandsForPostStart processes a list of DevWorkspace commands
 // and generates a corev1.LifecycleHandler for the PostStart lifecycle hook.
-func processCommandsForPostStart(commands []dw.Command, postStartTimeout string) (*corev1.LifecycleHandler, error) {
+func processCommandsForPostStart(commands []dw.Command, postStartTimeout string, enableDebugStart bool) (*corev1.LifecycleHandler, error) {
 	if postStartTimeout == "" {
 		// use the fallback if no timeout propagated
-		return processCommandsWithoutTimeoutFallback(commands)
+		return processCommandsWithoutTimeoutFallback(enableDebugStart, commands)
 	}
 
 	originalUserScript, err := buildUserScript(commands)
@@ -101,7 +101,7 @@ func processCommandsForPostStart(commands []dw.Command, postStartTimeout string)
 	scriptToExecute := "set -e\n" + originalUserScript
 	escapedUserScriptForTimeoutWrapper := strings.ReplaceAll(scriptToExecute, "'", `'\''`)
 
-	fullScriptWithTimeout := generateScriptWithTimeout(escapedUserScriptForTimeoutWrapper, postStartTimeout)
+	fullScriptWithTimeout := generateScriptWithTimeout(enableDebugStart, escapedUserScriptForTimeoutWrapper, postStartTimeout)
 
 	finalScriptForHook := fmt.Sprintf(redirectOutputFmt, fullScriptWithTimeout)
 
@@ -128,8 +128,13 @@ func processCommandsForPostStart(commands []dw.Command, postStartTimeout string)
 //	  - |
 //	    cd <workingDir>
 //	    <commandline>
-func processCommandsWithoutTimeoutFallback(commands []dw.Command) (*corev1.LifecycleHandler, error) {
+func processCommandsWithoutTimeoutFallback(debugEnabled bool, commands []dw.Command) (*corev1.LifecycleHandler, error) {
 	var dwCommands []string
+	if debugEnabled {
+		dwCommands = append(dwCommands, "set -e")
+		// TODO: Make sleep configurable?
+		dwCommands = append(dwCommands, "trap 'echo \"[postStart] failure encountered, sleep for debugging\"; sleep 3600' ERR")
+	}
 	for _, command := range commands {
 		execCmd := command.Exec
 		if len(execCmd.Env) > 0 {
@@ -142,6 +147,15 @@ func processCommandsWithoutTimeoutFallback(commands []dw.Command) (*corev1.Lifec
 	}
 
 	joinedCommands := strings.Join(dwCommands, "\n")
+	if debugEnabled {
+		joinedCommands = fmt.Sprintf(`cat << 'EOF' > /tmp/poststart.sh
+#!/bin/sh
+%s
+EOF
+chmod +x /tmp/poststart.sh
+/tmp/poststart.sh
+`, joinedCommands)
+	}
 
 	handler := &corev1.LifecycleHandler{
 		Exec: &corev1.ExecAction{
@@ -187,7 +201,7 @@ func buildUserScript(commands []dw.Command) (string, error) {
 // environment variable exports, and specific exit code handling.
 // The killAfterDurationSeconds is hardcoded to 5s within this generated script.
 // It conditionally prefixes the user script with the timeout command if available.
-func generateScriptWithTimeout(escapedUserScript string, postStartTimeout string) string {
+func generateScriptWithTimeout(debugEnabled bool, escapedUserScript string, postStartTimeout string) string {
 	// Convert `postStartTimeout` into the `timeout` format
 	var timeoutSeconds int64
 	if postStartTimeout != "" && postStartTimeout != "0" {
@@ -203,6 +217,7 @@ func generateScriptWithTimeout(escapedUserScript string, postStartTimeout string
 	return fmt.Sprintf(`
 export POSTSTART_TIMEOUT_DURATION="%d"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="%t"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -219,6 +234,11 @@ fi
 ${_TIMEOUT_COMMAND_PART} /bin/sh -c '%s'
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -239,5 +259,5 @@ else
 fi
 
 exit $exit_code
-`, timeoutSeconds, escapedUserScript)
+`, timeoutSeconds, debugEnabled, escapedUserScript)
 }

pkg/library/lifecycle/poststart_test.go

@@ -33,8 +33,9 @@ type postStartTestCase struct {
 }
 
 type postStartTestInput struct {
-	Devfile    *dw.DevWorkspaceTemplateSpec `json:"devfile,omitempty"`
-	Containers []corev1.Container           `json:"containers,omitempty"`
+	Devfile      *dw.DevWorkspaceTemplateSpec `json:"devfile,omitempty"`
+	DebugEnabled bool                         `json:"debugEnabled,omitempty"`
+	Containers   []corev1.Container           `json:"containers,omitempty"`
 }
 
 type postStartTestOutput struct {
@@ -76,7 +77,7 @@ func TestAddPostStartLifecycleHooks(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(fmt.Sprintf("%s (%s)", tt.Name, tt.testPath), func(t *testing.T) {
 			var timeout string
-			err := AddPostStartLifecycleHooks(tt.Input.Devfile, tt.Input.Containers, timeout)
+			err := AddPostStartLifecycleHooks(tt.Input.Devfile, tt.Input.Containers, timeout, tt.Input.DebugEnabled)
 			if tt.Output.ErrRegexp != nil && assert.Error(t, err) {
 				assert.Regexp(t, *tt.Output.ErrRegexp, err.Error(), "Error message should match")
 			} else {
@@ -299,15 +300,18 @@ func TestGenerateScriptWithTimeout(t *testing.T) {
 		name              string
 		escapedUserScript string
 		timeout           string
+		debugEnabled      bool
 		expectedScript    string
 	}{
 		{
 			name:              "Basic script with timeout",
 			escapedUserScript: "echo 'hello world'\nsleep 1",
 			timeout:           "10s",
+			debugEnabled:      false,
 			expectedScript: `
 export POSTSTART_TIMEOUT_DURATION="10"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="false"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -325,6 +329,11 @@ ${_TIMEOUT_COMMAND_PART} /bin/sh -c 'echo 'hello world'
 sleep 1'
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -351,9 +360,11 @@ exit $exit_code
 			name:              "Script with zero timeout (no timeout)",
 			escapedUserScript: "echo 'running indefinitely...'",
 			timeout:           "0s",
+			debugEnabled:      false,
 			expectedScript: `
 export POSTSTART_TIMEOUT_DURATION="0"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="false"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -370,6 +381,11 @@ fi
 ${_TIMEOUT_COMMAND_PART} /bin/sh -c 'echo 'running indefinitely...''
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -396,9 +412,11 @@ exit $exit_code
 			name:              "Empty user script",
 			escapedUserScript: "",
 			timeout:           "5s",
+			debugEnabled:      false,
 			expectedScript: `
 export POSTSTART_TIMEOUT_DURATION="5"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="false"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -415,6 +433,11 @@ fi
 ${_TIMEOUT_COMMAND_PART} /bin/sh -c ''
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -441,9 +464,11 @@ exit $exit_code
 			name:              "User script with already escaped single quotes",
 			escapedUserScript: "echo 'it'\\''s complex'",
 			timeout:           "30s",
+			debugEnabled:      false,
 			expectedScript: `
 export POSTSTART_TIMEOUT_DURATION="30"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="false"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -460,6 +485,11 @@ fi
 ${_TIMEOUT_COMMAND_PART} /bin/sh -c 'echo 'it'\''s complex''
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -486,9 +516,11 @@ exit $exit_code
 			name:              "User script with minute timeout",
 			escapedUserScript: "echo 'wait for it...'",
 			timeout:           "2m",
+			debugEnabled:      false,
 			expectedScript: `
 export POSTSTART_TIMEOUT_DURATION="120"
 export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="false"
 
 _TIMEOUT_COMMAND_PART=""
 _WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
@@ -505,6 +537,64 @@ fi
 ${_TIMEOUT_COMMAND_PART} /bin/sh -c 'echo 'wait for it...''
 exit_code=$?
 
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
+# Check the exit code based on whether timeout was attempted
+if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
+  if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
+    echo "[postStart hook] Commands terminated by SIGTERM (likely timed out after ${POSTSTART_TIMEOUT_DURATION}s). Exit code 143." >&2
+  elif [ $exit_code -eq 137 ]; then # 128 + 9 (SIGKILL)
+    echo "[postStart hook] Commands forcefully killed by SIGKILL (likely after --kill-after ${POSTSTART_KILL_AFTER_DURATION}s expired). Exit code 137." >&2
+  elif [ $exit_code -ne 0 ]; then # Catches any other non-zero exit code
+    echo "[postStart hook] Commands failed with exit code $exit_code." >&2
+  else
+    echo "[postStart hook] Commands completed successfully within the time limit." >&2
+  fi
+else
+  if [ $exit_code -ne 0 ]; then
+    echo "[postStart hook] Commands failed with exit code $exit_code (no timeout)." >&2
+  else
+    echo "[postStart hook] Commands completed successfully (no timeout)." >&2
+  fi
+fi
+
+exit $exit_code
+`,
+		},
+		{
+			name:              "Basic script with timeout and debug enabled",
+			escapedUserScript: "echo 'hello world'\nsleep 1",
+			timeout:           "10s",
+			debugEnabled:      true,
+			expectedScript: `
+export POSTSTART_TIMEOUT_DURATION="10"
+export POSTSTART_KILL_AFTER_DURATION="5"
+export DEBUG_ENABLED="true"
+
+_TIMEOUT_COMMAND_PART=""
+_WAS_TIMEOUT_USED="false" # Use strings "true" or "false" for shell boolean
+
+if command -v timeout >/dev/null 2>&1; then
+  echo "[postStart hook] Executing commands with timeout: ${POSTSTART_TIMEOUT_DURATION} seconds, kill after: ${POSTSTART_KILL_AFTER_DURATION} seconds" >&2
+  _TIMEOUT_COMMAND_PART="timeout --preserve-status --kill-after=${POSTSTART_KILL_AFTER_DURATION} ${POSTSTART_TIMEOUT_DURATION}"
+  _WAS_TIMEOUT_USED="true"
+else
+  echo "[postStart hook] WARNING: 'timeout' utility not found. Executing commands without timeout." >&2
+fi
+
+# Execute the user's script
+${_TIMEOUT_COMMAND_PART} /bin/sh -c 'echo 'hello world'
+sleep 1'
+exit_code=$?
+
+if [ "$DEBUG_ENABLED" = "true" ] && [ $exit_code -ne 0 ]; then
+  echo "[postStart] failure encountered, sleep for debugging" >&2
+  sleep 3600
+fi
+
 # Check the exit code based on whether timeout was attempted
 if [ "$_WAS_TIMEOUT_USED" = "true" ]; then
   if [ $exit_code -eq 143 ]; then # 128 + 15 (SIGTERM)
@@ -531,7 +621,7 @@ exit $exit_code
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			script := generateScriptWithTimeout(tt.escapedUserScript, tt.timeout)
+			script := generateScriptWithTimeout(tt.debugEnabled, tt.escapedUserScript, tt.timeout)
 			assert.Equal(t, tt.expectedScript, script)
 		})
 	}

pkg/library/lifecycle/testdata/postStart/adds_all_postStart_commands.yaml

@@ -1,6 +1,7 @@
 name: "Adds all postStart events to containers"
 
 input:
+  debugEnabled: false
   devfile:
     commands:
       - id: test-postStart-1

pkg/library/lifecycle/testdata/postStart/basic_postStart.yaml

@@ -1,6 +1,7 @@
 name: "Should add postStart lifecycle hook for basic event"
 
 input:
+  debugEnabled: false
   devfile:
     commands:
       - id: test-postStart