Skip to content

fix(analyzer): detect progress from committed work and embedded RALPH_STATUS #181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
enlego wants to merge 1 commit into from
Closed

fix(analyzer): detect progress from committed work and embedded RALPH_STATUS #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions lib/response_analyzer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,32 @@ analyze_response() {
has_progress=true
files_modified=$git_files
fi
# Also detect progress via recent commits (Claude may commit after each task)
local recent_commits=$(git log --oneline -1 --since="20 minutes ago" 2>/dev/null | wc -l)
if [[ $recent_commits -gt 0 ]]; then
has_progress=true
if [[ $files_modified -eq 0 ]]; then
files_modified=$(git diff HEAD~1 --name-only 2>/dev/null | wc -l)
fi
fi
fi

# Check TASKS_COMPLETED_THIS_LOOP and FILES_MODIFIED from embedded RALPH_STATUS block
# The JSON path only extracts EXIT_SIGNAL from embedded RALPH_STATUS; this supplements it
# Convert literal \n sequences to real newlines so grep/cut work correctly in both
# production (jq decodes JSON escapes to real newlines) and test environments
# (printf '%s' with '\n' produces literal backslash-n characters).
local embedded_result=$(jq -r '.result // ""' "$output_file" 2>/dev/null | sed 's/\\n/\n/g')
if [[ -n "$embedded_result" ]] && echo "$embedded_result" | grep -q -- "---RALPH_STATUS---"; then
local embedded_tasks=$(echo "$embedded_result" | grep "TASKS_COMPLETED_THIS_LOOP:" | cut -d: -f2 | xargs)
local embedded_files=$(echo "$embedded_result" | grep "FILES_MODIFIED:" | cut -d: -f2 | xargs)
if [[ -n "$embedded_tasks" ]] && [[ "$embedded_tasks" =~ ^[0-9]+$ ]] && [[ $embedded_tasks -gt 0 ]]; then
has_progress=true
[[ "${VERBOSE_PROGRESS:-}" == "true" ]] && echo "DEBUG: Progress from TASKS_COMPLETED_THIS_LOOP=$embedded_tasks" >&2
fi
if [[ -n "$embedded_files" ]] && [[ "$embedded_files" =~ ^[0-9]+$ ]] && [[ $files_modified -eq 0 ]]; then
files_modified=$embedded_files
fi
fi

# Write analysis results for JSON path using jq for safe construction
Expand Down
129 changes: 129 additions & 0 deletions tests/unit/test_json_parsing.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1109,3 +1109,132 @@ EOF
local has_denials=$(jq -r '.has_permission_denials' "$result_file")
assert_equal "$has_denials" "true"
}

# =============================================================================
# PROGRESS DETECTION WITH COMMITTED WORK TESTS
# =============================================================================
# When Claude commits work after each task, git diff shows 0 uncommitted changes.
# Progress must be detected via recent commits and the embedded RALPH_STATUS block.

@test "analyze_response detects progress from TASKS_COMPLETED_THIS_LOOP in embedded RALPH_STATUS" {
local output_file="$LOG_DIR/test_output.log"

local ralph_status
ralph_status=$(printf '%s' \
'Implemented notifications context.\n\n' \
'---RALPH_STATUS---\n' \
'STATUS: IN_PROGRESS\n' \
'TASKS_COMPLETED_THIS_LOOP: 1\n' \
'FILES_MODIFIED: 2\n' \
'TESTS_STATUS: NOT_RUN\n' \
'WORK_TYPE: IMPLEMENTATION\n' \
'EXIT_SIGNAL: false\n' \
'---END_RALPH_STATUS---')

jq -n \
--arg result "$ralph_status" \
--arg session "session-tasks-completed" \
'{type: "result", subtype: "success", result: $result, session_id: $session, permission_denials: []}' \
> "$output_file"

# No uncommitted changes, no recent commits — progress must come from RALPH_STATUS block
analyze_response "$output_file" 1

local has_progress
has_progress=$(jq -r '.analysis.has_progress' "$RALPH_DIR/.response_analysis")
assert_equal "$has_progress" "true"
}

@test "analyze_response detects progress via recent git commit when git diff is zero" {
local output_file="$LOG_DIR/test_output.log"

local ralph_status
ralph_status=$(printf '%s' \
'Module created and committed.\n\n' \
'---RALPH_STATUS---\n' \
'STATUS: IN_PROGRESS\n' \
'TASKS_COMPLETED_THIS_LOOP: 1\n' \
'FILES_MODIFIED: 1\n' \
'TESTS_STATUS: NOT_RUN\n' \
'WORK_TYPE: IMPLEMENTATION\n' \
'EXIT_SIGNAL: false\n' \
'---END_RALPH_STATUS---')

jq -n \
--arg result "$ralph_status" \
--arg session "session-committed" \
'{type: "result", subtype: "success", result: $result, session_id: $session, permission_denials: []}' \
> "$output_file"

# Commit a file so git log --since shows a recent commit but git diff shows nothing
echo "defmodule Foo do end" > foo.ex
git add foo.ex
git commit -m "feat: implement Foo context" > /dev/null 2>&1

analyze_response "$output_file" 1

local has_progress
has_progress=$(jq -r '.analysis.has_progress' "$RALPH_DIR/.response_analysis")
assert_equal "$has_progress" "true"
}

@test "analyze_response extracts FILES_MODIFIED from RALPH_STATUS when git diff is zero" {
local output_file="$LOG_DIR/test_output.log"

local ralph_status
ralph_status=$(printf '%s' \
'Three files created.\n\n' \
'---RALPH_STATUS---\n' \
'STATUS: IN_PROGRESS\n' \
'TASKS_COMPLETED_THIS_LOOP: 1\n' \
'FILES_MODIFIED: 3\n' \
'TESTS_STATUS: NOT_RUN\n' \
'WORK_TYPE: IMPLEMENTATION\n' \
'EXIT_SIGNAL: false\n' \
'---END_RALPH_STATUS---')

jq -n \
--arg result "$ralph_status" \
--arg session "session-files-modified" \
'{type: "result", subtype: "success", result: $result, session_id: $session, permission_denials: []}' \
> "$output_file"

# Commit everything so git diff returns 0
echo "defmodule Bar do end" > bar.ex
git add bar.ex
git commit -m "feat: add Bar module" > /dev/null 2>&1

analyze_response "$output_file" 1

local files_modified
files_modified=$(jq -r '.analysis.files_modified' "$RALPH_DIR/.response_analysis")
[[ "$files_modified" -ge 1 ]]
}

@test "analyze_response does not flag progress when TASKS_COMPLETED_THIS_LOOP is zero" {
local output_file="$LOG_DIR/test_output.log"

local ralph_status
ralph_status=$(printf '%s' \
'Reviewed code, no changes needed.\n\n' \
'---RALPH_STATUS---\n' \
'STATUS: IN_PROGRESS\n' \
'TASKS_COMPLETED_THIS_LOOP: 0\n' \
'FILES_MODIFIED: 0\n' \
'TESTS_STATUS: NOT_RUN\n' \
'WORK_TYPE: IMPLEMENTATION\n' \
'EXIT_SIGNAL: false\n' \
'---END_RALPH_STATUS---')

jq -n \
--arg result "$ralph_status" \
--arg session "session-no-progress" \
'{type: "result", subtype: "success", result: $result, session_id: $session, permission_denials: []}' \
> "$output_file"

analyze_response "$output_file" 1

local has_progress
has_progress=$(jq -r '.analysis.has_progress' "$RALPH_DIR/.response_analysis")
assert_equal "$has_progress" "false"
}
Loading