Merge pull request #141 from conductor-oss/fix/agent-e2e-llm-budget

test: skip agent e2e when LLM budget is exhausted

Author: ling-senpeng13

e2e/src/test/java/BaseTest.java

@@ -19,6 +19,7 @@
 import java.util.Map;
 
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.ExtendWith;
 
 import io.orkes.conductor.client.model.agent.CompileResponse;
 
@@ -37,6 +38,7 @@
  *   <li>Helper to extract agentDef from a plan() result</li>
  * </ul>
  */
+@ExtendWith(LlmBudgetExhaustionExtension.class)
 public abstract class BaseTest {
 
     /** API URL for the Conductor server (includes /api suffix). */

e2e/src/test/java/LlmBudgetExhaustionExtension.java

@@ -0,0 +1,29 @@
+import java.util.Locale;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
+import org.opentest4j.TestAbortedException;
+
+/** Skips live LLM tests when a provider's configured API usage budget is exhausted. */
+final class LlmBudgetExhaustionExtension implements TestExecutionExceptionHandler {
+
+    static final String USAGE_LIMIT_MESSAGE = "reached your specified api usage limits";
+
+    @Override
+    public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
+        if (isUsageLimitError(throwable)) {
+            throw new TestAbortedException("LLM API usage budget is exhausted", throwable);
+        }
+        throw throwable;
+    }
+
+    static boolean isUsageLimitError(Throwable throwable) {
+        for (Throwable current = throwable; current != null; current = current.getCause()) {
+            String message = current.getMessage();
+            if (message != null && message.toLowerCase(Locale.ROOT).contains(USAGE_LIMIT_MESSAGE)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java

@@ -0,0 +1,42 @@
+import org.junit.jupiter.api.Test;
+import org.opentest4j.TestAbortedException;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class LlmBudgetExhaustionExtensionTest {
+
+    @Test
+    void detectsUsageLimitMessageInCauseChain() {
+        Throwable failure = new AssertionError(
+                "agent failed",
+                new IllegalStateException("You have reached your specified API usage limits. Try again later."));
+
+        assertTrue(LlmBudgetExhaustionExtension.isUsageLimitError(failure));
+    }
+
+    @Test
+    void doesNotMaskOtherLlmFailures() {
+        Throwable failure = new AssertionError("Agent failed: invalid model");
+
+        assertFalse(LlmBudgetExhaustionExtension.isUsageLimitError(failure));
+    }
+
+    @Test
+    void abortsUsageLimitFailuresAndRethrowsOtherFailures() {
+        LlmBudgetExhaustionExtension extension = new LlmBudgetExhaustionExtension();
+        Throwable budgetFailure =
+                new AssertionError("You have reached your specified API usage limits.");
+        Throwable otherFailure = new AssertionError("Agent failed: invalid model");
+
+        assertThrows(
+                TestAbortedException.class,
+                () -> extension.handleTestExecutionException(null, budgetFailure));
+        Throwable rethrown = assertThrows(
+                Throwable.class,
+                () -> extension.handleTestExecutionException(null, otherFailure));
+        assertSame(otherFailure, rethrown);
+    }
+}