Skip to content

Further E2E tests for resume #665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SteveSandersonMS merged 2 commits into from
Mar 4, 2026
Merged

Further E2E tests for resume #665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c6e73f
Further E2E tests for resume
SteveSandersonMS Mar 4, 2026
aaac89c
Address PR review feedback
SteveSandersonMS Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 10 additions & 62 deletions dotnet/test/SessionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,11 @@ public async Task Should_Resume_A_Session_Using_The_Same_Client()
var answer2 = await TestHelper.GetFinalAssistantMessageAsync(session2);
Assert.NotNull(answer2);
Assert.Contains("2", answer2!.Data.Content ?? string.Empty);

// Can continue the conversation statefully
var answer3 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
Assert.NotNull(answer3);
Assert.Contains("4", answer3!.Data.Content ?? string.Empty);
}

[Fact]
Expand All @@ -187,6 +192,11 @@ public async Task Should_Resume_A_Session_Using_A_New_Client()
var messages = await session2.GetMessagesAsync();
Assert.Contains(messages, m => m is UserMessageEvent);
Assert.Contains(messages, m => m is SessionResumeEvent);

// Can continue the conversation statefully
var answer2 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
Assert.NotNull(answer2);
Assert.Contains("4", answer2!.Data.Content ?? string.Empty);
}

[Fact]
Expand Down Expand Up @@ -231,68 +241,6 @@ await session.SendAsync(new MessageOptions
Assert.Contains("4", answer!.Data.Content ?? string.Empty);
}

// TODO: This test requires the session-events.schema.json to include assistant.message_delta.
// The CLI v0.0.376 emits delta events at runtime, but the schema hasn't been updated yet.
// Once the schema is updated and types are regenerated, this test can be enabled.
[Fact(Skip = "Requires schema update for AssistantMessageDeltaEvent type")]
public async Task Should_Receive_Streaming_Delta_Events_When_Streaming_Is_Enabled()
{
var session = await CreateSessionAsync(new SessionConfig { Streaming = true });

var deltaContents = new List<string>();
var doneEvent = new TaskCompletionSource<bool>();

session.On(evt =>
{
switch (evt)
{
// TODO: Uncomment once AssistantMessageDeltaEvent is generated
// case AssistantMessageDeltaEvent delta:
// if (!string.IsNullOrEmpty(delta.Data.DeltaContent))
// deltaContents.Add(delta.Data.DeltaContent);
// break;
case SessionIdleEvent:
doneEvent.TrySetResult(true);
break;
}
});

await session.SendAsync(new MessageOptions { Prompt = "What is 2+2?" });

// Wait for completion
var completed = await Task.WhenAny(doneEvent.Task, Task.Delay(TimeSpan.FromSeconds(60)));
Assert.Equal(doneEvent.Task, completed);

// Should have received delta events
Assert.NotEmpty(deltaContents);

// Get the final message to compare
var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session);
Assert.NotNull(assistantMessage);

// Accumulated deltas should equal the final message
var accumulated = string.Join("", deltaContents);
Assert.Equal(assistantMessage!.Data.Content, accumulated);

// Final message should contain the answer
Assert.Contains("4", assistantMessage.Data.Content ?? string.Empty);
}

[Fact]
public async Task Should_Pass_Streaming_Option_To_Session_Creation()
{
// Verify that the streaming option is accepted without errors
var session = await CreateSessionAsync(new SessionConfig { Streaming = true });

Assert.Matches(@"^[a-f0-9-]+$", session.SessionId);

// Session should still work normally
await session.SendAsync(new MessageOptions { Prompt = "What is 1+1?" });
var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session);
Assert.NotNull(assistantMessage);
Assert.Contains("2", assistantMessage!.Data.Content);
}

[Fact]
public async Task Should_Receive_Session_Events()
{
Expand Down
99 changes: 99 additions & 0 deletions dotnet/test/StreamingFidelityTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
*--------------------------------------------------------------------------------------------*/

using GitHub.Copilot.SDK.Test.Harness;
using Xunit;
using Xunit.Abstractions;

namespace GitHub.Copilot.SDK.Test;

public class StreamingFidelityTests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "streaming_fidelity", output)
{
[Fact]
public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled()
{
var session = await CreateSessionAsync(new SessionConfig { Streaming = true });

var events = new List<SessionEvent>();
session.On(evt => events.Add(evt));

await session.SendAndWaitAsync(new MessageOptions { Prompt = "Count from 1 to 5, separated by commas." });

var types = events.Select(e => e.Type).ToList();

// Should have streaming deltas before the final message
var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();
Assert.NotEmpty(deltaEvents);

// Deltas should have content
foreach (var delta in deltaEvents)
{
Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent));
}

// Should still have a final assistant.message
Assert.Contains("assistant.message", types);

// Deltas should come before the final message
var firstDeltaIdx = types.IndexOf("assistant.message_delta");
var lastAssistantIdx = types.LastIndexOf("assistant.message");
Assert.True(firstDeltaIdx < lastAssistantIdx);

await session.DisposeAsync();
}

[Fact]
public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled()
{
var session = await CreateSessionAsync(new SessionConfig { Streaming = false });

var events = new List<SessionEvent>();
session.On(evt => events.Add(evt));

await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say 'hello world'." });

var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();

// No deltas when streaming is off
Assert.Empty(deltaEvents);

// But should still have a final assistant.message
var assistantEvents = events.OfType<AssistantMessageEvent>().ToList();
Assert.NotEmpty(assistantEvents);

await session.DisposeAsync();
}

[Fact]
public async Task Should_Produce_Deltas_After_Session_Resume()
{
var session = await CreateSessionAsync(new SessionConfig { Streaming = false });
await session.SendAndWaitAsync(new MessageOptions { Prompt = "What is 3 + 6?" });
await session.DisposeAsync();

// Resume using a new client
using var newClient = Ctx.CreateClient();
var session2 = await newClient.ResumeSessionAsync(session.SessionId,
new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = true });

var events = new List<SessionEvent>();
session2.On(evt => events.Add(evt));

var answer = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" });
Assert.NotNull(answer);
Assert.Contains("18", answer!.Data.Content ?? string.Empty);

// Should have streaming deltas before the final message
var deltaEvents = events.OfType<AssistantMessageDeltaEvent>().ToList();
Assert.NotEmpty(deltaEvents);

// Deltas should have content
foreach (var delta in deltaEvents)
{
Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent));
}

await session2.DisposeAsync();
}
}
111 changes: 18 additions & 93 deletions go/internal/e2e/session_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,15 @@ func TestSession(t *testing.T) {
if answer2.Data.Content == nil || !strings.Contains(*answer2.Data.Content, "2") {
t.Errorf("Expected resumed session answer to contain '2', got %v", answer2.Data.Content)
}

// Can continue the conversation statefully
answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"})
if err != nil {
t.Fatalf("Failed to send follow-up message: %v", err)
}
if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") {
t.Errorf("Expected follow-up answer to contain '4', got %v", answer3)
}
})

t.Run("should resume a session using a new client", func(t *testing.T) {
Expand Down Expand Up @@ -432,6 +441,15 @@ func TestSession(t *testing.T) {
if !hasSessionResume {
t.Error("Expected messages to contain 'session.resume'")
}

// Can continue the conversation statefully
answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"})
if err != nil {
t.Fatalf("Failed to send follow-up message: %v", err)
}
if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") {
t.Errorf("Expected follow-up answer to contain '4', got %v", answer3)
}
})

t.Run("should throw error when resuming non-existent session", func(t *testing.T) {
Expand Down Expand Up @@ -565,99 +583,6 @@ func TestSession(t *testing.T) {
}
})

t.Run("should receive streaming delta events when streaming is enabled", func(t *testing.T) {
ctx.ConfigureForTest(t)

session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
Streaming: true,
})
if err != nil {
t.Fatalf("Failed to create session with streaming: %v", err)
}

var deltaContents []string
done := make(chan bool)

session.On(func(event copilot.SessionEvent) {
switch event.Type {
case "assistant.message_delta":
if event.Data.DeltaContent != nil {
deltaContents = append(deltaContents, *event.Data.DeltaContent)
}
case "session.idle":
close(done)
}
})

_, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 2+2?"})
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}

// Wait for completion
select {
case <-done:
case <-time.After(60 * time.Second):
t.Fatal("Timed out waiting for session.idle")
}

// Should have received delta events
if len(deltaContents) == 0 {
t.Error("Expected to receive delta events, got none")
}

// Get the final message to compare
assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session)
if err != nil {
t.Fatalf("Failed to get assistant message: %v", err)
}

// Accumulated deltas should equal the final message
accumulated := strings.Join(deltaContents, "")
if assistantMessage.Data.Content != nil && accumulated != *assistantMessage.Data.Content {
t.Errorf("Accumulated deltas don't match final message.\nAccumulated: %q\nFinal: %q", accumulated, *assistantMessage.Data.Content)
}

// Final message should contain the answer
if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "4") {
t.Errorf("Expected assistant message to contain '4', got %v", assistantMessage.Data.Content)
}
})

t.Run("should pass streaming option to session creation", func(t *testing.T) {
ctx.ConfigureForTest(t)

// Verify that the streaming option is accepted without errors
session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
Streaming: true,
})
if err != nil {
t.Fatalf("Failed to create session with streaming: %v", err)
}

matched, _ := regexp.MatchString(`^[a-f0-9-]+$`, session.SessionID)
if !matched {
t.Errorf("Expected session ID to match UUID pattern, got %q", session.SessionID)
}

// Session should still work normally
_, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"})
if err != nil {
t.Fatalf("Failed to send message: %v", err)
}

assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session)
if err != nil {
t.Fatalf("Failed to get assistant message: %v", err)
}

if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "2") {
t.Errorf("Expected assistant message to contain '2', got %v", assistantMessage.Data.Content)
}
})

t.Run("should receive session events", func(t *testing.T) {
ctx.ConfigureForTest(t)

Expand Down
Loading
Loading