Conversation API

.changes/connection-credentials

@@ -0,0 +1 @@
+patch type="added" "Abstract token source for easier token fetching in production and faster integration with sandbox environment"

Package.swift

@@ -23,10 +23,9 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.6.2"),
         .package(url: "https://github.com/apple/swift-collections.git", from: "1.1.0"),
+        .package(url: "https://github.com/vapor/jwt-kit.git", from: "4.13.5"),
         // Only used for DocC generation
         .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.3.0"),
-        // Only used for Testing
-        .package(url: "https://github.com/vapor/jwt-kit.git", from: "4.13.4"),
     ],
     targets: [
         .target(
@@ -41,6 +40,7 @@ let package = Package(
                 .product(name: "DequeModule", package: "swift-collections"),
                 .product(name: "OrderedCollections", package: "swift-collections"),
                 .product(name: "Logging", package: "swift-log"),
+                .product(name: "JWTKit", package: "jwt-kit"),
                 "LKObjCHelpers",
             ],
             exclude: [
@@ -57,14 +57,12 @@ let package = Package(
             name: "LiveKitTests",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
             ]
         ),
         .testTarget(
             name: "LiveKitTestsObjC",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
             ]
         ),
     ],

Package@swift-6.0.swift

@@ -24,10 +24,9 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.6.2"),
         .package(url: "https://github.com/apple/swift-collections.git", from: "1.1.0"),
+        .package(url: "https://github.com/vapor/jwt-kit.git", from: "4.13.5"),
         // Only used for DocC generation
         .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.3.0"),
-        // Only used for Testing
-        .package(url: "https://github.com/vapor/jwt-kit.git", from: "4.13.4"),
     ],
     targets: [
         .target(
@@ -42,6 +41,7 @@ let package = Package(
                 .product(name: "DequeModule", package: "swift-collections"),
                 .product(name: "OrderedCollections", package: "swift-collections"),
                 .product(name: "Logging", package: "swift-log"),
+                .product(name: "JWTKit", package: "jwt-kit"),
                 "LKObjCHelpers",
             ],
             exclude: [
@@ -58,14 +58,12 @@ let package = Package(
             name: "LiveKitTests",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
             ]
         ),
         .testTarget(
             name: "LiveKitTestsObjC",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
             ]
         ),
     ],

Sources/LiveKit/Agent/Agent.swift

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+@MainActor
+open class Agent: ObservableObject {
+    @Published public private(set) var state: AgentState = .idle
+
+    @Published public private(set) var audioTrack: (any AudioTrack)?
+    @Published public private(set) var avatarVideoTrack: (any VideoTrack)?
+
+    public let participant: Participant
+
+    public init(participant: Participant) {
+        self.participant = participant
+        observe(participant)
+    }
+
+    private func observe(_ participant: Participant) {
+        Task { [weak self] in
+            for try await _ in participant.changes {
+                guard let self else { return }
+
+                state = participant.agentState
+                updateTracks(of: participant)
+            }
+        }
+    }
+
+    private func updateTracks(of participant: Participant) {
+        audioTrack = participant.audioTracks.first(where: { $0.source == .microphone })?.track as? AudioTrack
+        avatarVideoTrack = participant.avatarWorker?.firstCameraVideoTrack
+    }
+}
+
+extension AgentState: CustomStringConvertible {
+    public var description: String {
+        rawValue.capitalized
+    }
+}

Sources/LiveKit/Agent/Chat/Message.swift

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+/// A message received from the agent.
+public struct ReceivedMessage: Identifiable, Equatable, Codable, Sendable {
+    public let id: String
+    public let timestamp: Date
+    public let content: Content
+
+    public enum Content: Equatable, Codable, Sendable {
+        case agentTranscript(String)
+        case userTranscript(String)
+        case userInput(String)
+    }
+}
+
+/// A message sent to the agent.
+public struct SentMessage: Identifiable, Equatable, Codable, Sendable {
+    public let id: String
+    public let timestamp: Date
+    public let content: Content
+
+    public enum Content: Equatable, Codable, Sendable {
+        case userInput(String)
+    }
+}

Sources/LiveKit/Agent/Chat/Receive/MessageReceiver.swift

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+/// A protocol that defines a message receiver.
+///
+/// A message receiver is responsible for creating a stream of messages from the agent.
+/// It is used to receive messages from the agent and update the message feed.
+///
+/// - SeeAlso: ``ReceivedMessage``
+public protocol MessageReceiver: Sendable {
+    func messages() async throws -> AsyncStream<ReceivedMessage>
+}

Sources/LiveKit/Agent/Chat/Receive/TranscriptionDelegateReceiver.swift

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+/// An actor that receives transcription messages from the room and yields them as messages.
+///
+/// Room delegate methods are called multiple times for each message, with a stable message ID
+/// that can be direcly used for diffing.
+///
+/// Example:
+/// ```
+/// { id: "1", content: "Hello" }
+/// { id: "1", content: "Hello world!" }
+/// ```
+@available(*, deprecated, message: "Use TranscriptionStreamReceiver compatible with livekit-agents 1.0")
+actor TranscriptionDelegateReceiver: MessageReceiver, RoomDelegate {
+    private let room: Room
+    private var continuation: AsyncStream<ReceivedMessage>.Continuation?
+
+    init(room: Room) {
+        self.room = room
+        room.add(delegate: self)
+    }
+
+    deinit {
+        room.remove(delegate: self)
+    }
+
+    /// Creates a new message stream for the transcription delegate receiver.
+    func messages() -> AsyncStream<ReceivedMessage> {
+        let (stream, continuation) = AsyncStream.makeStream(of: ReceivedMessage.self)
+        self.continuation = continuation
+        return stream
+    }
+
+    nonisolated func room(_: Room, participant: Participant, trackPublication _: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) {
+        segments
+            .filter { !$0.text.isEmpty }
+            .forEach { segment in
+                let message = ReceivedMessage(
+                    id: segment.id,
+                    timestamp: segment.lastReceivedTime,
+                    content: participant.isAgent ? .agentTranscript(segment.text) : .userTranscript(segment.text)
+                )
+                Task {
+                    await yield(message)
+                }
+            }
+    }
+
+    private func yield(_ message: ReceivedMessage) {
+        continuation?.yield(message)
+    }
+}