Skip to content

Commit d2e72df

Browse files
VinayGuthalrlazodaymxn
authored
Bidirectional Streaming Android (#6759)
Bidirectional streaming for android. Creates a bunch of helper classes for the same. The main classes which handle the bidirectional streaming are LiveGenerativeModel and LiveSession --------- Co-authored-by: VinayGuthal <[email protected]> Co-authored-by: Rodrigo Lazo Paz <[email protected]> Co-authored-by: Rodrigo Lazo <[email protected]> Co-authored-by: Daymon <[email protected]>
1 parent e65e93f commit d2e72df

25 files changed

+1620
-9
lines changed

‎firebase-vertexai/CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
`GenerativeModel` or `ImagenModel`.
44
* [changed] Added new exception type for quota exceeded scenarios.
55
* [feature] `CountTokenRequest` now includes `GenerationConfig` from the model.
6+
* [feature] Added preliminary support for bidirectional streaming. This feature is not yet fully supported.
67
* [changed] **Breaking Change**: `ImagenInlineImage.data` now returns the raw
78
image bytes (in JPEG or PNG format, as specified in
89
`ImagenInlineImage.mimeType`) instead of Base64-encoded data. (#6800)

‎firebase-vertexai/api.txt

+162
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ package com.google.firebase.vertexai {
2929
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null);
3030
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null);
3131
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
32+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName);
33+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null);
34+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null);
35+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null, com.google.firebase.vertexai.type.Content? systemInstruction = null);
36+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.vertexai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.vertexai.type.Tool>? tools = null, com.google.firebase.vertexai.type.Content? systemInstruction = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
3237
property public static final com.google.firebase.vertexai.FirebaseVertexAI instance;
3338
field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion;
3439
}
@@ -63,6 +68,10 @@ package com.google.firebase.vertexai {
6368
method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>>);
6469
}
6570

71+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class LiveGenerativeModel {
72+
method public suspend Object? connect(kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.LiveSession>);
73+
}
74+
6675
}
6776

6877
package com.google.firebase.vertexai.java {
@@ -105,10 +114,42 @@ package com.google.firebase.vertexai.java {
105114
method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
106115
}
107116

117+
@com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class LiveModelFutures {
118+
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.vertexai.type.LiveSession> connect();
119+
method public static final com.google.firebase.vertexai.java.LiveModelFutures from(com.google.firebase.vertexai.LiveGenerativeModel model);
120+
field public static final com.google.firebase.vertexai.java.LiveModelFutures.Companion Companion;
121+
}
122+
123+
public static final class LiveModelFutures.Companion {
124+
method public com.google.firebase.vertexai.java.LiveModelFutures from(com.google.firebase.vertexai.LiveGenerativeModel model);
125+
}
126+
127+
@com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class LiveSessionFutures {
128+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> close();
129+
method public static final com.google.firebase.vertexai.java.LiveSessionFutures from(com.google.firebase.vertexai.type.LiveSession session);
130+
method public abstract org.reactivestreams.Publisher<com.google.firebase.vertexai.type.LiveContentResponse> receive();
131+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(com.google.firebase.vertexai.type.Content content);
132+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> send(String text);
133+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendFunctionResponse(java.util.List<com.google.firebase.vertexai.type.FunctionResponsePart> functionList);
134+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendMediaStream(java.util.List<com.google.firebase.vertexai.type.MediaData> mediaChunks);
135+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.FunctionCallPart,com.google.firebase.vertexai.type.FunctionResponsePart>? functionCallHandler);
136+
method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> stopAudioConversation();
137+
method public abstract void stopReceiving();
138+
field public static final com.google.firebase.vertexai.java.LiveSessionFutures.Companion Companion;
139+
}
140+
141+
public static final class LiveSessionFutures.Companion {
142+
method public com.google.firebase.vertexai.java.LiveSessionFutures from(com.google.firebase.vertexai.type.LiveSession session);
143+
}
144+
108145
}
109146

110147
package com.google.firebase.vertexai.type {
111148

149+
public final class AudioRecordInitializationFailedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
150+
ctor public AudioRecordInitializationFailedException(String message);
151+
}
152+
112153
public final class BlockReason {
113154
method public String getName();
114155
method public int getOrdinal();
@@ -520,6 +561,85 @@ package com.google.firebase.vertexai.type {
520561
public final class InvalidStateException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
521562
}
522563

564+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class LiveContentResponse {
565+
method public com.google.firebase.vertexai.type.Content? getData();
566+
method public java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>? getFunctionCalls();
567+
method public int getStatus();
568+
method public String? getText();
569+
property public final com.google.firebase.vertexai.type.Content? data;
570+
property public final java.util.List<com.google.firebase.vertexai.type.FunctionCallPart>? functionCalls;
571+
property public final int status;
572+
property public final String? text;
573+
}
574+
575+
@kotlin.jvm.JvmInline public static final value class LiveContentResponse.Status {
576+
field public static final com.google.firebase.vertexai.type.LiveContentResponse.Status.Companion Companion;
577+
}
578+
579+
public static final class LiveContentResponse.Status.Companion {
580+
method public int getINTERRUPTED();
581+
method public int getNORMAL();
582+
method public int getTURN_COMPLETE();
583+
property public final int INTERRUPTED;
584+
property public final int NORMAL;
585+
property public final int TURN_COMPLETE;
586+
}
587+
588+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class LiveGenerationConfig {
589+
field public static final com.google.firebase.vertexai.type.LiveGenerationConfig.Companion Companion;
590+
}
591+
592+
public static final class LiveGenerationConfig.Builder {
593+
ctor public LiveGenerationConfig.Builder();
594+
method public com.google.firebase.vertexai.type.LiveGenerationConfig build();
595+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setCandidateCount(Integer? candidateCount);
596+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setFrequencyPenalty(Float? frequencyPenalty);
597+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setMaxOutputTokens(Integer? maxOutputTokens);
598+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setPresencePenalty(Float? presencePenalty);
599+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setResponseModalities(com.google.firebase.vertexai.type.ResponseModality? responseModalities);
600+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setSpeechConfig(com.google.firebase.vertexai.type.SpeechConfig? speechConfig);
601+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTemperature(Float? temperature);
602+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTopK(Integer? topK);
603+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder setTopP(Float? topP);
604+
field public Integer? candidateCount;
605+
field public Float? frequencyPenalty;
606+
field public Integer? maxOutputTokens;
607+
field public Float? presencePenalty;
608+
field public com.google.firebase.vertexai.type.ResponseModality? responseModality;
609+
field public com.google.firebase.vertexai.type.SpeechConfig? speechConfig;
610+
field public Float? temperature;
611+
field public Integer? topK;
612+
field public Float? topP;
613+
}
614+
615+
public static final class LiveGenerationConfig.Companion {
616+
method public com.google.firebase.vertexai.type.LiveGenerationConfig.Builder builder();
617+
}
618+
619+
public final class LiveGenerationConfigKt {
620+
method public static com.google.firebase.vertexai.type.LiveGenerationConfig liveGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.LiveGenerationConfig.Builder,kotlin.Unit> init);
621+
}
622+
623+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class LiveSession {
624+
method public suspend Object? close(kotlin.coroutines.Continuation<? super kotlin.Unit>);
625+
method public kotlinx.coroutines.flow.Flow<com.google.firebase.vertexai.type.LiveContentResponse> receive();
626+
method public suspend Object? send(com.google.firebase.vertexai.type.Content content, kotlin.coroutines.Continuation<? super kotlin.Unit>);
627+
method public suspend Object? send(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
628+
method public suspend Object? sendFunctionResponse(java.util.List<com.google.firebase.vertexai.type.FunctionResponsePart> functionList, kotlin.coroutines.Continuation<? super kotlin.Unit>);
629+
method public suspend Object? sendMediaStream(java.util.List<com.google.firebase.vertexai.type.MediaData> mediaChunks, kotlin.coroutines.Continuation<? super kotlin.Unit>);
630+
method public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.FunctionCallPart,com.google.firebase.vertexai.type.FunctionResponsePart>? functionCallHandler = null, kotlin.coroutines.Continuation<? super kotlin.Unit>);
631+
method public void stopAudioConversation();
632+
method public void stopReceiving();
633+
}
634+
635+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class MediaData {
636+
ctor public MediaData(byte[] data, String mimeType);
637+
method public byte[] getData();
638+
method public String getMimeType();
639+
property public final byte[] data;
640+
property public final String mimeType;
641+
}
642+
523643
public final class ModalityTokenCount {
524644
method public operator com.google.firebase.vertexai.type.ContentModality component1();
525645
method public operator int component2();
@@ -568,6 +688,19 @@ package com.google.firebase.vertexai.type {
568688
public final class RequestTimeoutException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
569689
}
570690

691+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ResponseModality {
692+
method public int getOrdinal();
693+
property public final int ordinal;
694+
field public static final com.google.firebase.vertexai.type.ResponseModality AUDIO;
695+
field public static final com.google.firebase.vertexai.type.ResponseModality.Companion Companion;
696+
field public static final com.google.firebase.vertexai.type.ResponseModality IMAGE;
697+
field public static final com.google.firebase.vertexai.type.ResponseModality TEXT;
698+
field public static final com.google.firebase.vertexai.type.ResponseModality UNSPECIFIED;
699+
}
700+
701+
public static final class ResponseModality.Companion {
702+
}
703+
571704
public final class ResponseStoppedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
572705
method public com.google.firebase.vertexai.type.GenerateContentResponse getResponse();
573706
property public final com.google.firebase.vertexai.type.GenerateContentResponse response;
@@ -679,9 +812,23 @@ package com.google.firebase.vertexai.type {
679812
public final class ServerException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
680813
}
681814

815+
public final class ServiceConnectionHandshakeFailedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
816+
ctor public ServiceConnectionHandshakeFailedException(String message, Throwable? cause = null);
817+
}
818+
682819
public final class ServiceDisabledException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
683820
}
684821

822+
public final class SessionAlreadyReceivingException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
823+
ctor public SessionAlreadyReceivingException();
824+
}
825+
826+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class SpeechConfig {
827+
ctor public SpeechConfig(com.google.firebase.vertexai.type.Voices voice);
828+
method public com.google.firebase.vertexai.type.Voices getVoice();
829+
property public final com.google.firebase.vertexai.type.Voices voice;
830+
}
831+
685832
public abstract class StringFormat {
686833
}
687834

@@ -728,5 +875,20 @@ package com.google.firebase.vertexai.type {
728875
property public final int totalTokenCount;
729876
}
730877

878+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class Voices {
879+
method public int getOrdinal();
880+
property public final int ordinal;
881+
field public static final com.google.firebase.vertexai.type.Voices AOEDE;
882+
field public static final com.google.firebase.vertexai.type.Voices CHARON;
883+
field public static final com.google.firebase.vertexai.type.Voices.Companion Companion;
884+
field public static final com.google.firebase.vertexai.type.Voices FENRIR;
885+
field public static final com.google.firebase.vertexai.type.Voices KORE;
886+
field public static final com.google.firebase.vertexai.type.Voices PUCK;
887+
field public static final com.google.firebase.vertexai.type.Voices UNSPECIFIED;
888+
}
889+
890+
public static final class Voices.Companion {
891+
}
892+
731893
}
732894

‎firebase-vertexai/firebase-vertexai.gradle.kts

+5-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,10 @@ android {
6363
isReturnDefaultValues = true
6464
}
6565
}
66-
lint { targetSdk = targetSdkVersion }
66+
lint {
67+
targetSdk = targetSdkVersion
68+
baseline = file("lint-baseline.xml")
69+
}
6770
sourceSets { getByName("test").java.srcDirs("src/testUtil") }
6871
}
6972

@@ -84,6 +87,7 @@ tasks.withType<KotlinCompile>().all {
8487
dependencies {
8588
implementation(libs.ktor.client.okhttp)
8689
implementation(libs.ktor.client.core)
90+
implementation(libs.ktor.client.websockets)
8791
implementation(libs.ktor.client.content.negotiation)
8892
implementation(libs.ktor.serialization.kotlinx.json)
8993
implementation(libs.ktor.client.logging)

‎firebase-vertexai/lint-baseline.xml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright 2025 Google LLC
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<issues format="6" by="lint 8.3.2" type="baseline" client="gradle" dependencies="false" name="AGP (8.3.2)" variant="all" version="8.3.2">
18+
19+
<issue
20+
id="MissingPermission"
21+
message="Missing permissions required by AudioHelper.startRecording: android.permission.RECORD_AUDIO"
22+
errorLine1=" CoroutineScope(backgroundDispatcher).launch {"
23+
errorLine2=" ^">
24+
<location
25+
file="src/main/kotlin/com/google/firebase/vertexai/type/LiveSession.kt"
26+
line="133"
27+
column="42"/>
28+
</issue>
29+
30+
</issues>

‎firebase-vertexai/src/main/AndroidManifest.xml

+5-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@
2020
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
2121
<uses-permission android:name="android.permission.INTERNET" />
2222

23-
<application>
23+
<!-- To use the audio conversation feature of Live Sessions, update your app's manifest -->
24+
<!-- to request audio recording permission -->
25+
<!-- <uses-permission android:name="android.permission.RECORD_AUDIO" /> -->
26+
27+
<application>
2428
<service android:name="com.google.firebase.components.ComponentDiscoveryService"
2529
android:exported="false">
2630
<meta-data

0 commit comments

Comments
 (0)