Skip to content

Commit 1b8ec71

Browse files
committed
refactor(bigquery): address PR 13943 review comments on channel lifecycle and schema resolution
1 parent e8ba1a8 commit 1b8ec71

2 files changed

Lines changed: 35 additions & 26 deletions

File tree

java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ArrowDeserializer.java

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,11 @@ private ArrowDeserializer() {}
8686
* @throws IOException if deserialization of the Arrow schema fails
8787
*/
8888
static Object deserializeSchema(byte[] schemaBytes) throws IOException {
89-
return MessageSerializer.deserializeSchema(
90-
new ReadChannel(new ByteArrayReadableSeekableByteChannel(schemaBytes)));
89+
try (ByteArrayReadableSeekableByteChannel byteChannel =
90+
new ByteArrayReadableSeekableByteChannel(schemaBytes);
91+
ReadChannel readChannel = new ReadChannel(byteChannel)) {
92+
return MessageSerializer.deserializeSchema(readChannel);
93+
}
9194
}
9295

9396
/**
@@ -121,12 +124,30 @@ static Object jsonToArrowSchema(String json) {
121124
}
122125
}
123126

127+
/**
128+
* Resolves an Apache Arrow Schema from either an in-memory Schema POJO or a serialized JSON
129+
* string.
130+
*
131+
* @param arrowSchema the Arrow schema POJO or JSON string representation
132+
* @return the resolved Apache Arrow Schema, or null if schema cannot be resolved
133+
* @throws IOException if parsing JSON fails
134+
*/
135+
private static org.apache.arrow.vector.types.pojo.Schema resolveArrowSchema(Object arrowSchema)
136+
throws IOException {
137+
if (arrowSchema instanceof org.apache.arrow.vector.types.pojo.Schema) {
138+
return (org.apache.arrow.vector.types.pojo.Schema) arrowSchema;
139+
}
140+
if (arrowSchema instanceof String) {
141+
return org.apache.arrow.vector.types.pojo.Schema.fromJSON((String) arrowSchema);
142+
}
143+
return null;
144+
}
145+
124146
/**
125147
* Reads and decodes a batch of Arrow rows from the provided stream iterator into the row batch.
126148
*
127149
* @param iterator the stream iterator providing ReadRowsResponse messages
128-
* @param arrowSchemaPojo the Arrow schema pojo (or null if restoring from json)
129-
* @param arrowSchemaJson the Arrow schema JSON representation
150+
* @param arrowSchema the Arrow schema POJO or serialized JSON representation
130151
* @param schema the BigQuery target Schema
131152
* @param rowBatch the destination list for decoded rows
132153
* @param pageSize the maximum number of rows to decode in this batch
@@ -137,28 +158,24 @@ static Object jsonToArrowSchema(String json) {
137158
*/
138159
static boolean loadArrowRows(
139160
Iterator<ReadRowsResponse> iterator,
140-
Object arrowSchemaPojo,
141-
String arrowSchemaJson,
161+
Object arrowSchema,
142162
Schema schema,
143163
List<FieldValueList> rowBatch,
144164
long pageSize,
145165
long totalRowsReturned,
146166
long maxResults)
147167
throws IOException {
148-
org.apache.arrow.vector.types.pojo.Schema arrowSchema =
149-
arrowSchemaPojo instanceof org.apache.arrow.vector.types.pojo.Schema
150-
? (org.apache.arrow.vector.types.pojo.Schema) arrowSchemaPojo
151-
: (arrowSchemaJson != null
152-
? org.apache.arrow.vector.types.pojo.Schema.fromJSON(arrowSchemaJson)
153-
: null);
168+
org.apache.arrow.vector.types.pojo.Schema resolvedSchema = resolveArrowSchema(arrowSchema);
154169

155-
if (arrowSchema == null) {
170+
if (resolvedSchema == null) {
156171
return false;
157172
}
158173

174+
org.apache.arrow.vector.types.pojo.Schema arrowSchemaFinal = resolvedSchema;
175+
159176
try (BufferAllocator childAllocator =
160177
AllocatorHolder.ALLOCATOR.newChildAllocator("loadArrowRows", 0, Long.MAX_VALUE);
161-
VectorSchemaRoot closedRoot = createVectorSchemaRoot(arrowSchema, childAllocator)) {
178+
VectorSchemaRoot closedRoot = createVectorSchemaRoot(arrowSchemaFinal, childAllocator)) {
162179
VectorLoader loader = new VectorLoader(closedRoot);
163180
boolean hasMore = false;
164181
while (rowBatch.size() < pageSize

java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowDeserializerTest.java

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,7 @@ public void testLoadArrowRows_multiBatchStream() throws IOException {
214214
List<FieldValueList> rowBatch = new ArrayList<>();
215215
boolean hasMore =
216216
ArrowDeserializer.loadArrowRows(
217-
Arrays.asList(r1, r2).iterator(),
218-
arrowSchema,
219-
null,
220-
bqSchema,
221-
rowBatch,
222-
10L,
223-
0L,
224-
10L);
217+
Arrays.asList(r1, r2).iterator(), arrowSchema, bqSchema, rowBatch, 10L, 0L, 10L);
225218

226219
assertFalse(hasMore);
227220
assertEquals(4, rowBatch.size());
@@ -246,7 +239,7 @@ public void testLoadArrowRows_respectsPageSize() throws IOException {
246239
List<FieldValueList> rowBatch = new ArrayList<>();
247240
boolean hasMore =
248241
ArrowDeserializer.loadArrowRows(
249-
Arrays.asList(r1, r2).iterator(), arrowSchema, null, bqSchema, rowBatch, 2L, 0L, 10L);
242+
Arrays.asList(r1, r2).iterator(), arrowSchema, bqSchema, rowBatch, 2L, 0L, 10L);
250243

251244
assertTrue(hasMore);
252245
assertEquals(2, rowBatch.size());
@@ -269,7 +262,7 @@ public void testLoadArrowRows_respectsMaxResults() throws IOException {
269262
List<FieldValueList> rowBatch = new ArrayList<>();
270263
boolean hasMore =
271264
ArrowDeserializer.loadArrowRows(
272-
Arrays.asList(r1, r2).iterator(), arrowSchema, null, bqSchema, rowBatch, 10L, 0L, 3L);
265+
Arrays.asList(r1, r2).iterator(), arrowSchema, bqSchema, rowBatch, 10L, 0L, 3L);
273266

274267
assertFalse(hasMore);
275268
assertEquals(3, rowBatch.size());
@@ -294,7 +287,7 @@ public void testLoadArrowRows_unconsumedBatchRowsSignalHasMore() throws IOExcept
294287
List<FieldValueList> rowBatch = new ArrayList<>();
295288
boolean hasMore =
296289
ArrowDeserializer.loadArrowRows(
297-
Arrays.asList(r1).iterator(), arrowSchema, null, bqSchema, rowBatch, 2L, 0L, 10L);
290+
Arrays.asList(r1).iterator(), arrowSchema, bqSchema, rowBatch, 2L, 0L, 10L);
298291

299292
assertTrue(hasMore);
300293
assertEquals(2, rowBatch.size());
@@ -310,7 +303,6 @@ public void testLoadArrowRows_nullSchemaReturnsFalse() throws IOException {
310303
ArrowDeserializer.loadArrowRows(
311304
Arrays.<ReadRowsResponse>asList().iterator(),
312305
null,
313-
null,
314306
Schema.of(),
315307
rowBatch,
316308
10L,

0 commit comments

Comments
 (0)