Skip to content

Commit 2775eb1

Browse files
authored
fix(bigquery): default Arrow pagination stream location to US instead of global (#14458)
When both jobId and BigQueryOptions locations are unspecified, default the Storage Read API stream resource location in ArrowQueryPageFetcher to US instead of global to prevent 404 errors. From https://docs.cloud.google.com/bigquery/docs/locations#default_location: > If the location isn't explicitly specified, and it can't be determined from the resources in the request, the default location is used. If default location isn't set, the job runs in the US multi-region.
1 parent a9fed00 commit 2775eb1

3 files changed

Lines changed: 73 additions & 12 deletions

File tree

java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -379,18 +379,19 @@ public Page<FieldValueList> getNextPage() {
379379
List<FieldValueList> rowBatch = new ArrayList<>((int) Math.min(pageSize, 10000L));
380380

381381
try {
382-
// Resolve job location in order: JobId location -> BigQueryOptions location -> "global"
382+
// Resolve job location in order: JobId location -> BigQueryOptions location -> "US"
383383
// default.
384384
// The Storage Read API stream resource name requires a location component (e.g.
385385
// projects/{project}/locations/{location}/jobs/{job}/streams/_default). If no specific
386-
// location was provided on the job or service options, defaulting to "global" allows
387-
// queries created without an explicit location to still stream results without failing.
386+
// location was provided on the job or service options, defaulting to "US" (the standard
387+
// BigQuery default multi-region) allows queries created without an explicit location
388+
// to stream results from default datasets without failing.
388389
String location = jobId.getLocation();
389390
if (location == null) {
390391
location = serviceOptions.getLocation();
391392
}
392393
if (location == null) {
393-
location = "global";
394+
location = "US";
394395
}
395396

396397
if (streamIterator == null) {
@@ -2919,12 +2920,13 @@ && getOptions().getOpenTelemetryTracer() != null) {
29192920
if (jobLocation == null) {
29202921
jobLocation = getOptions().getLocation();
29212922
}
2922-
if (jobLocation != null) {
2923-
streamName =
2924-
String.format(
2925-
"projects/%s/locations/%s/jobs/%s/streams/_default",
2926-
jobProject, jobLocation, actualJobId.getJob());
2923+
if (jobLocation == null) {
2924+
jobLocation = "US";
29272925
}
2926+
streamName =
2927+
String.format(
2928+
"projects/%s/locations/%s/jobs/%s/streams/_default",
2929+
jobProject, jobLocation, actualJobId.getJob());
29282930
}
29292931

29302932
BigQueryReadClient client = null;

java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ArrowQueryPageFetcherTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,8 @@ void testGetNextPage_respectsMaxResults() throws IOException {
289289
}
290290

291291
@Test
292-
void testGetNextPage_missingLocationDefaultsToGlobal() throws IOException {
293-
byte[] batchBytes = createBatchBytes(ImmutableList.of(10L));
292+
void testGetNextPage_missingLocationDefaultsToUS() throws IOException {
293+
byte[] batchBytes = createBatchBytes(ImmutableList.of(1L));
294294
ReadRowsResponse response =
295295
ReadRowsResponse.newBuilder()
296296
.setArrowRecordBatch(
@@ -331,7 +331,7 @@ void testGetNextPage_missingLocationDefaultsToGlobal() throws IOException {
331331
Page<FieldValueList> page = fetcher.getNextPage();
332332
assertNotNull(page);
333333
assertEquals(
334-
"projects/" + PROJECT + "/locations/global/jobs/" + JOB + "/streams/_default",
334+
"projects/" + PROJECT + "/locations/US/jobs/" + JOB + "/streams/_default",
335335
requestCapture.getValue().getReadStream());
336336
}
337337

java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2954,6 +2954,65 @@ void testQueryArrowDefaultsToJobCreationOptional() throws IOException, Interrupt
29542954
assertEquals("ARROW", requestPb.getQueryResultsFormat());
29552955
}
29562956

2957+
@Test
2958+
void testQueryArrowDefaultsToUSLocationWhenUnspecified() throws Exception {
2959+
org.apache.arrow.vector.types.pojo.Schema arrowSchema =
2960+
new org.apache.arrow.vector.types.pojo.Schema(
2961+
ImmutableList.of(
2962+
org.apache.arrow.vector.types.pojo.Field.nullable(
2963+
"id", new ArrowType.Int(64, true))));
2964+
2965+
byte[] schemaBytes;
2966+
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
2967+
MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), arrowSchema);
2968+
schemaBytes = out.toByteArray();
2969+
}
2970+
2971+
QueryJobConfiguration config =
2972+
QueryJobConfiguration.newBuilder("SELECT 1")
2973+
.setQueryResultsFormat(QueryResultsFormat.ARROW)
2974+
.build();
2975+
com.google.api.services.bigquery.model.JobReference jobRef =
2976+
new com.google.api.services.bigquery.model.JobReference()
2977+
.setProjectId(PROJECT)
2978+
.setJobId(JOB); // No location set
2979+
com.google.api.services.bigquery.model.QueryResponse queryResponsePb =
2980+
new com.google.api.services.bigquery.model.QueryResponse()
2981+
.setJobReference(jobRef)
2982+
.setJobComplete(true)
2983+
.setTotalRows(BigInteger.ONE)
2984+
.setArrowSchema(
2985+
new com.google.api.services.bigquery.model.ArrowSchema()
2986+
.setSerializedSchema(BaseEncoding.base64().encode(schemaBytes)));
2987+
2988+
BigQueryReadClient mockReadClient =
2989+
mock(BigQueryReadClient.class, withSettings().withoutAnnotations());
2990+
@SuppressWarnings("unchecked")
2991+
ServerStreamingCallable<ReadRowsRequest, ReadRowsResponse> mockCallable =
2992+
mock(ServerStreamingCallable.class, withSettings().withoutAnnotations());
2993+
@SuppressWarnings("unchecked")
2994+
ServerStream<ReadRowsResponse> mockServerStream =
2995+
mock(ServerStream.class, withSettings().withoutAnnotations());
2996+
ArgumentCaptor<ReadRowsRequest> requestCapture = ArgumentCaptor.forClass(ReadRowsRequest.class);
2997+
when(mockCallable.call(requestCapture.capture())).thenReturn(mockServerStream);
2998+
when(mockServerStream.iterator()).thenReturn(Collections.emptyIterator());
2999+
when(mockReadClient.readRowsCallable()).thenReturn(mockCallable);
3000+
3001+
when(bigqueryRpcMock.queryRpcSkipExceptionTranslation(eq(PROJECT), any(QueryRequest.class)))
3002+
.thenReturn(queryResponsePb);
3003+
3004+
bigquery = options.getService();
3005+
((BigQueryImpl) bigquery).setBigQueryReadClient(mockReadClient);
3006+
3007+
ArrowQueryResult result = bigquery.queryArrow(config);
3008+
assertNotNull(result);
3009+
result.iterator().hasNext();
3010+
3011+
assertEquals(
3012+
"projects/" + PROJECT + "/locations/US/jobs/" + JOB + "/streams/_default",
3013+
requestCapture.getValue().getReadStream());
3014+
}
3015+
29573016
@Test
29583017
void testQueryWithArrowFormatSlowPathFallback() throws Exception {
29593018
JobId queryJob = JobId.of(PROJECT, JOB).toBuilder().setLocation(LOCATION).build();

0 commit comments

Comments
 (0)