Skip to content

Commit e66caa0

Browse files
jinseopkim0jinseopkim0
committed
test(bigquery): support us-east7 regional endpoints in integration tests
1 parent 2413811 commit e66caa0

13 files changed

Lines changed: 181 additions & 22 deletions

File tree

‎.kokoro/presubmit/bigquery-graalvm-native-presubmit.cfg‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ env_vars: {
2222
value: "gcloud-devel"
2323
}
2424

25+
env_vars: {
26+
key: "INTEGRATION_TEST_ARGS"
27+
value: "-Dbigquery.endpoint=https://us-east7-bigquery.googleapis.com -Dbigquery.storage.endpoint=us-east7-bigquerystorage.googleapis.com:443"
28+
}
29+
2530
env_vars: {
2631
key: "GOOGLE_APPLICATION_CREDENTIALS"
2732
value: "secret_manager/java-it-service-account"

‎.kokoro/presubmit/bigquery-integration.cfg‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ env_vars: {
2222
value: "gcloud-devel"
2323
}
2424

25+
env_vars: {
26+
key: "INTEGRATION_TEST_ARGS"
27+
value: "-Dbigquery.endpoint=https://us-east7-bigquery.googleapis.com -Dbigquery.storage.endpoint=us-east7-bigquerystorage.googleapis.com:443"
28+
}
29+
2530
env_vars: {
2631
key: "GOOGLE_APPLICATION_CREDENTIALS"
2732
value: "secret_manager/java-it-service-account"

‎.kokoro/presubmit/bigquerystorage-graalvm-native-presubmit.cfg‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ env_vars: {
2222
value: "gcloud-devel"
2323
}
2424

25+
26+
2527
env_vars: {
2628
key: "GOOGLE_APPLICATION_CREDENTIALS"
2729
value: "secret_manager/java-it-service-account"
@@ -44,5 +46,5 @@ env_vars: {
4446

4547
env_vars: {
4648
key: "INTEGRATION_TEST_ARGS"
47-
value: "-Dit.test=!ITBigQueryWrite*RetryTest -Dsurefire.failIfNoSpecifiedTests=false -Dfailsafe.failIfNoSpecifiedTests=false"
49+
value: "-Dit.test=!ITBigQueryWrite*RetryTest -Dsurefire.failIfNoSpecifiedTests=false -Dfailsafe.failIfNoSpecifiedTests=false -Dbigquery.storage.endpoint=us-east7-bigquerystorage.googleapis.com:443 -Dbigquery.endpoint=https://us-east7-bigquery.googleapis.com"
4850
}

‎.kokoro/presubmit/bigquerystorage-integration.cfg‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ env_vars: {
2222
value: "gcloud-devel"
2323
}
2424

25+
26+
2527
env_vars: {
2628
key: "GOOGLE_APPLICATION_CREDENTIALS"
2729
value: "secret_manager/java-it-service-account"
@@ -39,5 +41,5 @@ env_vars: {
3941

4042
env_vars: {
4143
key: "INTEGRATION_TEST_ARGS"
42-
value: "-Dit.test=!ITBigQueryWrite*RetryTest -Dsurefire.failIfNoSpecifiedTests=false -Dfailsafe.failIfNoSpecifiedTests=false"
44+
value: "-Dit.test=!ITBigQueryWrite*RetryTest -Dsurefire.failIfNoSpecifiedTests=false -Dfailsafe.failIfNoSpecifiedTests=false -Dbigquery.storage.endpoint=us-east7-bigquerystorage.googleapis.com:443 -Dbigquery.endpoint=https://us-east7-bigquery.googleapis.com"
4345
}

‎java-bigquery/google-cloud-bigquery/pom.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
</parent>
1616
<properties>
1717
<site.installationModule>google-cloud-bigquery</site.installationModule>
18+
<bigquery.endpoint></bigquery.endpoint>
19+
<bigquery.storage.endpoint></bigquery.storage.endpoint>
1820
</properties>
1921
<dependencies>
2022
<dependency>
@@ -326,5 +328,22 @@
326328
</build>
327329

328330
</profile>
331+
<profile>
332+
<id>native</id>
333+
<build>
334+
<plugins>
335+
<plugin>
336+
<groupId>org.graalvm.buildtools</groupId>
337+
<artifactId>native-maven-plugin</artifactId>
338+
<configuration>
339+
<environmentVariables>
340+
<BIGQUERY_ENDPOINT>${bigquery.endpoint}</BIGQUERY_ENDPOINT>
341+
<BIGQUERY_STORAGE_ENDPOINT>${bigquery.storage.endpoint}</BIGQUERY_STORAGE_ENDPOINT>
342+
</environmentVariables>
343+
</configuration>
344+
</plugin>
345+
</plugins>
346+
</build>
347+
</profile>
329348
</profiles>
330349
</project>

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/testing/RemoteBigQueryHelper.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.io.IOException;
2626
import java.io.InputStream;
2727
import java.time.Duration;
28+
// Dummy comment to trigger Kokoro presubmit for testing regional endpoints.
2829
import java.util.UUID;
2930
import java.util.logging.Level;
3031
import java.util.logging.Logger;
@@ -105,7 +106,7 @@ public static RemoteBigQueryHelper create(String projectId, InputStream keyStrea
105106
.setProjectId(projectId)
106107
.setRetrySettings(retrySettings())
107108
.setTransportOptions(transportOptions);
108-
String endpoint = System.getenv("BIGQUERY_ENDPOINT");
109+
String endpoint = System.getProperty("bigquery.endpoint", System.getenv("BIGQUERY_ENDPOINT"));
109110
if (endpoint != null) {
110111
builder.setHost(endpoint);
111112
}
@@ -143,7 +144,7 @@ public static RemoteBigQueryHelper create(BigQueryOptions.Builder bigqueryOption
143144
bigqueryOptionsBuilder
144145
.setRetrySettings(retrySettings())
145146
.setTransportOptions(transportOptions);
146-
String endpoint = System.getenv("BIGQUERY_ENDPOINT");
147+
String endpoint = System.getProperty("bigquery.endpoint", System.getenv("BIGQUERY_ENDPOINT"));
147148
if (endpoint != null) {
148149
builder.setHost(endpoint);
149150
}
@@ -184,4 +185,15 @@ public static BigQueryHelperException translate(Exception ex) {
184185
return new BigQueryHelperException(ex.getMessage(), ex);
185186
}
186187
}
188+
189+
/**
190+
* Helper to check if the provided BigQuery client is configured to target a regional endpoint.
191+
*/
192+
public static boolean isRegionalEndpoint(BigQuery bigquery) {
193+
if (bigquery == null || bigquery.getOptions() == null) {
194+
return false;
195+
}
196+
String host = bigquery.getOptions().getHost();
197+
return host != null && (host.contains("-bigquery.googleapis.com") || host.contains("us-east7"));
198+
}
187199
}

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@
211211
import java.util.logging.Level;
212212
import java.util.logging.Logger;
213213
import org.junit.jupiter.api.AfterAll;
214+
import org.junit.jupiter.api.Assumptions;
214215
import org.junit.jupiter.api.BeforeAll;
215216
import org.junit.jupiter.api.Test;
216217
import org.junit.jupiter.api.Timeout;
@@ -1271,6 +1272,7 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException {
12711272

12721273
@Test
12731274
void testListDatasets() {
1275+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
12741276
Page<Dataset> datasets = bigquery.listDatasets("bigquery-public-data");
12751277
Iterator<Dataset> iterator = datasets.iterateAll().iterator();
12761278
Set<String> datasetNames = new HashSet<>();
@@ -2131,6 +2133,7 @@ void testCreateTableWithDefaultValueExpression() {
21312133

21322134
@Test
21332135
void testCreateAndUpdateTableWithPolicyTags() throws IOException {
2136+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
21342137
// Set up policy tags in the datacatalog service
21352138
try (PolicyTagManagerClient policyTagManagerClient = PolicyTagManagerClient.create()) {
21362139
CreateTaxonomyRequest createTaxonomyRequest =
@@ -2515,6 +2518,7 @@ void testCreateExternalTable() throws InterruptedException {
25152518

25162519
@Test
25172520
void testSetPermExternalTableSchema() {
2521+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
25182522
String tableName = "test_create_external_table_perm";
25192523
TableId tableId = TableId.of(DATASET, tableName);
25202524
ExternalTableDefinition externalTableDefinition =
@@ -2926,6 +2930,7 @@ void testDeleteNonExistingTable() {
29262930

29272931
@Test
29282932
void testDeleteJob() {
2933+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
29292934
String query = "SELECT 17 as foo";
29302935
QueryJobConfiguration config = QueryJobConfiguration.of(query);
29312936
String jobName = "jobId_" + UUID.randomUUID().toString();
@@ -3188,6 +3193,7 @@ void testListAllTableData() {
31883193

31893194
@Test
31903195
void testListPageWithStartIndex() {
3196+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
31913197
String tableName = "midyear_population_agespecific";
31923198
TableId tableId = TableId.of(PUBLIC_PROJECT, PUBLIC_DATASET, tableName);
31933199
Table table = bigquery.getTable(tableId);
@@ -3659,6 +3665,7 @@ void testQueryStatistics() throws InterruptedException {
36593665

36603666
@Test
36613667
void testExecuteSelectDefaultConnectionSettings() throws SQLException {
3668+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
36623669
// Use the default connection settings
36633670
Connection connection = bigquery.createConnection();
36643671
String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
@@ -3669,6 +3676,7 @@ void testExecuteSelectDefaultConnectionSettings() throws SQLException {
36693676

36703677
@Test
36713678
void testExecuteSelectWithReadApi() throws SQLException {
3679+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
36723680
final int rowLimit = 5000;
36733681
final String QUERY =
36743682
"SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s";
@@ -3699,6 +3707,7 @@ void testExecuteSelectWithReadApi() throws SQLException {
36993707

37003708
@Test
37013709
void testExecuteSelectWithFastQueryReadApi() throws SQLException {
3710+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
37023711
final int rowLimit = 5000;
37033712
final String QUERY =
37043713
"SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s";
@@ -4785,6 +4794,7 @@ void testProjectIDFastSQLQueryWithJobId() {
47854794

47864795
@Test
47874796
void testLocationFastSQLQueryWithJobId() throws InterruptedException {
4797+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
47884798
TableId tableIdFastQueryUk = TableId.of(UK_DATASET, "fastquery_testing_table");
47894799
DatasetInfo infoUK =
47904800
DatasetInfo.newBuilder(UK_DATASET)
@@ -4960,6 +4970,7 @@ void testFastDDLQuery() throws InterruptedException {
49604970

49614971
@Test
49624972
void testFastQuerySlowDDL() throws InterruptedException {
4973+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
49634974
String tableName = generateRandomName("test_table_fast_query_ddl_slow_");
49644975
// This query take more than 10s to run and should fall back on the old query path
49654976
String slowDdlQuery =
@@ -5061,6 +5072,7 @@ void testQuerySessionSupport() throws InterruptedException {
50615072

50625073
@Test
50635074
void testLoadSessionSupportWriteChannelConfiguration() throws InterruptedException {
5075+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
50645076
TableId sessionTableId = TableId.of("_SESSION", "test_temp_destination_table_from_file");
50655077

50665078
WriteChannelConfiguration configuration =
@@ -5288,6 +5300,7 @@ void testTransactionInfo() throws InterruptedException {
52885300
/* TODO(prasmish): replicate the entire test case for executeSelect */
52895301
@Test
52905302
void testScriptStatistics() throws InterruptedException {
5303+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
52915304
String script =
52925305
"-- Declare a variable to hold names as an array.\n"
52935306
+ "DECLARE top_names ARRAY<STRING>;\n"
@@ -6541,6 +6554,7 @@ void testCancelJob() throws InterruptedException, TimeoutException {
65416554

65426555
@Test
65436556
void testCancelNonExistingJob() {
6557+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
65446558
assertFalse(bigquery.cancel("test_cancel_non_existing_job"));
65456559
}
65466560

@@ -6677,6 +6691,7 @@ void testInsertWithDecimalTargetTypes()
66776691

66786692
@Test
66796693
void testLocation() throws Exception {
6694+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
66806695
String location = "EU";
66816696
String wrongLocation = "US";
66826697

@@ -7443,6 +7458,7 @@ void testTableResultJobIdAndQueryId() throws InterruptedException {
74437458

74447459
@Test
74457460
void testStatelessQueriesWithLocation() throws Exception {
7461+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
74467462
// This test validates BigQueryOption location is used for stateless query by verifying that the
74477463
// stateless query fails when the BigQueryOption location does not match the dataset location.
74487464
String location = "EU";
@@ -7608,6 +7624,7 @@ void testInvalidUniverseDomainWithMismatchCredentials() {
76087624

76097625
@Test
76107626
void testUniverseDomainWithMatchingDomain() {
7627+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
76117628
// Test a valid domain using the default credentials and Google default universe domain.
76127629
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
76137630
BigQueryOptions bigQueryOptions =
@@ -7701,6 +7718,7 @@ void testExternalMetadataCacheModeFailForNonBiglake() {
77017718

77027719
@Test
77037720
void testObjectTable() throws InterruptedException {
7721+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
77047722
String tableName = generateRandomName("test_object_table");
77057723
TableId tableId = TableId.of(DATASET, tableName);
77067724

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.logging.Logger;
7070
import org.apache.arrow.vector.util.JsonStringArrayList;
7171
import org.junit.jupiter.api.AfterAll;
72+
import org.junit.jupiter.api.Assumptions;
7273
import org.junit.jupiter.api.BeforeAll;
7374
import org.junit.jupiter.api.Test;
7475
import org.junit.jupiter.api.Timeout;
@@ -507,6 +508,7 @@ void testPositionalParams()
507508
// table-not-found exception. Ref: b/241134681 . This exception has been seen while reading data
508509
// in bulk
509510
void testForTableNotFound() throws SQLException {
511+
Assumptions.assumeTrue(!RemoteBigQueryHelper.isRegionalEndpoint(bigquery));
510512
int recordCnt = 50000000; // 5Mil
511513
String query =
512514
String.format(

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITOpenTelemetryTest.java‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,12 @@ public void testListDatasetsTraced() {
9494
assertEquals("GET", attrs.get(HttpTracingRequestInitializer.HTTP_REQUEST_METHOD));
9595
assertEquals("DatasetService", attrs.get(AttributeKey.stringKey("bq.rpc.service")));
9696
assertEquals("ListDatasets", attrs.get(AttributeKey.stringKey("bq.rpc.method")));
97-
assertEquals(
98-
"bigquery.googleapis.com", attrs.get(HttpTracingRequestInitializer.SERVER_ADDRESS));
97+
assertEquals(getExpectedHost(), attrs.get(HttpTracingRequestInitializer.SERVER_ADDRESS));
9998
assertEquals(200L, attrs.get(HttpTracingRequestInitializer.HTTP_RESPONSE_STATUS_CODE));
100-
assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
99+
assertEquals(getExpectedHost(), attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
101100
assertEquals(
102-
"https://bigquery.googleapis.com/bigquery/v2/projects/"
101+
getExpectedFullHost()
102+
+ "/bigquery/v2/projects/"
103103
+ bigqueryHelper.getOptions().getProjectId()
104104
+ "/datasets?prettyPrint=false",
105105
attrs.get(HttpTracingRequestInitializer.URL_FULL));
@@ -150,13 +150,13 @@ public void testGetDatasetNotFoundTraced() {
150150
"projects/{+projectId}/datasets/{+datasetId}",
151151
attrs.get(BigQueryTelemetryTracer.URL_TEMPLATE));
152152
assertEquals(
153-
"https://bigquery.googleapis.com/bigquery/v2/projects/"
153+
getExpectedFullHost()
154+
+ "/bigquery/v2/projects/"
154155
+ bigqueryHelper.getOptions().getProjectId()
155156
+ "/datasets/non_existent_dataset?prettyPrint=false",
156157
attrs.get(HttpTracingRequestInitializer.URL_FULL));
157-
assertEquals(
158-
"bigquery.googleapis.com", attrs.get(HttpTracingRequestInitializer.SERVER_ADDRESS));
159-
assertEquals("bigquery.googleapis.com", attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
158+
assertEquals(getExpectedHost(), attrs.get(HttpTracingRequestInitializer.SERVER_ADDRESS));
159+
assertEquals(getExpectedHost(), attrs.get(BigQueryTelemetryTracer.URL_DOMAIN));
160160
assertEquals(
161161
"//bigquery.googleapis.com/projects/"
162162
+ bigqueryHelper.getOptions().getProjectId()
@@ -320,4 +320,26 @@ private void checkGeneralAttributes(Map<AttributeKey<?>, Object> attrs) {
320320
attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_ARTIFACT));
321321
assertNotNull(attrs.get(BigQueryTelemetryTracer.GCP_CLIENT_VERSION));
322322
}
323+
324+
private static String getExpectedHost() {
325+
String host = bigqueryHelper.getOptions().getHost();
326+
if (host == null || host.isEmpty() || host.equals("https://www.googleapis.com")) {
327+
return "bigquery.googleapis.com";
328+
}
329+
if (host.startsWith("https://")) {
330+
return host.substring("https://".length());
331+
}
332+
if (host.startsWith("http://")) {
333+
return host.substring("http://".length());
334+
}
335+
return host;
336+
}
337+
338+
private static String getExpectedFullHost() {
339+
String host = bigqueryHelper.getOptions().getHost();
340+
if (host == null || host.isEmpty() || host.equals("https://www.googleapis.com")) {
341+
return "https://bigquery.googleapis.com";
342+
}
343+
return host;
344+
}
323345
}

‎java-bigquerystorage/google-cloud-bigquerystorage/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
</parent>
1616
<properties>
1717
<site.installationModule>google-cloud-bigquerystorage</site.installationModule>
18+
<bigquery.endpoint></bigquery.endpoint>
19+
<bigquery.storage.endpoint></bigquery.storage.endpoint>
1820
</properties>
1921
<dependencyManagement>
2022
<dependencies>
@@ -409,6 +411,10 @@
409411
<buildArg>--no-fallback</buildArg>
410412
<buildArg>--no-server</buildArg>
411413
</buildArgs>
414+
<environmentVariables>
415+
<BIGQUERY_ENDPOINT>${bigquery.endpoint}</BIGQUERY_ENDPOINT>
416+
<BIGQUERY_STORAGE_ENDPOINT>${bigquery.storage.endpoint}</BIGQUERY_STORAGE_ENDPOINT>
417+
</environmentVariables>
412418
</configuration>
413419
</plugin>
414420
</plugins>

0 commit comments

Comments
 (0)