feat(storage): Parameterized test setup for RCU Integration Testing

Author: nidhiii-27

java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java

@@ -158,11 +158,26 @@ public HttpStorageRpc(StorageOptions options, JsonFactory jsonFactory) {
     initializer = censusHttpModule.getHttpRequestInitializer(initializer);
     initializer = new InvocationIdInitializer(initializer, applicationName, tm);
     batchRequestInitializer = censusHttpModule.getHttpRequestInitializer(null);
-    storage =
+    String host = options.getHost();
+    Storage.Builder storageBuilder =
         new Storage.Builder(transport, jsonFactory, initializer)
-            .setRootUrl(options.getHost())
-            .setApplicationName(applicationName)
-            .build();
+            .setApplicationName(applicationName);
+    if (host != null) {
+      java.net.URI uri = java.net.URI.create(host);
+      String path = uri.getPath();
+      if (path != null && !path.isEmpty() && !"/".equals(path)) {
+        String rootUrl = host.substring(0, host.indexOf(path));
+        String servicePath = path.startsWith("/") ? path.substring(1) : path;
+        if (!servicePath.endsWith("/")) {
+          servicePath += "/";
+        }
+        storageBuilder.setRootUrl(rootUrl);
+        storageBuilder.setServicePath(servicePath);
+      } else {
+        storageBuilder.setRootUrl(host);
+      }
+    }
+    storage = storageBuilder.build();
   }
 
   public Storage getStorage() {

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java

@@ -20,10 +20,14 @@
 
 import com.google.cloud.storage.TransportCompatibility.Transport;
 import com.google.cloud.storage.it.runner.annotations.Backend;
+import com.google.cloud.storage.it.runner.annotations.Colocation;
 import com.google.cloud.storage.it.runner.annotations.CrossRun;
+import com.google.cloud.storage.it.runner.annotations.LocationType;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
 import java.util.Locale;
+import java.util.Set;
 import java.util.Objects;
 import javax.annotation.concurrent.Immutable;
 import javax.annotation.concurrent.ThreadSafe;
@@ -39,10 +43,18 @@ public final class CrossRunIntersection {
 
   private final @Nullable Backend backend;
   private final @Nullable Transport transport;
+  private final @Nullable LocationType locationType;
+  private final @Nullable Colocation colocation;
 
-  private CrossRunIntersection(@Nullable Backend backend, @Nullable Transport transport) {
+  private CrossRunIntersection(
+      @Nullable Backend backend,
+      @Nullable Transport transport,
+      @Nullable LocationType locationType,
+      @Nullable Colocation colocation) {
     this.backend = backend;
     this.transport = transport;
+    this.locationType = locationType;
+    this.colocation = colocation;
   }
 
   @Nullable
@@ -55,19 +67,45 @@ public Transport getTransport() {
     return transport;
   }
 
+  @Nullable
+  public LocationType getLocationType() {
+    return locationType;
+  }
+
+  @Nullable
+  public Colocation getColocation() {
+    return colocation;
+  }
+
   public CrossRunIntersection clearBackend() {
     if (backend == null) {
       return this;
     } else {
-      return new CrossRunIntersection(null, transport);
+      return new CrossRunIntersection(null, transport, locationType, colocation);
     }
   }
 
   public CrossRunIntersection clearTransport() {
     if (transport == null) {
       return this;
     } else {
-      return new CrossRunIntersection(backend, null);
+      return new CrossRunIntersection(backend, null, locationType, colocation);
+    }
+  }
+
+  public CrossRunIntersection clearLocationType() {
+    if (locationType == null) {
+      return this;
+    } else {
+      return new CrossRunIntersection(backend, transport, null, colocation);
+    }
+  }
+
+  public CrossRunIntersection clearColocation() {
+    if (colocation == null) {
+      return this;
+    } else {
+      return new CrossRunIntersection(backend, transport, locationType, null);
     }
   }
 
@@ -76,7 +114,7 @@ public CrossRunIntersection withBackend(Backend backend) {
     if (this.backend == backend) {
       return this;
     } else {
-      return new CrossRunIntersection(backend, transport);
+      return new CrossRunIntersection(backend, transport, locationType, colocation);
     }
   }
 
@@ -85,7 +123,25 @@ public CrossRunIntersection withTransport(Transport transport) {
     if (this.transport == transport) {
       return this;
     } else {
-      return new CrossRunIntersection(backend, transport);
+      return new CrossRunIntersection(backend, transport, locationType, colocation);
+    }
+  }
+
+  public CrossRunIntersection withLocationType(LocationType locationType) {
+    requireNonNull(locationType, "locationType must be non null");
+    if (this.locationType == locationType) {
+      return this;
+    } else {
+      return new CrossRunIntersection(backend, transport, locationType, colocation);
+    }
+  }
+
+  public CrossRunIntersection withColocation(Colocation colocation) {
+    requireNonNull(colocation, "colocation must be non null");
+    if (this.colocation == colocation) {
+      return this;
+    } else {
+      return new CrossRunIntersection(backend, transport, locationType, colocation);
     }
   }
 
@@ -107,6 +163,20 @@ public boolean anyMatch(CrossRunIntersection other) {
       l = l.clearTransport();
     }
 
+    if (l.locationType == null) {
+      r = r.clearLocationType();
+    }
+    if (r.locationType == null) {
+      l = l.clearLocationType();
+    }
+
+    if (l.colocation == null) {
+      r = r.clearColocation();
+    }
+    if (r.colocation == null) {
+      l = l.clearColocation();
+    }
+
     return l.equals(r);
   }
 
@@ -119,7 +189,9 @@ public boolean anyMatch(CrossRunIntersection other) {
   public String fmtSuiteName() {
     String t = transport != null ? transport.toString() : "NULL_TRANSPORT";
     String b = backend != null ? backend.toString() : "NULL_BACKEND";
-    return String.format(Locale.US, "[%s][%s]", t, b);
+    String lt = locationType != null ? locationType.toString() : "NULL_LOCATION";
+    String c = colocation != null ? colocation.toString() : "NULL_COLOCATION";
+    return String.format(Locale.US, "[%s][%s][%s][%s]", t, b, lt, c);
   }
 
   @Override
@@ -131,54 +203,90 @@ public boolean equals(Object o) {
       return false;
     }
     CrossRunIntersection crossRunIntersection = (CrossRunIntersection) o;
-    return backend == crossRunIntersection.backend && transport == crossRunIntersection.transport;
+    return backend == crossRunIntersection.backend
+        && transport == crossRunIntersection.transport
+        && locationType == crossRunIntersection.locationType
+        && colocation == crossRunIntersection.colocation;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(backend, transport);
+    return Objects.hash(backend, transport, locationType, colocation);
   }
 
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
         .add("backend", backend)
         .add("transport", transport)
+        .add("locationType", locationType)
+        .add("colocation", colocation)
         .toString();
   }
 
-  public static CrossRunIntersection of(@Nullable Backend t, @Nullable Transport s) {
-    return new CrossRunIntersection(t, s);
+  public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t) {
+    return new CrossRunIntersection(b, t, null, null);
+  }
+
+  public static CrossRunIntersection of(
+      @Nullable Backend b,
+      @Nullable Transport t,
+      @Nullable LocationType lt,
+      @Nullable Colocation c) {
+    return new CrossRunIntersection(b, t, lt, c);
   }
 
   public static ImmutableSet<CrossRunIntersection> expand(CrossRun.Ignore i) {
     ImmutableSet<Backend> backends = ImmutableSet.copyOf(i.backends());
     ImmutableSet<Transport> transports = ImmutableSet.copyOf(i.transports());
-    return expand(backends, transports);
+    ImmutableSet<LocationType> locations = ImmutableSet.copyOf(i.locations());
+    ImmutableSet<Colocation> colocations = ImmutableSet.copyOf(i.colocations());
+    return expand(backends, transports, locations, colocations);
   }
 
   public static ImmutableSet<CrossRunIntersection> expand(CrossRun.Exclude i) {
     ImmutableSet<Backend> backends = ImmutableSet.copyOf(i.backends());
     ImmutableSet<Transport> transports = ImmutableSet.copyOf(i.transports());
-    return expand(backends, transports);
+    ImmutableSet<LocationType> locations = ImmutableSet.copyOf(i.locations());
+    ImmutableSet<Colocation> colocations = ImmutableSet.copyOf(i.colocations());
+    return expand(backends, transports, locations, colocations);
   }
 
   public static ImmutableSet<CrossRunIntersection> expand(
-      ImmutableSet<Backend> backends, ImmutableSet<@Nullable Transport> transports) {
-    if (backends.isEmpty() && transports.isEmpty()) {
+      ImmutableSet<Backend> backends,
+      ImmutableSet<@Nullable Transport> transports,
+      ImmutableSet<@Nullable LocationType> locations,
+      ImmutableSet<@Nullable Colocation> colocations) {
+    if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty() && colocations.isEmpty()) {
       return ImmutableSet.of();
-    } else if (!backends.isEmpty() && !transports.isEmpty()) {
-      return backends.stream()
-          .flatMap(t -> transports.stream().map(s -> new CrossRunIntersection(t, s)))
-          .collect(ImmutableSet.toImmutableSet());
-    } else if (!backends.isEmpty()) {
-      return backends.stream()
-          .map(t -> new CrossRunIntersection(t, null))
-          .collect(ImmutableSet.toImmutableSet());
-    } else {
-      return transports.stream()
-          .map(s -> new CrossRunIntersection(null, s))
-          .collect(ImmutableSet.toImmutableSet());
     }
+
+    Set<@Nullable Backend> bSet =
+        backends.isEmpty() ? Collections.singleton((Backend) null) : backends;
+    Set<@Nullable Transport> tSet =
+        transports.isEmpty() ? Collections.singleton((Transport) null) : transports;
+    Set<@Nullable LocationType> lSet =
+        locations.isEmpty() ? Collections.singleton((LocationType) null) : locations;
+    Set<@Nullable Colocation> cSet =
+        colocations.isEmpty() ? Collections.singleton((Colocation) null) : colocations;
+
+    return bSet.stream()
+        .flatMap(
+            b ->
+                tSet.stream()
+                    .flatMap(
+                        t ->
+                            lSet.stream()
+                                .flatMap(
+                                    l ->
+                                        cSet.stream()
+                                            .map(c -> new CrossRunIntersection(b, t, l, c)))))
+        .filter(
+            i ->
+                !(i.backend == null
+                    && i.transport == null
+                    && i.locationType == null
+                    && i.colocation == null))
+        .collect(ImmutableSet.toImmutableSet());
   }
 }

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java

@@ -23,6 +23,8 @@
 import com.google.cloud.storage.it.runner.annotations.Parameterized;
 import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter;
 import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider;
+import com.google.cloud.storage.it.runner.annotations.Colocation;
+import com.google.cloud.storage.it.runner.annotations.LocationType;
 import com.google.cloud.storage.it.runner.annotations.SingleBackend;
 import com.google.cloud.storage.it.runner.registry.Registry;
 import com.google.common.collect.ImmutableList;
@@ -166,7 +168,16 @@ private static List<Runner> computeRunners(Class<?> klass, Registry registry)
                   .flatMap(
                       b ->
                           ImmutableSet.copyOf(crossRun.transports()).stream()
-                              .map(t -> CrossRunIntersection.of(b, t)))
+                              .flatMap(
+                                  t ->
+                                      ImmutableSet.copyOf(crossRun.locations()).stream()
+                                          .flatMap(
+                                              l ->
+                                                  ImmutableSet.copyOf(crossRun.colocations()).stream()
+                                                      .map(
+                                                          c ->
+                                                              CrossRunIntersection.of(
+                                                                  b, t, l, c)))))
                   .flatMap(
                       c -> {
                         TestInitializer ti = registry.newTestInitializerForCell(c);
@@ -187,23 +198,38 @@ private static List<Runner> computeRunners(Class<?> klass, Registry registry)
                   .collect(ImmutableList.toImmutableList()));
     } else {
       Backend backend = singleBackend.value();
-      CrossRunIntersection crossRunIntersection = CrossRunIntersection.of(backend, null);
-      TestInitializer ti = registry.newTestInitializerForCell(crossRunIntersection);
-      if (parameters != null) {
-        return SneakyException.unwrap(
-            () ->
-                parameters.stream()
-                    .map(
-                        param ->
-                            StorageITLeafRunner.unsafeOf(
-                                testClass,
-                                crossRunIntersection,
-                                fmtParam(param),
-                                ti.andThen(setFieldTo(testClass, param))))
-                    .collect(ImmutableList.toImmutableList()));
-      } else {
-        return ImmutableList.of(StorageITLeafRunner.of(testClass, crossRunIntersection, null, ti));
-      }
+      boolean isDefault =
+          singleBackend.locations().length == 1
+              && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD
+              && singleBackend.colocations().length == 1
+              && singleBackend.colocations()[0] == Colocation.CO_LOCATED;
+
+      return SneakyException.unwrap(
+          () ->
+              ImmutableSet.copyOf(singleBackend.locations()).stream()
+                  .flatMap(
+                      l ->
+                          ImmutableSet.copyOf(singleBackend.colocations()).stream()
+                              .map(c -> CrossRunIntersection.of(backend, null, l, c)))
+                  .flatMap(
+                      c -> {
+                        TestInitializer ti = registry.newTestInitializerForCell(c);
+                        if (parameters != null) {
+                          return parameters.stream()
+                              .map(
+                                  param ->
+                                      StorageITLeafRunner.unsafeOf(
+                                          testClass,
+                                          c,
+                                          isDefault ? fmtParam(param) : fmtParam(c, param),
+                                          ti.andThen(setFieldTo(testClass, param))));
+                        } else {
+                          return Stream.of(
+                              StorageITLeafRunner.unsafeOf(
+                                  testClass, c, isDefault ? null : c.fmtSuiteName(), ti));
+                        }
+                      })
+                  .collect(ImmutableList.toImmutableList()));
     }
   }
 

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java

@@ -20,6 +20,8 @@
 public enum Backend {
   /** Use the "Production" GCS endpoints */
   PROD,
+  /** Use the GCS Pre-prod (Staging) endpoints */
+  PREPROD,
   /** Use the test bench as a backend */
   TEST_BENCH
 }