From 8d5aef64b28e763bd3d8561d6e01099e04729ace Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 17 Sep 2025 16:22:45 +0000 Subject: [PATCH 01/27] squashing last two commits --- google-cloud-storage/pom.xml | 5 +- .../cloud/storage/HttpStorageOptions.java | 14 +- .../cloud/storage/MultipartUploadClient.java | 59 +++++ .../storage/MultipartUploadClientImpl.java | 244 ++++++++++++++++++ .../storage/MultipartUploadSettings.java | 32 +++ .../cloud/storage/MultipartUploadUtility.java | 75 ++++++ .../com/google/cloud/storage/RequestBody.java | 67 +++++ .../cloud/storage/RewindableContent.java | 25 ++ .../google/cloud/storage/StorageOptions.java | 11 + .../model/AbortMultipartUploadRequest.java | 71 +++++ .../model/AbortMultipartUploadResponse.java | 20 ++ .../model/CompleteMultipartUploadRequest.java | 118 +++++++++ .../CompleteMultipartUploadResponse.java | 21 ++ .../model/CompletedMultipartUpload.java | 82 ++++++ .../multipartupload/model/CompletedPart.java | 63 +++++ .../model/CreateMultipartUploadRequest.java | 89 +++++++ .../model/CreateMultipartUploadResponse.java | 82 ++++++ .../model/UploadPartRequest.java | 118 +++++++++ .../model/UploadPartResponse.java | 74 ++++++ pom.xml | 5 + 20 files changed, 1262 insertions(+), 13 deletions(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/AbortMultipartUploadRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/AbortMultipartUploadResponse.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompletedMultipartUpload.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompletedPart.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index fae2986b44..886ad83b41 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -140,7 +140,10 @@ io.opentelemetry opentelemetry-api - + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + io.opentelemetry opentelemetry-sdk-metrics diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java index b1400bcb62..fc69333792 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java @@ -26,15 +26,13 @@ import com.google.api.gax.rpc.HeaderProvider; import com.google.api.gax.tracing.ApiTracerFactory; import com.google.auth.Credentials; +import com.google.cloud.storage.Retrying.RetryingDependencies; import com.google.cloud.ServiceFactory; import com.google.cloud.ServiceRpc; import com.google.cloud.TransportOptions; import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.spi.ServiceRpcFactory; import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory; -import com.google.cloud.storage.Retrying.DefaultRetrier; -import com.google.cloud.storage.Retrying.HttpRetrier; -import com.google.cloud.storage.Retrying.RetryingDependencies; import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.spi.StorageRpcFactory; @@ -407,15 +405,7 @@ public Storage create(StorageOptions options) { blobWriteSessionConfig = HttpStorageOptions.defaults().getDefaultStorageWriterConfig(); } WriterFactory factory = blobWriteSessionConfig.createFactory(clock); - StorageImpl storage = - new StorageImpl( - httpStorageOptions, - factory, - new HttpRetrier( - new DefaultRetrier( - OtelStorageDecorator.retryContextDecorator(otel), - RetryingDependencies.simple( - options.getClock(), options.getRetrySettings())))); + StorageImpl storage = new StorageImpl(httpStorageOptions, factory, options.createRetrier()); return OtelStorageDecorator.decorate(storage, otel, Transport.HTTP); } catch (IOException e) { throw new IllegalStateException( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java new file mode 100644 index 0000000000..068d4810c2 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -0,0 +1,59 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalExtensionOnly; +import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.UploadPartRequest; +import com.google.cloud.storage.multipartupload.model.UploadPartResponse; +import java.io.IOException; +import java.net.URI; +import java.security.NoSuchAlgorithmException; + +@BetaApi +@InternalExtensionOnly +public abstract class MultipartUploadClient { + + MultipartUploadClient() {} + + public abstract CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) + throws IOException; + + public abstract UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) + throws IOException; + + public abstract CompleteMultipartUploadResponse completeMultipartUpload( + CompleteMultipartUploadRequest request) + throws NoSuchAlgorithmException, IOException; + + public abstract AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) + throws IOException; + + public static MultipartUploadClient create(MultipartUploadSettings config) { + HttpStorageOptions options = config.getOptions(); + return new MultipartUploadClientImpl( + URI.create(options.getHost()), + options.getStorageRpcV1().getStorage().getRequestFactory(), + options.createRetrier()); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java new file mode 100644 index 0000000000..b10a3dc21c --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -0,0 +1,244 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import static com.google.cloud.storage.MultipartUploadUtility.getRfc1123Date; +import static com.google.cloud.storage.MultipartUploadUtility.readStream; +import static com.google.cloud.storage.MultipartUploadUtility.signRequest; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.client.http.AbstractHttpContent; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.util.ObjectParser; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.UnimplementedException; +import com.google.cloud.storage.Retrying.Retrier; +import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.UploadPartRequest; +import com.google.cloud.storage.multipartupload.model.UploadPartResponse; +import io.grpc.Status; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.lang.reflect.Type; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +public class MultipartUploadClientImpl extends MultipartUploadClient { + + // Add HMAC keys from GCS Settings > Interoperability + + // --- End Configuration --- + private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; + + public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { + } + + public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) + throws IOException { + String resourcePath = "/" + request.bucket() + "/" + request.key(); + String uri = GCS_ENDPOINT + resourcePath + "?uploads"; + String date = getRfc1123Date(); + String contentType = "application/x-www-form-urlencoded"; + // GCS Signature Rule #1: The '?uploads' query string IS included for the initiate request. + String signature = signRequest("POST", "", contentType, date, resourcePath + "?uploads", GOOGLE_SECRET_KEY); + String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Date", date); + connection.setRequestProperty("Authorization", authHeader); + connection.setRequestProperty("Content-Type", contentType); + connection.setFixedLengthStreamingMode(0); + connection.setDoOutput(true); + + if (connection.getResponseCode() != 200) { + String error = readStream(connection.getErrorStream()); + throw new RuntimeException("Failed to initiate upload: " + connection.getResponseCode() + " " + error); + } + + XmlMapper xmlMapper = new XmlMapper(); + return xmlMapper.readValue( + connection.getInputStream(), CreateMultipartUploadResponse.class); + } + + public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) + throws IOException { + String resourcePath = "/" + request.bucket() + "/" + request.key(); + String queryString = "?partNumber=" + request.partNumber() + "&uploadId=" + request.uploadId(); + String uri = GCS_ENDPOINT + resourcePath + queryString; + String date = getRfc1123Date(); + String contentType = "application/octet-stream"; + // GCS Signature Rule #2: The query string IS NOT included for the PUT part request. + String signature = signRequest("PUT", "", contentType, date, resourcePath, GOOGLE_SECRET_KEY); + + String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); + connection.setRequestMethod("PUT"); + connection.setRequestProperty("Date", date); + connection.setRequestProperty("Authorization", authHeader); + connection.setRequestProperty("Content-Type", contentType); + connection.setFixedLengthStreamingMode(requestBody.getPartData().length); + connection.setDoOutput(true); + + try (OutputStream os = connection.getOutputStream()) { + os.write(requestBody.getPartData()); + } + + if (connection.getResponseCode() != 200) { + String error = readStream(connection.getErrorStream()); + throw new RuntimeException("Failed to upload part " + request.partNumber() + ": " + connection.getResponseCode() + " " + error); + } + String eTag = connection.getHeaderField("ETag"); + return UploadPartResponse.builder().eTag(eTag).build(); + } + + public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request) + throws NoSuchAlgorithmException, IOException { + String resourcePath = "/" + request.bucket() + "/" + request.key(); + String queryString = "?uploadId=" + request.uploadId(); + String uri = GCS_ENDPOINT + resourcePath + queryString; + + XmlMapper xmlMapper = new XmlMapper(); + byte[] xmlBodyBytes = xmlMapper.writeValueAsBytes(request.multipartUpload()); + + MessageDigest md = MessageDigest.getInstance("MD5"); + String contentMd5 = Base64.getEncoder().encodeToString(md.digest(xmlBodyBytes)); + String date = getRfc1123Date(); + String contentType = "application/xml"; + + // GCS Signature Rule #3: The query string IS NOT included for the POST complete request. + String signature = signRequest("POST", contentMd5, contentType, date, resourcePath, GOOGLE_SECRET_KEY); + String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Date", date); + connection.setRequestProperty("Authorization", authHeader); + connection.setRequestProperty("Content-Type", contentType); + connection.setRequestProperty("Content-MD5", contentMd5); + connection.setFixedLengthStreamingMode(xmlBodyBytes.length); + connection.setDoOutput(true); + + try (OutputStream os = connection.getOutputStream()) { + os.write(xmlBodyBytes); + } + + if (connection.getResponseCode() != 200) { + String error = readStream(connection.getErrorStream()); + throw new RuntimeException("Failed to complete upload: " + connection.getResponseCode() + " " + error); + } + return null; + } + + @Override + public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) throws IOException{ + String resourcePath = "/" + request.bucket() + "/" + request.key(); + String queryString = "?uploadId=" + request.uploadId(); + String uri = GCS_ENDPOINT + resourcePath + queryString; + String date = getRfc1123Date(); + + // GCS Signature Rule #4: The query string IS NOT included for the DELETE abort request. + String signature = signRequest("DELETE", "", "", date, resourcePath, GOOGLE_SECRET_KEY); + + String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); + connection.setRequestMethod("DELETE"); + connection.setRequestProperty("Date", date); + connection.setRequestProperty("Authorization", authHeader); + + if (connection.getResponseCode() != 204) { + String error = readStream(connection.getErrorStream()); + throw new RuntimeException("Failed to abort upload: " + connection.getResponseCode() + " " + error); + } + return new AbortMultipartUploadResponse(); + } + + private static final class Utf8StringRequestContent extends AbstractHttpContent { + + private final byte[] xml; + + private Utf8StringRequestContent(byte[] xml) { + // https://www.ietf.org/rfc/rfc2376.txt#:~:text=6.1%20text/xml%20with%20UTF%2D8%20Charset + super("text/xml;charset=utf-8"); + this.xml = xml; + } + + @Override + public long getLength() throws IOException { + return super.getLength(); + } + + @Override + public void writeTo(OutputStream out) throws IOException { + out.write(xml); + } + + public static Utf8StringRequestContent of(String xml) { + return new Utf8StringRequestContent(xml.getBytes(StandardCharsets.UTF_8)); + } + } + + private static class XmlObjectParser implements ObjectParser { + + @Override + public T parseAndClose(InputStream in, Charset charset, Class dataClass) + throws IOException { + try (InputStream is = in) { + return todo(); + } + } + + @Override + public Object parseAndClose(InputStream in, Charset charset, Type dataType) throws IOException { + try (InputStream is = in) { + return todo(); + } + } + + @Override + public T parseAndClose(Reader reader, Class dataClass) throws IOException { + try (Reader r = reader) { + return todo(); + } + } + + @Override + public Object parseAndClose(Reader reader, Type dataType) throws IOException { + try (Reader r = reader) { + return todo(); + } + } + + private static T todo() { + throw new UnimplementedException("todo", null, GrpcStatusCode.of(Status.Code.UNIMPLEMENTED), false); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java new file mode 100644 index 0000000000..d3941a383c --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +public final class MultipartUploadSettings { + private final HttpStorageOptions options; + + private MultipartUploadSettings(HttpStorageOptions options) { + this.options = options; + } + + public HttpStorageOptions getOptions() { + return options; + } + + public static MultipartUploadSettings of(HttpStorageOptions options) { + return new MultipartUploadSettings(options); + } +} \ No newline at end of file diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java new file mode 100644 index 0000000000..7982ea55cb --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Base64; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +public class MultipartUploadUtility { + public static String readStream(InputStream inputStream) throws IOException { + if (inputStream == null) return ""; + StringBuilder response = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + } + return response.toString(); + } + + public static String signRequest(String httpVerb, String contentMd5, String contentType, String date, String canonicalizedResource, String googleSecretKey) { + try { + String stringToSign = httpVerb + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedResource; + Mac sha1Hmac = Mac.getInstance("HmacSHA1"); + SecretKeySpec secretKey = new SecretKeySpec(googleSecretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA1"); + sha1Hmac.init(secretKey); + byte[] signatureBytes = sha1Hmac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(signatureBytes); + } catch (Exception e) { + throw new RuntimeException("Failed to sign request", e); + } + } + + public static byte[] readPart(File file, long position, int size) throws IOException { + byte[] buffer = new byte[size]; + try (FileInputStream fis = new FileInputStream(file)) { + fis.getChannel().position(position); + int bytesRead = fis.read(buffer, 0, size); + if (bytesRead != size) { + byte[] smallerBuffer = new byte[bytesRead]; + System.arraycopy(buffer, 0, smallerBuffer, 0, bytesRead); + return smallerBuffer; + } + } + return buffer; + } + + public static String getRfc1123Date() { + return DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneId.of("GMT")).format(ZonedDateTime.now()); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java new file mode 100644 index 0000000000..5f1a315457 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java @@ -0,0 +1,67 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalExtensionOnly; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; + +@BetaApi +@InternalExtensionOnly +public final class RequestBody { + + private final RewindableContent content; + private static byte[] byteArray; + + private RequestBody(RewindableContent content) { + this.content = content; + } + + RewindableContent getContent() { + return content; + } + + public static RequestBody empty() { + return new RequestBody(RewindableContent.empty()); + } + + public static RequestBody of(ByteBuffer... buffers) { + return new RequestBody(RewindableContent.of(buffers)); + } + + public static RequestBody fromByteBuffer(ByteBuffer buffer) { + byteArray = new byte[buffer.remaining()]; + // The get() method copies the bytes from the buffer into the array. + // This operation advances the buffer's position. + buffer.get(byteArray); + return new RequestBody(RewindableContent.of(buffer)); + } + + public static RequestBody of(ByteBuffer[] srcs, int srcsOffset, int srcsLength) { + return new RequestBody(RewindableContent.of(srcs, srcsOffset, srcsLength)); + } + + public static RequestBody of(Path path) throws IOException { + return new RequestBody(RewindableContent.of(path)); + } + + public byte[] getPartData() { + return byteArray; + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java index 8d299bfb54..7cf3dfe797 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java @@ -20,6 +20,7 @@ import com.google.api.client.http.HttpMediaType; import com.google.common.base.Preconditions; import com.google.common.io.ByteStreams; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -50,6 +51,30 @@ private RewindableContent() { abstract void flagDirty(); + /** + * Returns the content as a byte array. + * + *

NOTE: This method will read the entire content into memory. If the content is large, + * this may cause an OutOfMemoryError. + * + * @return The byte array representation of the content. + * @throws IOException if an I/O error occurs. + */ + public byte[] asByteArray() { + if (getLength() == 0) { + return new byte[0]; + } + Preconditions.checkState( + getLength() <= Integer.MAX_VALUE, "Content is too large to be represented as a byte array."); + ByteArrayOutputStream baos = new ByteArrayOutputStream((int) getLength()); + try { + writeTo(baos); + } catch (IOException e) { + throw new RuntimeException(e); + } + return baos.toByteArray(); + } + @Override public final boolean retrySupported() { return false; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index 4dac2b43ef..3a2d21ced9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -35,6 +35,10 @@ import java.util.Locale; import java.util.Properties; import org.checkerframework.checker.nullness.qual.NonNull; +import com.google.cloud.storage.Retrying.DefaultRetrier; +import com.google.cloud.storage.Retrying.HttpRetrier; +import com.google.cloud.storage.Retrying.Retrier; +import com.google.cloud.storage.Retrying.RetryingDependencies; public abstract class StorageOptions extends ServiceOptions { @@ -68,6 +72,13 @@ public abstract class StorageOptions extends ServiceOptions completedPartList; + + private CompletedMultipartUpload(Builder builder) { + this.completedPartList = builder.parts; + } + + public List parts() { + return completedPartList; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CompletedMultipartUpload)) { + return false; + } + CompletedMultipartUpload that = (CompletedMultipartUpload) o; + return Objects.equals(completedPartList, that.completedPartList); + } + + @Override + public int hashCode() { + return Objects.hash(completedPartList); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("completedPartList", completedPartList) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private List parts; + + private Builder() {} + + public Builder parts(List completedPartList) { + this.parts = completedPartList; + return this; + } + + public CompletedMultipartUpload build() { + return new CompletedMultipartUpload(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompletedPart.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompletedPart.java new file mode 100644 index 0000000000..fab67c5f6e --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompletedPart.java @@ -0,0 +1,63 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.multipartupload.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +public final class CompletedPart { + + @JacksonXmlProperty(localName = "PartNumber") + private final int partNumber; + + @JacksonXmlProperty(localName = "ETag") + private final String eTag; + + private CompletedPart(int partNumber, String eTag) { + this.partNumber = partNumber; + this.eTag = eTag; + } + + public static Builder builder() { + return new Builder(); + } + + public int partNumber() { + return partNumber; + } + + public String eTag() { + return eTag; + } + + public static class Builder { + private int partNumber; + private String etag; + + public Builder partNumber(int partNumber) { + this.partNumber = partNumber; + return this; + } + + public Builder eTag(String etag) { + this.etag = etag; + return this; + } + + public CompletedPart build() { + return new CompletedPart(partNumber, etag); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java new file mode 100644 index 0000000000..15abcf2522 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public class CreateMultipartUploadRequest { + private final String bucket; + + private final String key; + + private CreateMultipartUploadRequest(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; + } + + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateMultipartUploadRequest)) { + return false; + } + CreateMultipartUploadRequest that = (CreateMultipartUploadRequest) o; + return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, key); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String bucket; + private String key; + + private Builder() {} + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder key(String key) { + this.key = key; + return this; + } + + public CreateMultipartUploadRequest build() { + return new CreateMultipartUploadRequest(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java new file mode 100644 index 0000000000..976fb99385 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -0,0 +1,82 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.google.common.base.MoreObjects; +import java.util.Objects; + +@JacksonXmlRootElement(localName = "InitiateMultipartUploadResult") +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreateMultipartUploadResponse { + + @JacksonXmlProperty(localName = "UploadId") + private String uploadId; + + private CreateMultipartUploadResponse(Builder builder) { + this.uploadId = builder.uploadId; + } + + private CreateMultipartUploadResponse() {} + + public String uploadId() { + return uploadId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateMultipartUploadResponse)) { + return false; + } + CreateMultipartUploadResponse that = (CreateMultipartUploadResponse) o; + return Objects.equals(uploadId, that.uploadId); + } + + @Override + public int hashCode() { + return Objects.hash(uploadId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("uploadId", uploadId).toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String uploadId; + + private Builder() {} + + public Builder uploadId(String uploadId) { + this.uploadId = uploadId; + return this; + } + + public CreateMultipartUploadResponse build() { + return new CreateMultipartUploadResponse(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartRequest.java new file mode 100644 index 0000000000..ed4cbd9d79 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartRequest.java @@ -0,0 +1,118 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public final class UploadPartRequest { + + private final String bucket; + private final String key; + private final int partNumber; + private final String uploadId; + + private UploadPartRequest(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; + this.partNumber = builder.partNumber; + this.uploadId = builder.uploadId; + } + + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + + public int partNumber() { + return partNumber; + } + + public String uploadId() { + return uploadId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof UploadPartRequest)) { + return false; + } + UploadPartRequest that = (UploadPartRequest) o; + return partNumber == that.partNumber + && Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(uploadId, that.uploadId); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, key, partNumber, uploadId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("partNumber", partNumber) + .add("uploadId", uploadId) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String bucket; + private String key; + private int partNumber; + private String uploadId; + + private Builder() {} + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder key(String key) { + this.key = key; + return this; + } + + public Builder partNumber(int partNumber) { + this.partNumber = partNumber; + return this; + } + + public Builder uploadId(String uploadId) { + this.uploadId = uploadId; + return this; + } + + public UploadPartRequest build() { + return new UploadPartRequest(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java new file mode 100644 index 0000000000..5acc044645 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java @@ -0,0 +1,74 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public final class UploadPartResponse { + + private final String eTag; + + private UploadPartResponse(Builder builder) { + this.eTag = builder.etag; + } + + public String eTag() { + return eTag; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof UploadPartResponse)) { + return false; + } + UploadPartResponse that = (UploadPartResponse) o; + return Objects.equals(eTag, that.eTag); + } + + @Override + public int hashCode() { + return Objects.hash(eTag); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("etag", eTag).toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String etag; + + private Builder() {} + + public Builder eTag(String etag) { + this.etag = etag; + return this; + } + + public UploadPartResponse build() { + return new UploadPartResponse(this); + } + } +} diff --git a/pom.xml b/pom.xml index 1410ff777f..f1bfe9c3e5 100644 --- a/pom.xml +++ b/pom.xml @@ -155,6 +155,11 @@ 4.5.14 test + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + 2.15.2 + org.apache.httpcomponents httpcore From 044b4837223f08220cbcc55393f3bdb1b4522126 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 17 Sep 2025 18:14:38 +0000 Subject: [PATCH 02/27] Added url encoding --- .../storage/MultipartUploadClientImpl.java | 93 ++++--------------- 1 file changed, 19 insertions(+), 74 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index b10a3dc21c..98e070ed0c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -20,11 +20,7 @@ import static com.google.cloud.storage.MultipartUploadUtility.signRequest; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.api.client.http.AbstractHttpContent; import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.util.ObjectParser; -import com.google.api.gax.grpc.GrpcStatusCode; -import com.google.api.gax.rpc.UnimplementedException; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; @@ -34,16 +30,14 @@ import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; -import io.grpc.Status; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.Reader; -import java.lang.reflect.Type; +import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; -import java.nio.charset.Charset; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -61,7 +55,9 @@ public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Ret public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) throws IOException { - String resourcePath = "/" + request.bucket() + "/" + request.key(); + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; String uri = GCS_ENDPOINT + resourcePath + "?uploads"; String date = getRfc1123Date(); String contentType = "application/x-www-form-urlencoded"; @@ -89,8 +85,10 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) throws IOException { - String resourcePath = "/" + request.bucket() + "/" + request.key(); - String queryString = "?partNumber=" + request.partNumber() + "&uploadId=" + request.uploadId(); + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String queryString = "?partNumber=" + request.partNumber() + "&uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String date = getRfc1123Date(); String contentType = "application/octet-stream"; @@ -121,8 +119,10 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request) throws NoSuchAlgorithmException, IOException { - String resourcePath = "/" + request.bucket() + "/" + request.key(); - String queryString = "?uploadId=" + request.uploadId(); + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; XmlMapper xmlMapper = new XmlMapper(); @@ -159,8 +159,10 @@ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipart @Override public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) throws IOException{ - String resourcePath = "/" + request.bucket() + "/" + request.key(); - String queryString = "?uploadId=" + request.uploadId(); + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String date = getRfc1123Date(); @@ -181,64 +183,7 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq return new AbortMultipartUploadResponse(); } - private static final class Utf8StringRequestContent extends AbstractHttpContent { - - private final byte[] xml; - - private Utf8StringRequestContent(byte[] xml) { - // https://www.ietf.org/rfc/rfc2376.txt#:~:text=6.1%20text/xml%20with%20UTF%2D8%20Charset - super("text/xml;charset=utf-8"); - this.xml = xml; - } - - @Override - public long getLength() throws IOException { - return super.getLength(); - } - - @Override - public void writeTo(OutputStream out) throws IOException { - out.write(xml); - } - - public static Utf8StringRequestContent of(String xml) { - return new Utf8StringRequestContent(xml.getBytes(StandardCharsets.UTF_8)); - } - } - - private static class XmlObjectParser implements ObjectParser { - - @Override - public T parseAndClose(InputStream in, Charset charset, Class dataClass) - throws IOException { - try (InputStream is = in) { - return todo(); - } - } - - @Override - public Object parseAndClose(InputStream in, Charset charset, Type dataType) throws IOException { - try (InputStream is = in) { - return todo(); - } - } - - @Override - public T parseAndClose(Reader reader, Class dataClass) throws IOException { - try (Reader r = reader) { - return todo(); - } - } - - @Override - public Object parseAndClose(Reader reader, Type dataType) throws IOException { - try (Reader r = reader) { - return todo(); - } - } - - private static T todo() { - throw new UnimplementedException("todo", null, GrpcStatusCode.of(Status.Code.UNIMPLEMENTED), false); - } + private String encode(String value) throws UnsupportedEncodingException { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } } From 3c15ef2a7206eaf04df0817d9f4265652aa97b51 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 17 Sep 2025 19:19:49 +0000 Subject: [PATCH 03/27] Added md5 checksumming --- .../cloud/storage/MultipartUploadClient.java | 2 +- .../storage/MultipartUploadClientImpl.java | 14 +++++---- .../CompleteMultipartUploadResponse.java | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index 068d4810c2..936292a743 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -40,7 +40,7 @@ public abstract CreateMultipartUploadResponse createMultipartUpload(CreateMultip throws IOException; public abstract UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) - throws IOException; + throws IOException, NoSuchAlgorithmException; public abstract CompleteMultipartUploadResponse completeMultipartUpload( CompleteMultipartUploadRequest request) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 98e070ed0c..d23bdb8813 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -84,7 +84,7 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload } public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) - throws IOException { + throws IOException, NoSuchAlgorithmException { String encodedBucket = encode(request.bucket()); String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; @@ -93,7 +93,10 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String date = getRfc1123Date(); String contentType = "application/octet-stream"; // GCS Signature Rule #2: The query string IS NOT included for the PUT part request. - String signature = signRequest("PUT", "", contentType, date, resourcePath, GOOGLE_SECRET_KEY); + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] partData = requestBody.getPartData(); + String contentMd5 = Base64.getEncoder().encodeToString(md.digest(partData)); + String signature = signRequest("PUT", contentMd5, contentType, date, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; @@ -102,11 +105,12 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ connection.setRequestProperty("Date", date); connection.setRequestProperty("Authorization", authHeader); connection.setRequestProperty("Content-Type", contentType); - connection.setFixedLengthStreamingMode(requestBody.getPartData().length); + connection.setRequestProperty("Content-MD5", contentMd5); + connection.setFixedLengthStreamingMode(partData.length); connection.setDoOutput(true); try (OutputStream os = connection.getOutputStream()) { - os.write(requestBody.getPartData()); + os.write(partData); } if (connection.getResponseCode() != 200) { @@ -154,7 +158,7 @@ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipart String error = readStream(connection.getErrorStream()); throw new RuntimeException("Failed to complete upload: " + connection.getResponseCode() + " " + error); } - return null; + return xmlMapper.readValue(connection.getInputStream(), CompleteMultipartUploadResponse.class); } @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java index 0559b8e827..c17ad31f7f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java @@ -16,6 +16,35 @@ package com.google.cloud.storage.multipartupload.model; +import com.fasterxml.jackson.annotation.JsonProperty; + public class CompleteMultipartUploadResponse { + @JsonProperty("Location") + private String location; + + @JsonProperty("Bucket") + private String bucket; + + @JsonProperty("Key") + private String key; + + @JsonProperty("ETag") + private String etag; + + public String getLocation() { + return location; + } + + public String getBucket() { + return bucket; + } + + public String getKey() { + return key; + } + + public String getEtag() { + return etag; + } } From e8eb531a318e3ed0249a965debb1a905a6b106d0 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 19 Sep 2025 15:00:03 +0000 Subject: [PATCH 04/27] Using HttpRequestFactory instead of HTTP url connection --- .../storage/MultipartUploadClientImpl.java | 170 +++++++++++------- .../cloud/storage/MultipartUploadUtility.java | 40 ++++- 2 files changed, 143 insertions(+), 67 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index d23bdb8813..3e76604644 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -16,11 +16,14 @@ package com.google.cloud.storage; import static com.google.cloud.storage.MultipartUploadUtility.getRfc1123Date; -import static com.google.cloud.storage.MultipartUploadUtility.readStream; import static com.google.cloud.storage.MultipartUploadUtility.signRequest; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; @@ -31,17 +34,15 @@ import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.UnsupportedEncodingException; -import java.net.HttpURLConnection; import java.net.URI; -import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Base64; +import java.util.HashMap; +import java.util.Map; public class MultipartUploadClientImpl extends MultipartUploadClient { @@ -50,7 +51,17 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { // --- End Configuration --- private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; + private final HttpRequestFactory requestFactory; + private final Map extensionHeaders; + public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { + this.requestFactory = requestFactory; + this.extensionHeaders = new HashMap<>(); + //TODO fix the hard coded header + this.extensionHeaders.put( + "x-goog-api-client", + "gl-java/11.0.27__OpenLogic-OpenJDK__OpenLogic-OpenJDK gccl/2.56.1-SNAPSHOT--protobuf-3.25.8 gax/2.70.0 protobuf/3.25.8"); + this.extensionHeaders.put("x-goog-user-project", "aipp-internal-testing"); } public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) @@ -62,25 +73,37 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String date = getRfc1123Date(); String contentType = "application/x-www-form-urlencoded"; // GCS Signature Rule #1: The '?uploads' query string IS included for the initiate request. - String signature = signRequest("POST", "", contentType, date, resourcePath + "?uploads", GOOGLE_SECRET_KEY); + String signature = + signRequest( + "POST", + "", + contentType, + date, + extensionHeaders, + resourcePath + "?uploads", + GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); - connection.setRequestMethod("POST"); - connection.setRequestProperty("Date", date); - connection.setRequestProperty("Authorization", authHeader); - connection.setRequestProperty("Content-Type", contentType); - connection.setFixedLengthStreamingMode(0); - connection.setDoOutput(true); - - if (connection.getResponseCode() != 200) { - String error = readStream(connection.getErrorStream()); - throw new RuntimeException("Failed to initiate upload: " + connection.getResponseCode() + " " + error); + HttpRequest httpRequest = + requestFactory.buildPostRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + HttpResponse response = httpRequest.execute(); + + if (!response.isSuccessStatusCode()) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to initiate upload: " + response.getStatusCode() + " " + error); } XmlMapper xmlMapper = new XmlMapper(); - return xmlMapper.readValue( - connection.getInputStream(), CreateMultipartUploadResponse.class); + return xmlMapper.readValue(response.getContent(), CreateMultipartUploadResponse.class); } public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) @@ -88,7 +111,8 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String encodedBucket = encode(request.bucket()); String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; - String queryString = "?partNumber=" + request.partNumber() + "&uploadId=" + encode(request.uploadId()); + String queryString = + "?partNumber=" + request.partNumber() + "&uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String date = getRfc1123Date(); String contentType = "application/octet-stream"; @@ -96,33 +120,41 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ MessageDigest md = MessageDigest.getInstance("MD5"); byte[] partData = requestBody.getPartData(); String contentMd5 = Base64.getEncoder().encodeToString(md.digest(partData)); - String signature = signRequest("PUT", contentMd5, contentType, date, resourcePath, GOOGLE_SECRET_KEY); + String signature = + signRequest( + "PUT", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); - connection.setRequestMethod("PUT"); - connection.setRequestProperty("Date", date); - connection.setRequestProperty("Authorization", authHeader); - connection.setRequestProperty("Content-Type", contentType); - connection.setRequestProperty("Content-MD5", contentMd5); - connection.setFixedLengthStreamingMode(partData.length); - connection.setDoOutput(true); - - try (OutputStream os = connection.getOutputStream()) { - os.write(partData); + HttpRequest httpRequest = + requestFactory.buildPutRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, partData)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + httpRequest.getHeaders().setContentMD5(contentMd5); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } - - if (connection.getResponseCode() != 200) { - String error = readStream(connection.getErrorStream()); - throw new RuntimeException("Failed to upload part " + request.partNumber() + ": " + connection.getResponseCode() + " " + error); + httpRequest.setThrowExceptionOnExecuteError(false); + HttpResponse response = httpRequest.execute(); + + if (!response.isSuccessStatusCode()) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to upload part " + + request.partNumber() + + ": " + + response.getStatusCode() + + " " + + error); } - String eTag = connection.getHeaderField("ETag"); + String eTag = response.getHeaders().getETag(); return UploadPartResponse.builder().eTag(eTag).build(); } - public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request) - throws NoSuchAlgorithmException, IOException { + public CompleteMultipartUploadResponse completeMultipartUpload( + CompleteMultipartUploadRequest request) throws NoSuchAlgorithmException, IOException { String encodedBucket = encode(request.bucket()); String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; @@ -138,31 +170,35 @@ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipart String contentType = "application/xml"; // GCS Signature Rule #3: The query string IS NOT included for the POST complete request. - String signature = signRequest("POST", contentMd5, contentType, date, resourcePath, GOOGLE_SECRET_KEY); + String signature = + signRequest( + "POST", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); - connection.setRequestMethod("POST"); - connection.setRequestProperty("Date", date); - connection.setRequestProperty("Authorization", authHeader); - connection.setRequestProperty("Content-Type", contentType); - connection.setRequestProperty("Content-MD5", contentMd5); - connection.setFixedLengthStreamingMode(xmlBodyBytes.length); - connection.setDoOutput(true); - - try (OutputStream os = connection.getOutputStream()) { - os.write(xmlBodyBytes); + HttpRequest httpRequest = + requestFactory.buildPostRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + httpRequest.getHeaders().setContentMD5(contentMd5); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } + httpRequest.setThrowExceptionOnExecuteError(false); + HttpResponse response = httpRequest.execute(); - if (connection.getResponseCode() != 200) { - String error = readStream(connection.getErrorStream()); - throw new RuntimeException("Failed to complete upload: " + connection.getResponseCode() + " " + error); + if (!response.isSuccessStatusCode()) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to complete upload: " + response.getStatusCode() + " " + error); } - return xmlMapper.readValue(connection.getInputStream(), CompleteMultipartUploadResponse.class); + return xmlMapper.readValue(response.getContent(), CompleteMultipartUploadResponse.class); } @Override - public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) throws IOException{ + public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) + throws IOException { String encodedBucket = encode(request.bucket()); String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; @@ -171,18 +207,24 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String date = getRfc1123Date(); // GCS Signature Rule #4: The query string IS NOT included for the DELETE abort request. - String signature = signRequest("DELETE", "", "", date, resourcePath, GOOGLE_SECRET_KEY); + String signature = + signRequest("DELETE", "", "", date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); - connection.setRequestMethod("DELETE"); - connection.setRequestProperty("Date", date); - connection.setRequestProperty("Authorization", authHeader); + HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + HttpResponse response = httpRequest.execute(); - if (connection.getResponseCode() != 204) { - String error = readStream(connection.getErrorStream()); - throw new RuntimeException("Failed to abort upload: " + connection.getResponseCode() + " " + error); + if (response.getStatusCode() != 204) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to abort upload: " + response.getStatusCode() + " " + error); } return new AbortMultipartUploadResponse(); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java index 7982ea55cb..20bc2386cf 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java @@ -26,6 +26,10 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Base64; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -42,11 +46,41 @@ public static String readStream(InputStream inputStream) throws IOException { return response.toString(); } - public static String signRequest(String httpVerb, String contentMd5, String contentType, String date, String canonicalizedResource, String googleSecretKey) { + public static String signRequest( + String httpVerb, + String contentMd5, + String contentType, + String date, + Map extensionHeaders, + String canonicalizedResource, + String googleSecretKey) { try { - String stringToSign = httpVerb + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedResource; + String canonicalizedExtensionHeaders = ""; + if (extensionHeaders != null && !extensionHeaders.isEmpty()) { + SortedMap sortedHeaders = new TreeMap<>(); + for (Map.Entry entry : extensionHeaders.entrySet()) { + sortedHeaders.put(entry.getKey().toLowerCase(), entry.getValue()); + } + canonicalizedExtensionHeaders = + sortedHeaders.entrySet().stream() + .map(entry -> entry.getKey() + ":" + entry.getValue().trim().replaceAll("\\s+", " ")) + .collect(Collectors.joining("\n")) + + "\n"; + } + String stringToSign = + httpVerb + + "\n" + + contentMd5 + + "\n" + + contentType + + "\n" + + date + + "\n" + + canonicalizedExtensionHeaders + + canonicalizedResource; Mac sha1Hmac = Mac.getInstance("HmacSHA1"); - SecretKeySpec secretKey = new SecretKeySpec(googleSecretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA1"); + SecretKeySpec secretKey = + new SecretKeySpec(googleSecretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA1"); sha1Hmac.init(secretKey); byte[] signatureBytes = sha1Hmac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); return Base64.getEncoder().encodeToString(signatureBytes); From 4fa6113f4dff11a7aaf6296be398d471fd9d8df3 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Thu, 25 Sep 2025 11:51:40 +0000 Subject: [PATCH 05/27] Fixed crc32 checksum issue --- .../cloud/storage/MultipartUploadClient.java | 2 +- .../storage/MultipartUploadClientImpl.java | 40 +++++++++++++++---- .../cloud/storage/MultipartUploadUtility.java | 5 ++- .../com/google/cloud/storage/RequestBody.java | 17 ++++---- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index 936292a743..b5d9b7b436 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -47,7 +47,7 @@ public abstract CompleteMultipartUploadResponse completeMultipartUpload( throws NoSuchAlgorithmException, IOException; public abstract AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) - throws IOException; + throws IOException, NoSuchAlgorithmException; public static MultipartUploadClient create(MultipartUploadSettings config) { HttpStorageOptions options = config.getOptions(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 3e76604644..622d06db05 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -33,10 +33,12 @@ import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; +import com.google.common.hash.Hashing; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLEncoder; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -52,16 +54,18 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; private final HttpRequestFactory requestFactory; - private final Map extensionHeaders; public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { this.requestFactory = requestFactory; - this.extensionHeaders = new HashMap<>(); - //TODO fix the hard coded header - this.extensionHeaders.put( + } + + private Map getExtensionHeader(){ + Map extensionHeaders = new HashMap<>(); + extensionHeaders.put( "x-goog-api-client", "gl-java/11.0.27__OpenLogic-OpenJDK__OpenLogic-OpenJDK gccl/2.56.1-SNAPSHOT--protobuf-3.25.8 gax/2.70.0 protobuf/3.25.8"); - this.extensionHeaders.put("x-goog-user-project", "aipp-internal-testing"); + extensionHeaders.put("x-goog-user-project", "aipp-internal-testing"); + return extensionHeaders; } public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) @@ -72,6 +76,7 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String uri = GCS_ENDPOINT + resourcePath + "?uploads"; String date = getRfc1123Date(); String contentType = "application/x-www-form-urlencoded"; + Map extensionHeaders = getExtensionHeader(); // GCS Signature Rule #1: The '?uploads' query string IS included for the initiate request. String signature = signRequest( @@ -120,6 +125,14 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ MessageDigest md = MessageDigest.getInstance("MD5"); byte[] partData = requestBody.getPartData(); String contentMd5 = Base64.getEncoder().encodeToString(md.digest(partData)); + String crc32cString = + Base64.getEncoder() + .encodeToString( + ByteBuffer.allocate(4) + .putInt(Hashing.crc32c().hashBytes(partData).asInt()) + .array()); + Map extensionHeaders = getExtensionHeader(); + extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); String signature = signRequest( "PUT", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); @@ -133,6 +146,7 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); httpRequest.getHeaders().setContentMD5(contentMd5); + httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } @@ -168,8 +182,16 @@ public CompleteMultipartUploadResponse completeMultipartUpload( String contentMd5 = Base64.getEncoder().encodeToString(md.digest(xmlBodyBytes)); String date = getRfc1123Date(); String contentType = "application/xml"; + String crc32cString = + Base64.getEncoder() + .encodeToString( + ByteBuffer.allocate(4) + .putInt(Hashing.crc32c().hashBytes(xmlBodyBytes).asInt()) + .array()); // GCS Signature Rule #3: The query string IS NOT included for the POST complete request. + Map extensionHeaders = getExtensionHeader(); + extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); String signature = signRequest( "POST", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); @@ -182,6 +204,7 @@ public CompleteMultipartUploadResponse completeMultipartUpload( httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); httpRequest.getHeaders().setContentMD5(contentMd5); + httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } @@ -198,23 +221,26 @@ public CompleteMultipartUploadResponse completeMultipartUpload( @Override public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) - throws IOException { + throws IOException, NoSuchAlgorithmException { String encodedBucket = encode(request.bucket()); String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String date = getRfc1123Date(); + String contentType = "application/x-www-form-urlencoded"; + Map extensionHeaders = getExtensionHeader(); // GCS Signature Rule #4: The query string IS NOT included for the DELETE abort request. String signature = - signRequest("DELETE", "", "", date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); + signRequest("DELETE", "", contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java index 20bc2386cf..4dd696f612 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java @@ -70,14 +70,15 @@ public static String signRequest( String stringToSign = httpVerb + "\n" - + contentMd5 + + (contentMd5 == null ? "" : contentMd5) + "\n" - + contentType + + (contentType == null ? "" : contentType) + "\n" + date + "\n" + canonicalizedExtensionHeaders + canonicalizedResource; + System.out.println(stringToSign); Mac sha1Hmac = Mac.getInstance("HmacSHA1"); SecretKeySpec secretKey = new SecretKeySpec(googleSecretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA1"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java index 5f1a315457..d6efab87aa 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java @@ -27,7 +27,7 @@ public final class RequestBody { private final RewindableContent content; - private static byte[] byteArray; + private byte[] byteArray; private RequestBody(RewindableContent content) { this.content = content; @@ -38,7 +38,9 @@ RewindableContent getContent() { } public static RequestBody empty() { - return new RequestBody(RewindableContent.empty()); + RequestBody requestBody = new RequestBody(RewindableContent.empty()); + requestBody.byteArray = new byte[0]; + return requestBody; } public static RequestBody of(ByteBuffer... buffers) { @@ -46,11 +48,12 @@ public static RequestBody of(ByteBuffer... buffers) { } public static RequestBody fromByteBuffer(ByteBuffer buffer) { - byteArray = new byte[buffer.remaining()]; - // The get() method copies the bytes from the buffer into the array. - // This operation advances the buffer's position. - buffer.get(byteArray); - return new RequestBody(RewindableContent.of(buffer)); + ByteBuffer duplicate = buffer.duplicate(); + byte[] arr = new byte[duplicate.remaining()]; + duplicate.get(arr); + RequestBody requestBody = new RequestBody(RewindableContent.of(buffer)); + requestBody.byteArray = arr; + return requestBody; } public static RequestBody of(ByteBuffer[] srcs, int srcsOffset, int srcsLength) { From 365453bf04c3622a4183e6deafb11b99151a07ed Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Thu, 25 Sep 2025 12:09:22 +0000 Subject: [PATCH 06/27] Refactored Http requestfactory out --- .../cloud/storage/HttpRequestManager.java | 113 ++++++++++++++++++ .../storage/MultipartUploadClientImpl.java | 82 +++++-------- 2 files changed, 143 insertions(+), 52 deletions(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java new file mode 100644 index 0000000000..4b89987055 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -0,0 +1,113 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import java.io.IOException; +import java.util.Map; + +public class HttpRequestManager { + + private final HttpRequestFactory requestFactory; + + public HttpRequestManager(HttpRequestFactory requestFactory) { + this.requestFactory = requestFactory; + } + + public HttpResponse sendCreateMultipartUploadRequest( + String uri, String date, String authHeader, String contentType, Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = + requestFactory.buildPostRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } + + public HttpResponse sendUploadPartRequest( + String uri, + byte[] partData, + String date, + String authHeader, + String contentType, + String contentMd5, + String crc32cString, + Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = + requestFactory.buildPutRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, partData)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + httpRequest.getHeaders().setContentMD5(contentMd5); + httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } + + public HttpResponse sendCompleteMultipartUploadRequest( + String uri, + byte[] xmlBodyBytes, + String date, + String authHeader, + String contentType, + String contentMd5, + String crc32cString, + Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = + requestFactory.buildPostRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + httpRequest.getHeaders().setContentMD5(contentMd5); + httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } + + public HttpResponse sendAbortMultipartUploadRequest( + String uri, String date, String authHeader, String contentType, Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + httpRequest.getHeaders().setContentType(contentType); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 622d06db05..54181f4299 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -19,9 +19,6 @@ import static com.google.cloud.storage.MultipartUploadUtility.signRequest; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.api.client.http.ByteArrayContent; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.cloud.storage.Retrying.Retrier; @@ -50,16 +47,17 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { // Add HMAC keys from GCS Settings > Interoperability + // --- End Configuration --- private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; - private final HttpRequestFactory requestFactory; + private final HttpRequestManager httpRequestManager; public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { - this.requestFactory = requestFactory; + this.httpRequestManager = new HttpRequestManager(requestFactory); } - private Map getExtensionHeader(){ + private Map getExtensionHeader() { Map extensionHeaders = new HashMap<>(); extensionHeaders.put( "x-goog-api-client", @@ -89,17 +87,9 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpRequest httpRequest = - requestFactory.buildPostRequest( - new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); - httpRequest.getHeaders().set("Date", date); - httpRequest.getHeaders().setAuthorization(authHeader); - httpRequest.getHeaders().setContentType(contentType); - for (Map.Entry entry : extensionHeaders.entrySet()) { - httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); - } - httpRequest.setThrowExceptionOnExecuteError(false); - HttpResponse response = httpRequest.execute(); + HttpResponse response = + httpRequestManager.sendCreateMultipartUploadRequest( + uri, date, authHeader, contentType, extensionHeaders); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -139,19 +129,16 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpRequest httpRequest = - requestFactory.buildPutRequest( - new GenericUrl(uri), new ByteArrayContent(contentType, partData)); - httpRequest.getHeaders().set("Date", date); - httpRequest.getHeaders().setAuthorization(authHeader); - httpRequest.getHeaders().setContentType(contentType); - httpRequest.getHeaders().setContentMD5(contentMd5); - httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - for (Map.Entry entry : extensionHeaders.entrySet()) { - httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); - } - httpRequest.setThrowExceptionOnExecuteError(false); - HttpResponse response = httpRequest.execute(); + HttpResponse response = + httpRequestManager.sendUploadPartRequest( + uri, + partData, + date, + authHeader, + contentType, + contentMd5, + crc32cString, + extensionHeaders); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -197,19 +184,16 @@ public CompleteMultipartUploadResponse completeMultipartUpload( "POST", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpRequest httpRequest = - requestFactory.buildPostRequest( - new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); - httpRequest.getHeaders().set("Date", date); - httpRequest.getHeaders().setAuthorization(authHeader); - httpRequest.getHeaders().setContentType(contentType); - httpRequest.getHeaders().setContentMD5(contentMd5); - httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - for (Map.Entry entry : extensionHeaders.entrySet()) { - httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); - } - httpRequest.setThrowExceptionOnExecuteError(false); - HttpResponse response = httpRequest.execute(); + HttpResponse response = + httpRequestManager.sendCompleteMultipartUploadRequest( + uri, + xmlBodyBytes, + date, + authHeader, + contentType, + contentMd5, + crc32cString, + extensionHeaders); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -237,15 +221,9 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; - HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); - httpRequest.getHeaders().set("Date", date); - httpRequest.getHeaders().setAuthorization(authHeader); - httpRequest.getHeaders().setContentType(contentType); - for (Map.Entry entry : extensionHeaders.entrySet()) { - httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); - } - httpRequest.setThrowExceptionOnExecuteError(false); - HttpResponse response = httpRequest.execute(); + HttpResponse response = + httpRequestManager.sendAbortMultipartUploadRequest( + uri, date, authHeader, contentType, extensionHeaders); if (response.getStatusCode() != 204) { String error = response.parseAsString(); From 0f5eb9ce17ee641c70661bc8584729ca114907a1 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 29 Sep 2025 05:19:03 +0000 Subject: [PATCH 07/27] Added support for ADC credentials --- .../storage/MultipartUploadClientImpl.java | 56 +++++++++---------- .../cloud/storage/MultipartUploadUtility.java | 52 ----------------- 2 files changed, 25 insertions(+), 83 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 54181f4299..24718e24de 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -16,11 +16,12 @@ package com.google.cloud.storage; import static com.google.cloud.storage.MultipartUploadUtility.getRfc1123Date; -import static com.google.cloud.storage.MultipartUploadUtility.signRequest; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; @@ -40,21 +41,26 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.Map; public class MultipartUploadClientImpl extends MultipartUploadClient { - // Add HMAC keys from GCS Settings > Interoperability - - - // --- End Configuration --- private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; private final HttpRequestManager httpRequestManager; + private final GoogleCredentials credentials; public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { this.httpRequestManager = new HttpRequestManager(requestFactory); + try { + this.credentials = + GoogleCredentials.getApplicationDefault() + .createScoped(Collections.singleton("https://www.googleapis.com/auth/devstorage.read_write")); + } catch (IOException e) { + throw new RuntimeException("Failed to get application default credentials", e); + } } private Map getExtensionHeader() { @@ -75,17 +81,10 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String date = getRfc1123Date(); String contentType = "application/x-www-form-urlencoded"; Map extensionHeaders = getExtensionHeader(); - // GCS Signature Rule #1: The '?uploads' query string IS included for the initiate request. - String signature = - signRequest( - "POST", - "", - contentType, - date, - extensionHeaders, - resourcePath + "?uploads", - GOOGLE_SECRET_KEY); - String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); HttpResponse response = httpRequestManager.sendCreateMultipartUploadRequest( @@ -111,7 +110,6 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String uri = GCS_ENDPOINT + resourcePath + queryString; String date = getRfc1123Date(); String contentType = "application/octet-stream"; - // GCS Signature Rule #2: The query string IS NOT included for the PUT part request. MessageDigest md = MessageDigest.getInstance("MD5"); byte[] partData = requestBody.getPartData(); String contentMd5 = Base64.getEncoder().encodeToString(md.digest(partData)); @@ -123,11 +121,10 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ .array()); Map extensionHeaders = getExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - String signature = - signRequest( - "PUT", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); - String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); HttpResponse response = httpRequestManager.sendUploadPartRequest( @@ -176,13 +173,12 @@ public CompleteMultipartUploadResponse completeMultipartUpload( .putInt(Hashing.crc32c().hashBytes(xmlBodyBytes).asInt()) .array()); - // GCS Signature Rule #3: The query string IS NOT included for the POST complete request. Map extensionHeaders = getExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - String signature = - signRequest( - "POST", contentMd5, contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); - String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); HttpResponse response = httpRequestManager.sendCompleteMultipartUploadRequest( @@ -215,11 +211,9 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String contentType = "application/x-www-form-urlencoded"; Map extensionHeaders = getExtensionHeader(); - // GCS Signature Rule #4: The query string IS NOT included for the DELETE abort request. - String signature = - signRequest("DELETE", "", contentType, date, extensionHeaders, resourcePath, GOOGLE_SECRET_KEY); - - String authHeader = "GOOG1 " + GOOGLE_ACCESS_KEY + ":" + signature; + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); HttpResponse response = httpRequestManager.sendAbortMultipartUploadRequest( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java index 4dd696f612..68288fb9d0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadUtility.java @@ -21,17 +21,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Base64; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.stream.Collectors; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; public class MultipartUploadUtility { public static String readStream(InputStream inputStream) throws IOException { @@ -46,50 +38,6 @@ public static String readStream(InputStream inputStream) throws IOException { return response.toString(); } - public static String signRequest( - String httpVerb, - String contentMd5, - String contentType, - String date, - Map extensionHeaders, - String canonicalizedResource, - String googleSecretKey) { - try { - String canonicalizedExtensionHeaders = ""; - if (extensionHeaders != null && !extensionHeaders.isEmpty()) { - SortedMap sortedHeaders = new TreeMap<>(); - for (Map.Entry entry : extensionHeaders.entrySet()) { - sortedHeaders.put(entry.getKey().toLowerCase(), entry.getValue()); - } - canonicalizedExtensionHeaders = - sortedHeaders.entrySet().stream() - .map(entry -> entry.getKey() + ":" + entry.getValue().trim().replaceAll("\\s+", " ")) - .collect(Collectors.joining("\n")) - + "\n"; - } - String stringToSign = - httpVerb - + "\n" - + (contentMd5 == null ? "" : contentMd5) - + "\n" - + (contentType == null ? "" : contentType) - + "\n" - + date - + "\n" - + canonicalizedExtensionHeaders - + canonicalizedResource; - System.out.println(stringToSign); - Mac sha1Hmac = Mac.getInstance("HmacSHA1"); - SecretKeySpec secretKey = - new SecretKeySpec(googleSecretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA1"); - sha1Hmac.init(secretKey); - byte[] signatureBytes = sha1Hmac.doFinal(stringToSign.getBytes(StandardCharsets.UTF_8)); - return Base64.getEncoder().encodeToString(signatureBytes); - } catch (Exception e) { - throw new RuntimeException("Failed to sign request", e); - } - } - public static byte[] readPart(File file, long position, int size) throws IOException { byte[] buffer = new byte[size]; try (FileInputStream fis = new FileInputStream(file)) { From ab9794c54d09b69777fdfef2d3e7f24906f253d5 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 29 Sep 2025 06:29:58 +0000 Subject: [PATCH 08/27] Refactored XmlMapper out --- .../com/google/cloud/storage/MultipartUploadClientImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 24718e24de..1f836c4d4a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -51,9 +51,11 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { private final HttpRequestManager httpRequestManager; private final GoogleCredentials credentials; + private final XmlMapper xmlMapper; public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { this.httpRequestManager = new HttpRequestManager(requestFactory); + this.xmlMapper = new XmlMapper(); try { this.credentials = GoogleCredentials.getApplicationDefault() @@ -96,7 +98,6 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload "Failed to initiate upload: " + response.getStatusCode() + " " + error); } - XmlMapper xmlMapper = new XmlMapper(); return xmlMapper.readValue(response.getContent(), CreateMultipartUploadResponse.class); } @@ -159,7 +160,6 @@ public CompleteMultipartUploadResponse completeMultipartUpload( String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; - XmlMapper xmlMapper = new XmlMapper(); byte[] xmlBodyBytes = xmlMapper.writeValueAsBytes(request.multipartUpload()); MessageDigest md = MessageDigest.getInstance("MD5"); From ff37829f54b0ed3171cd080f50dd716ae2395d38 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 29 Sep 2025 07:11:47 +0000 Subject: [PATCH 09/27] Removed hard coded headers --- .../cloud/storage/MultipartUploadClient.java | 3 ++- .../cloud/storage/MultipartUploadClientImpl.java | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index b5d9b7b436..e3ebf21ca9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -54,6 +54,7 @@ public static MultipartUploadClient create(MultipartUploadSettings config) { return new MultipartUploadClientImpl( URI.create(options.getHost()), options.getStorageRpcV1().getStorage().getRequestFactory(), - options.createRetrier()); + options.createRetrier(), + options); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 1f836c4d4a..1b15143cbd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -52,10 +52,13 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { private final HttpRequestManager httpRequestManager; private final GoogleCredentials credentials; private final XmlMapper xmlMapper; + private final HttpStorageOptions options; - public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Retrier retrier) { + public MultipartUploadClientImpl( + URI uri, HttpRequestFactory requestFactory, Retrier retrier, HttpStorageOptions options) { this.httpRequestManager = new HttpRequestManager(requestFactory); this.xmlMapper = new XmlMapper(); + this.options = options; try { this.credentials = GoogleCredentials.getApplicationDefault() @@ -67,10 +70,12 @@ public MultipartUploadClientImpl(URI uri, HttpRequestFactory requestFactory, Ret private Map getExtensionHeader() { Map extensionHeaders = new HashMap<>(); - extensionHeaders.put( - "x-goog-api-client", - "gl-java/11.0.27__OpenLogic-OpenJDK__OpenLogic-OpenJDK gccl/2.56.1-SNAPSHOT--protobuf-3.25.8 gax/2.70.0 protobuf/3.25.8"); - extensionHeaders.put("x-goog-user-project", "aipp-internal-testing"); + if (options.getClientLibToken() != null) { + extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); + } + if (options.getProjectId() != null) { + extensionHeaders.put("x-goog-user-project", options.getProjectId()); + } return extensionHeaders; } From 45c0a242f3c502ac395fabc287e84158b7a640de Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 29 Sep 2025 08:54:45 +0000 Subject: [PATCH 10/27] Added list API for Java XML MPU --- .../model/ListPartsRequest.java | 135 ++++++++++++++++++ .../model/ListPartsResponse.java | 122 ++++++++++++++++ .../storage/multipartupload/model/Part.java | 82 +++++++++++ 3 files changed, 339 insertions(+) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsRequest.java new file mode 100644 index 0000000000..0e05338665 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsRequest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public class ListPartsRequest { + private final String bucket; + + private final String key; + + private final String uploadId; + + private final Integer maxParts; + + private final Integer partNumberMarker; + + private ListPartsRequest(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; + this.uploadId = builder.uploadId; + this.maxParts = builder.maxParts; + this.partNumberMarker = builder.partNumberMarker; + } + + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + + public String uploadId() { + return uploadId; + } + + public Integer getMaxParts() { + return maxParts; + } + + public Integer getPartNumberMarker() { + return partNumberMarker; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ListPartsRequest)) { + return false; + } + ListPartsRequest that = (ListPartsRequest) o; + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(uploadId, that.uploadId) + && Objects.equals(maxParts, that.maxParts) + && Objects.equals(partNumberMarker, that.partNumberMarker); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, key, uploadId, maxParts, partNumberMarker); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("uploadId", uploadId) + .add("maxParts", maxParts) + .add("partNumberMarker", partNumberMarker) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String bucket; + private String key; + private String uploadId; + private Integer maxParts; + private Integer partNumberMarker; + + private Builder() {} + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder key(String key) { + this.key = key; + return this; + } + + public Builder uploadId(String uploadId) { + this.uploadId = uploadId; + return this; + } + + public Builder maxParts(Integer maxParts) { + this.maxParts = maxParts; + return this; + } + + public Builder partNumberMarker(Integer partNumberMarker) { + this.partNumberMarker = partNumberMarker; + return this; + } + + public ListPartsRequest build() { + return new ListPartsRequest(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java new file mode 100644 index 0000000000..16eeb8007e --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java @@ -0,0 +1,122 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import java.util.Objects; + +public final class ListPartsResponse { + + @JacksonXmlProperty(localName = "Bucket") + private String bucket; + + @JacksonXmlProperty(localName = "Key") + private String key; + + @JacksonXmlProperty(localName = "UploadId") + private String uploadId; + + @JacksonXmlProperty(localName = "IsTruncated") + private boolean isTruncated; + + @JacksonXmlProperty(localName = "NextPartNumberMarker") + private String nextPartNumberMarker; + + @JacksonXmlProperty(localName = "Owner") + private String owner; + + @JacksonXmlProperty(localName = "StorageClass") + private String storageClass; + + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "Part") + private List parts; + + public String getBucket() { + return bucket; + } + + public String getKey() { + return key; + } + + public String getUploadId() { + return uploadId; + } + + public boolean isTruncated() { + return isTruncated; + } + + public String getNextPartNumberMarker() { + return nextPartNumberMarker; + } + + public String getOwner() { + return owner; + } + + public String getStorageClass() { + return storageClass; + } + + public List getParts() { + return parts; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ListPartsResponse)) { + return false; + } + ListPartsResponse that = (ListPartsResponse) o; + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(uploadId, that.uploadId) + && Objects.equals(isTruncated, that.isTruncated) + && Objects.equals(nextPartNumberMarker, that.nextPartNumberMarker) + && Objects.equals(owner, that.owner) + && Objects.equals(storageClass, that.storageClass) + && Objects.equals(parts, that.parts); + } + + @Override + public int hashCode() { + return Objects.hash( + bucket, key, uploadId, isTruncated, nextPartNumberMarker, owner, storageClass, parts); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("uploadId", uploadId) + .add("isTruncated", isTruncated) + .add("nextPartNumberMarker", nextPartNumberMarker) + .add("owner", owner) + .add("storageClass", storageClass) + .add("parts", parts) + .toString(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java new file mode 100644 index 0000000000..e945a9df7f --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java @@ -0,0 +1,82 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public final class Part { + + @JacksonXmlProperty(localName = "PartNumber") + private int partNumber; + + @JacksonXmlProperty(localName = "ETag") + private String eTag; + + @JacksonXmlProperty(localName = "Size") + private long size; + + @JacksonXmlProperty(localName = "LastModified") + private String lastModified; + + public int getPartNumber() { + return partNumber; + } + + public String getETag() { + return eTag; + } + + public long getSize() { + return size; + } + + public String getLastModified() { + return lastModified; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Part)) { + return false; + } + Part that = (Part) o; + return Objects.equals(partNumber, that.partNumber) + && Objects.equals(eTag, that.eTag) + && Objects.equals(size, that.size) + && Objects.equals(lastModified, that.lastModified); + } + + @Override + public int hashCode() { + return Objects.hash(partNumber, eTag, size, lastModified); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("partNumber", partNumber) + .add("eTag", eTag) + .add("size", size) + .add("lastModified", lastModified) + .toString(); + } +} From bb840c08519e6b052459f6889b027adb18968519 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 29 Sep 2025 08:55:04 +0000 Subject: [PATCH 11/27] Added list API for Java XML MPU --- .../cloud/storage/HttpRequestManager.java | 13 ++++++ .../cloud/storage/MultipartUploadClient.java | 4 ++ .../storage/MultipartUploadClientImpl.java | 34 ++++++++++++++ .../model/ListPartsResponse.java | 45 +++++++++++++++---- 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 4b89987055..0788a91501 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -110,4 +110,17 @@ public HttpResponse sendAbortMultipartUploadRequest( httpRequest.setThrowExceptionOnExecuteError(false); return httpRequest.execute(); } + + public HttpResponse sendListPartsRequest( + String uri, String date, String authHeader, Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = requestFactory.buildGetRequest(new GenericUrl(uri)); + httpRequest.getHeaders().set("Date", date); + httpRequest.getHeaders().setAuthorization(authHeader); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index e3ebf21ca9..4a6d8c93f2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -24,6 +24,8 @@ import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.ListPartsRequest; +import com.google.cloud.storage.multipartupload.model.ListPartsResponse; import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; import java.io.IOException; @@ -49,6 +51,8 @@ public abstract CompleteMultipartUploadResponse completeMultipartUpload( public abstract AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) throws IOException, NoSuchAlgorithmException; + public abstract ListPartsResponse listParts(ListPartsRequest listPartsRequest) throws IOException; + public static MultipartUploadClient create(MultipartUploadSettings config) { HttpStorageOptions options = config.getOptions(); return new MultipartUploadClientImpl( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 1b15143cbd..8fad445d8f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -29,6 +29,8 @@ import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.ListPartsRequest; +import com.google.cloud.storage.multipartupload.model.ListPartsResponse; import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; import com.google.common.hash.Hashing; @@ -232,6 +234,38 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq return new AbortMultipartUploadResponse(); } + @Override + public ListPartsResponse listParts(ListPartsRequest request) throws IOException { + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String queryString = "?uploadId=" + encode(request.uploadId()); + if (request.getMaxParts() != null) { + queryString += "&max-parts=" + request.getMaxParts(); + } + if (request.getPartNumberMarker() != null) { + queryString += "&part-number-marker=" + request.getPartNumberMarker(); + } + String uri = GCS_ENDPOINT + resourcePath + queryString; + String date = getRfc1123Date(); + Map extensionHeaders = getExtensionHeader(); + + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); + + HttpResponse response = + httpRequestManager.sendListPartsRequest(uri, date, authHeader, extensionHeaders); + + if (!response.isSuccessStatusCode()) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to list parts: " + response.getStatusCode() + " " + error); + } + + return xmlMapper.readValue(response.getContent(), ListPartsResponse.class); + } + private String encode(String value) throws UnsupportedEncodingException { return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java index 16eeb8007e..feca04a03c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java @@ -33,11 +33,17 @@ public final class ListPartsResponse { @JacksonXmlProperty(localName = "UploadId") private String uploadId; - @JacksonXmlProperty(localName = "IsTruncated") - private boolean isTruncated; + @JacksonXmlProperty(localName = "PartNumberMarker") + private Integer partNumberMarker; @JacksonXmlProperty(localName = "NextPartNumberMarker") - private String nextPartNumberMarker; + private Integer nextPartNumberMarker; + + @JacksonXmlProperty(localName = "MaxParts") + private Integer maxParts; + + @JacksonXmlProperty(localName = "IsTruncated") + private boolean isTruncated; @JacksonXmlProperty(localName = "Owner") private String owner; @@ -61,14 +67,22 @@ public String getUploadId() { return uploadId; } - public boolean isTruncated() { - return isTruncated; + public Integer getPartNumberMarker() { + return partNumberMarker; } - public String getNextPartNumberMarker() { + public Integer getNextPartNumberMarker() { return nextPartNumberMarker; } + public Integer getMaxParts() { + return maxParts; + } + + public boolean isTruncated() { + return isTruncated; + } + public String getOwner() { return owner; } @@ -93,8 +107,10 @@ public boolean equals(Object o) { return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key) && Objects.equals(uploadId, that.uploadId) - && Objects.equals(isTruncated, that.isTruncated) + && Objects.equals(partNumberMarker, that.partNumberMarker) && Objects.equals(nextPartNumberMarker, that.nextPartNumberMarker) + && Objects.equals(maxParts, that.maxParts) + && Objects.equals(isTruncated, that.isTruncated) && Objects.equals(owner, that.owner) && Objects.equals(storageClass, that.storageClass) && Objects.equals(parts, that.parts); @@ -103,7 +119,16 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - bucket, key, uploadId, isTruncated, nextPartNumberMarker, owner, storageClass, parts); + bucket, + key, + uploadId, + partNumberMarker, + nextPartNumberMarker, + maxParts, + isTruncated, + owner, + storageClass, + parts); } @Override @@ -112,8 +137,10 @@ public String toString() { .add("bucket", bucket) .add("key", key) .add("uploadId", uploadId) - .add("isTruncated", isTruncated) + .add("partNumberMarker", partNumberMarker) .add("nextPartNumberMarker", nextPartNumberMarker) + .add("maxParts", maxParts) + .add("isTruncated", isTruncated) .add("owner", owner) .add("storageClass", storageClass) .add("parts", parts) From f467ab95ec0b451d786d122a04271969c2b98c52 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 30 Sep 2025 14:36:00 +0000 Subject: [PATCH 12/27] Added pending paramteres for CreateMultipart upload --- .../cloud/storage/HttpRequestManager.java | 18 ++- .../storage/MultipartUploadClientImpl.java | 25 +++- .../model/CreateMultipartUploadRequest.java | 114 +++++++++++++++++- 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 0788a91501..3aedb028f1 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -32,7 +32,14 @@ public HttpRequestManager(HttpRequestFactory requestFactory) { } public HttpResponse sendCreateMultipartUploadRequest( - String uri, String date, String authHeader, String contentType, Map extensionHeaders) + String uri, + String date, + String authHeader, + String contentType, + String contentDisposition, + String contentEncoding, + String contentLanguage, + Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildPostRequest( @@ -40,6 +47,15 @@ public HttpResponse sendCreateMultipartUploadRequest( httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); + if (contentDisposition != null) { + httpRequest.getHeaders().set("Content-Disposition", contentDisposition); + } + if (contentEncoding != null) { + httpRequest.getHeaders().setContentEncoding(contentEncoding); + } + if (contentLanguage != null) { + httpRequest.getHeaders().set("Content-Language", contentLanguage); + } for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 8fad445d8f..38edf78dca 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -88,8 +88,22 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String resourcePath = "/" + encodedBucket + "/" + encodedKey; String uri = GCS_ENDPOINT + resourcePath + "?uploads"; String date = getRfc1123Date(); - String contentType = "application/x-www-form-urlencoded"; + String contentType = + request.getContentType() == null + ? "application/x-www-form-urlencoded" + : request.getContentType(); Map extensionHeaders = getExtensionHeader(); + if (request.getCannedAcl() != null) { + extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); + } + if (request.getMetadata() != null) { + for (Map.Entry entry : request.getMetadata().entrySet()) { + extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); + } + } + if (request.getStorageClass() != null) { + extensionHeaders.put("x-goog-storage-class", request.getStorageClass()); + } credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); @@ -97,7 +111,14 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload HttpResponse response = httpRequestManager.sendCreateMultipartUploadRequest( - uri, date, authHeader, contentType, extensionHeaders); + uri, + date, + authHeader, + contentType, + request.getContentDisposition(), + request.getContentEncoding(), + request.getContentLanguage(), + extensionHeaders); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 15abcf2522..f53c532a3c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -16,17 +16,33 @@ package com.google.cloud.storage.multipartupload.model; +import com.google.cloud.storage.Storage.PredefinedAcl; import com.google.common.base.MoreObjects; +import java.util.Map; import java.util.Objects; public class CreateMultipartUploadRequest { private final String bucket; private final String key; + private final PredefinedAcl cannedAcl; + private final String contentDisposition; + private final String contentEncoding; + private final String contentLanguage; + private final String contentType; + private final Map metadata; + private final String storageClass; private CreateMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; this.key = builder.key; + this.cannedAcl = builder.cannedAcl; + this.contentDisposition = builder.contentDisposition; + this.contentEncoding = builder.contentEncoding; + this.contentLanguage = builder.contentLanguage; + this.contentType = builder.contentType; + this.metadata = builder.metadata; + this.storageClass = builder.storageClass; } public String bucket() { @@ -37,6 +53,34 @@ public String key() { return key; } + public PredefinedAcl getCannedAcl() { + return cannedAcl; + } + + public String getContentDisposition() { + return contentDisposition; + } + + public String getContentEncoding() { + return contentEncoding; + } + + public String getContentLanguage() { + return contentLanguage; + } + + public String getContentType() { + return contentType; + } + + public Map getMetadata() { + return metadata; + } + + public String getStorageClass() { + return storageClass; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -46,12 +90,29 @@ public boolean equals(Object o) { return false; } CreateMultipartUploadRequest that = (CreateMultipartUploadRequest) o; - return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key); + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && cannedAcl == that.cannedAcl + && Objects.equals(contentDisposition, that.contentDisposition) + && Objects.equals(contentEncoding, that.contentEncoding) + && Objects.equals(contentLanguage, that.contentLanguage) + && Objects.equals(contentType, that.contentType) + && Objects.equals(metadata, that.metadata) + && Objects.equals(storageClass, that.storageClass); } @Override public int hashCode() { - return Objects.hash(bucket, key); + return Objects.hash( + bucket, + key, + cannedAcl, + contentDisposition, + contentEncoding, + contentLanguage, + contentType, + metadata, + storageClass); } @Override @@ -59,6 +120,13 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("bucket", bucket) .add("key", key) + .add("cannedAcl", cannedAcl) + .add("contentDisposition", contentDisposition) + .add("contentEncoding", contentEncoding) + .add("contentLanguage", contentLanguage) + .add("contentType", contentType) + .add("metadata", metadata) + .add("storageClass", storageClass) .toString(); } @@ -69,6 +137,13 @@ public static Builder builder() { public static class Builder { private String bucket; private String key; + private PredefinedAcl cannedAcl; + private String contentDisposition; + private String contentEncoding; + private String contentLanguage; + private String contentType; + private Map metadata; + private String storageClass; private Builder() {} @@ -82,6 +157,41 @@ public Builder key(String key) { return this; } + public Builder cannedAcl(PredefinedAcl cannedAcl) { + this.cannedAcl = cannedAcl; + return this; + } + + public Builder contentDisposition(String contentDisposition) { + this.contentDisposition = contentDisposition; + return this; + } + + public Builder contentEncoding(String contentEncoding) { + this.contentEncoding = contentEncoding; + return this; + } + + public Builder contentLanguage(String contentLanguage) { + this.contentLanguage = contentLanguage; + return this; + } + + public Builder contentType(String contentType) { + this.contentType = contentType; + return this; + } + + public Builder metadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public Builder storageClass(String storageClass) { + this.storageClass = storageClass; + return this; + } + public CreateMultipartUploadRequest build() { return new CreateMultipartUploadRequest(this); } From 9971089cbc437c00eb3980d4f94d5c2da024be11 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 30 Sep 2025 17:26:34 +0000 Subject: [PATCH 13/27] Added remaining fields for CreateMultipartUpload Response --- .../model/CreateMultipartUploadResponse.java | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java index 976fb99385..2aa6400aea 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -23,18 +23,33 @@ import java.util.Objects; @JacksonXmlRootElement(localName = "InitiateMultipartUploadResult") -@JsonIgnoreProperties(ignoreUnknown = true) public class CreateMultipartUploadResponse { + @JacksonXmlProperty(localName = "Bucket") + private String bucket; + + @JacksonXmlProperty(localName = "Key") + private String key; + @JacksonXmlProperty(localName = "UploadId") private String uploadId; private CreateMultipartUploadResponse(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; this.uploadId = builder.uploadId; } private CreateMultipartUploadResponse() {} + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + public String uploadId() { return uploadId; } @@ -48,17 +63,23 @@ public boolean equals(Object o) { return false; } CreateMultipartUploadResponse that = (CreateMultipartUploadResponse) o; - return Objects.equals(uploadId, that.uploadId); + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(uploadId, that.uploadId); } @Override public int hashCode() { - return Objects.hash(uploadId); + return Objects.hash(bucket, key, uploadId); } @Override public String toString() { - return MoreObjects.toStringHelper(this).add("uploadId", uploadId).toString(); + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("uploadId", uploadId) + .toString(); } public static Builder builder() { @@ -66,10 +87,22 @@ public static Builder builder() { } public static class Builder { + private String bucket; + private String key; private String uploadId; private Builder() {} + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder key(String key) { + this.key = key; + return this; + } + public Builder uploadId(String uploadId) { this.uploadId = uploadId; return this; From 7672ca1e118f3bb5741da18ef4805af3f54b17d0 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 03:01:51 +0000 Subject: [PATCH 14/27] Added Complete Multipart request and response --- .../storage/MultipartUploadClientImpl.java | 6 + .../model/CompleteMultipartUploadRequest.java | 32 ++++- .../CompleteMultipartUploadResponse.java | 109 +++++++++++++++--- 3 files changed, 129 insertions(+), 18 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 38edf78dca..232c6b944c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -203,6 +203,12 @@ public CompleteMultipartUploadResponse completeMultipartUpload( Map extensionHeaders = getExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); + if (request.requestPayer() != null) { + extensionHeaders.put("x-amz-request-payer", request.requestPayer()); + } + if (request.expectedBucketOwner() != null) { + extensionHeaders.put("x-amz-expected-bucket-owner", request.expectedBucketOwner()); + } credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java index 5f2955548a..7b08c7f553 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java @@ -25,12 +25,16 @@ public final class CompleteMultipartUploadRequest { private final String key; private final String uploadId; private final CompletedMultipartUpload multipartUpload; + private final String requestPayer; + private final String expectedBucketOwner; private CompleteMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; this.key = builder.key; this.uploadId = builder.uploadId; this.multipartUpload = builder.multipartUpload; + this.requestPayer = builder.requestPayer; + this.expectedBucketOwner = builder.expectedBucketOwner; } public String bucket() { @@ -49,6 +53,14 @@ public CompletedMultipartUpload multipartUpload() { return multipartUpload; } + public String requestPayer() { + return requestPayer; + } + + public String expectedBucketOwner() { + return expectedBucketOwner; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -61,12 +73,14 @@ public boolean equals(Object o) { return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key) && Objects.equals(uploadId, that.uploadId) - && Objects.equals(multipartUpload, that.multipartUpload); + && Objects.equals(multipartUpload, that.multipartUpload) + && Objects.equals(requestPayer, that.requestPayer) + && Objects.equals(expectedBucketOwner, that.expectedBucketOwner); } @Override public int hashCode() { - return Objects.hash(bucket, key, uploadId, multipartUpload); + return Objects.hash(bucket, key, uploadId, multipartUpload, requestPayer, expectedBucketOwner); } @Override @@ -76,6 +90,8 @@ public String toString() { .add("key", key) .add("uploadId", uploadId) .add("completedMultipartUpload", multipartUpload) + .add("requestPayer", requestPayer) + .add("expectedBucketOwner", expectedBucketOwner) .toString(); } @@ -88,6 +104,8 @@ public static class Builder { private String key; private String uploadId; private CompletedMultipartUpload multipartUpload; + private String requestPayer; + private String expectedBucketOwner; private Builder() {} @@ -111,6 +129,16 @@ public Builder multipartUpload(CompletedMultipartUpload completedMultipartUpload return this; } + public Builder requestPayer(String requestPayer) { + this.requestPayer = requestPayer; + return this; + } + + public Builder expectedBucketOwner(String expectedBucketOwner) { + this.expectedBucketOwner = expectedBucketOwner; + return this; + } + public CompleteMultipartUploadRequest build() { return new CompleteMultipartUploadRequest(this); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java index c17ad31f7f..6a73911135 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java @@ -17,34 +17,111 @@ package com.google.cloud.storage.multipartupload.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.google.common.base.MoreObjects; +import java.util.Objects; -public class CompleteMultipartUploadResponse { +@JsonDeserialize(builder = CompleteMultipartUploadResponse.Builder.class) +public final class CompleteMultipartUploadResponse { - @JsonProperty("Location") - private String location; + private final String location; + private final String bucket; + private final String key; + private final String etag; - @JsonProperty("Bucket") - private String bucket; + private CompleteMultipartUploadResponse(Builder builder) { + this.location = builder.location; + this.bucket = builder.bucket; + this.key = builder.key; + this.etag = builder.etag; + } - @JsonProperty("Key") - private String key; + public String getLocation() { + return location; + } - @JsonProperty("ETag") + public String getBucket() { + return bucket; + } + + public String getKey() { + return key; + } + + public String getEtag() { + return etag; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CompleteMultipartUploadResponse)) { + return false; + } + CompleteMultipartUploadResponse that = (CompleteMultipartUploadResponse) o; + return Objects.equals(location, that.location) + && Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(etag, that.etag); + } + + @Override + public int hashCode() { + return Objects.hash(location, bucket, key, etag); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("location", location) + .add("bucket", bucket) + .add("key", key) + .add("etag", etag) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") + public static class Builder { + private String location; + private String bucket; + private String key; private String etag; - public String getLocation() { - return location; + private Builder() {} + + @JsonProperty("Location") + public Builder setLocation(String location) { + this.location = location; + return this; + } + + @JsonProperty("Bucket") + public Builder setBucket(String bucket) { + this.bucket = bucket; + return this; } - public String getBucket() { - return bucket; + @JsonProperty("Key") + public Builder setKey(String key) { + this.key = key; + return this; } - public String getKey() { - return key; + @JsonProperty("ETag") + public Builder setEtag(String etag) { + this.etag = etag; + return this; } - public String getEtag() { - return etag; + public CompleteMultipartUploadResponse build() { + return new CompleteMultipartUploadResponse(this); } + } } From d11a9d6880e54b57555ca6faa556e59c9fc81070 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 05:22:57 +0000 Subject: [PATCH 15/27] Added remainig parameters for CreateMultipart Upload --- .../cloud/storage/HttpRequestManager.java | 30 +++---- .../storage/MultipartUploadClientImpl.java | 86 +++++++++++-------- .../model/CreateMultipartUploadRequest.java | 79 ++++++++++++++++- 3 files changed, 137 insertions(+), 58 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 3aedb028f1..8cd716f293 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -15,11 +15,14 @@ */ package com.google.cloud.storage; +import static com.google.cloud.storage.MultipartUploadUtility.getRfc1123Date; + import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import java.io.IOException; import java.util.Map; @@ -33,28 +36,21 @@ public HttpRequestManager(HttpRequestFactory requestFactory) { public HttpResponse sendCreateMultipartUploadRequest( String uri, - String date, String authHeader, String contentType, - String contentDisposition, - String contentEncoding, - String contentLanguage, + CreateMultipartUploadRequest request, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildPostRequest( new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); - httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); - if (contentDisposition != null) { - httpRequest.getHeaders().set("Content-Disposition", contentDisposition); - } - if (contentEncoding != null) { - httpRequest.getHeaders().setContentEncoding(contentEncoding); + if (request.getContentEncoding() != null && !request.getContentEncoding().isEmpty()) { + httpRequest.getHeaders().setContentEncoding(request.getContentEncoding()); } - if (contentLanguage != null) { - httpRequest.getHeaders().set("Content-Language", contentLanguage); + if (request.getCacheControl() != null && !request.getCacheControl().isEmpty()) { + httpRequest.getHeaders().setCacheControl(request.getCacheControl()); } for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); @@ -66,7 +62,6 @@ public HttpResponse sendCreateMultipartUploadRequest( public HttpResponse sendUploadPartRequest( String uri, byte[] partData, - String date, String authHeader, String contentType, String contentMd5, @@ -76,7 +71,6 @@ public HttpResponse sendUploadPartRequest( HttpRequest httpRequest = requestFactory.buildPutRequest( new GenericUrl(uri), new ByteArrayContent(contentType, partData)); - httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); httpRequest.getHeaders().setContentMD5(contentMd5); @@ -91,7 +85,6 @@ public HttpResponse sendUploadPartRequest( public HttpResponse sendCompleteMultipartUploadRequest( String uri, byte[] xmlBodyBytes, - String date, String authHeader, String contentType, String contentMd5, @@ -101,7 +94,6 @@ public HttpResponse sendCompleteMultipartUploadRequest( HttpRequest httpRequest = requestFactory.buildPostRequest( new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); - httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); httpRequest.getHeaders().setContentMD5(contentMd5); @@ -114,10 +106,9 @@ public HttpResponse sendCompleteMultipartUploadRequest( } public HttpResponse sendAbortMultipartUploadRequest( - String uri, String date, String authHeader, String contentType, Map extensionHeaders) + String uri, String authHeader, String contentType, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); - httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { @@ -128,10 +119,9 @@ public HttpResponse sendAbortMultipartUploadRequest( } public HttpResponse sendListPartsRequest( - String uri, String date, String authHeader, Map extensionHeaders) + String uri, String authHeader, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildGetRequest(new GenericUrl(uri)); - httpRequest.getHeaders().set("Date", date); httpRequest.getHeaders().setAuthorization(authHeader); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 232c6b944c..8902caf5d3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -78,6 +78,7 @@ private Map getExtensionHeader() { if (options.getProjectId() != null) { extensionHeaders.put("x-goog-user-project", options.getProjectId()); } + extensionHeaders.put("Date", getRfc1123Date()); return extensionHeaders; } @@ -87,38 +88,23 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String encodedKey = encode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; String uri = GCS_ENDPOINT + resourcePath + "?uploads"; - String date = getRfc1123Date(); - String contentType = - request.getContentType() == null - ? "application/x-www-form-urlencoded" - : request.getContentType(); - Map extensionHeaders = getExtensionHeader(); - if (request.getCannedAcl() != null) { - extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); - } - if (request.getMetadata() != null) { - for (Map.Entry entry : request.getMetadata().entrySet()) { - extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); - } - } - if (request.getStorageClass() != null) { - extensionHeaders.put("x-goog-storage-class", request.getStorageClass()); - } credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); String authHeader = "Bearer " + accessToken.getTokenValue(); + String contentType = + request.getContentType() == null + ? "application/x-www-form-urlencoded" + : request.getContentType(); + HttpResponse response = httpRequestManager.sendCreateMultipartUploadRequest( uri, - date, authHeader, contentType, - request.getContentDisposition(), - request.getContentEncoding(), - request.getContentLanguage(), - extensionHeaders); + request, + getExtensionHeadersForCreateMultipartUpload(request)); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -129,6 +115,46 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload return xmlMapper.readValue(response.getContent(), CreateMultipartUploadResponse.class); } + private Map getExtensionHeadersForCreateMultipartUpload(CreateMultipartUploadRequest request){ + Map extensionHeaders = getExtensionHeader(); + if (request.getContentDisposition() != null) { + extensionHeaders.put("Content-Disposition", request.getContentDisposition()); + } + if (request.getContentLanguage() != null && !request.getContentLanguage().isEmpty()) { + extensionHeaders.put("Content-Language", request.getContentLanguage()); + } + if (request.getCannedAcl() != null) { + extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); + } + if (request.getMetadata() != null) { + for (Map.Entry entry : request.getMetadata().entrySet()) { + if (entry.getKey() != null || entry.getValue() != null) { + extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); + } + } + } + if (request.getStorageClass() != null && !request.getStorageClass().isEmpty()) { + extensionHeaders.put("x-goog-storage-class", request.getStorageClass()); + } + if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { + extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); + } + // x-goog-object-lock-mode and x-goog-object-lock-retain-until-date should be specified together + // Refer: https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers + if (request.getObjectLockMode() != null + && !request.getObjectLockMode().isEmpty() + && request.getObjectLockRetainUntilDate() != null + && !request.getObjectLockRetainUntilDate().isEmpty()) { + extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode()); + extensionHeaders.put("x-goog-object-lock-retain-until-date", + request.getObjectLockRetainUntilDate()); + } + if (request.getCustomTime() != null && !request.getCustomTime().isEmpty()) { + extensionHeaders.put("x-goog-custom-time", request.getCustomTime()); + } + return extensionHeaders; + } + public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) throws IOException, NoSuchAlgorithmException { String encodedBucket = encode(request.bucket()); @@ -137,7 +163,6 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String queryString = "?partNumber=" + request.partNumber() + "&uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; - String date = getRfc1123Date(); String contentType = "application/octet-stream"; MessageDigest md = MessageDigest.getInstance("MD5"); byte[] partData = requestBody.getPartData(); @@ -159,7 +184,6 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ httpRequestManager.sendUploadPartRequest( uri, partData, - date, authHeader, contentType, contentMd5, @@ -192,7 +216,6 @@ public CompleteMultipartUploadResponse completeMultipartUpload( MessageDigest md = MessageDigest.getInstance("MD5"); String contentMd5 = Base64.getEncoder().encodeToString(md.digest(xmlBodyBytes)); - String date = getRfc1123Date(); String contentType = "application/xml"; String crc32cString = Base64.getEncoder() @@ -203,12 +226,6 @@ public CompleteMultipartUploadResponse completeMultipartUpload( Map extensionHeaders = getExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - if (request.requestPayer() != null) { - extensionHeaders.put("x-amz-request-payer", request.requestPayer()); - } - if (request.expectedBucketOwner() != null) { - extensionHeaders.put("x-amz-expected-bucket-owner", request.expectedBucketOwner()); - } credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); @@ -218,7 +235,6 @@ public CompleteMultipartUploadResponse completeMultipartUpload( httpRequestManager.sendCompleteMultipartUploadRequest( uri, xmlBodyBytes, - date, authHeader, contentType, contentMd5, @@ -241,7 +257,6 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String resourcePath = "/" + encodedBucket + "/" + encodedKey; String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; - String date = getRfc1123Date(); String contentType = "application/x-www-form-urlencoded"; Map extensionHeaders = getExtensionHeader(); @@ -251,7 +266,7 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq HttpResponse response = httpRequestManager.sendAbortMultipartUploadRequest( - uri, date, authHeader, contentType, extensionHeaders); + uri, authHeader, contentType, extensionHeaders); if (response.getStatusCode() != 204) { String error = response.parseAsString(); @@ -274,7 +289,6 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException queryString += "&part-number-marker=" + request.getPartNumberMarker(); } String uri = GCS_ENDPOINT + resourcePath + queryString; - String date = getRfc1123Date(); Map extensionHeaders = getExtensionHeader(); credentials.refreshIfExpired(); @@ -282,7 +296,7 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException String authHeader = "Bearer " + accessToken.getTokenValue(); HttpResponse response = - httpRequestManager.sendListPartsRequest(uri, date, authHeader, extensionHeaders); + httpRequestManager.sendListPartsRequest(uri, authHeader, extensionHeaders); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index f53c532a3c..58dc9c4dfe 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -32,6 +32,11 @@ public class CreateMultipartUploadRequest { private final String contentType; private final Map metadata; private final String storageClass; + private final String cacheControl; + private final String customTime; + private final String kmsKeyName; + private final String objectLockMode; + private final String objectLockRetainUntilDate; private CreateMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; @@ -43,6 +48,11 @@ private CreateMultipartUploadRequest(Builder builder) { this.contentType = builder.contentType; this.metadata = builder.metadata; this.storageClass = builder.storageClass; + this.cacheControl = builder.cacheControl; + this.customTime = builder.customTime; + this.kmsKeyName = builder.kmsKeyName; + this.objectLockMode = builder.objectLockMode; + this.objectLockRetainUntilDate = builder.objectLockRetainUntilDate; } public String bucket() { @@ -81,6 +91,26 @@ public String getStorageClass() { return storageClass; } + public String getCacheControl() { + return cacheControl; + } + + public String getCustomTime() { + return customTime; + } + + public String getKmsKeyName() { + return kmsKeyName; + } + + public String getObjectLockMode() { + return objectLockMode; + } + + public String getObjectLockRetainUntilDate() { + return objectLockRetainUntilDate; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -98,7 +128,12 @@ public boolean equals(Object o) { && Objects.equals(contentLanguage, that.contentLanguage) && Objects.equals(contentType, that.contentType) && Objects.equals(metadata, that.metadata) - && Objects.equals(storageClass, that.storageClass); + && Objects.equals(storageClass, that.storageClass) + && Objects.equals(cacheControl, that.cacheControl) + && Objects.equals(customTime, that.customTime) + && Objects.equals(kmsKeyName, that.kmsKeyName) + && Objects.equals(objectLockMode, that.objectLockMode) + && Objects.equals(objectLockRetainUntilDate, that.objectLockRetainUntilDate); } @Override @@ -112,7 +147,12 @@ public int hashCode() { contentLanguage, contentType, metadata, - storageClass); + storageClass, + cacheControl, + customTime, + kmsKeyName, + objectLockMode, + objectLockRetainUntilDate); } @Override @@ -127,6 +167,11 @@ public String toString() { .add("contentType", contentType) .add("metadata", metadata) .add("storageClass", storageClass) + .add("cacheControl", cacheControl) + .add("customTime", customTime) + .add("kmsKeyName", kmsKeyName) + .add("objectLockMode", objectLockMode) + .add("objectLockRetainUntilDate", objectLockRetainUntilDate) .toString(); } @@ -144,6 +189,11 @@ public static class Builder { private String contentType; private Map metadata; private String storageClass; + private String cacheControl; + private String customTime; + private String kmsKeyName; + private String objectLockMode; + private String objectLockRetainUntilDate; private Builder() {} @@ -192,6 +242,31 @@ public Builder storageClass(String storageClass) { return this; } + public Builder cacheControl(String cacheControl) { + this.cacheControl = cacheControl; + return this; + } + + public Builder customTime(String customTime) { + this.customTime = customTime; + return this; + } + + public Builder kmsKeyName(String kmsKeyName) { + this.kmsKeyName = kmsKeyName; + return this; + } + + public Builder objectLockMode(String objectLockMode) { + this.objectLockMode = objectLockMode; + return this; + } + + public Builder objectLockRetainUntilDate(String objectLockRetainUntilDate) { + this.objectLockRetainUntilDate = objectLockRetainUntilDate; + return this; + } + public CreateMultipartUploadRequest build() { return new CreateMultipartUploadRequest(this); } From a5434fc1dd96f4597c5655c4a4bf50d2115ba026 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 09:27:42 +0000 Subject: [PATCH 16/27] Add validations --- .../storage/MultipartUploadClientImpl.java | 46 ++--- .../google/cloud/storage/ObjectLockMode.java | 76 +++++++++ .../model/CreateMultipartUploadRequest.java | 161 +++++++++--------- 3 files changed, 182 insertions(+), 101 deletions(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 8902caf5d3..fc744fb178 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -42,10 +42,13 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; import java.util.Base64; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.TimeZone; public class MultipartUploadClientImpl extends MultipartUploadClient { @@ -70,7 +73,7 @@ public MultipartUploadClientImpl( } } - private Map getExtensionHeader() { + private Map getGenericExtensionHeader() { Map extensionHeaders = new HashMap<>(); if (options.getClientLibToken() != null) { extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); @@ -116,13 +119,7 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload } private Map getExtensionHeadersForCreateMultipartUpload(CreateMultipartUploadRequest request){ - Map extensionHeaders = getExtensionHeader(); - if (request.getContentDisposition() != null) { - extensionHeaders.put("Content-Disposition", request.getContentDisposition()); - } - if (request.getContentLanguage() != null && !request.getContentLanguage().isEmpty()) { - extensionHeaders.put("Content-Language", request.getContentLanguage()); - } + Map extensionHeaders = getGenericExtensionHeader(); if (request.getCannedAcl() != null) { extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); } @@ -133,24 +130,20 @@ private Map getExtensionHeadersForCreateMultipartUpload(CreateMu } } } - if (request.getStorageClass() != null && !request.getStorageClass().isEmpty()) { - extensionHeaders.put("x-goog-storage-class", request.getStorageClass()); + if (request.getStorageClass() != null) { + extensionHeaders.put("x-goog-storage-class", request.getStorageClass().toString()); } if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); } // x-goog-object-lock-mode and x-goog-object-lock-retain-until-date should be specified together // Refer: https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers - if (request.getObjectLockMode() != null - && !request.getObjectLockMode().isEmpty() - && request.getObjectLockRetainUntilDate() != null - && !request.getObjectLockRetainUntilDate().isEmpty()) { - extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode()); - extensionHeaders.put("x-goog-object-lock-retain-until-date", - request.getObjectLockRetainUntilDate()); + if (request.getObjectLockMode() != null && request.getObjectLockRetainUntilDate() != null) { + extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); + extensionHeaders.put("x-goog-object-lock-retain-until-date", toRfc3339String(request.getObjectLockRetainUntilDate())); } - if (request.getCustomTime() != null && !request.getCustomTime().isEmpty()) { - extensionHeaders.put("x-goog-custom-time", request.getCustomTime()); + if (request.getCustomTime() != null) { + extensionHeaders.put("x-goog-custom-time", toRfc3339String(request.getCustomTime())); } return extensionHeaders; } @@ -173,7 +166,7 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ ByteBuffer.allocate(4) .putInt(Hashing.crc32c().hashBytes(partData).asInt()) .array()); - Map extensionHeaders = getExtensionHeader(); + Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); credentials.refreshIfExpired(); @@ -224,7 +217,7 @@ public CompleteMultipartUploadResponse completeMultipartUpload( .putInt(Hashing.crc32c().hashBytes(xmlBodyBytes).asInt()) .array()); - Map extensionHeaders = getExtensionHeader(); + Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); credentials.refreshIfExpired(); @@ -258,7 +251,7 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String queryString = "?uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String contentType = "application/x-www-form-urlencoded"; - Map extensionHeaders = getExtensionHeader(); + Map extensionHeaders = getGenericExtensionHeader(); credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); @@ -289,7 +282,7 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException queryString += "&part-number-marker=" + request.getPartNumberMarker(); } String uri = GCS_ENDPOINT + resourcePath + queryString; - Map extensionHeaders = getExtensionHeader(); + Map extensionHeaders = getGenericExtensionHeader(); credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); @@ -310,4 +303,11 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException private String encode(String value) throws UnsupportedEncodingException { return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } + + private String toRfc3339String(Date date) { + TimeZone tz = TimeZone.getTimeZone("UTC"); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + df.setTimeZone(tz); + return df.format(date); + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java new file mode 100644 index 0000000000..c47582b97f --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java @@ -0,0 +1,76 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import com.google.api.core.ApiFunction; +import com.google.cloud.StringEnumType; +import com.google.cloud.StringEnumValue; + +/** + * Represents the object lock mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ +public final class ObjectLockMode extends StringEnumValue { + private static final long serialVersionUID = -1882734434792102329L; + + private ObjectLockMode(String constant) { + super(constant); + } + + private static final ApiFunction CONSTRUCTOR = + new ApiFunction() { + @Override + public ObjectLockMode apply(String constant) { + return new ObjectLockMode(constant); + } + }; + + private static final StringEnumType type = + new StringEnumType(ObjectLockMode.class, CONSTRUCTOR); + + /** + * Governance mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ + public static final ObjectLockMode GOVERNANCE = type.createAndRegister("GOVERNANCE"); + + /** + * Compliance mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ + public static final ObjectLockMode COMPLIANCE = type.createAndRegister("COMPLIANCE"); + + /** + * Get the ObjectLockMode for the given String constant, and throw an exception if the constant is + * not recognized. + */ + public static ObjectLockMode valueOfStrict(String constant) { + return type.valueOfStrict(constant); + } + + /** Get the ObjectLockMode for the given String constant, and allow unrecognized values. */ + public static ObjectLockMode valueOf(String constant) { + return type.valueOf(constant); + } + + /** Return the known values for ObjectLockMode. */ + public static ObjectLockMode[] values() { + return type.values(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 58dc9c4dfe..b415561cf8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -16,39 +16,33 @@ package com.google.cloud.storage.multipartupload.model; +import com.google.cloud.storage.ObjectLockMode; import com.google.cloud.storage.Storage.PredefinedAcl; +import com.google.cloud.storage.StorageClass; import com.google.common.base.MoreObjects; +import java.util.Date; import java.util.Map; import java.util.Objects; public class CreateMultipartUploadRequest { private final String bucket; - private final String key; private final PredefinedAcl cannedAcl; - private final String contentDisposition; - private final String contentEncoding; - private final String contentLanguage; private final String contentType; private final Map metadata; - private final String storageClass; - private final String cacheControl; - private final String customTime; + private final StorageClass storageClass; + private final Date customTime; private final String kmsKeyName; - private final String objectLockMode; - private final String objectLockRetainUntilDate; + private final ObjectLockMode objectLockMode; + private final Date objectLockRetainUntilDate; private CreateMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; this.key = builder.key; this.cannedAcl = builder.cannedAcl; - this.contentDisposition = builder.contentDisposition; - this.contentEncoding = builder.contentEncoding; - this.contentLanguage = builder.contentLanguage; this.contentType = builder.contentType; this.metadata = builder.metadata; this.storageClass = builder.storageClass; - this.cacheControl = builder.cacheControl; this.customTime = builder.customTime; this.kmsKeyName = builder.kmsKeyName; this.objectLockMode = builder.objectLockMode; @@ -67,18 +61,6 @@ public PredefinedAcl getCannedAcl() { return cannedAcl; } - public String getContentDisposition() { - return contentDisposition; - } - - public String getContentEncoding() { - return contentEncoding; - } - - public String getContentLanguage() { - return contentLanguage; - } - public String getContentType() { return contentType; } @@ -87,15 +69,11 @@ public Map getMetadata() { return metadata; } - public String getStorageClass() { + public StorageClass getStorageClass() { return storageClass; } - public String getCacheControl() { - return cacheControl; - } - - public String getCustomTime() { + public Date getCustomTime() { return customTime; } @@ -103,11 +81,11 @@ public String getKmsKeyName() { return kmsKeyName; } - public String getObjectLockMode() { + public ObjectLockMode getObjectLockMode() { return objectLockMode; } - public String getObjectLockRetainUntilDate() { + public Date getObjectLockRetainUntilDate() { return objectLockRetainUntilDate; } @@ -123,16 +101,12 @@ public boolean equals(Object o) { return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key) && cannedAcl == that.cannedAcl - && Objects.equals(contentDisposition, that.contentDisposition) - && Objects.equals(contentEncoding, that.contentEncoding) - && Objects.equals(contentLanguage, that.contentLanguage) && Objects.equals(contentType, that.contentType) && Objects.equals(metadata, that.metadata) && Objects.equals(storageClass, that.storageClass) - && Objects.equals(cacheControl, that.cacheControl) && Objects.equals(customTime, that.customTime) && Objects.equals(kmsKeyName, that.kmsKeyName) - && Objects.equals(objectLockMode, that.objectLockMode) + && objectLockMode == that.objectLockMode && Objects.equals(objectLockRetainUntilDate, that.objectLockRetainUntilDate); } @@ -142,13 +116,9 @@ public int hashCode() { bucket, key, cannedAcl, - contentDisposition, - contentEncoding, - contentLanguage, contentType, metadata, storageClass, - cacheControl, customTime, kmsKeyName, objectLockMode, @@ -161,13 +131,9 @@ public String toString() { .add("bucket", bucket) .add("key", key) .add("cannedAcl", cannedAcl) - .add("contentDisposition", contentDisposition) - .add("contentEncoding", contentEncoding) - .add("contentLanguage", contentLanguage) .add("contentType", contentType) .add("metadata", metadata) .add("storageClass", storageClass) - .add("cacheControl", cacheControl) .add("customTime", customTime) .add("kmsKeyName", kmsKeyName) .add("objectLockMode", objectLockMode) @@ -183,86 +149,125 @@ public static class Builder { private String bucket; private String key; private PredefinedAcl cannedAcl; - private String contentDisposition; - private String contentEncoding; - private String contentLanguage; private String contentType; private Map metadata; - private String storageClass; - private String cacheControl; - private String customTime; + private StorageClass storageClass; + private Date customTime; private String kmsKeyName; - private String objectLockMode; - private String objectLockRetainUntilDate; + private ObjectLockMode objectLockMode; + private Date objectLockRetainUntilDate; private Builder() {} + /** + * The bucket to which the object is being uploaded. + * + * @param bucket The bucket name + * @return this builder + */ public Builder bucket(String bucket) { this.bucket = bucket; return this; } + /** + * The name of the object. + * + * @param key The object name + * @return this builder + */ public Builder key(String key) { this.key = key; return this; } + /** + * A canned ACL to apply to the object. + * + * @param cannedAcl The canned ACL + * @return this builder + */ public Builder cannedAcl(PredefinedAcl cannedAcl) { this.cannedAcl = cannedAcl; return this; } - public Builder contentDisposition(String contentDisposition) { - this.contentDisposition = contentDisposition; - return this; - } - - public Builder contentEncoding(String contentEncoding) { - this.contentEncoding = contentEncoding; - return this; - } - - public Builder contentLanguage(String contentLanguage) { - this.contentLanguage = contentLanguage; - return this; - } - + /** + * The MIME type of the data you are uploading. + * + * @param contentType The Content-Type + * @return this builder + */ public Builder contentType(String contentType) { this.contentType = contentType; return this; } + /** + * The custom metadata of the object. + * + * @param metadata The custom metadata + * @return this builder + */ public Builder metadata(Map metadata) { this.metadata = metadata; return this; } - public Builder storageClass(String storageClass) { + /** + * Gives each part of the upload and the resulting object a storage class besides the default + * storage class of the associated bucket. + * + * @param storageClass The Storage-Class + * @return this builder + */ + public Builder storageClass(StorageClass storageClass) { this.storageClass = storageClass; return this; } - public Builder cacheControl(String cacheControl) { - this.cacheControl = cacheControl; - return this; - } - - public Builder customTime(String customTime) { + /** + * A user-specified date and time. + * + * @param customTime The custom time + * @return this builder + */ + public Builder customTime(Date customTime) { this.customTime = customTime; return this; } + /** + * The customer-managed encryption key to use to encrypt the object. Refer: Customer Managed Keys + * + * @param kmsKeyName The Cloud KMS key + * @return this builder + */ public Builder kmsKeyName(String kmsKeyName) { this.kmsKeyName = kmsKeyName; return this; } - public Builder objectLockMode(String objectLockMode) { + /** + * Mode of the object's retention configuration. GOVERNANCE corresponds to unlocked mode, and + * COMPLIANCE corresponds to locked mode. + * + * @param objectLockMode The object lock mode + * @return this builder + */ + public Builder objectLockMode(ObjectLockMode objectLockMode) { this.objectLockMode = objectLockMode; return this; } - public Builder objectLockRetainUntilDate(String objectLockRetainUntilDate) { + /** + * Date that determines the time until which the object is retained as immutable. + * + * @param objectLockRetainUntilDate The object lock retention until date + * @return this builder + */ + public Builder objectLockRetainUntilDate(Date objectLockRetainUntilDate) { this.objectLockRetainUntilDate = objectLockRetainUntilDate; return this; } From c6bf25be9df9831682bb62f6a2982eeefe6ef29b Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 09:36:29 +0000 Subject: [PATCH 17/27] Added pending validation --- .../cloud/storage/MultipartUploadClientImpl.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index fc744fb178..fd2ae40556 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -34,6 +34,7 @@ import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; import com.google.common.hash.Hashing; +import com.google.common.net.MediaType; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; @@ -96,10 +97,17 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload AccessToken accessToken = credentials.getAccessToken(); String authHeader = "Bearer " + accessToken.getTokenValue(); - String contentType = - request.getContentType() == null - ? "application/x-www-form-urlencoded" - : request.getContentType(); + String contentType; + if (request.getContentType() == null) { + contentType = "application/x-www-form-urlencoded"; + } else { + try { + contentType = MediaType.parse(request.getContentType()).toString(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid Content-Type header provided: " + request.getContentType(), e); + } + } HttpResponse response = httpRequestManager.sendCreateMultipartUploadRequest( From a37072dcfe5130174a5004408db7ed581a9e448f Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 09:45:07 +0000 Subject: [PATCH 18/27] Added Java docs for CompleteMultipartResponse --- .../cloud/storage/HttpRequestManager.java | 6 --- .../model/CreateMultipartUploadResponse.java | 49 +++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 8cd716f293..9c717977ed 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -46,12 +46,6 @@ public HttpResponse sendCreateMultipartUploadRequest( new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); - if (request.getContentEncoding() != null && !request.getContentEncoding().isEmpty()) { - httpRequest.getHeaders().setContentEncoding(request.getContentEncoding()); - } - if (request.getCacheControl() != null && !request.getCacheControl().isEmpty()) { - httpRequest.getHeaders().setCacheControl(request.getCacheControl()); - } for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java index 2aa6400aea..3343f51f92 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -22,6 +22,10 @@ import com.google.common.base.MoreObjects; import java.util.Objects; +/** + * Represents the response from a CreateMultipartUpload request. This class encapsulates the details + * of the initiated multipart upload, including the bucket, key, and the unique upload ID. + */ @JacksonXmlRootElement(localName = "InitiateMultipartUploadResult") public class CreateMultipartUploadResponse { @@ -42,14 +46,30 @@ private CreateMultipartUploadResponse(Builder builder) { private CreateMultipartUploadResponse() {} + /** + * Returns the name of the bucket where the multipart upload was initiated. + * + * @return The bucket name. + */ public String bucket() { return bucket; } + /** + * Returns the key (object name) for which the multipart upload was initiated. + * + * @return The object key. + */ public String key() { return key; } + /** + * Returns the unique identifier for this multipart upload. This ID must be included in all + * subsequent requests related to this upload (e.g., uploading parts, completing the upload). + * + * @return The upload ID. + */ public String uploadId() { return uploadId; } @@ -82,10 +102,16 @@ public String toString() { .toString(); } + /** + * Creates a new builder for {@link CreateMultipartUploadResponse}. + * + * @return A new builder. + */ public static Builder builder() { return new Builder(); } + /** A builder for {@link CreateMultipartUploadResponse} objects. */ public static class Builder { private String bucket; private String key; @@ -93,21 +119,44 @@ public static class Builder { private Builder() {} + /** + * Sets the bucket name for the multipart upload. + * + * @param bucket The bucket name. + * @return This builder. + */ public Builder bucket(String bucket) { this.bucket = bucket; return this; } + /** + * Sets the key (object name) for the multipart upload. + * + * @param key The object key. + * @return This builder. + */ public Builder key(String key) { this.key = key; return this; } + /** + * Sets the upload ID for the multipart upload. + * + * @param uploadId The upload ID. + * @return This builder. + */ public Builder uploadId(String uploadId) { this.uploadId = uploadId; return this; } + /** + * Builds a new {@link CreateMultipartUploadResponse} object. + * + * @return A new {@link CreateMultipartUploadResponse} object. + */ public CreateMultipartUploadResponse build() { return new CreateMultipartUploadResponse(this); } From b63ea922039ce08ac42001cea0836c7edf4b7bc1 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 1 Oct 2025 10:00:15 +0000 Subject: [PATCH 19/27] Added java docs --- .../model/CompleteMultipartUploadRequest.java | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java index 7b08c7f553..5f2955548a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadRequest.java @@ -25,16 +25,12 @@ public final class CompleteMultipartUploadRequest { private final String key; private final String uploadId; private final CompletedMultipartUpload multipartUpload; - private final String requestPayer; - private final String expectedBucketOwner; private CompleteMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; this.key = builder.key; this.uploadId = builder.uploadId; this.multipartUpload = builder.multipartUpload; - this.requestPayer = builder.requestPayer; - this.expectedBucketOwner = builder.expectedBucketOwner; } public String bucket() { @@ -53,14 +49,6 @@ public CompletedMultipartUpload multipartUpload() { return multipartUpload; } - public String requestPayer() { - return requestPayer; - } - - public String expectedBucketOwner() { - return expectedBucketOwner; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -73,14 +61,12 @@ public boolean equals(Object o) { return Objects.equals(bucket, that.bucket) && Objects.equals(key, that.key) && Objects.equals(uploadId, that.uploadId) - && Objects.equals(multipartUpload, that.multipartUpload) - && Objects.equals(requestPayer, that.requestPayer) - && Objects.equals(expectedBucketOwner, that.expectedBucketOwner); + && Objects.equals(multipartUpload, that.multipartUpload); } @Override public int hashCode() { - return Objects.hash(bucket, key, uploadId, multipartUpload, requestPayer, expectedBucketOwner); + return Objects.hash(bucket, key, uploadId, multipartUpload); } @Override @@ -90,8 +76,6 @@ public String toString() { .add("key", key) .add("uploadId", uploadId) .add("completedMultipartUpload", multipartUpload) - .add("requestPayer", requestPayer) - .add("expectedBucketOwner", expectedBucketOwner) .toString(); } @@ -104,8 +88,6 @@ public static class Builder { private String key; private String uploadId; private CompletedMultipartUpload multipartUpload; - private String requestPayer; - private String expectedBucketOwner; private Builder() {} @@ -129,16 +111,6 @@ public Builder multipartUpload(CompletedMultipartUpload completedMultipartUpload return this; } - public Builder requestPayer(String requestPayer) { - this.requestPayer = requestPayer; - return this; - } - - public Builder expectedBucketOwner(String expectedBucketOwner) { - this.expectedBucketOwner = expectedBucketOwner; - return this; - } - public CompleteMultipartUploadRequest build() { return new CompleteMultipartUploadRequest(this); } From ed63e19de4dee61eefada01f880bfbdbe0d0a556 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 3 Oct 2025 13:30:46 +0000 Subject: [PATCH 20/27] md5 and crc32c added in response --- .../storage/MultipartUploadClientImpl.java | 18 ++++++++++- .../model/UploadPartResponse.java | 32 +++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index fd2ae40556..f10d83030d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -202,7 +202,23 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ + error); } String eTag = response.getHeaders().getETag(); - return UploadPartResponse.builder().eTag(eTag).build(); + String crc32cFromHeader = null; + String md5FromHeader = null; + String hashHeader = response.getHeaders().getFirstHeaderStringValue("x-goog-hash"); + if (hashHeader != null) { + String[] hashes = hashHeader.split(","); + for (String hash : hashes) { + String[] kv = hash.trim().split("=", 2); + if (kv.length == 2) { + if ("crc32c".equalsIgnoreCase(kv[0])) { + crc32cFromHeader = kv[1]; + } else if ("md5".equalsIgnoreCase(kv[0])) { + md5FromHeader = kv[1]; + } + } + } + } + return UploadPartResponse.builder().eTag(eTag).crc32c(crc32cFromHeader).md5(md5FromHeader).build(); } public CompleteMultipartUploadResponse completeMultipartUpload( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java index 5acc044645..184ba883e7 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/UploadPartResponse.java @@ -22,15 +22,27 @@ public final class UploadPartResponse { private final String eTag; + private final String crc32c; + private final String md5; private UploadPartResponse(Builder builder) { this.eTag = builder.etag; + this.crc32c = builder.crc32c; + this.md5 = builder.md5; } public String eTag() { return eTag; } + public String crc32c() { + return crc32c; + } + + public String md5() { + return md5; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -40,25 +52,29 @@ public boolean equals(Object o) { return false; } UploadPartResponse that = (UploadPartResponse) o; - return Objects.equals(eTag, that.eTag); + return Objects.equals(eTag, that.eTag) && Objects.equals(crc32c, that.crc32c) && Objects.equals(md5, that.md5); } @Override public int hashCode() { - return Objects.hash(eTag); + return Objects.hash(eTag, crc32c, md5); } @Override public String toString() { - return MoreObjects.toStringHelper(this).add("etag", eTag).toString(); + return MoreObjects.toStringHelper(this).add("etag", eTag).add("crc32c", crc32c).add("md5", md5).toString(); } public static Builder builder() { return new Builder(); } + + public static class Builder { private String etag; + private String crc32c; + private String md5; private Builder() {} @@ -67,6 +83,16 @@ public Builder eTag(String etag) { return this; } + public Builder crc32c(String crc32c) { + this.crc32c = crc32c; + return this; + } + + public Builder md5(String md5) { + this.md5 = md5; + return this; + } + public UploadPartResponse build() { return new UploadPartResponse(this); } From 86ea017c338ceddea687be3ca86c21d78b46a70f Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 3 Oct 2025 16:42:17 +0000 Subject: [PATCH 21/27] Added retrier implementation --- .../storage/MultipartUploadClientImpl.java | 88 +++++++++++-------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index f10d83030d..0598247469 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -22,6 +22,7 @@ import com.google.api.client.http.HttpResponse; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.storage.Conversions.Decoder; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse; @@ -50,6 +51,7 @@ import java.util.HashMap; import java.util.Map; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; public class MultipartUploadClientImpl extends MultipartUploadClient { @@ -59,12 +61,14 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { private final GoogleCredentials credentials; private final XmlMapper xmlMapper; private final HttpStorageOptions options; + private final Retrier retrier; public MultipartUploadClientImpl( URI uri, HttpRequestFactory requestFactory, Retrier retrier, HttpStorageOptions options) { this.httpRequestManager = new HttpRequestManager(requestFactory); this.xmlMapper = new XmlMapper(); this.options = options; + this.retrier = retrier; try { this.credentials = GoogleCredentials.getApplicationDefault() @@ -177,19 +181,16 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - - HttpResponse response = - httpRequestManager.sendUploadPartRequest( - uri, - partData, - authHeader, - contentType, - contentMd5, - crc32cString, - extensionHeaders); + HttpResponse response = retrier.run( + Retrying.alwaysRetry(), + () -> { + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); + return httpRequestManager.sendUploadPartRequest(uri, partData, authHeader, contentType, + contentMd5, crc32cString, extensionHeaders); + }, + Decoder.identity()); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -244,19 +245,22 @@ public CompleteMultipartUploadResponse completeMultipartUpload( Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - - HttpResponse response = - httpRequestManager.sendCompleteMultipartUploadRequest( - uri, - xmlBodyBytes, - authHeader, - contentType, - contentMd5, - crc32cString, - extensionHeaders); + HttpResponse response = retrier.run( + Retrying.alwaysRetry(), + () -> { + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); + return httpRequestManager.sendCompleteMultipartUploadRequest( + uri, + xmlBodyBytes, + authHeader, + contentType, + contentMd5, + crc32cString, + extensionHeaders); + }, + Decoder.identity()); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); @@ -277,13 +281,16 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq String contentType = "application/x-www-form-urlencoded"; Map extensionHeaders = getGenericExtensionHeader(); - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - - HttpResponse response = - httpRequestManager.sendAbortMultipartUploadRequest( - uri, authHeader, contentType, extensionHeaders); + HttpResponse response = retrier.run( + Retrying.alwaysRetry(), + () -> { + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); + return httpRequestManager.sendAbortMultipartUploadRequest( + uri, authHeader, contentType, extensionHeaders); + }, + Decoder.identity()); if (response.getStatusCode() != 204) { String error = response.parseAsString(); @@ -308,12 +315,15 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException String uri = GCS_ENDPOINT + resourcePath + queryString; Map extensionHeaders = getGenericExtensionHeader(); - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - - HttpResponse response = - httpRequestManager.sendListPartsRequest(uri, authHeader, extensionHeaders); + HttpResponse response = retrier.run( + Retrying.alwaysRetry(), + () -> { + credentials.refreshIfExpired(); + AccessToken accessToken = credentials.getAccessToken(); + String authHeader = "Bearer " + accessToken.getTokenValue(); + return httpRequestManager.sendListPartsRequest(uri, authHeader, extensionHeaders); + }, + Decoder.identity()); if (!response.isSuccessStatusCode()) { String error = response.parseAsString(); From c7f641dd26f09d0990abb741f4eae156cf841389 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 3 Oct 2025 18:37:10 +0000 Subject: [PATCH 22/27] Added fix for reuquest body --- .../cloud/storage/HttpRequestManager.java | 5 ++- .../storage/MultipartUploadClientImpl.java | 13 ++----- .../com/google/cloud/storage/RequestBody.java | 13 +------ .../cloud/storage/RewindableContent.java | 37 ++++++++++++++++++- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 9c717977ed..9f002dc7a6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -19,6 +19,7 @@ import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpContent; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; @@ -55,7 +56,7 @@ public HttpResponse sendCreateMultipartUploadRequest( public HttpResponse sendUploadPartRequest( String uri, - byte[] partData, + HttpContent content, String authHeader, String contentType, String contentMd5, @@ -64,7 +65,7 @@ public HttpResponse sendUploadPartRequest( throws IOException { HttpRequest httpRequest = requestFactory.buildPutRequest( - new GenericUrl(uri), new ByteArrayContent(contentType, partData)); + new GenericUrl(uri), content); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); httpRequest.getHeaders().setContentMD5(contentMd5); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 0598247469..1cec197526 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -169,15 +169,8 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ "?partNumber=" + request.partNumber() + "&uploadId=" + encode(request.uploadId()); String uri = GCS_ENDPOINT + resourcePath + queryString; String contentType = "application/octet-stream"; - MessageDigest md = MessageDigest.getInstance("MD5"); - byte[] partData = requestBody.getPartData(); - String contentMd5 = Base64.getEncoder().encodeToString(md.digest(partData)); - String crc32cString = - Base64.getEncoder() - .encodeToString( - ByteBuffer.allocate(4) - .putInt(Hashing.crc32c().hashBytes(partData).asInt()) - .array()); + String contentMd5 = requestBody.getContent().getMd5(); + String crc32cString = requestBody.getContent().getCrc32c(); Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); @@ -187,7 +180,7 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); String authHeader = "Bearer " + accessToken.getTokenValue(); - return httpRequestManager.sendUploadPartRequest(uri, partData, authHeader, contentType, + return httpRequestManager.sendUploadPartRequest(uri, requestBody.getContent(), authHeader, contentType, contentMd5, crc32cString, extensionHeaders); }, Decoder.identity()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java index d6efab87aa..18bf01d659 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java @@ -27,7 +27,6 @@ public final class RequestBody { private final RewindableContent content; - private byte[] byteArray; private RequestBody(RewindableContent content) { this.content = content; @@ -38,9 +37,7 @@ RewindableContent getContent() { } public static RequestBody empty() { - RequestBody requestBody = new RequestBody(RewindableContent.empty()); - requestBody.byteArray = new byte[0]; - return requestBody; + return new RequestBody(RewindableContent.empty()); } public static RequestBody of(ByteBuffer... buffers) { @@ -51,9 +48,7 @@ public static RequestBody fromByteBuffer(ByteBuffer buffer) { ByteBuffer duplicate = buffer.duplicate(); byte[] arr = new byte[duplicate.remaining()]; duplicate.get(arr); - RequestBody requestBody = new RequestBody(RewindableContent.of(buffer)); - requestBody.byteArray = arr; - return requestBody; + return new RequestBody(RewindableContent.of(buffer)); } public static RequestBody of(ByteBuffer[] srcs, int srcsOffset, int srcsLength) { @@ -63,8 +58,4 @@ public static RequestBody of(ByteBuffer[] srcs, int srcsOffset, int srcsLength) public static RequestBody of(Path path) throws IOException { return new RequestBody(RewindableContent.of(path)); } - - public byte[] getPartData() { - return byteArray; - } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java index 7cf3dfe797..9e8f9f481c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java @@ -18,7 +18,10 @@ import com.google.api.client.http.AbstractHttpContent; import com.google.api.client.http.HttpMediaType; +import com.google.api.client.util.Base64; import com.google.common.base.Preconditions; +import com.google.common.hash.Hashing; +import com.google.common.hash.HashingOutputStream; import com.google.common.io.ByteStreams; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -31,11 +34,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Locale; abstract class RewindableContent extends AbstractHttpContent { + private String md5; + private String crc32c; + private RewindableContent() { super((HttpMediaType) null); } @@ -58,7 +67,6 @@ private RewindableContent() { * this may cause an OutOfMemoryError. * * @return The byte array representation of the content. - * @throws IOException if an I/O error occurs. */ public byte[] asByteArray() { if (getLength() == 0) { @@ -80,6 +88,31 @@ public final boolean retrySupported() { return false; } + public String getMd5() throws IOException { + if (md5 == null) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + writeTo(new DigestOutputStream(ByteStreams.nullOutputStream(), md)); + md5 = Base64.encodeBase64String(md.digest()); + } catch (NoSuchAlgorithmException e) { + throw new IOException(e); + } + } + return md5; + } + + public String getCrc32c() throws IOException { + if (crc32c == null) { + HashingOutputStream hashingOutputStream = + new HashingOutputStream(Hashing.crc32c(), ByteStreams.nullOutputStream()); + writeTo(hashingOutputStream); + byte[] bytes = + ByteBuffer.allocate(4).putInt(hashingOutputStream.hash().asInt()).array(); + crc32c = Base64.encodeBase64String(bytes); + } + return crc32c; + } + static RewindableContent empty() { return EmptyRewindableContent.INSTANCE; } @@ -121,6 +154,8 @@ public void writeTo(OutputStream out) throws IOException { out.flush(); } + + @Override long writeTo(WritableByteChannel gbc) { return 0; From d62c3f69e4d25dcc4006b3d703b16df93b83e6cd Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 7 Oct 2025 16:32:20 +0000 Subject: [PATCH 23/27] Requestbody change --- .../main/java/com/google/cloud/storage/RequestBody.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java index 18bf01d659..53d3672d18 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RequestBody.java @@ -44,13 +44,6 @@ public static RequestBody of(ByteBuffer... buffers) { return new RequestBody(RewindableContent.of(buffers)); } - public static RequestBody fromByteBuffer(ByteBuffer buffer) { - ByteBuffer duplicate = buffer.duplicate(); - byte[] arr = new byte[duplicate.remaining()]; - duplicate.get(arr); - return new RequestBody(RewindableContent.of(buffer)); - } - public static RequestBody of(ByteBuffer[] srcs, int srcsOffset, int srcsLength) { return new RequestBody(RewindableContent.of(srcs, srcsOffset, srcsLength)); } From 15b37b44b33bc91a1f8053ac888b01c4fa41bd50 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 10 Oct 2025 08:27:25 +0000 Subject: [PATCH 24/27] fixed rewindable content --- .../cloud/storage/HttpRequestManager.java | 8 -- .../storage/MultipartUploadClientImpl.java | 6 +- .../cloud/storage/RewindableContent.java | 114 ++++++++++-------- pom.xml | 17 ++- 4 files changed, 83 insertions(+), 62 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 9f002dc7a6..5f19d8ef95 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -59,8 +59,6 @@ public HttpResponse sendUploadPartRequest( HttpContent content, String authHeader, String contentType, - String contentMd5, - String crc32cString, Map extensionHeaders) throws IOException { HttpRequest httpRequest = @@ -68,8 +66,6 @@ public HttpResponse sendUploadPartRequest( new GenericUrl(uri), content); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); - httpRequest.getHeaders().setContentMD5(contentMd5); - httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } @@ -82,8 +78,6 @@ public HttpResponse sendCompleteMultipartUploadRequest( byte[] xmlBodyBytes, String authHeader, String contentType, - String contentMd5, - String crc32cString, Map extensionHeaders) throws IOException { HttpRequest httpRequest = @@ -91,8 +85,6 @@ public HttpResponse sendCompleteMultipartUploadRequest( new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); - httpRequest.getHeaders().setContentMD5(contentMd5); - httpRequest.getHeaders().set("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 1cec197526..bdadc89050 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -173,15 +173,15 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ String crc32cString = requestBody.getContent().getCrc32c(); Map extensionHeaders = getGenericExtensionHeader(); extensionHeaders.put("x-goog-hash", "crc32c=" + crc32cString + ",md5=" + contentMd5); - HttpResponse response = retrier.run( Retrying.alwaysRetry(), () -> { + requestBody.getContent().rewindTo(0); credentials.refreshIfExpired(); AccessToken accessToken = credentials.getAccessToken(); String authHeader = "Bearer " + accessToken.getTokenValue(); return httpRequestManager.sendUploadPartRequest(uri, requestBody.getContent(), authHeader, contentType, - contentMd5, crc32cString, extensionHeaders); + extensionHeaders); }, Decoder.identity()); @@ -249,8 +249,6 @@ public CompleteMultipartUploadResponse completeMultipartUpload( xmlBodyBytes, authHeader, contentType, - contentMd5, - crc32cString, extensionHeaders); }, Decoder.identity()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java index 9e8f9f481c..ae47cd6efc 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java @@ -42,9 +42,6 @@ abstract class RewindableContent extends AbstractHttpContent { - private String md5; - private String crc32c; - private RewindableContent() { super((HttpMediaType) null); } @@ -60,58 +57,14 @@ private RewindableContent() { abstract void flagDirty(); - /** - * Returns the content as a byte array. - * - *

NOTE: This method will read the entire content into memory. If the content is large, - * this may cause an OutOfMemoryError. - * - * @return The byte array representation of the content. - */ - public byte[] asByteArray() { - if (getLength() == 0) { - return new byte[0]; - } - Preconditions.checkState( - getLength() <= Integer.MAX_VALUE, "Content is too large to be represented as a byte array."); - ByteArrayOutputStream baos = new ByteArrayOutputStream((int) getLength()); - try { - writeTo(baos); - } catch (IOException e) { - throw new RuntimeException(e); - } - return baos.toByteArray(); - } - @Override public final boolean retrySupported() { return false; } - public String getMd5() throws IOException { - if (md5 == null) { - try { - MessageDigest md = MessageDigest.getInstance("MD5"); - writeTo(new DigestOutputStream(ByteStreams.nullOutputStream(), md)); - md5 = Base64.encodeBase64String(md.digest()); - } catch (NoSuchAlgorithmException e) { - throw new IOException(e); - } - } - return md5; - } + abstract String getMd5() throws IOException; - public String getCrc32c() throws IOException { - if (crc32c == null) { - HashingOutputStream hashingOutputStream = - new HashingOutputStream(Hashing.crc32c(), ByteStreams.nullOutputStream()); - writeTo(hashingOutputStream); - byte[] bytes = - ByteBuffer.allocate(4).putInt(hashingOutputStream.hash().asInt()).array(); - crc32c = Base64.encodeBase64String(bytes); - } - return crc32c; - } + abstract String getCrc32c() throws IOException; static RewindableContent empty() { return EmptyRewindableContent.INSTANCE; @@ -171,6 +124,16 @@ protected void rewindTo(long offset) {} @Override void flagDirty() {} + + @Override + String getMd5() throws IOException { + return ""; + } + + @Override + String getCrc32c() throws IOException { + return ""; + } } private static final class PathRewindableContent extends RewindableContent { @@ -225,6 +188,31 @@ long writeTo(GatheringByteChannel gbc) throws IOException { @Override void flagDirty() {} + + @Override + String getMd5() throws IOException { + try (SeekableByteChannel in = Files.newByteChannel(path, StandardOpenOption.READ)) { + in.position(readOffset); + HashingOutputStream hos = + new HashingOutputStream(Hashing.md5(), ByteStreams.nullOutputStream()); + ByteStreams.copy(in, Channels.newChannel(hos)); + return Base64.encodeBase64String(hos.hash().asBytes()); + } + } + + @Override + String getCrc32c() throws IOException { + try (SeekableByteChannel in = Files.newByteChannel(path, StandardOpenOption.READ)) { + in.position(readOffset); + HashingOutputStream hos = + new HashingOutputStream(Hashing.crc32c(), ByteStreams.nullOutputStream()); + ByteStreams.copy(in, Channels.newChannel(hos)); + int crc32cInt = hos.hash().asInt(); + ByteBuffer bb = ByteBuffer.allocate(4); + bb.putInt(crc32cInt); + return Base64.encodeBase64String(bb.array()); + } + } } private static final class ByteBufferContent extends RewindableContent { @@ -320,5 +308,33 @@ void rewindTo(long offset) { void flagDirty() { this.dirty = true; } + + @Override + String getMd5() throws IOException { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + // important to use duplicate, so we don't consume the buffer for the actual write + for (ByteBuffer buffer : buffers) { + md.update(buffer.duplicate()); + } + return Base64.encodeBase64String(md.digest()); + } catch (NoSuchAlgorithmException e) { + // should not happen + throw new IOException(e); + } + } + + @Override + String getCrc32c() throws IOException { + com.google.common.hash.Hasher crc32cHasher = Hashing.crc32c().newHasher(); + // important to use duplicate, so we don't consume the buffer for the actual write + for (ByteBuffer buffer : buffers) { + crc32cHasher.putBytes(buffer.duplicate()); + } + int crc32cInt = crc32cHasher.hash().asInt(); + ByteBuffer bb = ByteBuffer.allocate(4); + bb.putInt(crc32cInt); + return Base64.encodeBase64String(bb.array()); + } } } diff --git a/pom.xml b/pom.xml index f1bfe9c3e5..3a34f6e955 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,22 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.15.2 + 2.19.2 + + + com.fasterxml.jackson.core + jackson-core + 2.19.2 + + + com.fasterxml.jackson.core + jackson-databind + 2.19.2 + + + com.fasterxml.jackson.core + jackson-annotations + 2.19.2 org.apache.httpcomponents From fb238549b6b3e8b831b30d25e63fa3976b01d7a8 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 10 Oct 2025 08:36:24 +0000 Subject: [PATCH 25/27] Removed Auth --- .../cloud/storage/HttpRequestManager.java | 14 ++------ .../storage/MultipartUploadClientImpl.java | 35 ++----------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java index 5f19d8ef95..1a7a4aa533 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRequestManager.java @@ -15,8 +15,6 @@ */ package com.google.cloud.storage; -import static com.google.cloud.storage.MultipartUploadUtility.getRfc1123Date; - import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpContent; @@ -37,7 +35,6 @@ public HttpRequestManager(HttpRequestFactory requestFactory) { public HttpResponse sendCreateMultipartUploadRequest( String uri, - String authHeader, String contentType, CreateMultipartUploadRequest request, Map extensionHeaders) @@ -45,7 +42,6 @@ public HttpResponse sendCreateMultipartUploadRequest( HttpRequest httpRequest = requestFactory.buildPostRequest( new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); - httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); @@ -57,14 +53,12 @@ public HttpResponse sendCreateMultipartUploadRequest( public HttpResponse sendUploadPartRequest( String uri, HttpContent content, - String authHeader, String contentType, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildPutRequest( new GenericUrl(uri), content); - httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); @@ -76,14 +70,12 @@ public HttpResponse sendUploadPartRequest( public HttpResponse sendCompleteMultipartUploadRequest( String uri, byte[] xmlBodyBytes, - String authHeader, String contentType, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildPostRequest( new GenericUrl(uri), new ByteArrayContent(contentType, xmlBodyBytes)); - httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); @@ -93,10 +85,9 @@ public HttpResponse sendCompleteMultipartUploadRequest( } public HttpResponse sendAbortMultipartUploadRequest( - String uri, String authHeader, String contentType, Map extensionHeaders) + String uri, String contentType, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildDeleteRequest(new GenericUrl(uri)); - httpRequest.getHeaders().setAuthorization(authHeader); httpRequest.getHeaders().setContentType(contentType); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); @@ -106,10 +97,9 @@ public HttpResponse sendAbortMultipartUploadRequest( } public HttpResponse sendListPartsRequest( - String uri, String authHeader, Map extensionHeaders) + String uri, Map extensionHeaders) throws IOException { HttpRequest httpRequest = requestFactory.buildGetRequest(new GenericUrl(uri)); - httpRequest.getHeaders().setAuthorization(authHeader); for (Map.Entry entry : extensionHeaders.entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index bdadc89050..bc130e9cc1 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -20,8 +20,6 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Conversions.Decoder; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; @@ -46,7 +44,6 @@ import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.Base64; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -58,7 +55,6 @@ public class MultipartUploadClientImpl extends MultipartUploadClient { private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; private final HttpRequestManager httpRequestManager; - private final GoogleCredentials credentials; private final XmlMapper xmlMapper; private final HttpStorageOptions options; private final Retrier retrier; @@ -69,13 +65,6 @@ public MultipartUploadClientImpl( this.xmlMapper = new XmlMapper(); this.options = options; this.retrier = retrier; - try { - this.credentials = - GoogleCredentials.getApplicationDefault() - .createScoped(Collections.singleton("https://www.googleapis.com/auth/devstorage.read_write")); - } catch (IOException e) { - throw new RuntimeException("Failed to get application default credentials", e); - } } private Map getGenericExtensionHeader() { @@ -97,10 +86,6 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload String resourcePath = "/" + encodedBucket + "/" + encodedKey; String uri = GCS_ENDPOINT + resourcePath + "?uploads"; - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - String contentType; if (request.getContentType() == null) { contentType = "application/x-www-form-urlencoded"; @@ -116,7 +101,6 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload HttpResponse response = httpRequestManager.sendCreateMultipartUploadRequest( uri, - authHeader, contentType, request, getExtensionHeadersForCreateMultipartUpload(request)); @@ -177,10 +161,7 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ Retrying.alwaysRetry(), () -> { requestBody.getContent().rewindTo(0); - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - return httpRequestManager.sendUploadPartRequest(uri, requestBody.getContent(), authHeader, contentType, + return httpRequestManager.sendUploadPartRequest(uri, requestBody.getContent(), contentType, extensionHeaders); }, Decoder.identity()); @@ -241,13 +222,9 @@ public CompleteMultipartUploadResponse completeMultipartUpload( HttpResponse response = retrier.run( Retrying.alwaysRetry(), () -> { - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); return httpRequestManager.sendCompleteMultipartUploadRequest( uri, xmlBodyBytes, - authHeader, contentType, extensionHeaders); }, @@ -275,11 +252,8 @@ public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadReq HttpResponse response = retrier.run( Retrying.alwaysRetry(), () -> { - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); return httpRequestManager.sendAbortMultipartUploadRequest( - uri, authHeader, contentType, extensionHeaders); + uri, contentType, extensionHeaders); }, Decoder.identity()); @@ -309,10 +283,7 @@ public ListPartsResponse listParts(ListPartsRequest request) throws IOException HttpResponse response = retrier.run( Retrying.alwaysRetry(), () -> { - credentials.refreshIfExpired(); - AccessToken accessToken = credentials.getAccessToken(); - String authHeader = "Bearer " + accessToken.getTokenValue(); - return httpRequestManager.sendListPartsRequest(uri, authHeader, extensionHeaders); + return httpRequestManager.sendListPartsRequest(uri, extensionHeaders); }, Decoder.identity()); From aae152ee6b7abd37beb23e8220a4c8b596faaee2 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 10 Oct 2025 12:00:40 +0000 Subject: [PATCH 26/27] Cleaned header implementation a bit --- .../storage/MultipartUploadClientImpl.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index bc130e9cc1..607c657d93 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -43,12 +43,16 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.Base64; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; public class MultipartUploadClientImpl extends MultipartUploadClient { @@ -177,23 +181,21 @@ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requ + error); } String eTag = response.getHeaders().getETag(); - String crc32cFromHeader = null; - String md5FromHeader = null; - String hashHeader = response.getHeaders().getFirstHeaderStringValue("x-goog-hash"); - if (hashHeader != null) { - String[] hashes = hashHeader.split(","); - for (String hash : hashes) { - String[] kv = hash.trim().split("=", 2); - if (kv.length == 2) { - if ("crc32c".equalsIgnoreCase(kv[0])) { - crc32cFromHeader = kv[1]; - } else if ("md5".equalsIgnoreCase(kv[0])) { - md5FromHeader = kv[1]; - } - } - } - } - return UploadPartResponse.builder().eTag(eTag).crc32c(crc32cFromHeader).md5(md5FromHeader).build(); + Map hashes = extractHashesFromHeader(response); + return UploadPartResponse.builder().eTag(eTag).crc32c(hashes.get("crc32c")).md5(hashes.get("md5")).build(); + } + + private Map extractHashesFromHeader(HttpResponse response) { + return Optional.ofNullable(response.getHeaders().getFirstHeaderStringValue("x-goog-hash")) + .map( + h -> + Arrays.stream(h.split(",")) + .map(s -> s.trim().split("=", 2)) + .filter(a -> a.length == 2) + .filter(a -> "crc32c".equalsIgnoreCase(a[0]) || "md5".equalsIgnoreCase(a[0])) + .collect( + Collectors.toMap(a -> a[0].toLowerCase(), a -> a[1], (v1, v2) -> v1))) + .orElse(Collections.emptyMap()); } public CompleteMultipartUploadResponse completeMultipartUpload( From 43758ab085068d5e3124d1c16ebe45f7211590ba Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Mon, 13 Oct 2025 10:32:30 +0000 Subject: [PATCH 27/27] Added Unit tests --- .../CompleteMultipartUploadResponse.java | 16 +- .../model/ListPartsResponse.java | 2 + .../storage/multipartupload/model/Part.java | 55 ++- .../cloud/storage/HttpRequestManagerTest.java | 171 +++++++ .../MultipartUploadClientImplTest.java | 434 ++++++++++++++++++ 5 files changed, 666 insertions(+), 12 deletions(-) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/HttpRequestManagerTest.java create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java index 6a73911135..1d47c319f8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CompleteMultipartUploadResponse.java @@ -37,19 +37,19 @@ private CompleteMultipartUploadResponse(Builder builder) { this.etag = builder.etag; } - public String getLocation() { + public String location() { return location; } - public String getBucket() { + public String bucket() { return bucket; } - public String getKey() { + public String key() { return key; } - public String getEtag() { + public String etag() { return etag; } @@ -97,25 +97,25 @@ public static class Builder { private Builder() {} @JsonProperty("Location") - public Builder setLocation(String location) { + public Builder location(String location) { this.location = location; return this; } @JsonProperty("Bucket") - public Builder setBucket(String bucket) { + public Builder bucket(String bucket) { this.bucket = bucket; return this; } @JsonProperty("Key") - public Builder setKey(String key) { + public Builder key(String key) { this.key = key; return this; } @JsonProperty("ETag") - public Builder setEtag(String etag) { + public Builder etag(String etag) { this.etag = etag; return this; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java index feca04a03c..62954e913f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ListPartsResponse.java @@ -16,6 +16,7 @@ package com.google.cloud.storage.multipartupload.model; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.google.common.base.MoreObjects; @@ -42,6 +43,7 @@ public final class ListPartsResponse { @JacksonXmlProperty(localName = "MaxParts") private Integer maxParts; + @JsonAlias("truncated") // S3 returns "truncated", GCS returns "IsTruncated" @JacksonXmlProperty(localName = "IsTruncated") private boolean isTruncated; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java index e945a9df7f..0fa7946a10 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/Part.java @@ -34,22 +34,36 @@ public final class Part { @JacksonXmlProperty(localName = "LastModified") private String lastModified; - public int getPartNumber() { + // for jackson + private Part() {} + + private Part(Builder builder) { + this.partNumber = builder.partNumber; + this.eTag = builder.eTag; + this.size = builder.size; + this.lastModified = builder.lastModified; + } + + public int partNumber() { return partNumber; } - public String getETag() { + public String eTag() { return eTag; } - public long getSize() { + public long size() { return size; } - public String getLastModified() { + public String lastModified() { return lastModified; } + public static Builder builder() { + return new Builder(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -79,4 +93,37 @@ public String toString() { .add("lastModified", lastModified) .toString(); } + + public static final class Builder { + private int partNumber; + private String eTag; + private long size; + private String lastModified; + + private Builder() {} + + public Builder partNumber(int partNumber) { + this.partNumber = partNumber; + return this; + } + + public Builder eTag(String eTag) { + this.eTag = eTag; + return this; + } + + public Builder size(long size) { + this.size = size; + return this; + } + + public Builder lastModified(String lastModified) { + this.lastModified = lastModified; + return this; + } + + public Part build() { + return new Part(this); + } + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpRequestManagerTest.java new file mode 100644 index 0000000000..7025546a91 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpRequestManagerTest.java @@ -0,0 +1,171 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.HttpContent; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class HttpRequestManagerTest { + + private static final String BUCKET = "bucket"; + private static final String KEY = "key"; + private static final String CONTENT_TYPE = "application/octet-stream"; + private static final String URI = "https://storage.googleapis.com/" + BUCKET + "/" + KEY; + + @Test + public void testSendCreateMultipartUploadRequest() throws IOException { + final AtomicReference capturedMethod = new AtomicReference<>(); + final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + capturedMethod.set(method); + lowLevelRequest.setUrl(url); + return lowLevelRequest; + } + }; + HttpRequestManager httpRequestManager = new HttpRequestManager(transport.createRequestFactory()); + + CreateMultipartUploadRequest createRequest = + CreateMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).build(); + Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); + + httpRequestManager.sendCreateMultipartUploadRequest(URI, CONTENT_TYPE, createRequest, headers); + + assertThat(capturedMethod.get()).isEqualTo("POST"); + assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); + assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); + assertThat(lowLevelRequest.getContentAsString()).isEqualTo(""); + } + + @Test + public void testSendUploadPartRequest() throws IOException { + final AtomicReference capturedMethod = new AtomicReference<>(); + final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + capturedMethod.set(method); + lowLevelRequest.setUrl(url); + return lowLevelRequest; + } + }; + HttpRequestManager httpRequestManager = new HttpRequestManager(transport.createRequestFactory()); + + byte[] contentBytes = "test content".getBytes(); + HttpContent content = new ByteArrayContent(CONTENT_TYPE, contentBytes); + Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); + + httpRequestManager.sendUploadPartRequest(URI, content, CONTENT_TYPE, headers); + + assertThat(capturedMethod.get()).isEqualTo("PUT"); + assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); + assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); + assertThat(lowLevelRequest.getContentAsString()).isEqualTo("test content"); + } + + @Test + public void testSendCompleteMultipartUploadRequest() throws IOException { + final AtomicReference capturedMethod = new AtomicReference<>(); + final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + capturedMethod.set(method); + lowLevelRequest.setUrl(url); + return lowLevelRequest; + } + }; + HttpRequestManager httpRequestManager = new HttpRequestManager(transport.createRequestFactory()); + + byte[] xmlBodyBytes = "".getBytes(); + Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); + + httpRequestManager.sendCompleteMultipartUploadRequest(URI, xmlBodyBytes, CONTENT_TYPE, headers); + + assertThat(capturedMethod.get()).isEqualTo("POST"); + assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); + assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); + assertThat(lowLevelRequest.getContentAsString()).isEqualTo(""); + } + + @Test + public void testSendAbortMultipartUploadRequest() throws IOException { + final AtomicReference capturedMethod = new AtomicReference<>(); + final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + capturedMethod.set(method); + lowLevelRequest.setUrl(url); + return lowLevelRequest; + } + }; + HttpRequestManager httpRequestManager = new HttpRequestManager(transport.createRequestFactory()); + + Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); + + httpRequestManager.sendAbortMultipartUploadRequest(URI, CONTENT_TYPE, headers); + + assertThat(capturedMethod.get()).isEqualTo("DELETE"); + assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); + assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); + } + + @Test + public void testSendListPartsRequest() throws IOException { + final AtomicReference capturedMethod = new AtomicReference<>(); + final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + capturedMethod.set(method); + lowLevelRequest.setUrl(url); + return lowLevelRequest; + } + }; + HttpRequestManager httpRequestManager = new HttpRequestManager(transport.createRequestFactory()); + + Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); + + httpRequestManager.sendListPartsRequest(URI, headers); + + assertThat(capturedMethod.get()).isEqualTo("GET"); + assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); + assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); + } +} \ No newline at end of file diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java new file mode 100644 index 0000000000..d1b97dec5e --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java @@ -0,0 +1,434 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.Json; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.cloud.storage.Retrying.Retrier; +import com.google.cloud.storage.Storage.PredefinedAcl; +import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.CompletedMultipartUpload; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.ListPartsRequest; +import com.google.cloud.storage.multipartupload.model.ListPartsResponse; +import com.google.cloud.storage.multipartupload.model.CompletedPart; +import com.google.cloud.storage.multipartupload.model.UploadPartRequest; +import com.google.cloud.storage.multipartupload.model.UploadPartResponse; +import java.io.IOException; +import java.net.URI; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.Map; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +@RunWith(JUnit4.class) +public class MultipartUploadClientImplTest { + + private static final String BUCKET = "bucket"; + private static final String KEY = "key"; + private static final String UPLOAD_ID = "uploadId"; + private static final String CONTENT_TYPE = "application/octet-stream"; + private static final String PROJECT_ID = "project-id"; + + private MultipartUploadClientImpl multipartUploadClient; + + @Mock private HttpRequestManager httpRequestManager; + + @Mock private Retrier retrier; + + @Captor private ArgumentCaptor> extensionHeadersCaptor; + + @Captor private ArgumentCaptor uriCaptor; + + private final XmlMapper xmlMapper = new XmlMapper(); + private AutoCloseable mocks; + + @Before + public void setUp() throws Exception { + mocks = MockitoAnnotations.openMocks(this); + HttpStorageOptions options = HttpStorageOptions.newBuilder().setProjectId(PROJECT_ID).build(); + multipartUploadClient = + new MultipartUploadClientImpl(new URI("https://storage.googleapis.com"), null, retrier, options); + // Replace the httpRequestManager with a mock + java.lang.reflect.Field field = + MultipartUploadClientImpl.class.getDeclaredField("httpRequestManager"); + field.setAccessible(true); + field.set(multipartUploadClient, httpRequestManager); + } + + @After + public void tearDown() throws Exception { + mocks.close(); + } + + @Test + public void testCreateMultipartUpload() throws IOException { + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).contentType(CONTENT_TYPE).build(); + CreateMultipartUploadResponse expectedResponse = + CreateMultipartUploadResponse.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); + String responseXml = xmlMapper.writeValueAsString(expectedResponse); + HttpResponse httpResponse = createHttpResponse(200, responseXml); + + when(httpRequestManager.sendCreateMultipartUploadRequest( + any(String.class), any(String.class), any(), any())) + .thenReturn(httpResponse); + + CreateMultipartUploadResponse actualResponse = + multipartUploadClient.createMultipartUpload(request); + + assertThat(actualResponse.bucket()).isEqualTo(expectedResponse.bucket()); + assertThat(actualResponse.key()).isEqualTo(expectedResponse.key()); + assertThat(actualResponse.uploadId()).isEqualTo(expectedResponse.uploadId()); + } + + @Test + public void testCreateMultipartUpload_withHeaders() throws IOException { + Map metadata = Collections.singletonMap("key", "value"); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder().bucket(BUCKET).key(KEY) + .contentType(CONTENT_TYPE) + .cannedAcl(PredefinedAcl.PRIVATE) + .metadata(metadata) + .storageClass(StorageClass.COLDLINE) + .build(); + CreateMultipartUploadResponse expectedResponse = + CreateMultipartUploadResponse.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); + String responseXml = xmlMapper.writeValueAsString(expectedResponse); + HttpResponse httpResponse = createHttpResponse(200, responseXml); + + when(httpRequestManager.sendCreateMultipartUploadRequest( + any(String.class), any(String.class), any(), extensionHeadersCaptor.capture())) + .thenReturn(httpResponse); + + multipartUploadClient.createMultipartUpload(request); + + Map capturedHeaders = extensionHeadersCaptor.getValue(); + assertThat(capturedHeaders).containsEntry("x-goog-acl", "PRIVATE"); + assertThat(capturedHeaders).containsEntry("x-goog-meta-key", "value"); + assertThat(capturedHeaders).containsEntry("x-goog-storage-class", "COLDLINE"); + } + + @Test + public void testCreateMultipartUpload_error() throws IOException { + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).contentType(CONTENT_TYPE).build(); + HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); + + when(httpRequestManager.sendCreateMultipartUploadRequest( + any(String.class), any(String.class), any(), any())) + .thenReturn(httpResponse); + + RuntimeException exception = assertThrows(RuntimeException.class, () -> multipartUploadClient.createMultipartUpload(request)); + assertThat(exception.getMessage()).isEqualTo("Failed to initiate upload: 500 Internal Server Error"); + } + + @Test + public void testUploadPart() throws IOException, NoSuchAlgorithmException { + UploadPartRequest request = + UploadPartRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).partNumber(1).build(); + UploadPartResponse expectedResponse = + UploadPartResponse.builder().eTag("etag").crc32c("crc32c").md5("md5").build(); + HttpResponse httpResponse = createHttpResponse(200, ""); + httpResponse + .getHeaders() + .setETag(expectedResponse.eTag()) + .set("x-goog-hash", "crc32c=" + expectedResponse.crc32c() + ",md5=" + expectedResponse.md5()); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendUploadPartRequest(any(String.class), any(), any(String.class), any())) + .thenReturn(httpResponse); + + UploadPartResponse actualResponse = + multipartUploadClient.uploadPart(request, RequestBody.empty()); + + assertThat(actualResponse.eTag()).isEqualTo(expectedResponse.eTag()); + assertThat(actualResponse.crc32c()).isEqualTo(expectedResponse.crc32c()); + assertThat(actualResponse.md5()).isEqualTo(expectedResponse.md5()); + } + + @Test + public void testUploadPart_error() throws IOException { + UploadPartRequest request = + UploadPartRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).partNumber(1).build(); + HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendUploadPartRequest(any(String.class), any(), any(String.class), any())) + .thenReturn(httpResponse); + + RuntimeException exception = assertThrows(RuntimeException.class, () -> multipartUploadClient.uploadPart(request, RequestBody.empty())); + assertThat(exception.getMessage()).isEqualTo("Failed to upload part 1: 500 Internal Server Error"); + } + + @Test + public void testCompleteMultipartUpload() throws IOException, NoSuchAlgorithmException { + + CompletedPart part = CompletedPart.builder().partNumber(1).eTag("etag").build(); + CompletedMultipartUpload multipartUpload = CompletedMultipartUpload.builder().parts(Collections.singletonList(part)).build(); + CompleteMultipartUploadRequest request = + CompleteMultipartUploadRequest.builder() + .bucket(BUCKET) + .key(KEY) + .uploadId(UPLOAD_ID) + .multipartUpload(multipartUpload) + .build(); + String responseXml = + "" + + "location" + + "" + + BUCKET + + "" + + "" + + KEY + + "" + + "etag" + + ""; + HttpResponse httpResponse = createHttpResponse(200, responseXml); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendCompleteMultipartUploadRequest( + any(String.class), any(byte[].class), any(String.class), any())) + .thenReturn(httpResponse); + + CompleteMultipartUploadResponse actualResponse = + multipartUploadClient.completeMultipartUpload(request); + + assertThat(actualResponse.bucket()).isEqualTo(BUCKET); + assertThat(actualResponse.key()).isEqualTo(KEY); + assertThat(actualResponse.etag()).isEqualTo("etag"); + assertThat(actualResponse.location()).isEqualTo("location"); + } + + @Test + public void testCompleteMultipartUpload_error() throws IOException { + CompletedPart part = CompletedPart.builder().partNumber(1).eTag("etag").build(); + CompletedMultipartUpload multipartUpload = CompletedMultipartUpload.builder().parts(Collections.singletonList(part)).build(); + CompleteMultipartUploadRequest request = + CompleteMultipartUploadRequest.builder() + .bucket(BUCKET) + .key(KEY) + .uploadId(UPLOAD_ID) + .multipartUpload(multipartUpload) + .build(); + HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendCompleteMultipartUploadRequest( + any(String.class), any(byte[].class), any(String.class), any())) + .thenReturn(httpResponse); + + RuntimeException exception = assertThrows(RuntimeException.class, () -> multipartUploadClient.completeMultipartUpload(request)); + assertThat(exception.getMessage()).isEqualTo("Failed to complete upload: 500 Internal Server Error"); + } + + @Test + public void testAbortMultipartUpload() throws IOException, NoSuchAlgorithmException { + AbortMultipartUploadRequest request = + AbortMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); + HttpResponse httpResponse = createHttpResponse(204, ""); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendAbortMultipartUploadRequest( + any(String.class), any(String.class), any())) + .thenReturn(httpResponse); + + multipartUploadClient.abortMultipartUpload(request); + } + + @Test + public void testAbortMultipartUpload_error() throws IOException { + AbortMultipartUploadRequest request = + AbortMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); + HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendAbortMultipartUploadRequest( + any(String.class), any(String.class), any())) + .thenReturn(httpResponse); + + RuntimeException exception = assertThrows(RuntimeException.class, () -> multipartUploadClient.abortMultipartUpload(request)); + assertThat(exception.getMessage()).isEqualTo("Failed to abort upload: 500 Internal Server Error"); + } + + @Test + public void testListParts() throws IOException { + ListPartsRequest request = + ListPartsRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).maxParts(10).build(); + ListPartsResponse expectedResponse = + xmlMapper.readValue( + "\n" + + " bucket\n" + + " key\n" + + " uploadId\n" + + " 10\n" + + " 0\n" + + " 1\n" + + " \n" + + " 1\n" + + " \"etag\"\n" + + " 1\n" + + " 2023-05-11T09:30:00Z\n" + + " \n" + + " STANDARD\n" + + "", + ListPartsResponse.class); + String responseXml = xmlMapper.writeValueAsString(expectedResponse); + HttpResponse httpResponse = createHttpResponse(200, responseXml); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendListPartsRequest(any(String.class), any())) + .thenReturn(httpResponse); + + ListPartsResponse actualResponse = multipartUploadClient.listParts(request); + + assertThat(actualResponse.getBucket()).isEqualTo(expectedResponse.getBucket()); + assertThat(actualResponse.getKey()).isEqualTo(expectedResponse.getKey()); + assertThat(actualResponse.getUploadId()).isEqualTo(expectedResponse.getUploadId()); + assertThat(actualResponse.getMaxParts()).isEqualTo(expectedResponse.getMaxParts()); + assertThat(actualResponse.getPartNumberMarker()) + .isEqualTo(expectedResponse.getPartNumberMarker()); + assertThat(actualResponse.getNextPartNumberMarker()) + .isEqualTo(expectedResponse.getNextPartNumberMarker()); + assertThat(actualResponse.getParts()).hasSize(1); + assertThat(actualResponse.getStorageClass()).isEqualTo(expectedResponse.getStorageClass()); + } + + @Test + public void testListParts_error() throws IOException { + ListPartsRequest request = + ListPartsRequest.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).maxParts(10).build(); + HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendListPartsRequest(any(String.class), any())) + .thenReturn(httpResponse); + + RuntimeException exception = assertThrows(RuntimeException.class, () -> multipartUploadClient.listParts(request)); + assertThat(exception.getMessage()).isEqualTo("Failed to list parts: 500 Internal Server Error"); + } + + @Test + public void testListParts_withQueryParameters() throws IOException { + ListPartsRequest request = + ListPartsRequest.builder() + .bucket(BUCKET) + .key(KEY) + .uploadId(UPLOAD_ID) + .maxParts(10) + .partNumberMarker(5) + .build(); + ListPartsResponse expectedResponse = + xmlMapper.readValue( + "\n" + + " bucket\n" + + " key\n" + + " uploadId\n" + + " 10\n" + + " 5\n" + + " 10\n" + + "", + ListPartsResponse.class); + String responseXml = xmlMapper.writeValueAsString(expectedResponse); + HttpResponse httpResponse = createHttpResponse(200, responseXml); + + when(retrier.run(any(), any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1, java.util.concurrent.Callable.class).call(); + return httpResponse; + }); + when(httpRequestManager.sendListPartsRequest(uriCaptor.capture(), any())) + .thenReturn(httpResponse); + + multipartUploadClient.listParts(request); + + String capturedUri = uriCaptor.getValue(); + assertThat(capturedUri).contains("&max-parts=10"); + assertThat(capturedUri).contains("&part-number-marker=5"); + } + + private HttpResponse createHttpResponse(int statusCode, String content) throws IOException { + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(statusCode); + response.setContentType(Json.MEDIA_TYPE); + response.setContent(content); + return response; + } + }; + } + }; + HttpRequestFactory requestFactory = transport.createRequestFactory(); + HttpRequest request = requestFactory.buildGetRequest(new GenericUrl("https://storage.googleapis.com")); + request.setThrowExceptionOnExecuteError(false); + return request.execute(); + } +}