diff --git a/generator/.DevConfigs/32a12d7c-afc6-4bcf-a2d8-9c49b335b935.json b/generator/.DevConfigs/32a12d7c-afc6-4bcf-a2d8-9c49b335b935.json new file mode 100644 index 000000000000..65fb80e5465d --- /dev/null +++ b/generator/.DevConfigs/32a12d7c-afc6-4bcf-a2d8-9c49b335b935.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Fixed issue calling UploadPart with an unseekable stream and disabling checksum failing." + ] + } + ] +} \ No newline at end of file diff --git a/sdk/src/Services/S3/Custom/Internal/AmazonS3PostMarshallHandler.cs b/sdk/src/Services/S3/Custom/Internal/AmazonS3PostMarshallHandler.cs index 5e2198b0e55b..bedc79a6f595 100644 --- a/sdk/src/Services/S3/Custom/Internal/AmazonS3PostMarshallHandler.cs +++ b/sdk/src/Services/S3/Custom/Internal/AmazonS3PostMarshallHandler.cs @@ -125,11 +125,11 @@ private static void SetStreamChecksum(UploadPartRequest uploadPartRequest, IRequ if (uploadPartRequest.InputStream != null) { // Wrap input stream in partial wrapper (to upload only part of the stream) - var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize.GetValueOrDefault()); + var partialStream = GetStreamWithLength(uploadPartRequest.InputStream, uploadPartRequest.PartSize.GetValueOrDefault(), chooseMin: true); if (partialStream.Length > 0 && !(uploadPartRequest.DisablePayloadSigning ?? false)) request.UseChunkEncoding = uploadPartRequest.UseChunkEncoding; if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) - request.Headers.Add(HeaderKeys.ContentLengthHeader, partialStream.Length.ToString(CultureInfo.InvariantCulture)); + request.Headers.Add(HeaderKeys.ContentLengthHeader, (partialStream.Length - partialStream.Position).ToString(CultureInfo.InvariantCulture)); request.DisablePayloadSigning = uploadPartRequest.DisablePayloadSigning; uploadPartRequest.InputStream = partialStream; @@ -159,7 +159,7 @@ private static void SetStreamChecksum(PutObjectRequest putObjectRequest, IReques if (putObjectRequest.InputStream != null) { // Wrap the stream in a stream that has a length - var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); + var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength, chooseMin: false); if (streamWithLength.Length > 0 && !(putObjectRequest.DisablePayloadSigning ?? false)) request.UseChunkEncoding = putObjectRequest.UseChunkEncoding; var length = streamWithLength.Length - streamWithLength.Position; @@ -190,7 +190,7 @@ private static void SetStreamChecksum(PutObjectRequest putObjectRequest, IReques /// If the stream supports seeking, returns stream. /// Otherwise, uses hintLength to create a read-only, non-seekable stream of given length /// - private static Stream GetStreamWithLength(Stream baseStream, long hintLength) + private static Stream GetStreamWithLength(Stream baseStream, long hintLength, bool chooseMin) { Stream result = baseStream; bool shouldWrapStream = false; @@ -198,6 +198,14 @@ private static Stream GetStreamWithLength(Stream baseStream, long hintLength) try { length = baseStream.Length - baseStream.Position; + + // If chooseMin is true that means we are uploading a part of a stream and the hintLength + // must be treated as the maximum length to read from the baseStream. + if (chooseMin && length > hintLength) + { + shouldWrapStream = true; + length = hintLength; + } } catch (NotSupportedException) { diff --git a/sdk/test/Services/S3/IntegrationTests/PutUnseekableStreamTests.cs b/sdk/test/Services/S3/IntegrationTests/PutUnseekableStreamTests.cs new file mode 100644 index 000000000000..76fbeae332c8 --- /dev/null +++ b/sdk/test/Services/S3/IntegrationTests/PutUnseekableStreamTests.cs @@ -0,0 +1,130 @@ +using System.IO; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Util; +using System.Threading.Tasks; + +namespace AWSSDK_DotNet.IntegrationTests.Tests.S3 +{ + [TestClass] + public class PutUnseekableStreamTests : TestBase + { + private static string bucketName; + + [ClassInitialize()] + public static void Initialize(TestContext a) + { + StreamWriter writer = File.CreateText("PutObjectFile.txt"); + writer.Write("This is some sample text.!!"); + writer.Close(); + + bucketName = S3TestUtils.CreateBucketWithWait(Client, true); + } + + [ClassCleanup] + public static void ClassCleanup() + { + AmazonS3Util.DeleteS3BucketWithObjects(Client, bucketName); + BaseClean(); + } + + [TestMethod] + [TestCategory("S3")] + public async Task TestPutObject() + { + var stream = new CustomStream(Encoding.UTF8.GetBytes("Hello, S3!")); + var putRequest = new PutObjectRequest + { + BucketName = bucketName, + Key = "put-object-unseekable-test.txt", + InputStream = stream, + DisablePayloadSigning = true + }; + + await Client.PutObjectAsync(putRequest); + + var getRequest = new GetObjectRequest + { + BucketName = bucketName, + Key = "put-object-unseekable-test.txt" + }; + using (var getResponse = await Client.GetObjectAsync(getRequest)) + { + using (var reader = new StreamReader(getResponse.ResponseStream)) + { + var content = reader.ReadToEnd(); + Assert.AreEqual("Hello, S3!", content); + } + } + } + + [TestMethod] + [TestCategory("S3")] + public async Task TestUploadPart() + { + var stream = new CustomStream(Encoding.UTF8.GetBytes("Hello, S3!")); + + var initiateMultipartUploadRequest = new InitiateMultipartUploadRequest + { + BucketName = bucketName, + Key = "upload-part-unseekable-test.txt" + }; + + var initiateMultipartUploadResponse = await Client.InitiateMultipartUploadAsync(initiateMultipartUploadRequest); + + var uploadPartRequest = new UploadPartRequest + { + BucketName = bucketName, + Key = "upload-part-unseekable-test.txt", + UploadId = initiateMultipartUploadResponse.UploadId, + PartNumber = 1, + PartSize = stream.Length, + InputStream = stream, + DisablePayloadSigning = true, + IsLastPart = true, + }; + + + var uploadPartResponse = await Client.UploadPartAsync(uploadPartRequest); + + var completeMultipartUploadRequest = new CompleteMultipartUploadRequest + { + BucketName = bucketName, + Key = "upload-part-unseekable-test.txt", + UploadId = initiateMultipartUploadResponse.UploadId + }; + + completeMultipartUploadRequest.AddPartETags(uploadPartResponse); + + await Client.CompleteMultipartUploadAsync(completeMultipartUploadRequest); + + var getRequest = new GetObjectRequest + { + BucketName = bucketName, + Key = "upload-part-unseekable-test.txt" + }; + using (var getResponse = await Client.GetObjectAsync(getRequest)) + { + using (var reader = new StreamReader(getResponse.ResponseStream)) + { + var content = reader.ReadToEnd(); + Assert.AreEqual("Hello, S3!", content); + } + } + } + + + public class CustomStream : MemoryStream + { + public CustomStream(byte[] buffer) : base(buffer) + { + } + + public override bool CanSeek => false; + + } + } +}