From 4423cc57e26efe8980e619f06147ff104b5d4477 Mon Sep 17 00:00:00 2001 From: Philip J Freeman Date: Sun, 7 Dec 2025 20:08:12 -0800 Subject: [PATCH] increase multipart chunksize seeing failures uploading a very lage object: botocore.exceptions.ClientError: An error occurred (InvalidArgument) when calling the UploadPart operation: Part number must be an integer between 1 and 10000, inclusive This should support uploading file object sizes of up to 2.4TB... --- ServerBackupS3/S3Upload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ServerBackupS3/S3Upload.py b/ServerBackupS3/S3Upload.py index e929114..2f555a3 100644 --- a/ServerBackupS3/S3Upload.py +++ b/ServerBackupS3/S3Upload.py @@ -39,7 +39,9 @@ def __init__(self, bucket_name, region, verbose=False): def multipart_from_process(self, backup_name, process, verbose=False): - self.s3.Object(self.bucket_name, backup_name).upload_fileobj(process.stdout) + config = boto3.s3.transfer.TransferConfig(multipart_chunksize=1024 * 1024 * 256) + + self.s3.Object(self.bucket_name, backup_name).upload_fileobj(process.stdout, Config=config) if self.verbose: print("DEBUG: done with upload of", backup_name)