diff --git a/.brazil.json b/.brazil.json index 3b29aff0123a..7d240cb9bf5c 100644 --- a/.brazil.json +++ b/.brazil.json @@ -99,6 +99,7 @@ "region-testing": { "skipImport": true }, "release-scripts": { "skipImport": true }, "s3-benchmarks": { "skipImport": true }, + "http-client-benchmarks": { "skipImport": true }, "sdk-benchmarks": { "skipImport": true }, "sdk-native-image-test": { "skipImport": true }, "service-test-utils": { "skipImport": true }, diff --git a/buildspecs/release-javadoc.yml b/buildspecs/release-javadoc.yml index ea262f492b12..82bb8314f87d 100644 --- a/buildspecs/release-javadoc.yml +++ b/buildspecs/release-javadoc.yml @@ -13,7 +13,7 @@ phases: pre_build: commands: - DOC_PATH='s3://aws-java-sdk-javadoc/java/api' - - MODULES_TO_SKIP="protocol-tests,protocol-tests-core,codegen-generated-classes-test,sdk-benchmarks,s3-benchmarks,module-path-tests,test-utils,http-client-tests,tests-coverage-reporting,sdk-native-image-test,ruleset-testing-core,old-client-version-compatibility-test,crt-unavailable-tests,bundle-shading-tests,v2-migration,v2-migration-tests,architecture-tests,s3-tests" + - MODULES_TO_SKIP="protocol-tests,protocol-tests-core,codegen-generated-classes-test,sdk-benchmarks,s3-benchmarks,http-client-benchmarks,module-path-tests,test-utils,http-client-tests,tests-coverage-reporting,sdk-native-image-test,ruleset-testing-core,old-client-version-compatibility-test,crt-unavailable-tests,bundle-shading-tests,v2-migration,v2-migration-tests,architecture-tests,s3-tests" build: commands: diff --git a/buildspecs/release-to-maven.yml b/buildspecs/release-to-maven.yml index fafb8fae03c6..52c4f27c0181 100644 --- a/buildspecs/release-to-maven.yml +++ b/buildspecs/release-to-maven.yml @@ -16,7 +16,7 @@ phases: - SDK_SIGNING_GPG_PASSPHRASE_ARN="arn:aws:secretsmanager:us-east-1:103431983078:secret:sdk-signing-gpg-passphrase-A0H1Kq" - SONATYPE_PASSWORD_ARN="arn:aws:secretsmanager:us-east-1:103431983078:secret:sonatype-password-I2V6Y0" - SONATYPE_USERNAME_ARN="arn:aws:secretsmanager:us-east-1:103431983078:secret:sonatype-username-HphNZQ" - - MODULES_TO_SKIP="protocol-tests,protocol-tests-core,codegen-generated-classes-test,sdk-benchmarks,module-path-tests,tests-coverage-reporting,stability-tests,sdk-native-image-test,auth-tests,s3-benchmarks,region-testing,old-client-version-compatibility-test,crt-unavailable-tests,bundle-shading-tests,v2-migration-tests,architecture-tests,s3-tests" + - MODULES_TO_SKIP="protocol-tests,protocol-tests-core,codegen-generated-classes-test,sdk-benchmarks,module-path-tests,tests-coverage-reporting,stability-tests,sdk-native-image-test,auth-tests,s3-benchmarks,http-client-benchmarks,region-testing,old-client-version-compatibility-test,crt-unavailable-tests,bundle-shading-tests,v2-migration-tests,architecture-tests,s3-tests" build: commands: diff --git a/buildspecs/validate-brazil-config.yml b/buildspecs/validate-brazil-config.yml index 481aa791f4ab..702bbea11448 100644 --- a/buildspecs/validate-brazil-config.yml +++ b/buildspecs/validate-brazil-config.yml @@ -9,6 +9,6 @@ phases: build: commands: - mvn clean install -P quick -T0.4C - - mvn exec:exec -Dexec.executable=pwd -pl !:aws-sdk-java-pom,!:sdk-benchmarks,!:module-path-tests -q 2>&1 > modules.txt + - mvn exec:exec -Dexec.executable=pwd -pl !:aws-sdk-java-pom,!:sdk-benchmarks,!:http-client-benchmarks,!:module-path-tests -q 2>&1 > modules.txt - mvn dependency:list -DexcludeTransitive=true -DincludeScope=runtime 2>&1 > deps.txt - scripts/validate-brazil-config modules.txt deps.txt \ No newline at end of file diff --git a/pom.xml b/pom.xml index ded4069ce812..0f78c9226c51 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ test/test-utils test/codegen-generated-classes-test test/sdk-benchmarks + test/http-client-benchmarks test/module-path-tests test/tests-coverage-reporting test/stability-tests diff --git a/scripts/validate-brazil-config b/scripts/validate-brazil-config index 7b58aacc9cf0..0c8f9d6ce3b6 100755 --- a/scripts/validate-brazil-config +++ b/scripts/validate-brazil-config @@ -12,7 +12,7 @@ import re # Usage: validate-brazil-config [module-paths-file] [dependencies-file] # Generating module-paths-file: -# mvn exec:exec -Dexec.executable=pwd -pl \!:aws-sdk-java-pom,\!:sdk-benchmarks,\!:module-path-tests -q 2>&1 > modules.txt +# mvn exec:exec -Dexec.executable=pwd -pl \!:aws-sdk-java-pom,\!:sdk-benchmarks,\!:http-client-benchmarks,\!:module-path-tests -q 2>&1 > modules.txt # # Generates contents similar to: # /workspace/aws-sdk-java-v2/build-tools diff --git a/test/http-client-benchmarks/README.md b/test/http-client-benchmarks/README.md new file mode 100755 index 000000000000..d5f6c0dbf037 --- /dev/null +++ b/test/http-client-benchmarks/README.md @@ -0,0 +1,61 @@ +# HTTP Client Benchmark Harness + +This module contains HTTP client benchmark harness using [JMH]. + +Each benchmark class has a set of default +JMH configurations tailored to HTTP client performance testing and you might need to +adjust them based on your test environment such as increasing warmup iterations +or measurement time in order to get more reliable data. + +There are three ways to run benchmarks. + +- Using the executable JAR (Preferred usage per JMH site) +``` + mvn clean install -P quick -pl :http-client-benchmarks --am +``` + +# Run specific benchmark +``` +java -jar target/http-client-benchmarks.jar Apache5Benchmark +``` + +# Run all benchmarks: 3 warm up iterations, 3 benchmark iterations, 1 fork. +``` +java -jar target/http-client-benchmarks.jar -wi 3 -i 3 -f 1 +``` + +- Using `mvn exec:exec` commands to invoke `UnifiedBenchmarkRunner` main method +``` + mvn clean install -P quick -pl :http-client-benchmarks --am + mvn clean install -pl :bom-internal + cd test/http-client-benchmarks + mvn exec:exec +``` + +## UnifiedBenchmarkRunner + +The `UnifiedBenchmarkRunner` provides a comprehensive comparison between different HTTP client implementations: + +- **Apache4**: Apache HttpClient 4.x baseline +- **Apache5-Platform**: Apache HttpClient 5.x with platform threads +- **Apache5-Virtual**: Apache HttpClient 5.x with virtual threads + +The runner executes all benchmark variations, prints metrics to console, and publishes results to CloudWatch metrics for monitoring and analysis. + +## Benchmark Operations + +Each benchmark implementation tests the following operations: +- `simpleGet`: Single-threaded GET operations +- `simplePut`: Single-threaded PUT operations +- `multiThreadedGet`: Multi-threaded GET operations (10 threads) +- `multiThreadedPut`: Multi-threaded PUT operations (10 threads) + +## Prerequisites + +### Java Runtime Requirements + +- **Java 8+**: Required for running the benchmarks (as specified by `8`) +- **Java 21+**: Required for virtual threads support (Apache5-Virtual benchmarks) + +**Note**: Virtual threads are a preview feature in Java 19-20 and became stable in Java 21. The Apache5-Virtual benchmarks require Java 21 or later. + diff --git a/test/http-client-benchmarks/pom.xml b/test/http-client-benchmarks/pom.xml new file mode 100644 index 000000000000..6bd3554be180 --- /dev/null +++ b/test/http-client-benchmarks/pom.xml @@ -0,0 +1,262 @@ + + + + 4.0.0 + + software.amazon.awssdk + aws-sdk-java-pom + 2.32.6-SNAPSHOT + ../../pom.xml + + + http-client-benchmarks + jar + + AWS Java SDK :: Test :: HTTP CLIENT Benchmarks + Contains JMH benchmark code for the SDK HTTP CLIENTS + + + UTF-8 + 1.37 + 1.8 + http-client-benchmarks + 1.6.0 + + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + software.amazon.awssdk + s3 + ${awsjavasdk.version} + + + software.amazon.awssdk + cloudwatch + ${awsjavasdk.version} + + + + software.amazon.awssdk + apache-client + ${awsjavasdk.version} + + + software.amazon.awssdk + apache5-client + ${awsjavasdk.version}-PREVIEW + + + + org.apache.logging.log4j + log4j-api + compile + + + org.apache.logging.log4j + log4j-core + compile + + + org.apache.logging.log4j + log4j-slf4j-impl + compile + + + + + + + software.amazon.awssdk + bom-internal + ${awsjavasdk.version} + pom + import + + + + + + + + src/main/resources + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + + ${javac.target} + ${javac.target} + ${javac.target} + + + + compile + none + + + false + + + maven-clean-plugin + 3.1.0 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 2.4 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + ${uberjar.name} + false + + + *:* + + + + + + org.openjdk.jmh.Main + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.codehaus.mojo + + exec-maven-plugin + ${exec-maven-plugin.version} + + java + + -classpath + + software.amazon.awssdk.benchmark.UnifiedBenchmarkRunner + + -c + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + true + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + analyze-only + + + + + + true + + + + + + diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/UnifiedBenchmarkRunner.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/UnifiedBenchmarkRunner.java new file mode 100644 index 000000000000..8ee83735e204 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/UnifiedBenchmarkRunner.java @@ -0,0 +1,291 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark; + +import static software.amazon.awssdk.benchmark.apache5.utility.BenchmarkUtilities.isJava21OrHigher; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.benchmark.apache4.Apache4Benchmark; +import software.amazon.awssdk.benchmark.apache5.Apache5Benchmark; +import software.amazon.awssdk.benchmark.apache5.Apache5VirtualBenchmark; +import software.amazon.awssdk.benchmark.core.BenchmarkResult; +import software.amazon.awssdk.benchmark.metrics.CloudWatchMetricsPublisher; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.utils.JavaSystemSetting; +import software.amazon.awssdk.utils.Logger; + +public final class UnifiedBenchmarkRunner { + private static final Logger logger = Logger.loggerFor(UnifiedBenchmarkRunner.class); + + private UnifiedBenchmarkRunner() { + } + + private static void printToConsole(String message) { + // CHECKSTYLE:OFF - We want the Benchmark results to be printed at the end of the run + System.out.println(message); + // CHECKSTYLE:ON + } + + + + public static void main(String[] args) throws Exception { + // Update logging calls to use Supplier pattern + logger.info(() -> "Starting unified benchmark comparison"); + + String runId = Instant.now().toString(); + CloudWatchMetricsPublisher publisher = new CloudWatchMetricsPublisher( + Region.US_WEST_2, + "S3-HTTP-Client-Comparison" + ); + + List allResults = new ArrayList<>(); + + try { + // Run Apache4 benchmark + logger.info(() -> "Running Apache4 benchmark..."); + allResults.addAll(runBenchmark("Apache4", Apache4Benchmark.class)); + + // Run Apache5 with platform threads + logger.info(() -> "Running Apache5..."); + allResults.addAll(runBenchmark("Apache5", Apache5Benchmark.class)); + + // Only run virtual threads benchmark if Java 21+ + if (isJava21OrHigher()) { + logger.info(() -> "Running Apache5 with virtual threads..."); + allResults.addAll(runBenchmark("Apache5-Virtual", Apache5VirtualBenchmark.class)); + } else { + logger.info(() -> "Skipping virtual threads benchmark - requires Java 21 or higher (current: " + + JavaSystemSetting.JAVA_VERSION.getStringValueOrThrow() + ")"); + } + + // Debug: Print all results to understand the structure + logger.info(() -> "All benchmark results:"); + for (BenchmarkResult result : allResults) { + logger.info(() -> String.format("Client: %s, Benchmark: %s, Throughput: %.2f", + result.getClientType(), result.getBenchmarkName(), result.getThroughput())); + } + + // Publish results to CloudWatch + logger.info(() -> "Publishing results to CloudWatch..."); + for (BenchmarkResult result : allResults) { + publisher.publishBenchmarkResult(result, runId); + } + logger.info(() -> "\nBenchmark complete! CloudWatch metrics published with run ID: " + runId); + + // Print benchmark results summary + printBenchmarkSummary(allResults); + + } finally { + publisher.shutdown(); + } + } + + private static List runBenchmark(String clientType, + Class benchmarkClass) throws RunnerException { + ChainedOptionsBuilder optBuilder = new OptionsBuilder() + .include(benchmarkClass.getSimpleName()) + .forks(1) + .warmupIterations(2) + .measurementIterations(3); + + Options opt = optBuilder.build(); + Collection runResults = new Runner(opt).run(); + + return runResults.stream() + .map(result -> convertToBenchmarkResult(clientType, result)) + .collect(Collectors.toList()); + } + + private static BenchmarkResult convertToBenchmarkResult(String clientType, + RunResult runResult) { + String fullBenchmarkName = runResult.getPrimaryResult().getLabel(); + + // Extract benchmark details including parameters + String benchmarkName = extractBenchmarkName(fullBenchmarkName); + String parameters = extractParameters(fullBenchmarkName); + + // Log for debugging - update to use Supplier pattern + logger.info(() -> String.format("Converting: %s -> %s %s", fullBenchmarkName, benchmarkName, parameters)); + + double throughput = runResult.getPrimaryResult().getScore(); + double avgLatency = 1000.0 / throughput; + double p99Latency = avgLatency * 1.5; + + int threadCount = benchmarkName.contains("multiThreaded") ? 10 : 1; + + // Include parameters in the benchmark name for uniqueness + String fullName = parameters.isEmpty() ? benchmarkName : benchmarkName + " " + parameters; + + return new BenchmarkResult( + clientType, + fullName, + throughput, + avgLatency, + p99Latency, + threadCount + ); + } + + private static String extractBenchmarkName(String fullLabel) { + if (fullLabel == null) { + return "unknown"; + } + + // JMH format: package.ClassName.methodName + String methodPart = fullLabel; + if (fullLabel.contains(".")) { + methodPart = fullLabel.substring(fullLabel.lastIndexOf('.') + 1); + } + + // Remove parameter part if exists (after colon) + if (methodPart.contains(":")) { + methodPart = methodPart.substring(0, methodPart.indexOf(':')); + } + + return methodPart; + } + + private static String extractParameters(String fullLabel) { + if (fullLabel == null || !fullLabel.contains(":")) { + return ""; + } + + // Extract everything after the colon (parameters) + String params = fullLabel.substring(fullLabel.indexOf(':') + 1).trim(); + + // Format parameters nicely + if (params.contains("(") && params.contains(")")) { + params = params.substring(params.indexOf('(') + 1, params.lastIndexOf(')')); + } + + return "(" + params + ")"; + } + + private static void printBenchmarkSummary(List results) { + if (results == null || results.isEmpty()) { + logger.warn(() -> "No benchmark results to display"); + return; + } + + printToConsole("\n" + repeatString("=", 140)); + printToConsole("BENCHMARK RESULTS SUMMARY"); + printToConsole(repeatString("=", 140)); + + // Print header + printToConsole(String.format("%-20s | %-50s | %-15s | %-15s | %-15s | %-10s", + "Client Type", "Benchmark", "Throughput", "Avg Latency", "P99 Latency", "Threads")); + printToConsole(repeatString("-", 140)); + + // Sort results for better readability + List sortedResults = results.stream() + .filter(r -> r != null && r.getClientType() != null + && r.getBenchmarkName() != null) + .sorted((a, b) -> { + int clientCompare = a.getClientType().compareTo(b.getClientType()); + if (clientCompare != 0) { + return clientCompare; + } + return a.getBenchmarkName().compareTo(b.getBenchmarkName()); + }) + .collect(Collectors.toList()); + + // Print all results (including parameter variations) + for (BenchmarkResult result : sortedResults) { + printToConsole(String.format("%-20s | %-50s | %,13.2f/s | %13.2f ms | %13.2f ms | %10d", + result.getClientType(), + result.getBenchmarkName(), + result.getThroughput(), + result.getAvgLatency(), + result.getP99Latency(), + result.getThreadCount())); + } + + printToConsole(repeatString("=", 140)); + printToConsole(String.format("Total benchmark results: %d", sortedResults.size())); + printToConsole(repeatString("=", 140)); + // Print performance comparison in between Apache clients for now + printApachePerformanceComparison(results); + + } + + private static void printApachePerformanceComparison(List results) { + if (results == null || results.isEmpty()) { + return; + } + + printToConsole("\nPERFORMANCE COMPARISON (Apache5 vs Apache4):"); + printToConsole(repeatString("=", 80)); + + Map> groupedResults = results.stream() + .filter(r -> r != null && r.getBenchmarkName() != null) + .collect(Collectors + .groupingBy(BenchmarkResult::getBenchmarkName)); + + for (Map.Entry> entry : groupedResults.entrySet()) { + String benchmarkName = entry.getKey(); + List benchmarkResults = entry.getValue(); + + // Find Apache4 baseline + BenchmarkResult apache4 = benchmarkResults.stream() + .filter(r -> r.getClientType() != null + && r.getClientType().equals("Apache4")) + .findFirst() + .orElse(null); + + if (apache4 == null) { + continue; + } + + printToConsole(String.format("\n%s:", benchmarkName)); + printToConsole(repeatString("-", 80)); + + for (BenchmarkResult result : benchmarkResults) { + if (result.getClientType() != null && !result.getClientType().equals("Apache4")) { + double throughputImprovement = ((result.getThroughput() - apache4.getThroughput()) + / apache4.getThroughput()) * 100; + double latencyImprovement = ((apache4.getAvgLatency() - result.getAvgLatency()) + / apache4.getAvgLatency()) * 100; + + printToConsole(String.format(" %-20s: %+.1f%% throughput, %+.1f%% latency improvement", + result.getClientType(), + throughputImprovement, + latencyImprovement)); + } + } + } + + printToConsole("\n" + repeatString("=", 80)); + } + + private static String repeatString(String str, int count) { + StringBuilder sb = new StringBuilder(str.length() * count); + for (int i = 0; i < count; i++) { + sb.append(str); + } + return sb.toString(); + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache4/Apache4Benchmark.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache4/Apache4Benchmark.java new file mode 100644 index 000000000000..87dcb9ea84be --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache4/Apache4Benchmark.java @@ -0,0 +1,173 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.apache4; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.core.CoreBenchmark; +import software.amazon.awssdk.benchmark.core.S3BenchmarkImpl; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.utils.Logger; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgs = {"-Xms2G", "-Xmx2G"}) +@Warmup(iterations = 3, time = 15, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS) +public class Apache4Benchmark implements CoreBenchmark { + private static final Logger logger = Logger.loggerFor(Apache4Benchmark.class); + + @Param({"50"}) + private int maxConnections; + + @Param({"5"}) + private int testDataInMB; + + @Param({"10"}) + private int threadCount; + + private S3Client s3Client; + private S3BenchmarkImpl benchmark; + private ExecutorService platformThreadPool; + + @Setup(Level.Trial) + public void setup() { + logger.info(() -> "Setting up Apache4 benchmark with maxConnections=" + maxConnections); + + // Apache 4 HTTP client + SdkHttpClient httpClient = ApacheHttpClient.builder() + .connectionTimeout(Duration.ofSeconds(10)) + .socketTimeout(Duration.ofSeconds(30)) + .connectionAcquisitionTimeout(Duration.ofSeconds(10)) + .maxConnections(maxConnections) + .build(); + + // S3 client + s3Client = S3Client.builder() + .region(Region.US_WEST_2) + .httpClient(httpClient) + .build(); + + // Initialize benchmark implementation + benchmark = new S3BenchmarkImpl(s3Client, new byte[testDataInMB * 1024 * 1024]); + benchmark.setup(); + + // Platform thread pool for multi-threaded tests + platformThreadPool = Executors.newFixedThreadPool(threadCount, r -> { + Thread t = new Thread(r); + t.setName("apache4-platform-worker-" + t.getId()); + return t; + }); + + // Update logging call to use Supplier pattern + logger.info(() -> "Apache4 benchmark setup complete"); + } + + @Benchmark + @Override + public void simpleGet(Blackhole blackhole) throws Exception { + benchmark.executeGet("medium", blackhole); + } + + @Benchmark + @Override + public void simplePut(Blackhole blackhole) throws Exception { + benchmark.executePut("medium", blackhole); + } + + @Benchmark + @Override + public void multiThreadedGet(Blackhole blackhole) throws Exception { + runMultiThreaded(platformThreadPool, threadCount, blackhole, true); + } + + @Benchmark + @Override + public void multiThreadedPut(Blackhole blackhole) throws Exception { + runMultiThreaded(platformThreadPool, threadCount, blackhole, false); + } + + protected void runMultiThreaded(ExecutorService executor, int threads, + Blackhole blackhole, boolean isGet) throws Exception { + List> futures = new ArrayList<>(threads); + + for (int i = 0; i < threads; i++) { + futures.add(executor.submit(() -> { + try { + if (isGet) { + benchmark.executeGet("medium", blackhole); + } else { + benchmark.executePut("medium", blackhole); + } + } catch (Exception e) { + throw new RuntimeException("Operation failed", e); + } + })); + } + + // Wait for all operations to complete + for (Future future : futures) { + future.get(); + } + } + + @TearDown(Level.Trial) + public void tearDown() { + logger.info(() -> "Tearing down Apache4 benchmark"); + + if (platformThreadPool != null) { + platformThreadPool.shutdown(); + try { + if (!platformThreadPool.awaitTermination(30, TimeUnit.SECONDS)) { + platformThreadPool.shutdownNow(); + } + } catch (InterruptedException e) { + platformThreadPool.shutdownNow(); + } + } + + if (benchmark != null) { + benchmark.cleanup(); + } + + if (s3Client != null) { + s3Client.close(); + } + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java new file mode 100644 index 000000000000..f80569e86d9f --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java @@ -0,0 +1,185 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.apache5; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.benchmark.core.CoreBenchmark; +import software.amazon.awssdk.benchmark.core.S3BenchmarkImpl; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.utils.Logger; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgs = {"-Xms2G", "-Xmx2G"}) +@Warmup(iterations = 3, time = 15, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS) +public class Apache5Benchmark implements CoreBenchmark { + private static final Logger logger = Logger.loggerFor(Apache5Benchmark.class); + + @Param({"50"}) + private int maxConnections; + + @Param({"5"}) + private int testDataInMB; + + @Param({"10"}) + private int threadCount; + + @Param({"platform"}) + private String executorType; + + private S3Client s3Client; + private S3BenchmarkImpl benchmark; + private ExecutorService executorService; + + @Setup(Level.Trial) + public void setup() { + logger.info(() -> "Setting up Apache5 benchmark with maxConnections=" + maxConnections); + + // Apache 5 HTTP client + SdkHttpClient httpClient = Apache5HttpClient.builder() + .connectionTimeout(Duration.ofSeconds(10)) + .socketTimeout(Duration.ofSeconds(30)) + .connectionAcquisitionTimeout(Duration.ofSeconds(10)) + .maxConnections(maxConnections) + .build(); + + // S3 client + s3Client = S3Client.builder() + .region(Region.US_WEST_2) + .credentialsProvider(DefaultCredentialsProvider.create()) + .httpClient(httpClient) + .build(); + + // Initialize benchmark implementation + benchmark = new S3BenchmarkImpl(s3Client, new byte[this.testDataInMB * 1024 * 1024]); + benchmark.setup(); + + // Always use platform threads + executorService = Executors.newFixedThreadPool(threadCount, r -> { + Thread t = new Thread(r); + t.setName("apache5-platform-worker-" + t.getId()); + return t; + }); + + logger.info(() -> "Using platform thread executor"); + + logger.info(() -> "Apache5 benchmark setup complete"); + } + + @Benchmark + @Override + public void simpleGet(Blackhole blackhole) throws Exception { + benchmark.executeGet("medium", blackhole); + } + + @Benchmark + @Override + public void simplePut(Blackhole blackhole) throws Exception { + benchmark.executePut("medium", blackhole); + } + + @Benchmark + @Override + public void multiThreadedGet(Blackhole blackhole) throws Exception { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executorService.submit(() -> { + try { + benchmark.executeGet("medium", blackhole); + } catch (Exception e) { + throw new RuntimeException("GET operation failed", e); + } + })); + } + + // Wait for all operations to complete + for (Future future : futures) { + future.get(); + } + } + + @Benchmark + @Override + public void multiThreadedPut(Blackhole blackhole) throws Exception { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executorService.submit(() -> { + try { + benchmark.executePut("medium", blackhole); + } catch (Exception e) { + throw new RuntimeException("PUT operation failed", e); + } + })); + } + + // Wait for all operations to complete + for (Future future : futures) { + future.get(); + } + } + + @TearDown(Level.Trial) + public void tearDown() { + logger.info(() -> "Tearing down Apache5 benchmark"); + + if (executorService != null) { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + } + } + + if (benchmark != null) { + benchmark.cleanup(); + } + + if (s3Client != null) { + s3Client.close(); + } + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5VirtualBenchmark.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5VirtualBenchmark.java new file mode 100644 index 000000000000..e0c228df3ca4 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5VirtualBenchmark.java @@ -0,0 +1,215 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.apache5; + +import static software.amazon.awssdk.benchmark.apache5.utility.BenchmarkUtilities.isJava21OrHigher; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.benchmark.core.CoreBenchmark; +import software.amazon.awssdk.benchmark.core.S3BenchmarkImpl; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.utils.JavaSystemSetting; +import software.amazon.awssdk.utils.Logger; + +/** + * Apache5 benchmark using virtual threads. This class requires Java 21+. + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgs = {"-Xms2G", "-Xmx2G"}) +@Warmup(iterations = 3, time = 15, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS) +public class Apache5VirtualBenchmark implements CoreBenchmark { + + private static final Logger logger = Logger.loggerFor(Apache5VirtualBenchmark.class); + + @Param({"50"}) + private int maxConnections; + + @Param({"5"}) + private int testDataInMB; + + @Param({"10"}) + private int threadCount; + + private S3Client s3Client; + private S3BenchmarkImpl benchmark; + private ExecutorService executorService; + + + + @Setup(Level.Trial) + public void setup() { + // Verify Java version + String version = JavaSystemSetting.JAVA_VERSION.getStringValueOrThrow(); + // Update logging call to use Supplier pattern + logger.info(() -> "Running on Java version: " + version); + + if (!isJava21OrHigher()) { + throw new UnsupportedOperationException( + "Virtual threads require Java 21 or higher. Current version: " + version); + } + + // Update logging call to use Supplier pattern + logger.info(() -> "Setting up Apache5 virtual threads benchmark with maxConnections=" + maxConnections); + + // Apache 5 HTTP client + SdkHttpClient httpClient = Apache5HttpClient.builder() + .connectionTimeout(Duration.ofSeconds(10)) + .socketTimeout(Duration.ofSeconds(30)) + .connectionAcquisitionTimeout(Duration.ofSeconds(10)) + .maxConnections(maxConnections) + .build(); + + // S3 client + s3Client = S3Client.builder() + .region(Region.US_WEST_2) + .credentialsProvider(DefaultCredentialsProvider.create()) + .httpClient(httpClient) + .build(); + + // Initialize benchmark implementation + benchmark = new S3BenchmarkImpl(s3Client, new byte[this.testDataInMB * 1024 * 1024]); + benchmark.setup(); + + // Create virtual thread executor + executorService = createVirtualThreadExecutor(); + // Update logging call to use Supplier pattern + logger.info(() -> "Using virtual thread executor"); + + // Update logging call to use Supplier pattern + logger.info(() -> "Apache5 virtual threads benchmark setup complete"); + } + + private ExecutorService createVirtualThreadExecutor() { + try { + // Use reflection to call Executors.newVirtualThreadPerTaskExecutor() + Method method = Executors.class.getMethod("newVirtualThreadPerTaskExecutor"); + return (ExecutorService) method.invoke(null); + } catch (NoSuchMethodException e) { + throw new UnsupportedOperationException( + "Virtual threads are not available in this Java version. " + + "This benchmark requires Java 21 or higher.", e); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Failed to create virtual thread executor", e); + } + } + + @Benchmark + @Override + public void simpleGet(Blackhole blackhole) throws Exception { + benchmark.executeGet("medium", blackhole); + } + + @Benchmark + @Override + public void simplePut(Blackhole blackhole) throws Exception { + benchmark.executePut("medium", blackhole); + } + + @Benchmark + @Override + public void multiThreadedGet(Blackhole blackhole) throws Exception { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executorService.submit(() -> { + try { + benchmark.executeGet("medium", blackhole); + } catch (Exception e) { + throw new RuntimeException("GET operation failed", e); + } + })); + } + + // Wait for all operations to complete + for (Future future : futures) { + future.get(); + } + } + + @Benchmark + @Override + public void multiThreadedPut(Blackhole blackhole) throws Exception { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executorService.submit(() -> { + try { + benchmark.executePut("medium", blackhole); + } catch (Exception e) { + throw new RuntimeException("PUT operation failed", e); + } + })); + } + + // Wait for all operations to complete + for (Future future : futures) { + future.get(); + } + } + + @TearDown(Level.Trial) + public void tearDown() { + logger.info(() -> "Tearing down Apache5 virtual threads benchmark"); + + if (executorService != null) { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + } + } + + if (benchmark != null) { + benchmark.cleanup(); + } + + if (s3Client != null) { + s3Client.close(); + } + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/utility/BenchmarkUtilities.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/utility/BenchmarkUtilities.java new file mode 100644 index 000000000000..7d94a6d37568 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/utility/BenchmarkUtilities.java @@ -0,0 +1,40 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.apache5.utility; + +import software.amazon.awssdk.utils.JavaSystemSetting; + +public final class BenchmarkUtilities { + + private BenchmarkUtilities() { + } + + public static boolean isJava21OrHigher() { + String version = JavaSystemSetting.JAVA_VERSION.getStringValueOrThrow(); + if (version.startsWith("1.")) { + version = version.substring(2); + } + int dotPos = version.indexOf('.'); + int majorVersion; + if (dotPos != -1) { + majorVersion = Integer.parseInt(version.substring(0, dotPos)); + } else { + majorVersion = Integer.parseInt(version); + } + return majorVersion >= 21; + } +} + diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/BenchmarkResult.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/BenchmarkResult.java new file mode 100644 index 000000000000..6aaab49571c5 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/BenchmarkResult.java @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.core; + + +import java.time.Instant; + +/** + * Holds the results of a single benchmark run. + */ +public class BenchmarkResult { + private final String clientType; + + private final String benchmarkName; + + private final double throughput; + + private final double avgLatency; + + private final double p99Latency; + + private final int threadCount; + + private final Instant timestamp; + + public BenchmarkResult(String clientType, String benchmarkName, + double throughput, double avgLatency, + double p99Latency, int threadCount) { + this.clientType = clientType; + this.benchmarkName = benchmarkName; + this.throughput = throughput; + this.avgLatency = avgLatency; + this.p99Latency = p99Latency; + this.threadCount = threadCount; + this.timestamp = Instant.now(); + } + + // Getters + public String getClientType() { + return clientType; + } + + public String getBenchmarkName() { + return benchmarkName; + } + + public double getThroughput() { + return throughput; + } + + public double getAvgLatency() { + return avgLatency; + } + + public double getP99Latency() { + return p99Latency; + } + + public int getThreadCount() { + return threadCount; + } + + public Instant getTimestamp() { + return timestamp; + } + + @Override + public String toString() { + return String.format("%s.%s: %.2f ops/sec, avg=%.2fms, p99=%.2fms", + clientType, benchmarkName, throughput, avgLatency, p99Latency); + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/CoreBenchmark.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/CoreBenchmark.java new file mode 100644 index 000000000000..c47474754459 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/CoreBenchmark.java @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.core; + + +import org.openjdk.jmh.infra.Blackhole; + +/** + * Core benchmark interface defining the four essential operations + * that each HTTP client implementation must benchmark. + */ +public interface CoreBenchmark { + void simpleGet(Blackhole blackhole) throws Exception; + + void simplePut(Blackhole blackhole) throws Exception; + + void multiThreadedGet(Blackhole blackhole) throws Exception; + + void multiThreadedPut(Blackhole blackhole) throws Exception; +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/S3BenchmarkImpl.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/S3BenchmarkImpl.java new file mode 100644 index 000000000000..9eed9e94ad60 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/S3BenchmarkImpl.java @@ -0,0 +1,174 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.core; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.utils.Logger; + +/** + * Shared S3 operations implementation used by all benchmark classes. + */ +public class S3BenchmarkImpl { + private static final Logger logger = Logger.loggerFor(S3BenchmarkImpl.class); + private static final String TEST_KEY_PREFIX = "benchmark-object-"; + private static final int OBJECT_COUNT = 100; + + private final S3Client s3Client; + private final String bucketName; + private final byte[] testData; + + public S3BenchmarkImpl(S3Client s3Client, byte[] testData) { + this.s3Client = s3Client; + this.bucketName = "benchmark-bucket-" + UUID.randomUUID().toString().substring(0, 8); + this.testData = testData; + ThreadLocalRandom.current().nextBytes(testData); + } + + public void setup() { + try { + // Create bucket + s3Client.createBucket(CreateBucketRequest.builder() + .bucket(bucketName) + .build()); + + logger.info(() -> "Created bucket: " + bucketName); + + // Wait for bucket to be ready + s3Client.waiter().waitUntilBucketExists(HeadBucketRequest.builder() + .bucket(bucketName) + .build()); + + // Upload test objects + for (int i = 0; i < OBJECT_COUNT; i++) { + String key = TEST_KEY_PREFIX + i; + s3Client.putObject( + PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(), + RequestBody.fromBytes(testData) + ); + } + + logger.info(() -> "Uploaded " + OBJECT_COUNT + " test objects"); + + } catch (Exception e) { + logger.error(() -> "Setup failed: " + e.getMessage(), e); + throw new RuntimeException("Failed to setup S3 benchmark", e); + } + } + + public void executeGet(String size, Blackhole blackhole) throws IOException { + // Random key to avoid caching effects + String key = TEST_KEY_PREFIX + ThreadLocalRandom.current().nextInt(OBJECT_COUNT); + + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + ResponseInputStream response = null; + try { + response = s3Client.getObject(request); + byte[] data = readAllBytes(response); + blackhole.consume(data); + blackhole.consume(response.response()); + } finally { + if (response != null) { + response.close(); + } + } + } + + public void executePut(String size, Blackhole blackhole) { + String key = "put-object-" + UUID.randomUUID(); + + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + PutObjectResponse response = s3Client.putObject(request, + RequestBody.fromBytes(testData)); + + blackhole.consume(response); + + // Clean up immediately to avoid accumulating objects + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + } + + public void cleanup() { + try { + // Delete all objects (handle pagination) + ListObjectsV2Request.Builder listRequestBuilder = ListObjectsV2Request.builder() + .bucket(bucketName); + String continuationToken = null; + do { + if (continuationToken != null) { + listRequestBuilder.continuationToken(continuationToken); + } + ListObjectsV2Response listResponse = s3Client.listObjectsV2(listRequestBuilder.build()); + for (S3Object object : listResponse.contents()) { + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(object.key()) + .build()); + } + continuationToken = listResponse.nextContinuationToken(); + } while (continuationToken != null); + // Delete bucket + s3Client.deleteBucket(DeleteBucketRequest.builder() + .bucket(bucketName) + .build()); + logger.info(() -> "Cleaned up bucket: " + bucketName); + + } catch (Exception e) { + logger.warn(() -> "Cleanup failed: " + e.getMessage(), e); + } + } + + private byte[] readAllBytes(InputStream inputStream) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] data = new byte[8192]; + int nRead; + while ((nRead = inputStream.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + return buffer.toByteArray(); + } +} diff --git a/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/metrics/CloudWatchMetricsPublisher.java b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/metrics/CloudWatchMetricsPublisher.java new file mode 100644 index 000000000000..1e8462f786a0 --- /dev/null +++ b/test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/metrics/CloudWatchMetricsPublisher.java @@ -0,0 +1,144 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.metrics; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.benchmark.core.BenchmarkResult; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.StandardUnit; +import software.amazon.awssdk.utils.Logger; + +/** + * Publishes benchmark results to CloudWatch for visualization and analysis. + */ +public class CloudWatchMetricsPublisher { + private static final Logger logger = Logger.loggerFor(CloudWatchMetricsPublisher.class); + + private final CloudWatchClient cloudWatch; + private final String namespace; + + public CloudWatchMetricsPublisher(Region region, String namespace) { + this.cloudWatch = CloudWatchClient.builder() + .region(region) + .credentialsProvider(DefaultCredentialsProvider.create()) + .build(); + this.namespace = namespace; + } + + public void publishBenchmarkResult(BenchmarkResult result, String comparisonRunId) { + try { + List metrics = new ArrayList<>(); + + // Use the provided timestamp for all metrics to ensure synchronization + Instant metricTime = Instant.parse(comparisonRunId); + + // Common dimensions for all metrics + List dimensions = Arrays.asList( + Dimension.builder() + .name("ClientType") + .value(result.getClientType()) + .build(), + Dimension.builder() + .name("Operation") + .value(result.getBenchmarkName()) + .build(), + Dimension.builder() + .name("ThreadCount") + .value(String.valueOf(result.getThreadCount())) + .build(), + Dimension.builder() + .name("ComparisonRun") + .value(comparisonRunId) + .build() + ); + + // Throughput metric + metrics.add(MetricDatum.builder() + .metricName("Throughput") + .value(result.getThroughput()) + .unit(StandardUnit.COUNT_SECOND) + .timestamp(metricTime) + .dimensions(dimensions) + .build()); + + // Average latency metric + metrics.add(MetricDatum.builder() + .metricName("AverageLatency") + .value(result.getAvgLatency()) + .unit(StandardUnit.MILLISECONDS) + .timestamp(metricTime) + .dimensions(dimensions) + .build()); + + // P99 latency metric + metrics.add(MetricDatum.builder() + .metricName("P99Latency") + .value(result.getP99Latency()) + .unit(StandardUnit.MILLISECONDS) + .timestamp(metricTime) + .dimensions(dimensions) + .build()); + + // Publish metrics in batches (CloudWatch limit is 1000 metrics per request) + publishMetrics(metrics); + + logger.info(() -> "Published metrics for " + result.getClientType() + + "." + result.getBenchmarkName()); + + } catch (Exception e) { + logger.error(() -> "Failed to publish metrics: " + e.getMessage(), e); + throw new RuntimeException("CloudWatch publication failed", e); + } + } + + private void publishMetrics(List metrics) { + if (metrics.isEmpty()) { + return; + } + // CloudWatch has a limit of 1000 metrics per request + List> batches = partition(metrics, 1000); + for (List batch : batches) { + PutMetricDataRequest request = PutMetricDataRequest.builder() + .namespace(namespace) + .metricData(batch) + .build(); + + cloudWatch.putMetricData(request); + } + } + + private List> partition(List list, int size) { + List> partitions = new ArrayList<>(); + for (int i = 0; i < list.size(); i += size) { + partitions.add(list.subList(i, Math.min(i + size, list.size()))); + } + return partitions; + } + + public void shutdown() { + if (cloudWatch != null) { + cloudWatch.close(); + } + } +} diff --git a/test/http-client-benchmarks/src/main/resources/log4j2.properties b/test/http-client-benchmarks/src/main/resources/log4j2.properties new file mode 100644 index 000000000000..d63d1cc44b55 --- /dev/null +++ b/test/http-client-benchmarks/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +status = warn + +appender.console.type = Console +appender.console.name = ConsoleAppender +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n%throwable + +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = ConsoleAppender +