Skip to content

Commit 5a96582

Browse files
Adds new API CosmosContainer.readAllItems to retrieve all documents of a logical partition avoiding IO to retrieve query plan (Azure#14481)
* Enabling query metrics by default * Adding null-value handling to quey metric validation * Fix for NPE in GatewayAddressCache * Adding comments explaining default value of QueryMetricEnabled * Adding support for readAllItems of a logical partition * Adding readAllItems for sync CosmosContainer as well * Adding basic unit test for ReadAllItems(PK) * Adding benchmark for ReadAllItems * Fixing cosmos-benchmark version as well * Fixing changelog for cosmos-encryption * Fixing versions * Fixing current version of azure-cosmos * Reverting azure-cosmos version bump * Bumping azure-cosmos version * Reverting unnecessary changelog and readme changes * Fixing azure-cosmos-benchmark to use dynamic number of precreated documents * Adding benchmark use case description * Fixing wrong StringUtils dependency * Remove unnecessary constants * Making readAllDocuments partition split safe * Moving new test in CosmosItemTest to ObjectNode instead of InternalObjectNode * Adding comment that ItemOperations readMany APIs are deprecated
1 parent 4791f8e commit 5a96582

File tree

21 files changed

+766
-199
lines changed

21 files changed

+766
-199
lines changed

eng/jacoco-test-coverage/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@
109109
<dependency>
110110
<groupId>com.azure</groupId>
111111
<artifactId>azure-cosmos</artifactId>
112-
<version>4.3.2-beta.3</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
112+
<version>4.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
113113
</dependency>
114114
<dependency>
115115
<groupId>com.azure</groupId>

eng/versioning/version_client.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ com.azure:azure-core-serializer-json-gson;1.0.0-beta.3;1.0.0-beta.4
2020
com.azure:azure-core-serializer-json-jackson;1.0.0-beta.3;1.0.0-beta.4
2121
com.azure:azure-core-test;1.4.0;1.5.0-beta.1
2222
com.azure:azure-core-tracing-opentelemetry;1.0.0-beta.6;1.0.0-beta.7
23-
com.azure:azure-cosmos;4.3.1;4.3.2-beta.3
23+
com.azure:azure-cosmos;4.3.1;4.4.0-beta.1
2424
com.azure:azure-cosmos-benchmark;4.0.1-beta.1;4.0.1-beta.1
2525
com.azure:azure-cosmos-encryption;1.0.0-beta.1;1.0.0-beta.1
2626
com.azure:azure-data-appconfiguration;1.1.4;1.2.0-beta.1

sdk/cosmos/azure-cosmos-benchmark/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ Copyright (c) Microsoft Corporation. All rights reserved.
33
Licensed under the MIT License.
44
-->
55
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
6-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
6+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
77
<modelVersion>4.0.0</modelVersion>
88
<parent>
99
<groupId>com.azure</groupId>
@@ -43,7 +43,7 @@ Licensed under the MIT License.
4343
<dependency>
4444
<groupId>com.azure</groupId>
4545
<artifactId>azure-cosmos</artifactId>
46-
<version>4.3.2-beta.3</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
46+
<version>4.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
4747
</dependency>
4848

4949
<dependency>

sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/AsyncQueryBenchmark.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,12 @@ protected void performWorkload(BaseSubscriber<FeedResponse<PojoizedJson>> baseSu
7676

7777
if (configuration.getOperationType() == Configuration.Operation.QueryCross) {
7878

79-
int index = r.nextInt(1000);
79+
int index = r.nextInt(this.configuration.getNumberOfPreCreatedDocuments());
8080
String sqlQuery = "Select * from c where c.id = \"" + docsToRead.get(index).getId() + "\"";
8181
obs = cosmosAsyncContainer.queryItems(sqlQuery, options, PojoizedJson.class).byPage();
8282
} else if (configuration.getOperationType() == Configuration.Operation.QuerySingle) {
8383

84-
int index = r.nextInt(1000);
84+
int index = r.nextInt(this.configuration.getNumberOfPreCreatedDocuments());
8585
String pk = docsToRead.get(index).getProperty(partitionKey);
8686
options.setPartitionKey(new PartitionKey(pk));
8787
String sqlQuery = "Select * from c where c." + partitionKey + " = \"" + pk + "\"";
@@ -123,6 +123,11 @@ protected void performWorkload(BaseSubscriber<FeedResponse<PojoizedJson>> baseSu
123123

124124
SqlQuerySpec query = queryBuilder.toSqlQuerySpec();
125125
obs = cosmosAsyncContainer.queryItems(query, options, PojoizedJson.class).byPage();
126+
} else if (configuration.getOperationType() == Configuration.Operation.ReadAllItemsOfLogicalPartition) {
127+
128+
int index = r.nextInt(this.configuration.getNumberOfPreCreatedDocuments());
129+
String pk = docsToRead.get(index).getProperty(partitionKey);
130+
obs = cosmosAsyncContainer.readAllItems(new PartitionKey(pk), options, PojoizedJson.class).byPage();
126131
} else {
127132
throw new IllegalArgumentException("Unsupported Operation: " + configuration.getOperationType());
128133
}

sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/Configuration.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ public class Configuration {
104104
+ "\tMixed - runa workload of 90 reads, 9 writes and 1 QueryTopOrderby per 100 operations *\n"
105105
+ "\tReadMyWrites - run a workflow of writes followed by reads and queries attempting to read the write.*\n"
106106
+ "\tCtlWorkload - run a ctl workflow.*\n"
107+
+ "\tReadAllItemsOfLogicalPartition - run a workload that uses readAllItems for a logical partition and prints throughput\n"
107108
+ "\n\t* writes 10k documents initially, which are used in the reads", converter = OperationTypeConverter.class)
108109
private Operation operation = Operation.WriteThroughput;
109110

@@ -172,7 +173,8 @@ public enum Operation {
172173
Mixed,
173174
ReadMyWrites,
174175
ReadThroughputWithMultipleClients,
175-
CtlWorkload;
176+
CtlWorkload,
177+
ReadAllItemsOfLogicalPartition;
176178

177179
static Operation fromString(String code) {
178180

sdk/cosmos/azure-cosmos-encryption/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ Licensed under the MIT License.
4343
<dependency>
4444
<groupId>com.azure</groupId>
4545
<artifactId>azure-cosmos</artifactId>
46-
<version>4.3.2-beta.3</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
46+
<version>4.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
4747
</dependency>
4848

4949
<dependency>

sdk/cosmos/azure-cosmos/CHANGELOG.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
# Release History
22

3-
## 4.3.2-beta.3 (Unreleased)
4-
3+
## 4.4.0-beta.1 (Unreleased)
4+
* Added new API to efficiently load many documents (via list of pk/id pairs or all documents for a set ok pk values)
5+
* Enabled query metrics by default
6+
* Fixed NPE in GatewayAddressCache
57

68
## 4.3.2-beta.2 (2020-08-17)
79
### Key Bug Fixes

sdk/cosmos/azure-cosmos/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ This project provides SDK library in Java for interacting with [SQL API][sql_api
1414
<dependency>
1515
<groupId>com.azure</groupId>
1616
<artifactId>azure-cosmos</artifactId>
17-
<version>4.3.2-beta.2</version>
17+
<version>4.4.0-beta.1</version>
1818
</dependency>
1919
```
2020
[//]: # ({x-version-update-end})

sdk/cosmos/azure-cosmos/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Licensed under the MIT License.
1313

1414
<groupId>com.azure</groupId>
1515
<artifactId>azure-cosmos</artifactId>
16-
<version>4.3.2-beta.3</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
16+
<version>4.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-cosmos;current} -->
1717
<name>Microsoft Azure SDK for SQL API of Azure Cosmos DB Service</name>
1818
<description>This Package contains Microsoft Azure Cosmos SDK (with Reactive Extension Reactor support) for Azure Cosmos DB SQL API</description>
1919
<packaging>jar</packaging>

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import com.azure.cosmos.implementation.RequestOptions;
1515
import com.azure.cosmos.implementation.TracerProvider;
1616
import com.azure.cosmos.implementation.Utils;
17+
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
18+
import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair;
1719
import com.azure.cosmos.implementation.query.QueryInfo;
1820
import com.azure.cosmos.models.CosmosConflictProperties;
1921
import com.azure.cosmos.models.CosmosContainerProperties;
@@ -28,8 +30,10 @@
2830
import com.azure.cosmos.models.SqlQuerySpec;
2931
import com.azure.cosmos.models.ThroughputProperties;
3032
import com.azure.cosmos.models.ThroughputResponse;
33+
import com.azure.cosmos.util.Beta;
3134
import com.azure.cosmos.util.CosmosPagedFlux;
3235
import com.azure.cosmos.util.UtilBridgeInternal;
36+
3337
import reactor.core.publisher.Flux;
3438
import reactor.core.publisher.Mono;
3539

@@ -530,6 +534,102 @@ public <T> Mono<CosmosItemResponse<T>> readItem(
530534
return withContext(context -> readItemInternal(itemId, requestOptions, itemType, context));
531535
}
532536

537+
/**
538+
* Reads many documents.
539+
*
540+
* @param <T> the type parameter
541+
* @param itemKeyList document id and partition key pair that needs to be read
542+
* @param classType class type
543+
* @return a Mono with feed response of cosmos items
544+
*/
545+
@Beta(Beta.SinceVersion.V4_4_0)
546+
public <T> Mono<FeedResponse<T>> readMany(
547+
List<Pair<String, PartitionKey>> itemKeyList,
548+
Class<T> classType) {
549+
550+
return this.readMany(itemKeyList, null, classType);
551+
}
552+
553+
/**
554+
* Reads many documents.
555+
*
556+
* @param <T> the type parameter
557+
* @param itemKeyList document id and partition key pair that needs to be read
558+
* @param sessionToken the optional Session token - null if the read can be made without specific session token
559+
* @param classType class type
560+
* @return a Mono with feed response of cosmos items
561+
*/
562+
@Beta(Beta.SinceVersion.V4_4_0)
563+
public <T> Mono<FeedResponse<T>> readMany(
564+
List<Pair<String, PartitionKey>> itemKeyList,
565+
String sessionToken,
566+
Class<T> classType) {
567+
568+
CosmosQueryRequestOptions options = new CosmosQueryRequestOptions();
569+
570+
if (!StringUtils.isNotEmpty(sessionToken)) {
571+
options = options.setSessionToken(sessionToken);
572+
}
573+
574+
options.setMaxDegreeOfParallelism(-1);
575+
return CosmosBridgeInternal
576+
.getAsyncDocumentClient(this.getDatabase())
577+
.readMany(itemKeyList, BridgeInternal.getLink(this), options, classType);
578+
}
579+
580+
/**
581+
* Reads all the items of a logical partition
582+
* <p>
583+
* After subscription the operation will be performed. The {@link CosmosPagedFlux} will
584+
* contain one or several feed responses of the read Cosmos items. In case of
585+
* failure the {@link CosmosPagedFlux} will error.
586+
*
587+
* @param <T> the type parameter.
588+
* @param partitionKey the partition key value of the documents that need to be read
589+
* @param classType the class type.
590+
* @return a {@link CosmosPagedFlux} containing one or several feed response pages
591+
* of the read Cosmos items or an error.
592+
*/
593+
@Beta(Beta.SinceVersion.V4_4_0)
594+
public <T> CosmosPagedFlux<T> readAllItems(
595+
PartitionKey partitionKey,
596+
Class<T> classType) {
597+
598+
return this.readAllItems(partitionKey, new CosmosQueryRequestOptions(), classType);
599+
}
600+
601+
/**
602+
* Reads all the items of a logical partition
603+
* <p>
604+
* After subscription the operation will be performed. The {@link CosmosPagedFlux} will
605+
* contain one or several feed responses of the read Cosmos items. In case of
606+
* failure the {@link CosmosPagedFlux} will error.
607+
*
608+
* @param <T> the type parameter.
609+
* @param partitionKey the partition key value of the documents that need to be read
610+
* @param options the feed options.
611+
* @param classType the class type.
612+
* @return a {@link CosmosPagedFlux} containing one or several feed response pages
613+
* of the read Cosmos items or an error.
614+
*/
615+
@Beta(Beta.SinceVersion.V4_4_0)
616+
public <T> CosmosPagedFlux<T> readAllItems(
617+
PartitionKey partitionKey,
618+
CosmosQueryRequestOptions options,
619+
Class<T> classType) {
620+
621+
return UtilBridgeInternal.createCosmosPagedFlux(pagedFluxOptions -> {
622+
pagedFluxOptions.setTracerInformation(this.getDatabase().getClient().getTracerProvider(),
623+
this.readAllItemsSpanName,
624+
this.getDatabase().getClient().getServiceEndpoint(), database.getId());
625+
setContinuationTokenAndMaxItemCount(pagedFluxOptions, options);
626+
return getDatabase()
627+
.getDocClientWrapper()
628+
.readAllDocuments(getLink(), partitionKey, options)
629+
.map(response -> prepareFeedResponse(response, classType));
630+
});
631+
}
632+
533633
/**
534634
* Replaces an item with the passed in item.
535635
* <p>

0 commit comments

Comments
 (0)