diff --git a/src/client-side-encryption/auto_encrypter.ts b/src/client-side-encryption/auto_encrypter.ts index 84ad34b520..da70bf5cba 100644 --- a/src/client-side-encryption/auto_encrypter.ts +++ b/src/client-side-encryption/auto_encrypter.ts @@ -403,7 +403,6 @@ export class AutoEncrypter { } const commandBuffer = Buffer.isBuffer(cmd) ? cmd : serialize(cmd, options); - const context = this._mongocrypt.makeEncryptionContext( MongoDBCollectionNamespace.fromString(ns).db, commandBuffer diff --git a/test/integration/change-streams/change_stream.test.ts b/test/integration/change-streams/change_stream.test.ts index 1e9ac09901..88246a136c 100644 --- a/test/integration/change-streams/change_stream.test.ts +++ b/test/integration/change-streams/change_stream.test.ts @@ -23,7 +23,8 @@ import { type ResumeToken } from '../../mongodb'; import * as mock from '../../tools/mongodb-mock/index'; -import { type FailPoint, sleep, TestBuilder, UnifiedTestSuiteBuilder } from '../../tools/utils'; +import { TestBuilder, UnifiedTestSuiteBuilder } from '../../tools/unified_suite_builder'; +import { type FailPoint, sleep } from '../../tools/utils'; import { delay, filterForCommands } from '../shared'; const initIteratorMode = async (cs: ChangeStream) => { diff --git a/test/integration/enumerate_collections.test.ts b/test/integration/enumerate_collections.test.ts index e93239be76..85c043c29f 100644 --- a/test/integration/enumerate_collections.test.ts +++ b/test/integration/enumerate_collections.test.ts @@ -1,4 +1,4 @@ -import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/utils'; +import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/unified_suite_builder'; describe('listCollections', () => { UnifiedTestSuiteBuilder.describe('comment option') diff --git a/test/integration/enumerate_databases.test.ts b/test/integration/enumerate_databases.test.ts index bcb6005be4..0b52ade130 100644 --- a/test/integration/enumerate_databases.test.ts +++ b/test/integration/enumerate_databases.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import { once } from 'events'; import { type MongoClient, MongoServerError } from '../mongodb'; -import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/utils'; +import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/unified_suite_builder'; const metadata: MongoDBMetadataUI = { requires: { diff --git a/test/integration/enumerate_indexes.test.ts b/test/integration/enumerate_indexes.test.ts index 4a6264d6a4..8e8793d604 100644 --- a/test/integration/enumerate_indexes.test.ts +++ b/test/integration/enumerate_indexes.test.ts @@ -1,4 +1,4 @@ -import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/utils'; +import { TestBuilder, UnifiedTestSuiteBuilder } from '../tools/unified_suite_builder'; describe('listIndexes()', () => { UnifiedTestSuiteBuilder.describe('comment option') diff --git a/test/integration/node-specific/comment_with_falsy_values.test.ts b/test/integration/node-specific/comment_with_falsy_values.test.ts index cb7c2bfd56..6d0819fd33 100644 --- a/test/integration/node-specific/comment_with_falsy_values.test.ts +++ b/test/integration/node-specific/comment_with_falsy_values.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { type Collection, type CommandStartedEvent, Long, type MongoClient } from '../../mongodb'; -import { TestBuilder, UnifiedTestSuiteBuilder } from '../../tools/utils'; +import { TestBuilder, UnifiedTestSuiteBuilder } from '../../tools/unified_suite_builder'; const falsyValues = [0, false, '', Long.ZERO, null, NaN] as const; const falsyToString = (value: (typeof falsyValues)[number]) => { diff --git a/test/integration/unified-test-format/unified_test_format.test.ts b/test/integration/unified-test-format/unified_test_format.test.ts index cb3b40f9f2..79a0341341 100644 --- a/test/integration/unified-test-format/unified_test_format.test.ts +++ b/test/integration/unified-test-format/unified_test_format.test.ts @@ -1,4 +1,8 @@ -import { type FailPoint, TestBuilder, UnifiedTestSuiteBuilder } from '../../tools/utils'; +import { + type FailPoint, + TestBuilder, + UnifiedTestSuiteBuilder +} from '../../tools/unified_suite_builder'; describe('Unified Test Runner', () => { UnifiedTestSuiteBuilder.describe('withTransaction error propagation') diff --git a/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.json b/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.json new file mode 100644 index 0000000000..0817508f8f --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.json @@ -0,0 +1,322 @@ +{ + "description": "fle2v2-BypassQueryAnalysis", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "bypassQueryAnalysis": true + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "BypassQueryAnalysis decrypts", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": "123" + } + ] + }, + { + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": { + "$$type": "binData" + }, + "__safeContent__": [ + { + "$binary": { + "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", + "subType": "00" + } + } + ] + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + ], + "ordered": true, + "encryptionInformation": { + "type": 1, + "schema": { + "default.default": { + "escCollection": "enxcol_.default.esc", + "ecocCollection": "enxcol_.default.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.yml b/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.yml new file mode 100644 index 0000000000..2b4a5ec114 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.yml @@ -0,0 +1,130 @@ +description: fle2v2-BypassQueryAnalysis + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "7.0.0" + # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Test has not run on Serverless. + # Serverless tests are planned for removal: DRIVERS-3115 + serverless: forbid + csfle: true + topologies: [ "replicaset", "sharded", "load-balanced" ] + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + keyVaultNamespace: keyvault.datakeys + bypassQueryAnalysis: true + observeEvents: [ commandStartedEvent ] + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + - client: + id: &client1 client1 + - database: + id: &unencryptedDB unencryptedDB + client: *client1 + databaseName: *encryptedDBName + - collection: + id: &unencryptedColl unencryptedColl + database: *unencryptedDB + collectionName: *encryptedCollName + +initialData: + - databaseName: &keyvaultDBName keyvault + collectionName: &datakeysCollName datakeys + documents: + - {'_id': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + encryptedFields: &encrypted_fields {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} + +tests: + - description: "BypassQueryAnalysis decrypts" + operations: + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0_encrypted { + "_id": 1, + "encryptedIndexed": { + "$binary": { + # Payload has an IndexKey of key1 and UserKey of key1. + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + - object: *encryptedColl + name: find + arguments: + filter: { "_id": 1 } + expectResult: [{"_id": 1, "encryptedIndexed": "123" }] + - object: *unencryptedColl + name: find + arguments: + filter: {} + expectResult: + - {"_id": 1, "encryptedIndexed": { "$$type": "binData" }, "__safeContent__": [{ "$binary" : { "base64" : "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", "subType" : "00" } }] } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + listCollections: 1 + filter: + name: *encryptedCollName + commandName: listCollections + - commandStartedEvent: + command: + insert: *encryptedCollName + documents: + - *doc0_encrypted + ordered: true + encryptionInformation: + type: 1 + schema: + "default.default": + # libmongocrypt applies escCollection and ecocCollection to outgoing command. + escCollection: "enxcol_.default.esc" + ecocCollection: "enxcol_.default.ecoc" + <<: *encrypted_fields + commandName: insert + - commandStartedEvent: + command: + find: *encryptedCollName + filter: { "_id": 1 } + commandName: find + - commandStartedEvent: + command: + find: *datakeysCollName + filter: { + "$or": [ + { + "_id": { + "$in": [ + {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}} + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + } + $db: *keyvaultDBName + readConcern: { level: "majority" } + commandName: find \ No newline at end of file diff --git a/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json new file mode 100644 index 0000000000..b5f848c080 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json @@ -0,0 +1,256 @@ +{ + "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "encryptedFieldsMap": { + "default.default": { + "fields": [] + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "encryptedFieldsMap is preferred over remote encryptedFields", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedUnindexed": "value123" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + }, + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml b/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml new file mode 100644 index 0000000000..67cca9b434 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml @@ -0,0 +1,114 @@ +description: fle2v2-EncryptedFields-vs-EncryptedFieldsMap + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "7.0.0" + # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Test has not run on Serverless. + # Serverless tests are planned for removal: DRIVERS-3115 + serverless: forbid + csfle: true + topologies: [ "replicaset", "sharded", "load-balanced" ] + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + keyVaultNamespace: keyvault.datakeys + encryptedFieldsMap: { + "default.default": { + "fields": [] + } + } + observeEvents: [ commandStartedEvent ] + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + +initialData: + - databaseName: &keyvaultDBName keyvault + collectionName: &datakeysCollName datakeys + documents: + - {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + encryptedFields: {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} + +tests: + - description: "encryptedFieldsMap is preferred over remote encryptedFields" + operations: + # EncryptedFieldsMap overrides remote encryptedFields. + # Automatic encryption does not occur on encryptedUnindexed. The value is validated on the server. + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { + _id: 1, + encryptedUnindexed: { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + - object: *encryptedColl + name: find + arguments: + filter: { "_id": 1 } + expectResult: + - {"_id": 1, "encryptedUnindexed": "value123" } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: insert + command: + insert: *encryptedCollName + documents: + - *doc0 + ordered: true + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: find + command: + find: *encryptedCollName + filter: { "_id": 1} + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: { + "$or": [ + { + "_id": { + "$in": [ + {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}} + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + } + $db: *keyvaultDBName + readConcern: { level: "majority" } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0 \ No newline at end of file diff --git a/test/spec/client-side-encryption/tests/unified/localSchema.json b/test/spec/client-side-encryption/tests/unified/localSchema.json new file mode 100644 index 0000000000..aee323d949 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/localSchema.json @@ -0,0 +1,343 @@ +{ + "description": "localSchema", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "4.1.10", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "schemaMap": { + "default.default": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + }, + "sessionToken": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "autoEncryptOpts": { + "schemaMap": { + "default.default": { + "properties": { + "test": { + "bsonType": "string" + } + }, + "bsonType": "object", + "required": [ + "test" + ] + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + }, + "sessionToken": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "database": { + "id": "encryptedDB2", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl2", + "database": "encryptedDB2", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [] + } + ], + "tests": [ + { + "description": "A local schema should override", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "readConcern": { + "level": "majority" + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + ] + }, + { + "description": "A local schema with no encryption is an error", + "operations": [ + { + "object": "encryptedColl2", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + }, + "expectError": { + "isError": true, + "errorContains": "JSON schema keyword 'required' is only allowed with a remote schema" + } + } + ] + } + ] +} diff --git a/test/spec/client-side-encryption/tests/unified/localSchema.yml b/test/spec/client-side-encryption/tests/unified/localSchema.yml new file mode 100644 index 0000000000..495b2774eb --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/localSchema.yml @@ -0,0 +1,103 @@ +description: localSchema + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "4.1.10" + csfle: true + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} + keyVaultNamespace: keyvault.datakeys + kmsProviders: + aws: { accessKeyId: { $$placeholder: 1 }, secretAccessKey: { $$placeholder: 1 }, sessionToken: { $$placeholder: 1 } } + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'test': {'bsonType': 'string'}}, 'bsonType': 'object', 'required': ['test']} + keyVaultNamespace: keyvault.datakeys + kmsProviders: + aws: { accessKeyId: { $$placeholder: 1 }, secretAccessKey: { $$placeholder: 1 }, sessionToken: { $$placeholder: 1 } } + observeEvents: [ commandStartedEvent ] + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + # intentionally the same DB and collection name as encryptedDB/Coll + - database: + id: &encryptedDB2 encryptedDB2 + client: *client1 + databaseName: *encryptedDBName + - collection: + id: &encryptedColl2 encryptedColl2 + database: *encryptedDB2 + collectionName: *encryptedDBName + +initialData: + - databaseName: &keyvaultDBName keyvault + collectionName: &datakeysCollName datakeys + documents: + - {'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + +tests: + - description: "A local schema should override" + operations: + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + - object: *encryptedColl + name: find + arguments: + filter: { _id: 1 } + expectResult: [*doc0] + expectEvents: + # Then key is fetched from the key vault. + - client: *client0 + events: + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + readConcern: { level: "majority" } + - commandStartedEvent: + commandName: insert + command: + insert: *encryptedCollName + documents: + - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } + ordered: true + - commandStartedEvent: + commandName: find + command: + find: *encryptedCollName + filter: { _id: 1 } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0_encrypted + - description: "A local schema with no encryption is an error" + operations: + - object: *encryptedColl2 + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + expectError: + isError: true + errorContains: "JSON schema keyword 'required' is only allowed with a remote schema" diff --git a/test/spec/client-side-encryption/tests/unified/maxWireVersion.json b/test/spec/client-side-encryption/tests/unified/maxWireVersion.json new file mode 100644 index 0000000000..d0af75ac99 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/maxWireVersion.json @@ -0,0 +1,101 @@ +{ + "description": "maxWireVersion", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + }, + "keyVaultNamespace": "keyvault.datakeys", + "extraOptions": { + "mongocryptdBypassSpawn": true + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + } + ], + "tests": [ + { + "description": "operation fails with maxWireVersion < 8", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "encrypted_string": "string0" + } + }, + "expectError": { + "errorContains": "Auto-encryption requires a minimum MongoDB version of 4.2" + } + } + ] + } + ] +} diff --git a/test/spec/client-side-encryption/tests/unified/maxWireVersion.yml b/test/spec/client-side-encryption/tests/unified/maxWireVersion.yml new file mode 100644 index 0000000000..75a51dd4e5 --- /dev/null +++ b/test/spec/client-side-encryption/tests/unified/maxWireVersion.yml @@ -0,0 +1,41 @@ +description: maxWireVersion + +schemaVersion: "1.23" + +runOnRequirements: + - maxServerVersion: "4.0.99" + csfle: true + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + kmsProviders: + aws: {} + keyVaultNamespace: keyvault.datakeys + extraOptions: + mongocryptdBypassSpawn: true # mongocryptd probably won't be on the path. mongocryptd was introduced in server 4.2. + - database: + id: &database0 database0 + client: *client0 + databaseName: default + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: default + +initialData: + - databaseName: keyvault + collectionName: datakeys + documents: + - {'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']} + +tests: + - description: "operation fails with maxWireVersion < 8" + operations: + - name: insertOne + object: *collection0 + arguments: + document: { encrypted_string: "string0" } + expectError: + errorContains: "Auto-encryption requires a minimum MongoDB version of 4.2" \ No newline at end of file diff --git a/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.json b/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.json new file mode 100644 index 0000000000..309d1d3b4b --- /dev/null +++ b/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.json @@ -0,0 +1,193 @@ +{ + "description": "poc-queryable-encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0", + "csfle": true, + "topologies": [ + "replicaset", + "load-balanced", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "encrypted" + } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "encrypted" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "poc-queryable-encryption", + "collectionName": "encrypted", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "tests": [ + { + "description": "insert, replace, and find with queryable encryption", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedInt": 11 + } + } + }, + { + "object": "encryptedColl", + "name": "replaceOne", + "arguments": { + "filter": { + "encryptedInt": 11 + }, + "replacement": { + "encryptedInt": 22 + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 22 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 22 + } + ] + }, + { + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": [ + { + "$binary": { + "base64": "rhS16TJojgDDBtbluxBokvcotP1mQTGeYpNt8xd3MJQ=", + "subType": "00" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.yml b/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.yml new file mode 100644 index 0000000000..797904ee95 --- /dev/null +++ b/test/spec/unified-test-format/valid-pass/poc-queryable-encryption.yml @@ -0,0 +1,86 @@ +description: poc-queryable-encryption + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "7.0" + csfle: true + # QE is not supported on standalone servers + topologies: [ replicaset, load-balanced, sharded ] + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName poc-queryable-encryption + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName encrypted + - client: + id: &client1 client1 + - database: + id: &unencryptedDB unencryptedDB + client: *client1 + databaseName: *encryptedDBName + - collection: + id: &unencryptedColl unencryptedColl + database: *unencryptedDB + collectionName: *encryptedCollName + +initialData: + - databaseName: keyvault + collectionName: datakeys + documents: + - _id: &keyid { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: + provider: local + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + encryptedFields: + fields: + - keyId: *keyid + path: 'encryptedInt' + bsonType: 'int' + queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} + +tests: + - description: insert, replace, and find with queryable encryption + operations: + - object: *encryptedColl + name: insertOne + arguments: + document: + _id: 1 + encryptedInt: 11 + - object: *encryptedColl + name: replaceOne + arguments: + filter: { encryptedInt: 11 } + replacement: { encryptedInt: 22 } + - object: *encryptedColl + name: find + arguments: + filter: { encryptedInt: 22 } + expectResult: + - _id: 1 + encryptedInt: 22 + - object: *unencryptedColl + name: find + arguments: + filter: {} + expectResult: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: [ { "$binary" : { "base64" : "rhS16TJojgDDBtbluxBokvcotP1mQTGeYpNt8xd3MJQ=", "subType" : "00" } } ] } \ No newline at end of file diff --git a/test/tools/runner/config.ts b/test/tools/runner/config.ts index b7f9951d57..46f41a02a8 100644 --- a/test/tools/runner/config.ts +++ b/test/tools/runner/config.ts @@ -538,6 +538,12 @@ export class AstrolabeTestConfiguration extends TestConfiguration { } export class AlpineTestConfiguration extends TestConfiguration { + get encryptDefaultExtraOptions(): MongoClientOptions['autoEncryption']['extraOptions'] { + return { + mongocryptdBypassSpawn: true, + mongocryptdURI: process.env.MONGOCRYPTD_URI + }; + } override newClient( urlOrQueryOptions?: string | Record, serverOptions?: MongoClientOptions @@ -547,8 +553,7 @@ export class AlpineTestConfiguration extends TestConfiguration { if (options.autoEncryption) { const extraOptions: MongoClientOptions['autoEncryption']['extraOptions'] = { ...options.autoEncryption.extraOptions, - mongocryptdBypassSpawn: true, - mongocryptdURI: process.env.MONGOCRYPTD_URI + ...this.encryptDefaultExtraOptions }; options.autoEncryption.extraOptions = extraOptions; } diff --git a/test/tools/unified-spec-runner/entities.ts b/test/tools/unified-spec-runner/entities.ts index 74f6242731..34d5aa886d 100644 --- a/test/tools/unified-spec-runner/entities.ts +++ b/test/tools/unified-spec-runner/entities.ts @@ -45,14 +45,16 @@ import { type TopologyOpeningEvent, WriteConcern } from '../../mongodb'; -import { getEnvironmentalOptions } from '../../tools/utils'; -import type { TestConfiguration } from '../runner/config'; +import { getEncryptExtraOptions, getEnvironmentalOptions } from '../../tools/utils'; +import { AlpineTestConfiguration, type TestConfiguration } from '../runner/config'; import { EntityEventRegistry } from './entity_event_registry'; import { trace } from './runner'; import type { ClientEntity, EntityDescription, ExpectedLogMessage } from './schema'; import { createClientEncryption, + getCSFLETestDataFromEnvironment, makeConnectionString, + mergeKMSProviders, patchCollectionOptions, patchDbOptions } from './unified-utils'; @@ -198,14 +200,7 @@ export class UnifiedMongoClient extends MongoClient { topology: 'MONGODB_LOG_TOPOLOGY' } as const; - constructor( - uri: string, - description: ClientEntity, - config: { - loggingEnabled?: boolean; - setupLogging?: (options: Record, id: string) => Record; - } - ) { + constructor(uri: string, description: ClientEntity, config: TestConfiguration) { const options: MongoClientOptions = { monitorCommands: true, __skipPingOnConnect: true, @@ -236,6 +231,36 @@ export class UnifiedMongoClient extends MongoClient { config.setupLogging?.(options, description.id); } + if (description.autoEncryptOpts) { + const { kmsProviders: kmsProvidersFromEnvironment } = getCSFLETestDataFromEnvironment( + process.env + ); + + let extraOptions = {}; + if (config instanceof AlpineTestConfiguration) { + extraOptions = { + ...config.encryptDefaultExtraOptions, + ...description.autoEncryptOpts.extraOptions + }; + } else if (getEncryptExtraOptions().cryptSharedLibPath != null) { + extraOptions = { + cryptSharedLibPath: getEncryptExtraOptions().cryptSharedLibPath, + ...description.autoEncryptOpts.extraOptions + }; + } + + const kmsProviders = mergeKMSProviders( + description.autoEncryptOpts.kmsProviders, + kmsProvidersFromEnvironment + ); + + options.autoEncryption = { + ...description.autoEncryptOpts, + kmsProviders, + extraOptions + }; + } + super(uri, options); this.observedEventEmitter.on('error', () => null); this.logCollector = logCollector; diff --git a/test/tools/unified-spec-runner/match.ts b/test/tools/unified-spec-runner/match.ts index 06a23388c0..a2cf880a28 100644 --- a/test/tools/unified-spec-runner/match.ts +++ b/test/tools/unified-spec-runner/match.ts @@ -774,6 +774,16 @@ function isMongoCryptError(err): boolean { if (err.constructor.name === 'MongoCryptError') { return true; } + + // TODO(NODE-7043): remove special handling for FLE errors in the UTR + if ( + err instanceof TypeError && + err.message.includes( + `csfle "analyze_query" failed: JSON schema keyword 'required' is only allowed with a remote schema` + ) + ) { + return true; + } return err.stack.includes('at ClientEncryption'); } diff --git a/test/tools/unified-spec-runner/runner.ts b/test/tools/unified-spec-runner/runner.ts index 20962618ac..aefe740747 100644 --- a/test/tools/unified-spec-runner/runner.ts +++ b/test/tools/unified-spec-runner/runner.ts @@ -81,7 +81,7 @@ async function runUnifiedTest( ctx.skip(); } - let utilClient; + let utilClient: MongoClient; if (ctx.configuration.isLoadBalanced) { // The util client can always point at the single mongos LB frontend. utilClient = ctx.configuration.newClient(ctx.configuration.singleMongosLoadBalancerUri); @@ -143,13 +143,26 @@ async function runUnifiedTest( trace('initialData'); for (const { databaseName, collectionName } of unifiedSuite.initialData) { const db = utilClient.db(databaseName); - const collection = db.collection(collectionName, { - writeConcern: { w: 'majority' } - }); trace('listCollections'); - const collectionList = await db.listCollections({ name: collectionName }).toArray(); - if (collectionList.length !== 0) { + const allCollections = new Set( + await db + .listCollections() + .map(({ name }) => name) + .toArray() + ); + + const collections = [ + collectionName, + `enxcol_.${collectionName}.esc`, + `enxcol_.${collectionName}.ecoc` + ].filter(allCollections.has.bind(allCollections)); + + for (const name of collections) { + const collection = db.collection(name, { + writeConcern: { w: 'majority' } + }); + trace('drop'); expect(await collection.drop()).to.be.true; } diff --git a/test/tools/unified-spec-runner/schema.ts b/test/tools/unified-spec-runner/schema.ts index cd2bd3ff3d..3fc60407a7 100644 --- a/test/tools/unified-spec-runner/schema.ts +++ b/test/tools/unified-spec-runner/schema.ts @@ -1,5 +1,6 @@ import type { Document, + MongoClientOptions, MongoLoggableComponent, ObjectId, ReadConcernLevel, @@ -152,7 +153,19 @@ export interface ClientEntity { observeSensitiveCommands?: boolean; // Was optionally scheduled for removal in NODE-6783, but opted to keep it for potential future use. storeEventsAsEntities?: StoreEventsAsEntity[]; + autoEncryptOpts?: Pick< + MongoClientOptions['autoEncryption'], + | 'keyVaultNamespace' + | 'bypassAutoEncryption' + | 'schemaMap' + | 'encryptedFieldsMap' + | 'extraOptions' + | 'bypassQueryAnalysis' + | 'keyExpirationMS' + > & + Pick; } + export interface DatabaseEntity { id: string; client: string; diff --git a/test/tools/unified_suite_builder.ts b/test/tools/unified_suite_builder.ts new file mode 100644 index 0000000000..f70706efb9 --- /dev/null +++ b/test/tools/unified_suite_builder.ts @@ -0,0 +1,187 @@ +import { runUnifiedSuite } from './unified-spec-runner/runner'; +import { + type CollectionData, + type EntityDescription, + type ExpectedEventsForClient, + type OperationDescription, + type RunOnRequirement, + type Test, + type UnifiedSuite +} from './unified-spec-runner/schema'; + +export class TestBuilder { + private _description: string; + private runOnRequirements: RunOnRequirement[] = []; + private _skipReason?: string; + private _operations: OperationDescription[] = []; + private _expectEvents?: ExpectedEventsForClient[] = []; + private _outcome?: CollectionData[] = []; + + static it(title: string) { + return new TestBuilder(title); + } + + constructor(description: string) { + this._description = description; + } + + operation(operation: OperationDescription): this { + this._operations.push({ + object: 'collection0', + arguments: {}, + ...operation + }); + return this; + } + + runOnRequirement(requirement: RunOnRequirement): this { + this.runOnRequirements.push(requirement); + return this; + } + + expectEvents(event: ExpectedEventsForClient): this { + this._expectEvents.push(event); + return this; + } + + toJSON(): Test { + const test: Test = { + description: this._description, + runOnRequirements: this.runOnRequirements, + operations: this._operations, + expectEvents: this._expectEvents, + outcome: this._outcome + }; + + if (this._skipReason != null) { + test.skipReason = this._skipReason; + } + + return test; + } +} + +export class UnifiedTestSuiteBuilder { + private _description = 'Default Description'; + private _schemaVersion = '1.0'; + private _createEntities: EntityDescription[]; + private _runOnRequirement: RunOnRequirement[] = []; + private _initialData: CollectionData[] = []; + private _tests: Test[] = []; + + static describe(title: string) { + return new UnifiedTestSuiteBuilder(title); + } + + /** + * Establish common defaults + * - id and name = client0, listens for commandStartedEvent + * - id and name = database0 + * - id and name = collection0 + */ + static get defaultEntities(): EntityDescription[] { + return [ + { + client: { + id: 'client0', + useMultipleMongoses: true, + observeEvents: ['commandStartedEvent'] + } + }, + { + database: { + id: 'database0', + client: 'client0', + databaseName: 'database0' + } + }, + { + collection: { + id: 'collection0', + database: 'database0', + collectionName: 'collection0' + } + } + ]; + } + + constructor(description: string) { + this._description = description; + this._createEntities = []; + } + + description(description: string): this { + this._description = description; + return this; + } + + test(test: Test): this; + test(test: Test[]): this; + test(test: Test | Test[]): this { + if (Array.isArray(test)) { + this._tests.push(...test); + } else { + this._tests.push(test); + } + return this; + } + + createEntities(entity: EntityDescription): this; + createEntities(entity: EntityDescription[]): this; + createEntities(entity: EntityDescription | EntityDescription[]): this { + if (Array.isArray(entity)) { + this._createEntities.push(...entity); + } else { + this._createEntities.push(entity); + } + return this; + } + + initialData(data: CollectionData): this; + initialData(data: CollectionData[]): this; + initialData(data: CollectionData | CollectionData[]): this { + if (Array.isArray(data)) { + this._initialData.push(...data); + } else { + this._initialData.push(data); + } + return this; + } + + runOnRequirement(requirement: RunOnRequirement): this; + runOnRequirement(requirement: RunOnRequirement[]): this; + runOnRequirement(requirement: RunOnRequirement | RunOnRequirement[]): this { + Array.isArray(requirement) + ? this._runOnRequirement.push(...requirement) + : this._runOnRequirement.push(requirement); + return this; + } + + schemaVersion(version: string): this { + this._schemaVersion = version; + return this; + } + + toJSON(): UnifiedSuite { + return { + description: this._description, + schemaVersion: this._schemaVersion, + runOnRequirements: this._runOnRequirement, + createEntities: this._createEntities, + initialData: this._initialData, + tests: this._tests + }; + } + + run(): void { + return runUnifiedSuite([this.toJSON()]); + } + + toMocha() { + return describe(this._description, () => runUnifiedSuite([this.toJSON()])); + } + + clone(): UnifiedSuite { + return JSON.parse(JSON.stringify(this)); + } +} diff --git a/test/tools/utils.ts b/test/tools/utils.ts index 2525c92d83..c4fa4ad6da 100644 --- a/test/tools/utils.ts +++ b/test/tools/utils.ts @@ -22,16 +22,6 @@ import { type TopologyOptions } from '../mongodb'; import { type TestConfiguration } from './runner/config'; -import { runUnifiedSuite } from './unified-spec-runner/runner'; -import { - type CollectionData, - type EntityDescription, - type ExpectedEventsForClient, - type OperationDescription, - type RunOnRequirement, - type Test, - type UnifiedSuite -} from './unified-spec-runner/schema'; export function ensureCalledWith(stub: any, args: any[]) { args.forEach((m: any) => expect(stub).to.have.been.calledWith(m)); @@ -233,58 +223,6 @@ export interface FailPoint { }; } -export class TestBuilder { - private _description: string; - private runOnRequirements: RunOnRequirement[] = []; - private _skipReason?: string; - private _operations: OperationDescription[] = []; - private _expectEvents?: ExpectedEventsForClient[] = []; - private _outcome?: CollectionData[] = []; - - static it(title: string) { - return new TestBuilder(title); - } - - constructor(description: string) { - this._description = description; - } - - operation(operation: OperationDescription): this { - this._operations.push({ - object: 'collection0', - arguments: {}, - ...operation - }); - return this; - } - - runOnRequirement(requirement: RunOnRequirement): this { - this.runOnRequirements.push(requirement); - return this; - } - - expectEvents(event: ExpectedEventsForClient): this { - this._expectEvents.push(event); - return this; - } - - toJSON(): Test { - const test: Test = { - description: this._description, - runOnRequirements: this.runOnRequirements, - operations: this._operations, - expectEvents: this._expectEvents, - outcome: this._outcome - }; - - if (this._skipReason != null) { - test.skipReason = this._skipReason; - } - - return test; - } -} - export function bufferToStream(buffer) { const stream = new Readable(); if (Array.isArray(buffer)) { @@ -315,131 +253,6 @@ export function generateOpMsgBuffer(document: Document): Buffer { return Buffer.concat([header, typeBuffer, docBuffer]); } -export class UnifiedTestSuiteBuilder { - private _description = 'Default Description'; - private _schemaVersion = '1.0'; - private _createEntities: EntityDescription[]; - private _runOnRequirement: RunOnRequirement[] = []; - private _initialData: CollectionData[] = []; - private _tests: Test[] = []; - - static describe(title: string) { - return new UnifiedTestSuiteBuilder(title); - } - - /** - * Establish common defaults - * - id and name = client0, listens for commandStartedEvent - * - id and name = database0 - * - id and name = collection0 - */ - static get defaultEntities(): EntityDescription[] { - return [ - { - client: { - id: 'client0', - useMultipleMongoses: true, - observeEvents: ['commandStartedEvent'] - } - }, - { - database: { - id: 'database0', - client: 'client0', - databaseName: 'database0' - } - }, - { - collection: { - id: 'collection0', - database: 'database0', - collectionName: 'collection0' - } - } - ]; - } - - constructor(description: string) { - this._description = description; - this._createEntities = []; - } - - description(description: string): this { - this._description = description; - return this; - } - - test(test: Test): this; - test(test: Test[]): this; - test(test: Test | Test[]): this { - if (Array.isArray(test)) { - this._tests.push(...test); - } else { - this._tests.push(test); - } - return this; - } - - createEntities(entity: EntityDescription): this; - createEntities(entity: EntityDescription[]): this; - createEntities(entity: EntityDescription | EntityDescription[]): this { - if (Array.isArray(entity)) { - this._createEntities.push(...entity); - } else { - this._createEntities.push(entity); - } - return this; - } - - initialData(data: CollectionData): this; - initialData(data: CollectionData[]): this; - initialData(data: CollectionData | CollectionData[]): this { - if (Array.isArray(data)) { - this._initialData.push(...data); - } else { - this._initialData.push(data); - } - return this; - } - - runOnRequirement(requirement: RunOnRequirement): this; - runOnRequirement(requirement: RunOnRequirement[]): this; - runOnRequirement(requirement: RunOnRequirement | RunOnRequirement[]): this { - Array.isArray(requirement) - ? this._runOnRequirement.push(...requirement) - : this._runOnRequirement.push(requirement); - return this; - } - - schemaVersion(version: string): this { - this._schemaVersion = version; - return this; - } - - toJSON(): UnifiedSuite { - return { - description: this._description, - schemaVersion: this._schemaVersion, - runOnRequirements: this._runOnRequirement, - createEntities: this._createEntities, - initialData: this._initialData, - tests: this._tests - }; - } - - run(): void { - return runUnifiedSuite([this.toJSON()]); - } - - toMocha() { - return describe(this._description, () => runUnifiedSuite([this.toJSON()])); - } - - clone(): UnifiedSuite { - return JSON.parse(JSON.stringify(this)); - } -} - export const alphabetically = (a: any, b: any) => { const res = `${a}`.localeCompare(`${b}`, 'en-US', { usage: 'sort',