diff --git a/.eslintrc.js b/.eslintrc.js index 002d1e7b8b9..91b38166932 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -15,7 +15,8 @@ module.exports = { '!.*', 'node_modules', '.git', - 'data' + 'data', + '.config' ], overrides: [ { diff --git a/docs/field-level-encryption.md b/docs/field-level-encryption.md index 3531fca0218..28ba0db9841 100644 --- a/docs/field-level-encryption.md +++ b/docs/field-level-encryption.md @@ -112,3 +112,51 @@ With the above connection, if you create a model named 'Test' that uses the 'tes const Model = mongoose.model('Test', mongoose.Schema({ name: String })); await Model.create({ name: 'super secret' }); ``` + +## Automatic FLE in Mongoose + +Mongoose supports the declaration of encrypted schemas - schemas that, when connected to a model, utilize MongoDB's Client Side +Field Level Encryption or Queryable Encryption under the hood. Mongoose automatically generates either an `encryptedFieldsMap` or a +`schemaMap` when instantiating a MongoClient and encrypts fields on write and decrypts fields on reads. + +### Encryption types + +MongoDB has two different automatic encryption implementations: client side field level encryption (CSFLE) and queryable encryption (QE). +See [choosing an in-use encryption approach](https://www.mongodb.com/docs/v7.3/core/queryable-encryption/about-qe-csfle/#choosing-an-in-use-encryption-approach). + +### Declaring Encrypted Schemas + +The following schema declares two properties, `name` and `ssn`. `ssn` is encrypted using queryable encryption, and +is configured for equality queries: + +```javascript +const encryptedUserSchema = new Schema({ + name: String, + ssn: { + type: String, + // 1 + encrypt: { + keyId: '', + queries: 'equality' + } + } + // 2 +}, { encryptionType: 'queryableEncryption' }); +``` + +To declare a field as encrypted, you must: + +1. Annotate the field with encryption metadata in the schema definition +2. Choose an encryption type for the schema and configure the schema for the encryption type + +Not all schematypes are supported for CSFLE and QE. For an overview of valid schema types, refer to MongoDB's documentation. + +### Registering Models + +Encrypted schemas must be registered on a connection, not the Mongoose global: + +```javascript + +const connection = mongoose.createConnection(); +const UserModel = connection.model('User', encryptedUserSchema); +``` diff --git a/lib/collection.js b/lib/collection.js index e6c365c9a13..9f60ba4c01b 100644 --- a/lib/collection.js +++ b/lib/collection.js @@ -81,7 +81,7 @@ Collection.prototype.onOpen = function() { * @api private */ -Collection.prototype.onClose = function() {}; +Collection.prototype.onClose = function() { }; /** * Queues a method for later execution when its diff --git a/lib/connection.js b/lib/connection.js index b747460083c..8ab0a76893c 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -607,7 +607,7 @@ Connection.prototype.bulkWrite = async function bulkWrite(ops, options) { Connection.prototype.createCollections = async function createCollections(options = {}) { const result = {}; - const errorsMap = { }; + const errorsMap = {}; const { continueOnError } = options; delete options.continueOnError; @@ -734,7 +734,7 @@ Connection.prototype.transaction = function transaction(fn, options) { throw err; }). finally(() => { - session.endSession().catch(() => {}); + session.endSession().catch(() => { }); }); }); }; @@ -1025,7 +1025,7 @@ Connection.prototype.openUri = async function openUri(uri, options) { for (const model of Object.values(this.models)) { // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); } // `createConnection()` calls this `openUri()` function without @@ -1061,7 +1061,7 @@ Connection.prototype.openUri = async function openUri(uri, options) { // to avoid uncaught exceptions when using `on('error')`. See gh-14377. Connection.prototype.on = function on(event, callback) { if (event === 'error' && this.$initialConnection) { - this.$initialConnection.catch(() => {}); + this.$initialConnection.catch(() => { }); } return EventEmitter.prototype.on.call(this, event, callback); }; @@ -1083,7 +1083,7 @@ Connection.prototype.on = function on(event, callback) { // to avoid uncaught exceptions when using `on('error')`. See gh-14377. Connection.prototype.once = function on(event, callback) { if (event === 'error' && this.$initialConnection) { - this.$initialConnection.catch(() => {}); + this.$initialConnection.catch(() => { }); } return EventEmitter.prototype.once.call(this, event, callback); }; @@ -1412,7 +1412,7 @@ Connection.prototype.model = function model(name, schema, collection, options) { } // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); return model; } @@ -1439,7 +1439,7 @@ Connection.prototype.model = function model(name, schema, collection, options) { } if (this === model.prototype.db - && (!collection || collection === model.collection.name)) { + && (!collection || collection === model.collection.name)) { // model already uses this connection. // only the first model with this name is cached to allow @@ -1626,8 +1626,8 @@ Connection.prototype.authMechanismDoesNotRequirePassword = function authMechanis */ Connection.prototype.optionsProvideAuthenticationData = function optionsProvideAuthenticationData(options) { return (options) && - (options.user) && - ((options.pass) || this.authMechanismDoesNotRequirePassword()); + (options.user) && + ((options.pass) || this.authMechanismDoesNotRequirePassword()); }; /** @@ -1689,7 +1689,7 @@ Connection.prototype.createClient = function createClient() { */ Connection.prototype.syncIndexes = async function syncIndexes(options = {}) { const result = {}; - const errorsMap = { }; + const errorsMap = {}; const { continueOnError } = options; delete options.continueOnError; diff --git a/lib/drivers/node-mongodb-native/connection.js b/lib/drivers/node-mongodb-native/connection.js index 0659ac4e647..28bdf50b946 100644 --- a/lib/drivers/node-mongodb-native/connection.js +++ b/lib/drivers/node-mongodb-native/connection.js @@ -12,6 +12,7 @@ const pkg = require('../../../package.json'); const processConnectionOptions = require('../../helpers/processConnectionOptions'); const setTimeout = require('../../helpers/timers').setTimeout; const utils = require('../../utils'); +const { Schema } = require('../../mongoose'); /** * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation. @@ -315,6 +316,16 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio }; } + const { schemaMap, encryptedFieldsMap } = this._buildEncryptionSchemas(); + + if (Object.keys(schemaMap).length > 0) { + options.autoEncryption.schemaMap = schemaMap; + } + + if (Object.keys(encryptedFieldsMap).length > 0) { + options.autoEncryption.encryptedFieldsMap = encryptedFieldsMap; + } + this.readyState = STATES.connecting; this._connectionString = uri; @@ -338,6 +349,55 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio return this; }; +/** + * Given a connection, which may or may not have encrypted models, build + * a schemaMap and/or an encryptedFieldsMap for the connection, combining all models + * into a single schemaMap and encryptedFields map. + * + * @returns a copy of the options object with a schemaMap and/or an encryptedFieldsMap added to the options' autoEncryption + * options. + */ +NativeConnection.prototype._buildEncryptionSchemas = function() { + const qeMappings = {}; + const csfleMappings = {}; + + // If discriminators are configured for the collection, there might be multiple models + // pointing to the same namespace. For this scenario, we merge all the schemas for each namespace + // into a single schema. + // Notably, this doesn't allow for discriminators to declare multiple values on the same fields. + for (const model of Object.values(this.models)) { + const { schema, collection: { collectionName } } = model; + const namespace = `${this.$dbName}.${collectionName}`; + if (schema.encryptionType() === 'csfle') { + csfleMappings[namespace] ??= new Schema({}, { encryptionType: 'csfle' }); + csfleMappings[namespace].add(schema); + } else if (schema.encryptionType() === 'queryableEncryption') { + qeMappings[namespace] ??= new Schema({}, { encryptionType: 'queryableEncryption' }); + qeMappings[namespace].add(schema); + } + } + + const schemaMap = Object.entries(csfleMappings).reduce( + (schemaMap, [namespace, schema]) => { + schemaMap[namespace] = schema._buildSchemaMap(); + return schemaMap; + }, + {} + ); + + const encryptedFieldsMap = Object.entries(qeMappings).reduce( + (encryptedFieldsMap, [namespace, schema]) => { + encryptedFieldsMap[namespace] = schema._buildEncryptedFields(); + return encryptedFieldsMap; + }, + {} + ); + + return { + schemaMap, encryptedFieldsMap + }; +}; + /*! * ignore */ @@ -358,7 +418,7 @@ NativeConnection.prototype.setClient = function setClient(client) { for (const model of Object.values(this.models)) { // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); } return this; @@ -401,9 +461,9 @@ function _setClient(conn, client, options, dbName) { }; const type = client && - client.topology && - client.topology.description && - client.topology.description.type || ''; + client.topology && + client.topology.description && + client.topology.description.type || ''; if (type === 'Single') { client.on('serverDescriptionChanged', ev => { diff --git a/lib/encryptionUtils.js b/lib/encryptionUtils.js new file mode 100644 index 00000000000..f0c46dee716 --- /dev/null +++ b/lib/encryptionUtils.js @@ -0,0 +1,72 @@ +'use strict'; + +const schemaTypes = require('./schema/index.js'); +const SchemaBigInt = require('./schema/bigint'); +const SchemaBoolean = require('./schema/boolean'); +const SchemaBuffer = require('./schema/buffer'); +const SchemaDate = require('./schema/date'); +const SchemaDecimal128 = require('./schema/decimal128'); +const SchemaDouble = require('./schema/double'); +const SchemaInt32 = require('./schema/int32'); +const SchemaObjectId = require('./schema/objectId'); +const SchemaString = require('./schema/string'); + +/** + * Given a schema and a path to a field in the schema, this returns the + * BSON type of the field, if it can be determined. This method specifically + * **only** handles BSON types that are used for CSFLE and QE - any other + * BSON types will return `null`. (example: MinKey and MaxKey). + * + * @param {import('.').Schema} schema + * @param {string} path + * @returns {string} + */ +function inferBSONType(schema, path) { + const type = schema.path(path); + + if (type instanceof SchemaString) { + return 'string'; + } + + if (type instanceof SchemaInt32) { + return 'int'; + } + + if (type instanceof SchemaBigInt) { + return 'long'; + } + + if (type instanceof SchemaBoolean) { + return 'bool'; + } + + if (type instanceof SchemaDate) { + return 'date'; + } + + if (type instanceof SchemaBuffer) { + return 'binData'; + } + + if (type instanceof SchemaObjectId) { + return 'objectId'; + } + + if (type instanceof SchemaDecimal128) { + return 'decimal'; + } + + if (type instanceof SchemaDouble) { + return 'double'; + } + + if (type instanceof schemaTypes.Array) { + return 'array'; + } + + return null; +} + +module.exports = { + inferBSONType +}; diff --git a/lib/encryption_utils.js b/lib/encryption_utils.js new file mode 100644 index 00000000000..1f17fa5032b --- /dev/null +++ b/lib/encryption_utils.js @@ -0,0 +1,72 @@ +'use strict'; + +const { Array } = require('./schema/index.js'); +const SchemaBigInt = require('./schema/bigint'); +const SchemaBoolean = require('./schema/boolean'); +const SchemaBuffer = require('./schema/buffer'); +const SchemaDate = require('./schema/date'); +const SchemaDecimal128 = require('./schema/decimal128'); +const SchemaDouble = require('./schema/double'); +const SchemaInt32 = require('./schema/int32'); +const SchemaObjectId = require('./schema/objectId'); +const SchemaString = require('./schema/string'); + +/** + * Given a schema and a path to a field in the schema, this returns the + * BSON type of the field, if it can be determined. This method specifically + * **only** handles BSON types that are used for CSFLE and QE - any other + * BSON types will return `null`. (example: MinKey and MaxKey). + * + * @param {import('.').Schema} schema + * @param {string} path + * @returns + */ +function inferBSONType(schema, path) { + const type = schema.path(path); + + if (type instanceof SchemaString) { + return 'string'; + } + + if (type instanceof SchemaInt32) { + return 'int'; + } + + if (type instanceof SchemaBigInt) { + return 'long'; + } + + if (type instanceof SchemaBoolean) { + return 'bool'; + } + + if (type instanceof SchemaDate) { + return 'date'; + } + + if (type instanceof SchemaBuffer) { + return 'binData'; + } + + if (type instanceof SchemaObjectId) { + return 'objectId'; + } + + if (type instanceof SchemaDecimal128) { + return 'decimal'; + } + + if (type instanceof SchemaDouble) { + return 'double'; + } + + if (type instanceof Array) { + return 'array'; + } + + return null; +} + +module.exports = { + inferBSONType +}; diff --git a/lib/schema.js b/lib/schema.js index 0204c6cc9c4..b0d992dd684 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -25,6 +25,7 @@ const setPopulatedVirtualValue = require('./helpers/populate/setPopulatedVirtual const setupTimestamps = require('./helpers/timestamps/setupTimestamps'); const utils = require('./utils'); const validateRef = require('./helpers/populate/validateRef'); +const { inferBSONType } = require('./encryptionUtils'); const hasNumericSubpathRegex = /\.\d+(\.|$)/; @@ -86,6 +87,7 @@ const numberRE = /^\d+$/; * - [pluginTags](https://mongoosejs.com/docs/guide.html#pluginTags): array of strings - defaults to `undefined`. If set and plugin called with `tags` option, will only apply that plugin to schemas with a matching tag. * - [virtuals](https://mongoosejs.com/docs/tutorials/virtuals.html#virtuals-via-schema-options): object - virtuals to define, alias for [`.virtual`](https://mongoosejs.com/docs/api/schema.html#Schema.prototype.virtual()) * - [collectionOptions]: object with options passed to [`createCollection()`](https://www.mongodb.com/docs/manual/reference/method/db.createCollection/) when calling `Model.createCollection()` or `autoCreate` set to true. + * - [encryptionType]: the encryption type for the schema. Valid options are `csfle` or `queryableEncryption`. See https://mongoosejs.com/docs/field-level-encryption. * * #### Options for Nested Schemas: * @@ -128,6 +130,7 @@ function Schema(obj, options) { // For internal debugging. Do not use this to try to save a schema in MDB. this.$id = ++id; this.mapPaths = []; + this.encryptedFields = {}; this.s = { hooks: new Kareem() @@ -166,7 +169,7 @@ function Schema(obj, options) { // ensure the documents get an auto _id unless disabled const auto_id = !this.paths['_id'] && - (this.options._id) && !_idSubDoc; + (this.options._id) && !_idSubDoc; if (auto_id) { addAutoId(this); @@ -463,6 +466,8 @@ Schema.prototype._clone = function _clone(Constructor) { s.aliases = Object.assign({}, this.aliases); + s.encryptedFields = clone(this.encryptedFields); + return s; }; @@ -495,7 +500,17 @@ Schema.prototype.pick = function(paths, options) { } for (const path of paths) { - if (this.nested[path]) { + if (path in this.encryptedFields) { + const encrypt = this.encryptedFields[path]; + const schemaType = this.path(path); + newSchema.add({ + [path]: { + encrypt, + [this.options.typeKey]: schemaType + } + }); + } + else if (this.nested[path]) { newSchema.add({ [path]: get(this.tree, path) }); } else { const schematype = this.path(path); @@ -506,6 +521,10 @@ Schema.prototype.pick = function(paths, options) { } } + if (!this._hasEncryptedFields()) { + newSchema.options.encryptionType = null; + } + return newSchema; }; @@ -534,9 +553,9 @@ Schema.prototype.omit = function(paths, options) { if (!Array.isArray(paths)) { throw new MongooseError( 'Schema#omit() only accepts an array argument, ' + - 'got "' + - typeof paths + - '"' + 'got "' + + typeof paths + + '"' ); } @@ -667,6 +686,20 @@ Schema.prototype._defaultToObjectOptions = function(json) { return defaultOptions; }; +/** + * Sets the encryption type of the schema, if a value is provided, otherwise + * returns the encryption type. + * + * @param {'csfle' | 'queryableEncryption' | undefined} encryptionType plain object with paths to add, or another schema + */ +Schema.prototype.encryptionType = function encryptionType(encryptionType) { + if (typeof encryptionType === 'string' || encryptionType === null) { + this.options.encryptionType = encryptionType; + } else { + return this.options.encryptionType; + } +}; + /** * Adds key path / schema type pairs to this schema. * @@ -735,7 +768,7 @@ Schema.prototype.add = function add(obj, prefix) { if ( key !== '_id' && ((typeof val !== 'object' && typeof val !== 'function' && !isMongooseTypeString) || - val == null) + val == null) ) { throw new TypeError(`Invalid schema configuration: \`${val}\` is not ` + `a valid type at path \`${key}\`. See ` + @@ -818,15 +851,128 @@ Schema.prototype.add = function add(obj, prefix) { } } } + + if (val.instanceOfSchema && val.encryptionType() != null) { + // schema.add({ field: }) + if (this.encryptionType() != val.encryptionType()) { + throw new Error('encryptionType of a nested schema must match the encryption type of the parent schema.'); + } + + for (const [encryptedField, encryptedFieldConfig] of Object.entries(val.encryptedFields)) { + const path = fullPath + '.' + encryptedField; + this._addEncryptedField(path, encryptedFieldConfig); + } + } + else if (typeof val === 'object' && 'encrypt' in val) { + // schema.add({ field: { type: , encrypt: { ... }}}) + const { encrypt } = val; + + if (this.encryptionType() == null) { + throw new Error('encryptionType must be provided'); + } + + this._addEncryptedField(fullPath, encrypt); + } else { + // if the field was already encrypted and we re-configure it to be unencrypted, remove + // the encrypted field configuration + this._removeEncryptedField(fullPath); + } } const aliasObj = Object.fromEntries( Object.entries(obj).map(([key]) => ([prefix + key, null])) ); aliasFields(this, aliasObj); + return this; }; +/** + * @param {string} path + * @param {object} fieldConfig + * + * @api private + */ +Schema.prototype._addEncryptedField = function _addEncryptedField(path, fieldConfig) { + const type = inferBSONType(this, path); + if (type == null) { + throw new Error('unable to determine bson type for field `' + path + '`'); + } + + this.encryptedFields[path] = clone(fieldConfig); +}; + +/** + * @api private + */ +Schema.prototype._removeEncryptedField = function _removeEncryptedField(path) { + delete this.encryptedFields[path]; +}; + +/** + * @api private + */ +Schema.prototype._hasEncryptedFields = function _hasEncryptedFields() { + return Object.keys(this.encryptedFields).length > 0; +}; + +Schema.prototype._buildEncryptedFields = function() { + const fields = Object.entries(this.encryptedFields).map( + ([path, config]) => { + const bsonType = inferBSONType(this, path); + // { path, bsonType, keyId, queries? } + return { path, bsonType, ...config }; + }); + + return { fields }; +}; + +Schema.prototype._buildSchemaMap = function() { + /** + * `schemaMap`s are JSON schemas, which use the following structure to represent objects: + * { field: { bsonType: 'object', properties: { ... } } } + * + * for example, a schema that looks like this `{ a: { b: int32 } }` would be encoded as + * `{ a: { bsonType: 'object', properties: { b: < encryption configuration > } } }` + * + * This function takes an array of path segments, an output object (that gets mutated) and + * a value to associated with the full path, and constructs a valid CSFLE JSON schema path for + * the object. This works for deeply nested properties as well. + * + * @param {string[]} path array of path components + * @param {object} object the object in which to build a JSON schema of `path`'s properties + * @param {object} value the value to associate with the path in object + */ + function buildNestedPath(path, object, value) { + let i = 0, component = path[i]; + for (; i < path.length - 1; ++i, component = path[i]) { + object[component] = object[component] == null ? { + bsonType: 'object', + properties: {} + } : object[component]; + object = object[component].properties; + } + object[component] = value; + } + + const schemaMapPropertyReducer = (accum, [path, propertyConfig]) => { + const bsonType = inferBSONType(this, path); + const pathComponents = path.split('.'); + const configuration = { encrypt: { ...propertyConfig, bsonType } }; + buildNestedPath(pathComponents, accum, configuration); + return accum; + }; + + const properties = Object.entries(this.encryptedFields).reduce( + schemaMapPropertyReducer, + {}); + + return { + bsonType: 'object', + properties + }; +}; + /** * Add an alias for `path`. This means getting or setting the `alias` * is equivalent to getting or setting the `path`. @@ -1008,23 +1154,23 @@ Schema.prototype.reserved = Schema.reserved; const reserved = Schema.reserved; // Core object reserved['prototype'] = -// EventEmitter -reserved.emit = -reserved.listeners = -reserved.removeListener = - -// document properties and functions -reserved.collection = -reserved.errors = -reserved.get = -reserved.init = -reserved.isModified = -reserved.isNew = -reserved.populated = -reserved.remove = -reserved.save = -reserved.toObject = -reserved.validate = 1; + // EventEmitter + reserved.emit = + reserved.listeners = + reserved.removeListener = + + // document properties and functions + reserved.collection = + reserved.errors = + reserved.get = + reserved.init = + reserved.isModified = + reserved.isNew = + reserved.populated = + reserved.remove = + reserved.save = + reserved.toObject = + reserved.validate = 1; reserved.collection = 1; /** @@ -1104,10 +1250,10 @@ Schema.prototype.path = function(path, obj) { } if (typeof branch[sub] !== 'object') { const msg = 'Cannot set nested path `' + path + '`. ' - + 'Parent path `' - + fullPath - + '` already set to type ' + branch[sub].name - + '.'; + + 'Parent path `' + + fullPath + + '` already set to type ' + branch[sub].name + + '.'; throw new Error(msg); } branch = branch[sub]; @@ -1375,6 +1521,16 @@ Schema.prototype.interpretAsType = function(path, obj, options) { let type = obj[options.typeKey] && (obj[options.typeKey] instanceof Function || options.typeKey !== 'type' || !obj.type.type) ? obj[options.typeKey] : {}; + + if (type instanceof SchemaType) { + if (type.path === path) { + return type; + } + const clone = type.clone(); + clone.path = path; + return clone; + } + let name; if (utils.isPOJO(type) || type === 'mixed') { @@ -1404,8 +1560,8 @@ Schema.prototype.interpretAsType = function(path, obj, options) { return new MongooseTypes.DocumentArray(path, cast, obj); } if (cast && - cast[options.typeKey] && - cast[options.typeKey].instanceOfSchema) { + cast[options.typeKey] && + cast[options.typeKey].instanceOfSchema) { if (!(cast[options.typeKey] instanceof Schema)) { if (this.options._isMerging) { cast[options.typeKey] = new Schema(cast[options.typeKey]); @@ -1739,7 +1895,7 @@ Schema.prototype.hasMixedParent = function(path) { for (let i = 0; i < subpaths.length; ++i) { path = i > 0 ? path + '.' + subpaths[i] : subpaths[i]; if (this.paths.hasOwnProperty(path) && - this.paths[path] instanceof MongooseTypes.Mixed) { + this.paths[path] instanceof MongooseTypes.Mixed) { return this.paths[path]; } } @@ -2520,6 +2676,8 @@ Schema.prototype.remove = function(path) { delete this.paths[name]; _deletePath(this, name); + + this._removeEncryptedField(name); }, this); } return this; @@ -2615,9 +2773,9 @@ Schema.prototype.removeVirtual = function(path) { Schema.prototype.loadClass = function(model, virtualsOnly) { // Stop copying when hit certain base classes if (model === Object.prototype || - model === Function.prototype || - model.prototype.hasOwnProperty('$isMongooseModelPrototype') || - model.prototype.hasOwnProperty('$isMongooseDocumentPrototype')) { + model === Function.prototype || + model.prototype.hasOwnProperty('$isMongooseModelPrototype') || + model.prototype.hasOwnProperty('$isMongooseDocumentPrototype')) { return this; } diff --git a/lib/utils.js b/lib/utils.js index 6fc5c335ef0..17ff140865a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -89,19 +89,19 @@ exports.deepEqual = function deepEqual(a, b) { } if ((isBsonType(a, 'ObjectId') && isBsonType(b, 'ObjectId')) || - (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) { + (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) { return a.toString() === b.toString(); } if (a instanceof RegExp && b instanceof RegExp) { return a.source === b.source && - a.ignoreCase === b.ignoreCase && - a.multiline === b.multiline && - a.global === b.global && - a.dotAll === b.dotAll && - a.unicode === b.unicode && - a.sticky === b.sticky && - a.hasIndices === b.hasIndices; + a.ignoreCase === b.ignoreCase && + a.multiline === b.multiline && + a.global === b.global && + a.dotAll === b.dotAll && + a.unicode === b.unicode && + a.sticky === b.sticky && + a.hasIndices === b.hasIndices; } if (a == null || b == null) { @@ -287,8 +287,8 @@ exports.merge = function merge(to, from, options, path) { // base schema has a given path as a single nested but discriminator schema // has the path as a document array, or vice versa (gh-9534) if (options.isDiscriminatorSchemaMerge && - (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) || - (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) { + (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) || + (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) { continue; } else if (from[key].instanceOfSchema) { if (to[key].instanceOfSchema) { @@ -995,7 +995,7 @@ exports.getOption = function(name) { * ignore */ -exports.noop = function() {}; +exports.noop = function() { }; exports.errorToPOJO = function errorToPOJO(error) { const isError = error instanceof Error; @@ -1025,3 +1025,13 @@ exports.injectTimestampsOption = function injectTimestampsOption(writeOperation, } writeOperation.timestamps = timestampsOption; }; + +exports.print = function(...args) { + const { inspect } = require('util'); + console.error( + inspect( + ...args, + { depth: Infinity } + ) + ); +}; diff --git a/package.json b/package.json index ecade1a0589..86cd719b83b 100644 --- a/package.json +++ b/package.json @@ -146,4 +146,4 @@ "target": "ES2017" } } -} +} \ No newline at end of file diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 8f366bc4bbc..2a28555f7bb 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -7,7 +7,7 @@ export CWD=$(pwd); # install extra dependency -npm install mongodb-client-encryption +npm install --no-save mongodb-client-encryption # set up mongodb cluster and encryption configuration if the data/ folder does not exist if [ ! -d "data" ]; then @@ -51,4 +51,4 @@ if [ ! -d "data" ]; then echo 'Cluster Configuration Finished!' cd .. -fi \ No newline at end of file +fi diff --git a/scripts/run-encryption-tests.sh b/scripts/run-encryption-tests.sh new file mode 100755 index 00000000000..0209292168d --- /dev/null +++ b/scripts/run-encryption-tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests + +export CWD=$(pwd); + +# set up mongodb cluster and encryption configuration if the data/ folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script + +if [ -d "data" ]; then + cd data +else + source $CWD/scripts/configure-cluster-with-encryption.sh +fi + +# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run +read -r -d '' SOURCE_SCRIPT << EOM +const fs = require('fs'); +const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) + .trim().split('\\n'); +const regex = /^(?.*): "(?.*)"$/; +const variables = file.map( + (line) => regex.exec(line.trim()).groups +).map( + ({key, value}) => \`export \${key}='\${value}'\` +).join('\n'); + +process.stdout.write(variables); +process.stdout.write('\n'); +EOM + +node --eval "$SOURCE_SCRIPT" | tee expansions.sh +source expansions.sh + +export MONGOOSE_TEST_URI=$MONGODB_URI + +# run encryption tests +cd .. +npx mocha --exit ./test/encryption/*.test.js diff --git a/test/encrypted_schema.test.js b/test/encrypted_schema.test.js new file mode 100644 index 00000000000..d5712aabe1a --- /dev/null +++ b/test/encrypted_schema.test.js @@ -0,0 +1,1091 @@ + +'use strict'; + +const assert = require('assert'); +const start = require('./common'); +const { ObjectId, Decimal128 } = require('../lib/types'); +const { Double, Int32, UUID } = require('bson'); + +const mongoose = start.mongoose; +const Schema = mongoose.Schema; + +/** + * + * @param {import('../lib').Schema} object + * @param {Array | string} path + * @returns + */ +function schemaHasEncryptedProperty(schema, path) { + path = [path].flat(); + path = path.join('.'); + + return path in schema.encryptedFields; +} + +const KEY_ID = '9fbdace3-4e48-412d-88df-3807e8009522'; +const algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'; + +describe('encrypted schema declaration', function() { + describe('schemaMap generation tests', function() { + for (const { type, name, encryptionType, schemaMap, encryptedFields } of primitiveSchemaMapTests()) { + describe(`When a schema is instantiated with an encrypted field of type ${name} for ${encryptionType}`, function() { + let schema; + const encrypt = { + keyId: KEY_ID + }; + encryptionType === 'csfle' && (encrypt.algorithm = algorithm); + + beforeEach(function() { + schema = new Schema({ + field: { + type, encrypt + } + }, { + encryptionType + }); + }); + + it(`Then the schema has an encrypted property of type ${name}`, function() { + assert.ok(schemaHasEncryptedProperty(schema, 'field')); + }); + + encryptionType === 'csfle' && it('then the generated schemaMap is correct', function() { + assert.deepEqual(schema._buildSchemaMap(), schemaMap); + }); + + encryptionType === 'qe' && it('then the generated encryptedFieldsMap is correct', function() { + assert.deepEqual(schema._buildEncryptedFields(), encryptedFields); + }); + }); + } + }); + + describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { + const tests = { + 'nested schema for csfle': + { + schemaFactory: () => { + const encryptedSchema = new Schema({ + encrypted: { + type: String, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + return new Schema({ + field: encryptedSchema + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + field: { + bsonType: 'object', + properties: { + encrypted: { encrypt: { bsonType: 'string', algorithm, keyId: KEY_ID } } + } + } + } + } + }, + 'nested schema for qe': { + schemaFactory: () => { + const encryptedSchema = new Schema({ + encrypted: { + type: String, encrypt: { keyId: KEY_ID } + } + }, { encryptionType: 'qe' }); + return new Schema({ + field: encryptedSchema + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + encryptedFields: { + fields: [ + { path: 'field.encrypted', keyId: KEY_ID, bsonType: 'string' } + ] + } + }, + 'nested object for csfle': + { + schemaFactory: () => { + return new Schema({ + field: { + encrypted: { + type: String, encrypt: { keyId: KEY_ID, algorithm } + } + } + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + field: { + bsonType: 'object', + properties: { + encrypted: { encrypt: { bsonType: 'string', algorithm, keyId: KEY_ID } } + } + } + } + } + }, + 'nested object for qe': { + schemaFactory: () => { + return new Schema({ + field: { + encrypted: { + type: String, encrypt: { keyId: KEY_ID } + } + } + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + encryptedFields: { + fields: [ + { path: 'field.encrypted', keyId: KEY_ID, bsonType: 'string' } + ] + } + }, + 'schema with encrypted array for csfle': { + schemaFactory: () => { + return new Schema({ + encrypted: { + type: [Number], + encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + encrypted: { + encrypt: { + bsonType: 'array', + keyId: KEY_ID, + algorithm + } + } + } + } + }, + 'schema with encrypted array for qe': { + schemaFactory: () => { + return new Schema({ + encrypted: { + type: [Number], + encrypt: { keyId: KEY_ID } + } + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['encrypted'])), + encryptedFields: { + fields: [ + { path: 'encrypted', keyId: KEY_ID, bsonType: 'array' } + ] + } + } + }; + + for (const [description, { schemaFactory, predicate, schemaMap, encryptedFields }] of Object.entries(tests)) { + it(description, function() { + const schema = schemaFactory(); + predicate(schema); + + schemaMap && assert.deepEqual(schema._buildSchemaMap(), schemaMap); + encryptedFields && assert.deepEqual(schema._buildEncryptedFields(), encryptedFields); + }); + } + }); + + describe('invalid schema types for encrypted schemas', function() { + describe('When a schema is instantiated with an encrypted field of type Number', function() { + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Number, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + describe('When a schema is instantiated with an encrypted field of type Mixed', function() { + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Schema.Types.Mixed, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + describe('When a schema is instantiated with a custom schema type plugin', function() { + class Int8 extends mongoose.SchemaType { + constructor(key, options) { + super(key, options, 'Int8'); + } + } + + beforeEach(function() { + // Don't forget to add `Int8` to the type registry + mongoose.Schema.Types.Int8 = Int8; + }); + afterEach(function() { + delete mongoose.Schema.Types.Int8; + }); + + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Int8, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + }); + + describe('options.encryptionType', function() { + describe('when an encrypted schema is instantiated and an encryptionType is not provided', function() { + it('an error is thrown', function() { + assert.throws( + () => { + new Schema({ + field: { + type: String, + encrypt: { keyId: KEY_ID, algorithm } + } + }); + }, /encryptionType must be provided/ + ); + + + }); + }); + + describe('when a nested encrypted schema is provided to schema constructor and the encryption types are different', function() { + it('then an error is thrown', function() { + const innerSchema = new Schema({ + field1: { + type: String, encrypt: { + keyId: KEY_ID, + queries: { type: 'equality' } + } + } + }, { encryptionType: 'csfle' }); + assert.throws(() => { + new Schema({ + field1: innerSchema + }, { encryptionType: 'queryableEncryption' }); + }, /encryptionType of a nested schema must match the encryption type of the parent schema/); + }); + }); + }); + + describe('tests for schema mutation methods', function() { + describe('Schema.prototype.add()', function() { + describe('Given a schema with no encrypted fields', function() { + describe('When an encrypted field is added', function() { + it('Then the encrypted field is added to the encrypted fields for the schema', function() { + const schema = new Schema({ + field1: Number + }); + schema.encryptionType('csfle'); + schema.add( + { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + ); + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + }); + }); + }); + + describe('Given a schema with an encrypted field', function() { + describe('when an encrypted field is added', function() { + describe('and the encryption type matches the existing encryption type', function() { + it('Then the encrypted field is added to the encrypted fields for the schema', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + ); + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + }); + }); + }); + }); + + describe('Given a schema with an encrypted field', function() { + describe('when an encrypted field is added with different encryption settings for the same field', function() { + it('The encryption settings for the field are overridden', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { name: { type: String, encrypt: { keyId: new UUID(), algorithm } } } + ); + assert.notEqual(schema.encryptedFields['name'].keyId, KEY_ID); + }); + + }); + + describe('When an unencrypted field is added for the same field', function() { + it('The field on the schema is overridden', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { field1: String } + ); + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + }); + + }); + }); + + describe('Given a schema', function() { + describe('When multiple encrypted fields are added to the schema in one call to add()', function() { + it('Then all the encrypted fields are added to the schema', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + } + ); + + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + assert.ok(schemaHasEncryptedProperty(schema, ['age'])); + }); + }); + }); + }); + + describe('Schema.prototype.remove()', function() { + describe('Given a schema with one encrypted field', function() { + describe('When the encrypted field is removed', function() { + it('Then the encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove('field1'); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + }); + }); + }); + + describe('Given a schema with multiple encrypted fields', function() { + describe('When one encrypted field is removed', function() { + it('The encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove(['field1']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema, ['age']), true); + }); + }); + + describe('When all encrypted fields are removed', function() { + it('The encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove(['field1', 'name', 'age']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['age']), false); + }); + }); + }); + + describe('when a nested encrypted property is removed', function() { + it('the encrypted field is removed from the schema', function() { + const schema = new Schema({ + field1: { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + }, { encryptionType: 'csfle' }); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1.name']), true); + + schema.remove(['field1.name']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1.name']), false); + }); + }); + }); + }); + + describe('tests for schema copying methods', function() { + describe('Schema.prototype.clone()', function() { + describe('Given a schema with encrypted fields', function() { + describe('When the schema is cloned', function() { + it('The resultant schema contains all the same encrypted fields as the original schema', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + }); + it('The encryption type of the cloned schema is the same as the original', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + describe('When the cloned schema is modified', function() { + it('The original is not modified', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + schema2.remove('name'); + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema1, ['name']), true); + }); + }); + }); + }); + }); + + describe('Schema.prototype.pick()', function() { + describe('When pick() is used with only unencrypted fields', function() { + it('Then the resultant schema has none of the original schema’s encrypted fields', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name1', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is set to the cloned schemas encryptionType', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name1', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When pick() is used with some unencrypted fields', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to pick().', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When pick() is used with nested paths', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to pick().', function() { + const originalSchema = new Schema({ + name: { + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name.name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name', 'name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + }); + + describe('Schema.prototype.omit()', function() { + describe('When omit() is used with only unencrypted fields', function() { + it('Then the resultant schema has all the original schema’s encrypted fields', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name1', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), true); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name1', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When omit() is used with some unencrypted fields', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to omit()', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), true); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When omit() is used with all the encrypted fields', function() { + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + }); + }); +}); + +function primitiveSchemaMapTests() { + return [ + { + name: 'string', + type: String, + encryptionType: 'csfle', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'string' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'string', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'string', + type: String, + encryptionType: 'qe', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'string' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'string', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'boolean', + type: Schema.Types.Boolean, + encryptionType: 'csfle', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'bool' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'bool', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'boolean', + encryptionType: 'qe', + type: Schema.Types.Boolean, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'bool' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'bool', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'buffer', + encryptionType: 'csfle', + type: Schema.Types.Buffer, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'binData' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'binData', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'buffer', + encryptionType: 'qe', + type: Schema.Types.Buffer, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'binData' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'binData', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'date', + encryptionType: 'csfle', + type: Date, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'date' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'date', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'date', + encryptionType: 'qe', + type: Date, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'date' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'date', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'objectid', + encryptionType: 'csfle', + type: ObjectId, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'objectId' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'objectId', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'objectid', + encryptionType: 'qe', + type: ObjectId, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'objectId' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'objectId', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'bigint', + encryptionType: 'csfle', + type: BigInt, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'long' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'long', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'bigint', + encryptionType: 'qe', + type: BigInt, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'long' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'long', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'Decimal128', + encryptionType: 'csfle', + type: Decimal128, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'decimal' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'decimal', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'Decimal128', + encryptionType: 'qe', + type: Decimal128, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'decimal' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'decimal', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'int32', + encryptionType: 'csfle', + type: Int32, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'int' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'int', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'int32', + encryptionType: 'qe', + type: Int32, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'int' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'int', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'double', + encryptionType: 'csfle', + type: Double, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'double' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'double', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'double', + encryptionType: 'qe', + type: Double, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'double' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'double', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + } + ]; +} diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index a3b562e80aa..4efe48fae6b 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -1,31 +1,24 @@ 'use strict'; const assert = require('assert'); -const mongodb = require('mongodb'); -const fs = require('fs'); +const mdb = require('mongodb'); const isBsonType = require('../../lib/helpers/isBsonType'); +const { Schema, createConnection } = require('../../lib'); +const { ObjectId, Double, Int32, Decimal128 } = require('bson'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); -describe('ci', () => { - - const cachedUri = process.env.MONGOOSE_TEST_URI; - const cachedLib = process.env.CRYPT_SHARED_LIB_PATH; - - before(function() { - const cwd = process.cwd(); - const file = fs.readFileSync(cwd + '/data/mo-expansion.yml', { encoding: 'utf-8' }).trim().split('\n'); - const regex = /^(?.*): "(?.*)"$/; - const variables = file.map((line) => regex.exec(line.trim()).groups).reduce((acc, { key, value }) => ({ ...acc, [key]: value }), {}); - process.env.CRYPT_SHARED_LIB_PATH = variables.CRYPT_SHARED_LIB_PATH; - process.env.MONGOOSE_TEST_URI = variables.MONGODB_URI; - }); - - after(function() { - process.env.CRYPT_SHARED_LIB_PATH = cachedLib; - process.env.MONGOOSE_TEST_URI = cachedUri; - }); +/** + * @param {object} object + * @param {string} property + */ +function isEncryptedValue(object, property) { + const value = object[property]; + assert.ok(isBsonType(value, 'Binary'), `auto encryption for property ${property} failed: not a BSON binary.`); + assert.ok(value.sub_type === 6, `auto encryption for property ${property} failed: not subtype 6.`); +} +describe('ci', () => { describe('environmental variables', () => { it('MONGOOSE_TEST_URI is set', async function() { const uri = process.env.MONGOOSE_TEST_URI; @@ -38,78 +31,739 @@ describe('ci', () => { }); }); - describe('basic integration', () => { - let keyVaultClient; - let dataKey; - let encryptedClient; - let unencryptedClient; - - beforeEach(async function() { - keyVaultClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); - await keyVaultClient.connect(); - await keyVaultClient.db('keyvault').collection('datakeys'); - const clientEncryption = new mongodb.ClientEncryption(keyVaultClient, { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } } + let keyId, keyId2, keyId3; + let utilClient; + + beforeEach(async function() { + const keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + keyId = await clientEncryption.createDataKey('local'); + keyId2 = await clientEncryption.createDataKey('local'); + keyId3 = await clientEncryption.createDataKey('local'); + await keyVaultClient.close(); + + utilClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); + + afterEach(async function() { + await utilClient.db('db').dropDatabase({ + w: 'majority' + }); + await utilClient.close(); + }); + + describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { + const algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'; + let connection; + let schema; + let model; + + const basicSchemaTypes = [ + { type: String, name: 'string', input: 3, expected: 3 }, + { type: Schema.Types.Boolean, name: 'boolean', input: true, expected: true }, + { type: Schema.Types.Buffer, name: 'buffer', input: Buffer.from([1, 2, 3]) }, + { type: Date, name: 'date', input: new Date(12, 12, 2012), expected: new Date(12, 12, 2012) }, + { type: ObjectId, name: 'objectid', input: new ObjectId() }, + { type: BigInt, name: 'bigint', input: 3n }, + { type: Decimal128, name: 'Decimal128', input: new Decimal128('1.5') }, + { type: Int32, name: 'int32', input: new Int32(5), expected: 5 }, + { type: Double, name: 'double', input: new Double(1.5) } + ]; + + for (const { type, name, input, expected } of basicSchemaTypes) { + this.afterEach(async function() { + await connection?.close(); }); - dataKey = await clientEncryption.createDataKey('local'); - encryptedClient = new mongodb.MongoClient( - process.env.MONGOOSE_TEST_URI, - { - autoEncryption: { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } }, - schemaMap: { - 'db.coll': { - bsonType: 'object', - encryptMetadata: { - keyId: [dataKey] - }, - properties: { - a: { - encrypt: { - bsonType: 'int', - algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [dataKey] - } + // eslint-disable-next-line no-inner-declarations + async function test() { + const [{ _id }] = await model.insertMany([{ field: input }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.field, 'Binary')); + assert.ok(encryptedDoc.field.sub_type === 6); + + const doc = await model.findOne({ _id }); + if (Buffer.isBuffer(input)) { + // mongoose's Buffer does not support deep equality - instead use the Buffer.equals method. + assert.ok(doc.field.equals(input)); + } else { + assert.deepEqual(doc.field, expected ?? input); + } + } + + describe('CSFLE', function() { + beforeEach(async function() { + schema = new Schema({ + field: { + type, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + + it(`${name} encrypts and decrypts`, test); + }); + + describe('queryableEncryption', function() { + beforeEach(async function() { + schema = new Schema({ + field: { + type, encrypt: { keyId: keyId } + } + }, { + encryptionType: 'queryableEncryption' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + + it(`${name} encrypts and decrypts`, test); + }); + } + + describe('nested object schemas', function() { + const tests = { + 'nested object schemas for CSFLE': { + modelFactory: () => { + const schema = new Schema({ + a: { + b: { + c: { + type: String, + encrypt: { keyId: [keyId], algorithm } + } + } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested object schemas for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + b: { + c: { + type: String, + encrypt: { keyId: keyId } } } } - }, + }, { + encryptionType: 'queryableEncryption' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested schemas for csfle': { + modelFactory: () => { + const nestedSchema = new Schema({ + b: { + c: { + type: String, + encrypt: { keyId: [keyId], algorithm } + } + } + }, { + encryptionType: 'csfle' + }); + + const schema = new Schema({ + a: nestedSchema + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested schemas for QE': { + modelFactory: () => { + const nestedSchema = new Schema({ + b: { + c: { + type: String, + encrypt: { keyId: keyId } + } + } + }, { + encryptionType: 'queryableEncryption' + }); + const schema = new Schema({ + a: nestedSchema + }, { + encryptionType: 'queryableEncryption' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: { b: { c: 'hello' } } }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a.b.c, 'Binary')); + assert.ok(encryptedDoc.a.b.c.sub_type === 6); + + const doc = await model.findOne({ _id }); + assert.deepEqual(doc.a.b.c, 'hello'); + }); + }); + } + }); + + describe('array encrypted fields', function() { + const tests = { + 'array fields for CSFLE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: [Int32], + encrypt: { + keyId: [keyId], + algorithm + } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + }, + 'array field for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: [Int32], + encrypt: { + keyId + } + } + }, { + encryptionType: 'queryableEncryption' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: [new Int32(3)] }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + + const doc = await model.findOne({ _id }); + assert.deepEqual(doc.a, [3]); + }); + }); + } + }); + + describe('multiple encrypted fields in a model', function() { + const tests = { + 'multiple fields in a schema for CSFLE': { + modelFactory: () => { + const encrypt = { + keyId: [keyId], + algorithm + }; + + const schema = new Schema({ + a: { + type: String, + encrypt + }, + b: { + type: BigInt + }, + c: { + d: { + type: String, + encrypt + } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + }, + 'multiple fields in a schema for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: String, + encrypt: { + keyId + } + }, + b: { + type: BigInt + }, + c: { + d: { + type: String, + encrypt: { + keyId: keyId2 + } + } + } + }, { + encryptionType: 'queryableEncryption' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: 'hello', b: 1n, c: { d: 'world' } }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + assert.ok(typeof encryptedDoc.b === 'number'); + assert.ok(isBsonType(encryptedDoc.c.d, 'Binary')); + assert.ok(encryptedDoc.c.d.sub_type === 6); + + const doc = await model.findOne({ _id }, {}); + assert.deepEqual(doc.a, 'hello'); + assert.deepEqual(doc.b, 1n); + assert.deepEqual(doc.c, { d: 'world' }); + }); + }); + } + }); + + describe('multiple schemas', function() { + const tests = { + 'multiple schemas for CSFLE': { + modelFactory: () => { + connection = createConnection(); + const encrypt = { + keyId: [keyId], + algorithm + }; + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt + } + }, { + encryptionType: 'csfle' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt + } + }, { + encryptionType: 'csfle' + })); + + return { model1, model2 }; + } + }, + 'multiple schemas for QE': { + modelFactory: () => { + connection = createConnection(); + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'queryableEncryption' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'queryableEncryption' + })); + + return { model1, model2 }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model1, model2 } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + { + const [{ _id }] = await model1.insertMany([{ a: 'hello' }]); + const encryptedDoc = await utilClient.db('db').collection('model1').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + + const doc = await model1.findOne({ _id }); + assert.deepEqual(doc.a, 'hello'); + } + + { + const [{ _id }] = await model2.insertMany([{ b: 'world' }]); + const encryptedDoc = await utilClient.db('db').collection('model2').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.b, 'Binary')); + assert.ok(encryptedDoc.b.sub_type === 6); + + const doc = await model2.findOne({ _id }); + assert.deepEqual(doc.b, 'world'); + } + }); + }); + } + }); + + describe('CSFLE and QE schemas on the same connection', function() { + it('encrypts and decrypts', async function() { + connection = createConnection(); + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'queryableEncryption' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt: { + keyId: [keyId], + algorithm + } + } + }, { + encryptionType: 'csfle' + })); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, extraOptions: { cryptdSharedLibRequired: true, cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } } + }); + + { + const [{ _id }] = await model1.insertMany([{ a: 'hello' }]); + const encryptedDoc = await utilClient.db('db').collection('model1').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + + const doc = await model1.findOne({ _id }); + assert.deepEqual(doc.a, 'hello'); } - ); - unencryptedClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); - }); + { + const [{ _id }] = await model2.insertMany([{ b: 'world' }]); + const encryptedDoc = await utilClient.db('db').collection('model2').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.b, 'Binary')); + assert.ok(encryptedDoc.b.sub_type === 6); - afterEach(async function() { - await keyVaultClient.close(); - await encryptedClient.close(); - await unencryptedClient.close(); + const doc = await model2.findOne({ _id }); + assert.deepEqual(doc.b, 'world'); + } + }); }); - it('ci set-up should support basic mongodb auto-encryption integration', async() => { - await encryptedClient.connect(); - const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + describe('Models with discriminators', function() { + let discrim1, discrim2, model; - // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); + describe('csfle', function() { + beforeEach(async function() { + connection = createConnection(); - assert.ok(encryptedResult); - assert.ok(encryptedResult.a); - assert.ok(isBsonType(encryptedResult.a, 'Binary')); - assert.ok(encryptedResult.a.sub_type === 6); + const schema = new Schema({ + name: { + type: String, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + }); + model = connection.model('Schema', schema); + discrim1 = model.discriminator('Test', new Schema({ + age: { + type: Int32, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + })); + + discrim2 = model.discriminator('Test2', new Schema({ + dob: { + type: Int32, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + })); + + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + it('encrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'age'); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'dob'); + } + }); + + it('decrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const decryptedDoc = await discrim1.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.age, 32); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const decryptedDoc = await discrim2.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.dob, 32); + } + }); + }); + + + describe('queryableEncryption', function() { + beforeEach(async function() { + connection = createConnection(); + + const schema = new Schema({ + name: { + type: String, encrypt: { keyId } + } + }, { + encryptionType: 'queryableEncryption' + }); + model = connection.model('Schema', schema); + discrim1 = model.discriminator('Test', new Schema({ + age: { + type: Int32, encrypt: { keyId: keyId2 } + } + }, { + encryptionType: 'queryableEncryption' + })); + + discrim2 = model.discriminator('Test2', new Schema({ + dob: { + type: Int32, encrypt: { keyId: keyId3 } + } + }, { + encryptionType: 'queryableEncryption' + })); + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + it('encrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'age'); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'dob'); + } + }); + + it('decrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const decryptedDoc = await discrim1.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.age, 32); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const decryptedDoc = await discrim2.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.dob, 32); + } + }); + }); - // when the encryptedClient runs a find, the original unencrypted value is returned - const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); - assert.ok(unencryptedResult); - assert.ok(unencryptedResult.a === 1); }); }); }); diff --git a/test/model.test.js b/test/model.test.js index da870125e0d..07cea0d5ab3 100644 --- a/test/model.test.js +++ b/test/model.test.js @@ -368,7 +368,7 @@ describe('Model', function() { assert.equal(post.get('comments')[0].comments[0].isNew, true); post.invalidate('title'); // force error - await post.save().catch(() => {}); + await post.save().catch(() => { }); assert.equal(post.isNew, true); assert.equal(post.get('comments')[0].isNew, true); assert.equal(post.get('comments')[0].comments[0].isNew, true); @@ -2479,7 +2479,7 @@ describe('Model', function() { const DefaultErr = db.model('Test', DefaultErrSchema); - new DefaultErr().save().catch(() => {}); + new DefaultErr().save().catch(() => { }); await new Promise(resolve => { DefaultErr.once('error', function(err) { @@ -3043,7 +3043,7 @@ describe('Model', function() { const Location = db.model('Test', LocationSchema); - await Location.collection.drop().catch(() => {}); + await Location.collection.drop().catch(() => { }); await Location.init(); await Location.create({ @@ -3512,7 +3512,7 @@ describe('Model', function() { listener = null; // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); changeStream.close(); changeStream = null; }); @@ -3664,7 +3664,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); await changeStream.close(); await db.close(); }); @@ -3682,7 +3682,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); const close = changeStream.close(); await db.asPromise(); @@ -3708,7 +3708,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); changeStream.close(); const closedData = await closed; @@ -5540,7 +5540,7 @@ describe('Model', function() { const Model = db.model('User', userSchema); - await Model.collection.drop().catch(() => {}); + await Model.collection.drop().catch(() => { }); await Model.createCollection(); const collectionName = Model.collection.name; @@ -5574,7 +5574,7 @@ describe('Model', function() { const Test = db.model('Test', schema, 'Test'); await Test.init(); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collections = await Test.db.db.listCollections().toArray(); @@ -5583,7 +5583,7 @@ describe('Model', function() { assert.equal(coll.type, 'timeseries'); assert.equal(coll.options.timeseries.timeField, 'timestamp'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); }); it('createCollection() enforces expireAfterSeconds (gh-11229)', async function() { @@ -5604,7 +5604,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var1', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expireAfterSeconds: 5 }); const collOptions = await Test.collection.options(); @@ -5632,7 +5632,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var2', schema, 'TestGH11229Var2'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expires: '5 seconds' }); const collOptions = await Test.collection.options(); @@ -5660,7 +5660,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var3', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collOptions = await Test.collection.options(); @@ -5688,7 +5688,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var4', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collOptions = await Test.collection.options(); @@ -5716,7 +5716,7 @@ describe('Model', function() { const Test = db.model('Test', schema, 'Test'); await Test.init(); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collections = await Test.db.db.listCollections().toArray(); @@ -5725,7 +5725,7 @@ describe('Model', function() { assert.deepEqual(coll.options.clusteredIndex.key, { _id: 1 }); assert.equal(coll.options.clusteredIndex.name, 'clustered test'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); }); it('mongodb actually removes expired documents (gh-11229)', async function() { @@ -5747,7 +5747,7 @@ describe('Model', function() { const Test = db.model('TestMongoDBExpireRemoval', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expireAfterSeconds: 5 }); await Test.insertMany([ @@ -5845,7 +5845,7 @@ describe('Model', function() { const Model = db.model('User', userSchema); - await Model.collection.drop().catch(() => {}); + await Model.collection.drop().catch(() => { }); await Model.createCollection(); await Model.createCollection(); @@ -6525,7 +6525,7 @@ describe('Model', function() { await User.bulkWrite([ { updateOne: { - filter: { }, + filter: {}, update: { friends: ['Sam'] }, upsert: true, setDefaultsOnInsert: true @@ -7002,7 +7002,7 @@ describe('Model', function() { }); it('insertMany should throw an error if there were operations that failed validation, ' + - 'but all operations that passed validation succeeded (gh-14572) (gh-13256)', async function() { + 'but all operations that passed validation succeeded (gh-14572) (gh-13256)', async function() { const userSchema = new Schema({ age: { type: Number } }); @@ -8020,7 +8020,7 @@ describe('Model', function() { decoratorSchema.loadClass(Decorator); // Define discriminated class before model is compiled - class Deco1 extends Decorator { whoAmI() { return 'I am Test1'; }} + class Deco1 extends Decorator { whoAmI() { return 'I am Test1'; } } const deco1Schema = new Schema({}); deco1Schema.loadClass(Deco1); decoratorSchema.discriminator('Test1', deco1Schema); @@ -8032,7 +8032,7 @@ describe('Model', function() { const shopModel = db.model('Test', shopSchema); // Define another discriminated class after the model is compiled - class Deco2 extends Decorator { whoAmI() { return 'I am Test2'; }} + class Deco2 extends Decorator { whoAmI() { return 'I am Test2'; } } const deco2Schema = new Schema({}); deco2Schema.loadClass(Deco2); decoratorSchema.discriminator('Test2', deco2Schema); @@ -8158,7 +8158,7 @@ describe('Model', function() { }); it('insertMany should throw an error if there were operations that failed validation, ' + - 'but all operations that passed validation succeeded (gh-13256)', async function() { + 'but all operations that passed validation succeeded (gh-13256)', async function() { const userSchema = new Schema({ age: { type: Number } }); diff --git a/types/query.d.ts b/types/query.d.ts index fbebf1b6467..77b3e47ad73 100644 --- a/types/query.d.ts +++ b/types/query.d.ts @@ -228,14 +228,14 @@ declare module 'mongoose' { type MergePopulatePaths> = QueryOp extends QueryOpThatReturnsDocument ? ResultType extends null - ? ResultType - : ResultType extends (infer U)[] - ? U extends Document - ? HydratedDocument, TDocOverrides, TQueryHelpers>[] - : (MergeType)[] - : ResultType extends Document - ? HydratedDocument, TDocOverrides, TQueryHelpers> - : MergeType + ? ResultType + : ResultType extends (infer U)[] + ? U extends Document + ? HydratedDocument, TDocOverrides, TQueryHelpers>[] + : (MergeType)[] + : ResultType extends Document + ? HydratedDocument, TDocOverrides, TQueryHelpers> + : MergeType : MergeType; class Query> implements SessionOperation { @@ -373,8 +373,8 @@ declare module 'mongoose' { ): QueryWithHelpers< Array< DocKey extends keyof WithLevel1NestedPaths - ? WithoutUndefined[DocKey]>> - : ResultType + ? WithoutUndefined[DocKey]>> + : ResultType >, DocType, THelpers, @@ -567,26 +567,26 @@ declare module 'mongoose' { val?: boolean | any ): QueryWithHelpers< ResultType extends null - ? GetLeanResultType | null - : GetLeanResultType, + ? GetLeanResultType | null + : GetLeanResultType, DocType, THelpers, RawDocType, QueryOp, TDocOverrides - >; + >; lean( val?: boolean | any ): QueryWithHelpers< ResultType extends null - ? LeanResultType | null - : LeanResultType, + ? LeanResultType | null + : LeanResultType, DocType, THelpers, RawDocType, QueryOp, TDocOverrides - >; + >; /** Specifies the maximum number of documents the query will return. */ limit(val: number): this; @@ -761,12 +761,12 @@ declare module 'mongoose' { {}, ResultType, ResultType extends any[] - ? ResultType extends HydratedDocument[] - ? HydratedDocument[] - : RawDocTypeOverride[] - : (ResultType extends HydratedDocument - ? HydratedDocument - : RawDocTypeOverride) | (null extends ResultType ? null : never) + ? ResultType extends HydratedDocument[] + ? HydratedDocument[] + : RawDocTypeOverride[] + : (ResultType extends HydratedDocument + ? HydratedDocument + : RawDocTypeOverride) | (null extends ResultType ? null : never) >, DocType, THelpers,