From fb1e120b8214771e96e7dcc6201d5478b9b166e6 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 25 Jun 2025 15:20:54 -0400 Subject: [PATCH 1/5] INTPYTHON-527 Add Queryable Encryption support --- .evergreen/config.yml | 22 + .evergreen/run-tests.sh | 1 + .github/workflows/mongodb_settings.py | 32 ++ .github/workflows/runtests.py | 145 ------- .github/workflows/test-python-atlas.yml | 1 + .github/workflows/test-python.yml | 1 + django_mongodb_backend/__init__.py | 2 + django_mongodb_backend/base.py | 5 +- django_mongodb_backend/features.py | 25 +- django_mongodb_backend/fields/__init__.py | 46 ++ django_mongodb_backend/fields/encryption.py | 122 ++++++ .../management/commands/showschemamap.py | 53 +++ django_mongodb_backend/routers.py | 26 +- django_mongodb_backend/schema.py | 67 ++- docs/source/faq.rst | 68 +++ docs/source/howto/index.rst | 1 + docs/source/howto/queryable-encryption.rst | 76 ++++ docs/source/index.rst | 1 + docs/source/ref/django-admin.rst | 17 + docs/source/ref/models/fields.rst | 89 ++++ docs/source/releases/5.2.x.rst | 1 + docs/source/topics/index.rst | 1 + docs/source/topics/known-issues.rst | 5 + docs/source/topics/queryable-encryption.rst | 67 +++ pyproject.toml | 3 +- tests/backend_/test_features.py | 80 ++++ tests/encryption_/__init__.py | 0 tests/encryption_/models.py | 87 ++++ tests/encryption_/routers.py | 24 ++ tests/encryption_/tests.py | 394 ++++++++++++++++++ 30 files changed, 1309 insertions(+), 153 deletions(-) create mode 100644 django_mongodb_backend/fields/encryption.py create mode 100644 django_mongodb_backend/management/commands/showschemamap.py create mode 100644 docs/source/howto/queryable-encryption.rst create mode 100644 docs/source/topics/queryable-encryption.rst create mode 100644 tests/encryption_/__init__.py create mode 100644 tests/encryption_/models.py create mode 100644 tests/encryption_/routers.py create mode 100644 tests/encryption_/tests.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d59bfe079..11695c462 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -90,6 +90,28 @@ buildvariants: tasks: - name: run-tests + - name: tests-7-noauth-nossl + display_name: Run Tests 7.0 NoAuth NoSSL + run_on: rhel87-small + expansions: + MONGODB_VERSION: "7.0" + TOPOLOGY: server + AUTH: "noauth" + SSL: "nossl" + tasks: + - name: run-tests + + - name: tests-7-auth-ssl + display_name: Run Tests 7.0 Auth SSL + run_on: rhel87-small + expansions: + MONGODB_VERSION: "7.0" + TOPOLOGY: server + AUTH: "auth" + SSL: "ssl" + tasks: + - name: run-tests + - name: tests-8-noauth-nossl display_name: Run Tests 8.0 NoAuth NoSSL run_on: rhel87-small diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f49a0e9a9..2170c6d5a 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -6,6 +6,7 @@ set -eux /opt/python/3.10/bin/python3 -m venv venv . venv/bin/activate python -m pip install -U pip +pip install ".[encryption]" pip install -e . # Install django and test dependencies diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 49d44a5fc..8b448ef14 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -1,5 +1,7 @@ import os +from pymongo.encryption_options import AutoEncryptionOpts + from django_mongodb_backend import parse_uri if mongodb_uri := os.getenv("MONGODB_URI"): @@ -27,6 +29,36 @@ }, } +DATABASES["encrypted"] = { + "ENGINE": "django_mongodb_backend", + "NAME": "djangotests-encrypted", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="my_encrypted_database.keyvault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_PROVIDERS": {}, + "KMS_CREDENTIALS": {}, +} + + +class EncryptedRouter: + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model, **hints): + return "local" + + +DATABASE_ROUTERS = [EncryptedRouter()] DEFAULT_AUTO_FIELD = "django_mongodb_backend.fields.ObjectIdAutoField" PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) SECRET_KEY = "django_tests_secret_key" diff --git a/.github/workflows/runtests.py b/.github/workflows/runtests.py index ebcc4876f..350ca0fc3 100755 --- a/.github/workflows/runtests.py +++ b/.github/workflows/runtests.py @@ -4,151 +4,6 @@ import sys test_apps = [ - "admin_changelist", - "admin_checks", - "admin_custom_urls", - "admin_docs", - "admin_filters", - "admin_inlines", - "admin_ordering", - "admin_scripts", - "admin_utils", - "admin_views", - "admin_widgets", - "aggregation", - "aggregation_regress", - "annotations", - "apps", - "async", - "auth_tests", - "backends", - "basic", - "bulk_create", - "cache", - "check_framework", - "constraints", - "contenttypes_tests", - "context_processors", - "custom_columns", - "custom_lookups", - "custom_managers", - "custom_pk", - "datatypes", - "dates", - "datetimes", - "db_functions", - "defer", - "defer_regress", - "delete", - "delete_regress", - "empty", - "empty_models", - "expressions", - "expressions_case", - "field_defaults", - "file_storage", - "file_uploads", - "fixtures", - "fixtures_model_package", - "fixtures_regress", - "flatpages_tests", - "force_insert_update", - "foreign_object", - "forms_tests", - "from_db_value", - "generic_inline_admin", - "generic_relations", - "generic_relations_regress", - "generic_views", - "get_earliest_or_latest", - "get_object_or_404", - "get_or_create", - "i18n", - "indexes", - "inline_formsets", - "introspection", - "invalid_models_tests", - "known_related_objects", - "lookup", - "m2m_and_m2o", - "m2m_intermediary", - "m2m_multiple", - "m2m_recursive", - "m2m_regress", - "m2m_signals", - "m2m_through", - "m2m_through_regress", - "m2o_recursive", - "managers_regress", - "many_to_many", - "many_to_one", - "many_to_one_null", - "max_lengths", - "messages_tests", - "migrate_signals", - "migration_test_data_persistence", - "migrations", - "model_fields", - "model_forms", - "model_formsets", - "model_formsets_regress", - "model_indexes", - "model_inheritance", - "model_inheritance_regress", - "model_options", - "model_package", - "model_regress", - "model_utils", - "modeladmin", - "multiple_database", - "mutually_referential", - "nested_foreign_keys", - "null_fk", - "null_fk_ordering", - "null_queries", - "one_to_one", - "or_lookups", - "order_with_respect_to", - "ordering", - "pagination", - "prefetch_related", - "proxy_model_inheritance", - "proxy_models", - "queries", - "queryset_pickle", - "redirects_tests", - "reserved_names", - "reverse_lookup", - "save_delete_hooks", - "schema", - "select_for_update", - "select_related", - "select_related_onetoone", - "select_related_regress", - "serializers", - "servers", - "sessions_tests", - "shortcuts", - "signals", - "sitemaps_tests", - "sites_framework", - "sites_tests", - "string_lookup", - "swappable_models", - "syndication_tests", - "test_client", - "test_client_regress", - "test_runner", - "test_utils", - "timezones", - "transactions", - "unmanaged_models", - "update", - "update_only_fields", - "user_commands", - "validation", - "view_tests", - "xor_lookups", # Add directories in django_mongodb_backend/tests *sorted( [ diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 6eab2b7e9..52f8b1aca 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -28,6 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip + pip3 install ".[encryption]" pip3 install -e . - name: Checkout Django uses: actions/checkout@v4 diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index ce7d300d0..93f0447f0 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -28,6 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip + pip3 install ".[encryption]" pip3 install -e . - name: Checkout Django uses: actions/checkout@v4 diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index 00700421a..25e431406 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -14,6 +14,7 @@ from .indexes import register_indexes # noqa: E402 from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 +from .routers import register_routers # noqa: E402 __all__ = ["parse_uri"] @@ -25,3 +26,4 @@ register_indexes() register_lookups() register_nodes() +register_routers() diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index 7f337cf82..180232c78 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -229,4 +229,7 @@ def cursor(self): def get_database_version(self): """Return a tuple of the database's version.""" - return tuple(self.connection.server_info()["versionArray"]) + # Avoid using PyMongo to check the database version or require + # pymongocrypt>=1.14.2 which will contain a fix for the `buildInfo` + # command. https://jira.mongodb.org/browse/PYTHON-5429 + return tuple(self.connection.admin.command("buildInfo")["versionArray"]) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 9f0245ec2..ec90bb4df 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -569,9 +569,17 @@ def django_test_expected_failures(self): }, } + @cached_property + def mongodb_version(self): + return self.connection.get_database_version() # e.g., (6, 3, 0) + @cached_property def is_mongodb_6_3(self): - return self.connection.get_database_version() >= (6, 3) + return self.mongodb_version >= (6, 3) + + @cached_property + def is_mongodb_7_0(self): + return self.mongodb_version >= (7, 0) @cached_property def supports_atlas_search(self): @@ -601,3 +609,18 @@ def _supports_transactions(self): hello = client.command("hello") # a replica set or a sharded cluster return "setName" in hello or hello.get("msg") == "isdbgrid" + + @cached_property + def supports_queryable_encryption(self): + """ + Queryable Encryption requires a MongoDB 7.0 or later replica set or sharded + cluster, as well as MonogDB Atlas or Enterprise. + """ + self.connection.ensure_connection() + build_info = self.connection.connection.admin.command("buildInfo") + is_enterprise = "enterprise" in build_info.get("modules") + return ( + (is_enterprise or self.supports_atlas_search) + and self._supports_transactions + and self.is_mongodb_7_0 + ) diff --git a/django_mongodb_backend/fields/__init__.py b/django_mongodb_backend/fields/__init__.py index 0c95afd69..2b4192098 100644 --- a/django_mongodb_backend/fields/__init__.py +++ b/django_mongodb_backend/fields/__init__.py @@ -3,6 +3,30 @@ from .duration import register_duration_field from .embedded_model import EmbeddedModelField from .embedded_model_array import EmbeddedModelArrayField +from .encryption import ( + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedEmailField, + EncryptedFieldMixin, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EqualityQuery, + RangeQuery, + has_encrypted_fields, +) from .json import register_json_field from .objectid import ObjectIdField from .polymorphic_embedded_model import PolymorphicEmbeddedModelField @@ -12,10 +36,32 @@ "ArrayField", "EmbeddedModelArrayField", "EmbeddedModelField", + "EncryptedBigIntegerField", + "EncryptedBinaryField", + "EncryptedBooleanField", + "EncryptedCharField", + "EncryptedDateField", + "EncryptedDateTimeField", + "EncryptedDecimalField", + "EncryptedEmailField", + "EncryptedFieldMixin", + "EncryptedFloatField", + "EncryptedGenericIPAddressField", + "EncryptedIntegerField", + "EncryptedPositiveBigIntegerField", + "EncryptedPositiveIntegerField", + "EncryptedPositiveSmallIntegerField", + "EncryptedSmallIntegerField", + "EncryptedTextField", + "EncryptedTimeField", + "EncryptedURLField", + "EqualityQuery", "ObjectIdAutoField", "ObjectIdField", "PolymorphicEmbeddedModelArrayField", "PolymorphicEmbeddedModelField", + "RangeQuery", + "has_encrypted_fields", "register_fields", ] diff --git a/django_mongodb_backend/fields/encryption.py b/django_mongodb_backend/fields/encryption.py new file mode 100644 index 000000000..29159dba7 --- /dev/null +++ b/django_mongodb_backend/fields/encryption.py @@ -0,0 +1,122 @@ +from django.db import models + + +def has_encrypted_fields(model): + return any(getattr(field, "encrypted", False) for field in model._meta.fields) + + +class EncryptedFieldMixin(models.Field): + encrypted = True + + def __init__(self, *args, queries=None, **kwargs): + self.queries = queries + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + + if self.queries is not None: + kwargs["queries"] = self.queries + + if path.startswith("django_mongodb_backend.fields.encrypted_model"): + path = path.replace( + "django_mongodb_backend.fields.encrypted_model", + "django_mongodb_backend.fields", + ) + + return name, path, args, kwargs + + +class EncryptedBigIntegerField(EncryptedFieldMixin, models.BigIntegerField): + pass + + +class EncryptedBinaryField(EncryptedFieldMixin, models.BinaryField): + pass + + +class EncryptedBooleanField(EncryptedFieldMixin, models.BooleanField): + pass + + +class EncryptedCharField(EncryptedFieldMixin, models.CharField): + pass + + +class EncryptedDateField(EncryptedFieldMixin, models.DateField): + pass + + +class EncryptedDateTimeField(EncryptedFieldMixin, models.DateTimeField): + pass + + +class EncryptedDecimalField(EncryptedFieldMixin, models.DecimalField): + pass + + +class EncryptedEmailField(EncryptedFieldMixin, models.EmailField): + pass + + +class EncryptedFloatField(EncryptedFieldMixin, models.FloatField): + pass + + +class EncryptedGenericIPAddressField(EncryptedFieldMixin, models.GenericIPAddressField): + pass + + +class EncryptedIntegerField(EncryptedFieldMixin, models.IntegerField): + pass + + +class EncryptedPositiveBigIntegerField(EncryptedFieldMixin, models.PositiveBigIntegerField): + pass + + +class EncryptedPositiveIntegerField(EncryptedFieldMixin, models.PositiveIntegerField): + pass + + +class EncryptedPositiveSmallIntegerField(EncryptedFieldMixin, models.PositiveSmallIntegerField): + pass + + +class EncryptedSmallIntegerField(EncryptedFieldMixin, models.SmallIntegerField): + pass + + +class EncryptedTimeField(EncryptedFieldMixin, models.TimeField): + pass + + +class EncryptedTextField(EncryptedFieldMixin, models.TextField): + pass + + +class EncryptedURLField(EncryptedFieldMixin, models.URLField): + pass + + +class EqualityQuery(dict): + def __init__(self, *, contention=None): + super().__init__(queryType="equality") + if contention is not None: + self["contention"] = contention + + +class RangeQuery(dict): + def __init__( + self, *, contention=None, max=None, min=None, precision=None, sparsity=None, trimFactor=None + ): + super().__init__(queryType="range") + options = { + "contention": contention, + "max": max, + "min": min, + "precision": precision, + "sparsity": sparsity, + "trimFactor": trimFactor, + } + self.update({k: v for k, v in options.items() if v is not None}) diff --git a/django_mongodb_backend/management/commands/showschemamap.py b/django_mongodb_backend/management/commands/showschemamap.py new file mode 100644 index 000000000..e7fcd4826 --- /dev/null +++ b/django_mongodb_backend/management/commands/showschemamap.py @@ -0,0 +1,53 @@ +from bson import json_util +from django.apps import apps +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS, connections, router +from pymongo.encryption import ClientEncryption + +from django_mongodb_backend.fields import has_encrypted_fields + + +class Command(BaseCommand): + help = "Generate a `schema_map` of encrypted fields for all encrypted" + " models in the database for use with `AutoEncryptionOpts` in" + " client configuration." + + def add_arguments(self, parser): + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + help="Specify the database to use for generating the encrypted" + "fields map. Defaults to the 'default' database.", + ) + parser.add_argument( + "--kms-provider", + default="local", + help="Specify the KMS provider to use for encryption. Defaults to 'local'.", + ) + + def handle(self, *args, **options): + db = options["database"] + connection = connections[db] + schema_map = {} + for app_config in apps.get_app_configs(): + for model in app_config.get_models(): + if has_encrypted_fields(model): + fields = connection.schema_editor()._get_encrypted_fields_map(model) + client = connection.connection + options = client._options.auto_encryption_opts + ce = ClientEncryption( + options._kms_providers, + options._key_vault_namespace, + client, + client.codec_options, + ) + kms_provider = router.kms_provider(model) + master_key = connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider) + for field in fields["fields"]: + data_key = ce.create_data_key( + kms_provider=kms_provider, + master_key=master_key, + ) + field["keyId"] = data_key + schema_map[model._meta.db_table] = fields + self.stdout.write(json_util.dumps(schema_map, indent=2)) diff --git a/django_mongodb_backend/routers.py b/django_mongodb_backend/routers.py index 60e54bbd8..15ed3b760 100644 --- a/django_mongodb_backend/routers.py +++ b/django_mongodb_backend/routers.py @@ -1,6 +1,8 @@ from django.apps import apps +from django.core.exceptions import ImproperlyConfigured +from django.db.utils import ConnectionRouter -from django_mongodb_backend.models import EmbeddedModel +from .fields import has_encrypted_fields class MongoRouter: @@ -9,10 +11,32 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): EmbeddedModels don't have their own collection and must be ignored by dumpdata. """ + from django_mongodb_backend.models import EmbeddedModel # noqa: PLC0415 + if not model_name: return None try: model = apps.get_model(app_label, model_name) except LookupError: return None + return False if issubclass(model, EmbeddedModel) else None + + +def kms_provider(self, model, *args, **kwargs): + for router in self.routers: + func = getattr(router, "kms_provider", None) + if func and callable(func): + result = func(model, *args, **kwargs) + if result is not None: + return result + if has_encrypted_fields(model): + raise ImproperlyConfigured("No kms_provider found in database router.") + return None + + +def register_routers(): + """ + Patch the ConnectionRouter to use the custom kms_provider method. + """ + ConnectionRouter.kms_provider = kms_provider diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index a12086a6e..702aae241 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -1,10 +1,12 @@ +from django.core.exceptions import ImproperlyConfigured +from django.db import router from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Index, UniqueConstraint +from pymongo.encryption import ClientEncryption from pymongo.operations import SearchIndexModel -from django_mongodb_backend.indexes import SearchIndex - -from .fields import EmbeddedModelField +from .fields import EmbeddedModelField, has_encrypted_fields +from .indexes import SearchIndex from .query import wrap_database_errors from .utils import OperationCollector @@ -41,7 +43,7 @@ def get_database(self): @wrap_database_errors @ignore_embedded_models def create_model(self, model): - self.get_database().create_collection(model._meta.db_table) + self._create_collection(model) self._create_model_indexes(model) # Make implicit M2M tables. for field in model._meta.local_many_to_many: @@ -418,3 +420,60 @@ def _field_should_have_unique(self, field): db_type = field.db_type(self.connection) # The _id column is automatically unique. return db_type and field.unique and field.column != "_id" + + def _create_collection(self, model): + """ + Create a collection for the model with the encrypted fields. If + provided, use the `_schema_map` in the client's + `auto_encryption_opts`. Otherwise, create the encrypted fields map + with `_get_encrypted_fields_map`. + """ + db = self.get_database() + db_table = model._meta.db_table + if has_encrypted_fields(model): + client = self.connection.connection + options = getattr(client._options, "auto_encryption_opts", None) + if options is not None: + if schema_map := getattr(options, "_schema_map", None): + db.create_collection(db_table, encryptedFields=schema_map[db_table]) + else: + ce = ClientEncryption( + options._kms_providers, + options._key_vault_namespace, + client, + client.codec_options, + ) + encrypted_fields_map = self._get_encrypted_fields_map(model) + provider = router.kms_provider(model) + credentials = self.connection.settings_dict.get("KMS_CREDENTIALS").get(provider) + ce.create_encrypted_collection( + db, + db_table, + encrypted_fields_map, + provider, + credentials, + ) + else: + raise ImproperlyConfigured( + "Encrypted fields found but the connection does not have " + "auto encryption options set. Please set `auto_encryption_opts` " + "in the connection settings." + ) + else: + db.create_collection(db_table) + + def _get_encrypted_fields_map(self, model): + connection = self.connection + fields = model._meta.fields + + return { + "fields": [ + { + "bsonType": field.db_type(connection), + "path": field.column, + **({"queries": field.queries} if getattr(field, "queries", None) else {}), + } + for field in fields + if getattr(field, "encrypted", False) + ] + } diff --git a/docs/source/faq.rst b/docs/source/faq.rst index bb52e1cde..3754b49b4 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -52,3 +52,71 @@ logging:: If running ``manage.py dumpdata`` results in ``CommandError: Unable to serialize database: 'EmbeddedModelManager' object has no attribute using'``, see :ref:`configuring-database-routers-setting`. + +.. _queryable-encryption: + +Queryable Encryption +==================== + +What about client side configuration? +------------------------------------- + +In the :doc:`Queryable Encryption how-to guide `, +server side Queryable Encryption configuration is covered. + +Client side Queryable Encryption configuration requires that the entire schema +for encrypted fields is known at the time of client connection. + +Schema Map +~~~~~~~~~~ + +In addition to the +:ref:`settings described in the how-to guide `, +you will need to provide a ``schema_map`` to the ``AutoEncryptionOpts``. + +Fortunately, this is easy to do with Django MongoDB Backend. You can use +the ``showschemamap`` management command to generate the schema map +for your encrypted fields, and then use the results in your settings. + +To generate the schema map, run the following command in your Django project: +:: + + python manage.py showschemamap + +.. note:: The ``showschemamap`` command is only available if you have the + ``django_mongodb_backend`` app included in the :setting:`INSTALLED_APPS` + setting. + +Settings +~~~~~~~~ + +Now include the generated schema map in your Django settings. + +:: + + … + DATABASES["encrypted"] = { + … + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + … + schema_map= { + "encryption__patientrecord": { + "fields": [ + { + "bsonType": "string", + "path": "ssn", + "queries": {"queryType": "equality"}, + "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), + }, + } + }, + # Add other models with encrypted fields here + }, + ), + … + }, + … + } + +You are now ready to use client side :doc:`Queryable Encryption ` in your Django project. diff --git a/docs/source/howto/index.rst b/docs/source/howto/index.rst index 95d7ef632..8451960ef 100644 --- a/docs/source/howto/index.rst +++ b/docs/source/howto/index.rst @@ -11,3 +11,4 @@ Project configuration :maxdepth: 1 contrib-apps + queryable-encryption diff --git a/docs/source/howto/queryable-encryption.rst b/docs/source/howto/queryable-encryption.rst new file mode 100644 index 000000000..94de44083 --- /dev/null +++ b/docs/source/howto/queryable-encryption.rst @@ -0,0 +1,76 @@ +================================ +Configuring Queryable Encryption +================================ + +Configuring Queryable Encryption in Django is similar to +`configuring Queryable Encryption in Python `_ +but with some additional steps required for Django. + +.. note:: This section describes how to configure server side Queryable Encryption in Django. + For configuration of client side Queryable Encryption, please refer to this :ref:`FAQ question `. + +Prerequisites +------------- + +In addition to :doc:`installing ` and +:doc:`configuring ` Django MongoDB Backend, +you will need to install PyMongo with Queryable Encryption support:: + + pip install django-mongodb-backend[encryption] + +.. note:: You can use Queryable Encryption on a MongoDB 7.0 or later replica + set or sharded cluster, but not a standalone instance. + `This table `_ + shows which MongoDB server products support which Queryable Encryption mechanisms. + +.. _server-side-queryable-encryption-settings: + +Settings +-------- + +Queryable Encryption in Django requires the use of an additional encrypted database +and Key Management Service (KMS) credentials as well as an encrypted database +router. Here's how to set it up in your Django settings. + +:: + + from django_mongodb_backend import parse_uri + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "default": parse_uri( + DATABASE_URL, + db_name="my_database", + ), + } + + DATABASES["encrypted"] = { + "ENGINE": "django_mongodb_backend", + "NAME": "my_encrypted_database", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="my_encrypted_database.keyvault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_PROVIDERS": {}, + "KMS_CREDENTIALS": {}, + } + + class EncryptedRouter: + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model, **hints): + return "local" + + DATABASE_ROUTERS = [EncryptedRouter()] + +You are now ready to use server side :doc:`Queryable Encryption ` in your Django project. diff --git a/docs/source/index.rst b/docs/source/index.rst index 9e0243487..2e584ad4b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -45,6 +45,7 @@ Models **Topic guides:** - :doc:`topics/embedded-models` +- :doc:`topics/queryable-encryption` Forms ===== diff --git a/docs/source/ref/django-admin.rst b/docs/source/ref/django-admin.rst index 93f90f9f6..4ee6d2b77 100644 --- a/docs/source/ref/django-admin.rst +++ b/docs/source/ref/django-admin.rst @@ -26,3 +26,20 @@ Available commands Specifies the database in which the cache collection(s) will be created. Defaults to ``default``. + + +``get_encrypted_fields_map`` +---------------------------- + +.. django-admin:: get_encrypted_fields_map + + Creates a schema map for encrypted fields that can be used with + :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure + an encrypted client. + + .. django-admin-option:: --database DATABASE + + Specifies the database to use. + Defaults to ``default``. + +.. TODO: Clarify how database specified could affect output. diff --git a/docs/source/ref/models/fields.rst b/docs/source/ref/models/fields.rst index 870d97061..796d3cc88 100644 --- a/docs/source/ref/models/fields.rst +++ b/docs/source/ref/models/fields.rst @@ -379,3 +379,92 @@ These indexes use 0-based indexing. .. admonition:: Forms are not supported ``PolymorphicEmbeddedModelArrayField``\s don't appear in model forms. + +.. _encrypted-fields: + +Encrypted fields +---------------- + +Encrypted fields are used to store sensitive data with MongoDB's Queryable +Encryption feature. They are subclasses of Django's built-in fields, and +they encrypt the data before storing it in the database. + ++----------------------------------------+------------------------------------------------------+ +| Encrypted Field | Django Field | ++========================================+======================================================+ +| ``EncryptedBigIntegerField`` | :class:`~django.db.models.BigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedBooleanField`` | :class:`~django.db.models.BooleanField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedCharField`` | :class:`~django.db.models.CharField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateField`` | :class:`~django.db.models.DateField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateTimeField`` | :class:`~django.db.models.DateTimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDecimalField`` | :class:`~django.db.models.DecimalField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedFloatField`` | :class:`~django.db.models.FloatField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedGenericIPAddressField`` | :class:`~django.db.models.GenericIPAddressField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedIntegerField`` | :class:`~django.db.models.IntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveBigIntegerField`` | :class:`~django.db.models.PositiveBigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveIntegerField`` | :class:`~django.db.models.PositiveIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveSmallIntegerField`` | :class:`~django.db.models.PositiveSmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedSmallIntegerField`` | :class:`~django.db.models.SmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTextField`` | :class:`~django.db.models.TextField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTimeField`` | :class:`~django.db.models.TimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedURLField`` | :class:`~django.db.models.URLField` | ++----------------------------------------+------------------------------------------------------+ + +.. _encrypted-fields-unsupported-fields: + +.. admonition:: Unsupported fields + + The following fields are supported by Django MongoDB Backend but are not supported by + Queryable Encryption. + + :class:`~django.db.models.SlugField` + +Query types +~~~~~~~~~~~ + +Django MongoDB Backend provides the following query type classes for use with +encrypted fields. + ++-------------------+----------------------------------------------------------------------------------------------+ +| ``EqualityQuery`` | This query type is used for equality checks. | ++-------------------+----------------------------------------------------------------------------------------------+ +| ``RangeQuery`` | This query type is used for range queries, such as greater than or less | +| | than checks. | ++-------------------+----------------------------------------------------------------------------------------------+ + +EncryptedFieldMixin +~~~~~~~~~~~~~~~~~~~ + +You can use the ``EncryptedFieldMixin`` to create your own encrypted fields. This mixin +supports the use of a ``queries`` argument in the field definition to specify query type +for the field:: + + from django.db import models + from django_mongodb_backend.fields import EncryptedFieldMixin, EqualityQuery + from .models import MyField + + + class MyEncryptedField(EncryptedFieldMixin, MyField): + pass + + + class MyModel(models.Model): + my_encrypted_field = MyEncryptedField( + queries=EqualityQuery(), + # Other field options... + ) diff --git a/docs/source/releases/5.2.x.rst b/docs/source/releases/5.2.x.rst index d7a86aca9..5ebe7461d 100644 --- a/docs/source/releases/5.2.x.rst +++ b/docs/source/releases/5.2.x.rst @@ -16,6 +16,7 @@ New features - Added :class:`~.fields.PolymorphicEmbeddedModelField` and :class:`~.fields.PolymorphicEmbeddedModelArrayField` for storing a model instance or list of model instances that may be of more than one model class. +- Added support for Queryable Encryption. Bug fixes --------- diff --git a/docs/source/topics/index.rst b/docs/source/topics/index.rst index 47e0c6dc0..f81c6630e 100644 --- a/docs/source/topics/index.rst +++ b/docs/source/topics/index.rst @@ -11,3 +11,4 @@ know: cache embedded-models known-issues + queryable-encryption diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 4b9edee70..3abab9ae2 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -102,3 +102,8 @@ Caching Secondly, you must use the :class:`django_mongodb_backend.cache.MongoDBCache` backend rather than Django's built-in database cache backend, ``django.core.cache.backends.db.DatabaseCache``. + +Queryable Encryption +==================== + +.. TODO: Add Django core limitations that affect Queryable Encryption. diff --git a/docs/source/topics/queryable-encryption.rst b/docs/source/topics/queryable-encryption.rst new file mode 100644 index 000000000..e9b3384c6 --- /dev/null +++ b/docs/source/topics/queryable-encryption.rst @@ -0,0 +1,67 @@ +Queryable Encryption +==================== + +Use :ref:`encrypted fields ` to store sensitive data in MongoDB +your data using `Queryable Encryption `_. + +.. _encrypted-field-example: + +The basics +---------- + +Let's consider this example:: + + from django.db import models + + from django_mongodb_backend.fields import EncryptedCharField, EqualityQuery + + + class Patient(models.Model): + ssn = EncryptedCharField(max_length=11, queries=EqualityQuery()) + + def __str__(self): + return self.ssn + +The API is similar to that of Django's relational fields, with some +security-related changes:: + + >>> bob = Patient(ssn="123-45-6789") + >>> bob.ssn + '123-45-6789' + +Represented in BSON, from an encrypted client connection, the patient data looks like this: + +.. code-block:: js + + { + _id: ObjectId('68825b066fac55353a8b2b41'), + ssn: '123-45-6789', + __safeContent__: [b'\xe0)NOFB\x9a,\x08\xd7\xdd\xb8\xa6\xba$…'] + } + +The ``ssn`` field is only visible from an encrypted client connection. From an unencrypted client connection, +the patient data looks like this: + +.. code-block:: js + + { + _id: ObjectId('6882566c586a440cd0649e8f'), + ssn: Binary.createFromBase64('DkrbD67ejkt2u…', 6), + } + +.. admonition:: List of encrypted fields + + See the full list of :ref:`encrypted fields ` in the :doc:`Model field reference `. + +Querying encrypted fields +------------------------- + +You can query encrypted fields using a `limited set of +query operators `_ +which must be specified in the field definition. For example, to query the ``ssn`` field for equality, you can use the +``EqualityQuery`` operator as shown in the example above. + + >>> Patient.objects.get(ssn="123-45-6789").ssn + '123-45-6789' + +If the ``ssn`` field provided in the query matches the encrypted value in the database, the query will succeed. diff --git a/pyproject.toml b/pyproject.toml index 7b412e1db..88e31897e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ classifiers = [ ] [project.optional-dependencies] -docs = [ "sphinx>=7"] +docs = ["sphinx>=7"] +encryption = ["pymongo[encryption]"] [project.urls] Homepage = "https://www.mongodb.org" diff --git a/tests/backend_/test_features.py b/tests/backend_/test_features.py index 05959fa70..dcdb93872 100644 --- a/tests/backend_/test_features.py +++ b/tests/backend_/test_features.py @@ -44,3 +44,83 @@ def mocked_command(command): with patch("pymongo.synchronous.database.Database.command", wraps=mocked_command): self.assertIs(connection.features._supports_transactions, False) + + +class SupportsQueryableEncryptionTests(TestCase): + def setUp(self): + # Clear the cached property. + connection.features.__dict__.pop("supports_queryable_encryption", None) + # Must initialize the feature before patching it. + connection.features._supports_transactions # noqa: B018 + + def tearDown(self): + del connection.features.supports_queryable_encryption + + @staticmethod + def enterprise_response(command): + if command == "buildInfo": + return {"modules": ["enterprise"]} + raise Exception("Unexpected command") + + @staticmethod + def non_enterprise_response(command): + if command == "buildInfo": + return {"modules": []} + raise Exception("Unexpected command") + + def test_supported_on_atlas(self): + """Supported on MongoDB 7.0+ Atlas replica set or sharded cluster.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", True), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_supported_on_enterprise(self): + """Supported on MongoDB 7.0+ Enterprise replica set or sharded cluster.""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_atlas_or_enterprise_required(self): + """Not supported on MongoDB Community Edition.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_transactions_required(self): + """ + Not supported if database isn't a replica set or sharded cluster + (i.e. DatabaseFeatures._supports_transactions = False). + """ + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", False), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_mongodb_7_0_required(self): + """Not supported on MongoDB < 7.0""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", False), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) diff --git a/tests/encryption_/__init__.py b/tests/encryption_/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/encryption_/models.py b/tests/encryption_/models.py new file mode 100644 index 000000000..35f627db5 --- /dev/null +++ b/tests/encryption_/models.py @@ -0,0 +1,87 @@ +from django.db import models + +from django_mongodb_backend.fields import ( + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedEmailField, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EqualityQuery, + RangeQuery, +) + + +class Appointment(models.Model): + time = EncryptedTimeField(queries=EqualityQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class Billing(models.Model): + cc_type = EncryptedCharField(max_length=20, queries=EqualityQuery()) + cc_number = EncryptedBigIntegerField(queries=EqualityQuery()) + account_balance = EncryptedDecimalField(max_digits=10, decimal_places=2, queries=RangeQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class PatientPortalUser(models.Model): + ip_address = EncryptedGenericIPAddressField(queries=EqualityQuery()) + url = EncryptedURLField(queries=EqualityQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class PatientRecord(models.Model): + ssn = EncryptedCharField(max_length=11, queries=EqualityQuery()) + birth_date = EncryptedDateField(queries=RangeQuery()) + profile_picture = EncryptedBinaryField(queries=EqualityQuery()) + patient_age = EncryptedIntegerField("patient_age", queries=RangeQuery()) + weight = EncryptedFloatField(queries=RangeQuery()) + + # TODO: Embed Billing model + # billing = + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class Patient(models.Model): + patient_id = EncryptedIntegerField("patient_id", queries=EqualityQuery()) + patient_name = EncryptedCharField(max_length=100) + patient_notes = EncryptedTextField(queries=EqualityQuery()) + registration_date = EncryptedDateTimeField(queries=EqualityQuery()) + is_active = EncryptedBooleanField(queries=EqualityQuery()) + email = EncryptedEmailField(max_length=254, queries=EqualityQuery()) + + # TODO: Embed PatientRecord model + # patient_record = + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class EncryptedNumbers(models.Model): + pos_bigint = EncryptedPositiveBigIntegerField(queries=EqualityQuery()) + + # FIXME: pymongo.errors.EncryptionError: Cannot encrypt element of type int + # because schema requires that type is one of: [ long ] + # pos_int = EncryptedPositiveIntegerField(queries=EqualityQuery()) + + pos_smallint = EncryptedPositiveSmallIntegerField(queries=EqualityQuery()) + smallint = EncryptedSmallIntegerField(queries=EqualityQuery()) diff --git a/tests/encryption_/routers.py b/tests/encryption_/routers.py new file mode 100644 index 000000000..941735091 --- /dev/null +++ b/tests/encryption_/routers.py @@ -0,0 +1,24 @@ +from django_mongodb_backend.fields import has_encrypted_fields + + +class TestEncryptedRouter: + """Router for testing encrypted models in Django. `kms_provider` + must be set on the global test router since table creation happens + at the start of the test suite, before @override_settings( + DATABASE_ROUTERS=[TestEncryptedRouter()]) takes effect. + """ + + def allow_migrate(self, db, app_label, model_name=None, model=None, **hints): + if model: + return db == ("encrypted" if has_encrypted_fields(model) else "default") + return db == "default" + + def db_for_read(self, model, **hints): + if has_encrypted_fields(model): + return "encrypted" + return "default" + + db_for_write = db_for_read + + def kms_provider(self, model, **hints): + return "local" diff --git a/tests/encryption_/tests.py b/tests/encryption_/tests.py new file mode 100644 index 000000000..463c93445 --- /dev/null +++ b/tests/encryption_/tests.py @@ -0,0 +1,394 @@ +import importlib +from datetime import datetime, time +from io import StringIO +from unittest.mock import patch + +import pymongo +from bson import json_util +from bson.binary import Binary +from django.core.management import call_command +from django.db import connections, models +from django.test import TransactionTestCase, modify_settings, override_settings + +from django_mongodb_backend.fields import EncryptedFieldMixin + +from .models import ( + Appointment, + Billing, + EncryptedNumbers, + Patient, + PatientPortalUser, + PatientRecord, +) +from .routers import TestEncryptedRouter + +EXPECTED_ENCRYPTED_FIELDS_MAP = { + "encryption__billing": { + "fields": [ + { + "bsonType": "string", + "path": "cc_type", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b" \x901\x89\x1f\xafAX\x9b*\xb1\xc7\xc5\xfdl\xa4", 4), + }, + { + "bsonType": "long", + "path": "cc_number", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x97\xb4\x9d\xb8\xd5\xa6Ay\x85\xfe\x00\xc0\xd4{\xa2\xff", 4), + }, + { + "bsonType": "decimal", + "path": "account_balance", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\xcc\x01-s\xea\xd9B\x8d\x80\xd7\xf8!n\xc6\xf5U", 4), + }, + ] + }, + "encryption__patientrecord": { + "fields": [ + { + "bsonType": "string", + "path": "ssn", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), + }, + { + "bsonType": "date", + "path": "birth_date", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"@\xdd\xb4\xd2%\xc2B\x94\xb5\x07\xbc(ER[s", 4), + }, + { + "bsonType": "binData", + "path": "profile_picture", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"Q\xa2\xebc!\xecD,\x8b\xe4$\xb6ul9\x9a", 4), + }, + { + "bsonType": "int", + "path": "patient_age", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\ro\x80\x1e\x8e1K\xde\xbc_\xc3bi\x95\xa6j", 4), + }, + { + "bsonType": "double", + "path": "weight", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\x9b\xfd:n\xe1\xd0N\xdd\xb3\xe7e)\x06\xea\x8a\x1d", 4), + }, + ] + }, + "encryption__patient": { + "fields": [ + { + "bsonType": "int", + "path": "patient_id", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x8ft\x16:\x8a\x91D\xc7\x8a\xdf\xe5O\n[\xfd\\", 4), + }, + { + "bsonType": "string", + "path": "patient_name", + # "keyId": Binary(b"<\x9b\xba\xeb:\xa4@m\x93\x0e\x0c\xcaN\x03\xfb\x05", 4), + }, + { + "bsonType": "string", + "path": "patient_notes", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x01\xe7\xd1isnB$\xa9(gwO\xca\x10\xbd", 4), + }, + { + "bsonType": "date", + "path": "registration_date", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"F\xfb\xae\x82\xd5\x9a@\xee\xbfJ\xaf#\x9c:-I", 4), + }, + { + "bsonType": "bool", + "path": "is_active", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\xb2\xb5\xc4K53A\xda\xb9V\xa6\xa9\x97\x94\xea;", 4), + }, + {"bsonType": "string", "path": "email", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__patientportaluser": { + "fields": [ + {"bsonType": "string", "path": "ip_address", "queries": {"queryType": "equality"}}, + {"bsonType": "string", "path": "url", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__encryptednumbers": { + "fields": [ + {"bsonType": "int", "path": "pos_bigint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "pos_smallint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "smallint", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__appointment": { + "fields": [{"bsonType": "date", "path": "time", "queries": {"queryType": "equality"}}] + }, +} + + +class EncryptedDurationField(EncryptedFieldMixin, models.DurationField): + """ + Unsupported by MongoDB when used with Queryable Encryption. + Included in tests until fix or wontfix. + """ + + +class EncryptedSlugField(EncryptedFieldMixin, models.SlugField): + """ + Unsupported by MongoDB when used with Queryable Encryption. + Included in tests until fix or wontfix. + """ + + +def reload_module(module): + """ + Reloads a module to ensure that any changes to environment variables + or other settings are applied without restarting the test runner. + """ + module = importlib.import_module(module) + importlib.reload(module) + return module + + +@modify_settings( + INSTALLED_APPS={"prepend": "django_mongodb_backend"}, +) +@override_settings(DATABASE_ROUTERS=[TestEncryptedRouter()]) +class EncryptedFieldTests(TransactionTestCase): + databases = {"default", "encrypted"} + available_apps = ["django_mongodb_backend", "encryption_"] + + def setUp(self): + self.appointment = Appointment(time="8:00") + self.appointment.save() + + self.billing = Billing(cc_type="Visa", cc_number=1234567890123456, account_balance=100.50) + self.billing.save() + + self.portal_user = PatientPortalUser( + ip_address="127.0.0.1", + url="https://example.com", + ) + self.portal_user.save() + + self.patientrecord = PatientRecord( + ssn="123-45-6789", + birth_date="1970-01-01", + profile_picture=b"image data", + weight=175.5, + patient_age=47, + ) + self.patientrecord.save() + + self.patient = Patient( + patient_id=1, + patient_name="John Doe", + patient_notes="patient notes " * 25, + registration_date=datetime(2023, 10, 1, 12, 0, 0), + is_active=True, + email="john.doe@example.com", + ) + self.patient.save() + + # TODO: Embed billing and patient_record models in patient model then add tests + + @classmethod + def setUpClass(cls): + super().setUpClass() + try: + from pymongo_auth_aws.auth import AwsCredential # noqa: PLC0415 + except ImportError: + cls.skipTest(cls, "pymongo_auth_aws not installed, skipping AWS credentials tests") + + cls.patch_aws = patch( + "pymongocrypt.synchronous.credentials.aws_temp_credentials", + return_value=AwsCredential(username="", password="", token=""), + ) + cls.patch_aws.start() + + cls.patch_azure = patch( + "pymongocrypt.synchronous.credentials._get_azure_credentials", return_value={} + ) + cls.patch_azure.start() + + cls.patch_gcp = patch( + "pymongocrypt.synchronous.credentials._get_gcp_credentials", return_value={} + ) + cls.patch_gcp.start() + + @classmethod + def tearDownClass(cls): + cls.patch_aws.stop() + cls.patch_azure.stop() + cls.patch_gcp.stop() + + def test_get_encrypted_fields_map(self): + """Test class method called by schema editor + and management command to get encrypted fields map for + `create_encrypted_collection` and `auto_encryption_opts` respectively. + There are no data keys in the results. + + Data keys for the schema editor are created by + `create_encrypted_collection` and data keys for the + management command are created by the management command + using code similar to the code in `create_encrypted_collection` + in Pymongo. + """ + expected_encrypted_fields_map = { + "encryption__patient": { + "fields": [ + { + "bsonType": "int", + "path": "patient_id", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "string", + "path": "patient_name", + }, + { + "bsonType": "string", + "path": "patient_notes", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "date", + "path": "registration_date", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "bool", + "path": "is_active", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "string", + "path": "email", + "queries": {"queryType": "equality"}, + }, + ] + }, + } + self.maxDiff = None + with connections["encrypted"].schema_editor() as editor: + db_table = self.patient._meta.db_table + self.assertEqual( + editor._get_encrypted_fields_map(self.patient), + expected_encrypted_fields_map[db_table], + ) + + def test_show_schema_map(self): + self.maxDiff = None + out = StringIO() + call_command( + "showschemamap", + "--database", + "encrypted", + verbosity=0, + stdout=out, + ) + # Remove keyIds since they are different for each run. + output_json = json_util.loads(out.getvalue()) + for table in output_json: + for field in output_json[table]["fields"]: + del field["keyId"] + # TODO: probably we don't need to test the entire mapping, otherwise it + # requires updates every time a new model or field is added! + self.assertEqual(EXPECTED_ENCRYPTED_FIELDS_MAP, output_json) + + def test_set_encrypted_fields_map_in_client(self): + # TODO: Create new client with and without schema map provided then + # sync database to ensure encrypted collections are created in both + pass + + def test_appointment(self): + self.assertEqual(Appointment.objects.get(time="8:00").time, time(8, 0)) + + def test_billing(self): + self.assertEqual( + Billing.objects.get(cc_number=1234567890123456).cc_number, 1234567890123456 + ) + self.assertEqual(Billing.objects.get(cc_type="Visa").cc_type, "Visa") + self.assertTrue(Billing.objects.filter(account_balance__gte=100.0).exists()) + + def test_patientportaluser(self): + self.assertEqual( + PatientPortalUser.objects.get(ip_address="127.0.0.1").ip_address, "127.0.0.1" + ) + + def test_patientrecord(self): + self.assertEqual(PatientRecord.objects.get(ssn="123-45-6789").ssn, "123-45-6789") + with self.assertRaises(PatientRecord.DoesNotExist): + PatientRecord.objects.get(ssn="000-00-0000") + self.assertTrue(PatientRecord.objects.filter(birth_date__gte="1969-01-01").exists()) + self.assertEqual( + PatientRecord.objects.get(ssn="123-45-6789").profile_picture, b"image data" + ) + with self.assertRaises(AssertionError): + self.assertEqual( + PatientRecord.objects.get(ssn="123-45-6789").profile_picture, b"bad image data" + ) + self.assertTrue(PatientRecord.objects.filter(patient_age__gte=40).exists()) + self.assertFalse(PatientRecord.objects.filter(patient_age__gte=200).exists()) + self.assertTrue(PatientRecord.objects.filter(weight__gte=175.0).exists()) + + # Test encrypted patient record in unencrypted database. + conn_params = connections["encrypted"].get_connection_params() + if conn_params.pop("auto_encryption_opts", False): + # Call MongoClient instead of get_new_connection because + # get_new_connection will return the encrypted connection + # from the connection pool. + with pymongo.MongoClient(**conn_params) as new_connection: + patientrecords = new_connection["test_encrypted"].encryption__patientrecord.find() + ssn = patientrecords[0]["ssn"] + self.assertTrue(isinstance(ssn, Binary)) + + def test_patient(self): + self.assertEqual( + Patient.objects.get(patient_notes="patient notes " * 25).patient_notes, + "patient notes " * 25, + ) + self.assertEqual( + Patient.objects.get( + registration_date=datetime(2023, 10, 1, 12, 0, 0) + ).registration_date, + datetime(2023, 10, 1, 12, 0, 0), + ) + self.assertTrue(Patient.objects.get(patient_id=1).is_active) + self.assertEqual( + Patient.objects.get(email="john.doe@example.com").email, "john.doe@example.com" + ) + + # Test decrypted patient record in encrypted database. + patients = connections["encrypted"].database.encryption__patient.find() + self.assertEqual(len(list(patients)), 1) + records = connections["encrypted"].database.encryption__patientrecord.find() + self.assertTrue("__safeContent__" in records[0]) + + +class EncryptedNumberFieldTests(EncryptedFieldTests): + def test_create_and_query(self): + EncryptedNumbers.objects.create( + pos_bigint=1000000, + # FIXME: pymongo.errors.EncryptionError: Cannot encrypt element of type int + # because schema requires that type is one of: [ long ] + # pos_int=1, + pos_smallint=12345, + smallint=-12345, + ) + + obj = EncryptedNumbers.objects.get(pos_bigint=1000000) + # obj = EncryptedNumbers.objects.get(pos_int=1) + obj = EncryptedNumbers.objects.get(pos_smallint=12345) + obj = EncryptedNumbers.objects.get(smallint=-12345) + + self.assertEqual(obj.pos_bigint, 1000000) + # self.assertEqual(obj.pos_int, 1) + self.assertEqual(obj.pos_smallint, 12345) + self.assertEqual(obj.smallint, -12345) From c7424292ca842e5101a216c51bd30054cbbc4e58 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Mon, 4 Aug 2025 07:37:30 -0400 Subject: [PATCH 2/5] Rename get_encrypted_fields_map -> showschemamap --- docs/source/ref/django-admin.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/ref/django-admin.rst b/docs/source/ref/django-admin.rst index 4ee6d2b77..13927af06 100644 --- a/docs/source/ref/django-admin.rst +++ b/docs/source/ref/django-admin.rst @@ -28,10 +28,10 @@ Available commands Defaults to ``default``. -``get_encrypted_fields_map`` +``showschemamap`` ---------------------------- -.. django-admin:: get_encrypted_fields_map +.. django-admin:: showschemamap Creates a schema map for encrypted fields that can be used with :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure @@ -41,5 +41,3 @@ Available commands Specifies the database to use. Defaults to ``default``. - -.. TODO: Clarify how database specified could affect output. From 5e0818dd73d86b4c79674a45060c2190f65ef2e5 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 4 Aug 2025 09:21:57 -0400 Subject: [PATCH 3/5] remove hardcoded db name --- tests/encryption_/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/encryption_/tests.py b/tests/encryption_/tests.py index 463c93445..409f844e7 100644 --- a/tests/encryption_/tests.py +++ b/tests/encryption_/tests.py @@ -6,6 +6,7 @@ import pymongo from bson import json_util from bson.binary import Binary +from django.conf import settings from django.core.management import call_command from django.db import connections, models from django.test import TransactionTestCase, modify_settings, override_settings @@ -340,12 +341,13 @@ def test_patientrecord(self): # Test encrypted patient record in unencrypted database. conn_params = connections["encrypted"].get_connection_params() + db_name = settings.DATABASES["encrypted"]["NAME"] if conn_params.pop("auto_encryption_opts", False): # Call MongoClient instead of get_new_connection because # get_new_connection will return the encrypted connection # from the connection pool. with pymongo.MongoClient(**conn_params) as new_connection: - patientrecords = new_connection["test_encrypted"].encryption__patientrecord.find() + patientrecords = new_connection[db_name].encryption__patientrecord.find() ssn = patientrecords[0]["ssn"] self.assertTrue(isinstance(ssn, Binary)) From 53dbf08a0714025856e2117485aee973d43b6c5c Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Mon, 4 Aug 2025 18:48:09 -0400 Subject: [PATCH 4/5] Code review fixes for tests and docs --- docs/source/howto/queryable-encryption.rst | 24 +-- docs/source/topics/known-issues.rst | 5 - docs/source/topics/queryable-encryption.rst | 54 +++---- tests/encryption_/{tests.py => test_base.py} | 158 ++----------------- tests/encryption_/test_management.py | 144 +++++++++++++++++ 5 files changed, 195 insertions(+), 190 deletions(-) rename tests/encryption_/{tests.py => test_base.py} (62%) create mode 100644 tests/encryption_/test_management.py diff --git a/docs/source/howto/queryable-encryption.rst b/docs/source/howto/queryable-encryption.rst index 94de44083..54a7c6231 100644 --- a/docs/source/howto/queryable-encryption.rst +++ b/docs/source/howto/queryable-encryption.rst @@ -3,11 +3,12 @@ Configuring Queryable Encryption ================================ Configuring Queryable Encryption in Django is similar to -`configuring Queryable Encryption in Python `_ -but with some additional steps required for Django. +:doc:`manual:core/queryable-encryption/quick-start` but with some additional +steps required for Django. -.. note:: This section describes how to configure server side Queryable Encryption in Django. - For configuration of client side Queryable Encryption, please refer to this :ref:`FAQ question `. +.. note:: This section describes how to configure server side Queryable + Encryption in Django. For configuration of client side Queryable Encryption, + please refer to this :ref:`FAQ question `. Prerequisites ------------- @@ -19,18 +20,18 @@ you will need to install PyMongo with Queryable Encryption support:: pip install django-mongodb-backend[encryption] .. note:: You can use Queryable Encryption on a MongoDB 7.0 or later replica - set or sharded cluster, but not a standalone instance. - `This table `_ - shows which MongoDB server products support which Queryable Encryption mechanisms. + set or sharded cluster, but not a standalone instance. + :ref:`This table ` shows which MongoDB + server products support which Queryable Encryption mechanisms. .. _server-side-queryable-encryption-settings: Settings -------- -Queryable Encryption in Django requires the use of an additional encrypted database -and Key Management Service (KMS) credentials as well as an encrypted database -router. Here's how to set it up in your Django settings. +Queryable Encryption in Django requires the use of an additional encrypted +database and Key Management Service (KMS) credentials as well as an encrypted +database router. Here's how to set it up in your Django settings. :: @@ -73,4 +74,5 @@ router. Here's how to set it up in your Django settings. DATABASE_ROUTERS = [EncryptedRouter()] -You are now ready to use server side :doc:`Queryable Encryption ` in your Django project. +You are now ready to use server side :doc:`Queryable Encryption +` in your Django project. diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 3abab9ae2..4b9edee70 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -102,8 +102,3 @@ Caching Secondly, you must use the :class:`django_mongodb_backend.cache.MongoDBCache` backend rather than Django's built-in database cache backend, ``django.core.cache.backends.db.DatabaseCache``. - -Queryable Encryption -==================== - -.. TODO: Add Django core limitations that affect Queryable Encryption. diff --git a/docs/source/topics/queryable-encryption.rst b/docs/source/topics/queryable-encryption.rst index e9b3384c6..9b2c56971 100644 --- a/docs/source/topics/queryable-encryption.rst +++ b/docs/source/topics/queryable-encryption.rst @@ -1,8 +1,8 @@ Queryable Encryption ==================== -Use :ref:`encrypted fields ` to store sensitive data in MongoDB -your data using `Queryable Encryption `_. +Use :ref:`encrypted fields ` to store sensitive data in +MongoDB with :doc:`manual:core/queryable-encryption`. .. _encrypted-field-example: @@ -22,25 +22,11 @@ Let's consider this example:: def __str__(self): return self.ssn -The API is similar to that of Django's relational fields, with some -security-related changes:: - - >>> bob = Patient(ssn="123-45-6789") - >>> bob.ssn - '123-45-6789' - -Represented in BSON, from an encrypted client connection, the patient data looks like this: - -.. code-block:: js - - { - _id: ObjectId('68825b066fac55353a8b2b41'), - ssn: '123-45-6789', - __safeContent__: [b'\xe0)NOFB\x9a,\x08\xd7\xdd\xb8\xa6\xba$…'] - } +Querying encrypted fields +------------------------- -The ``ssn`` field is only visible from an encrypted client connection. From an unencrypted client connection, -the patient data looks like this: +The ``ssn`` field is only visible from an encrypted client connection. From an +unencrypted client connection, the patient data looks like this: .. code-block:: js @@ -51,17 +37,27 @@ the patient data looks like this: .. admonition:: List of encrypted fields - See the full list of :ref:`encrypted fields ` in the :doc:`Model field reference `. + See the full list of :ref:`encrypted fields ` in the + :doc:`Model field reference `. -Querying encrypted fields -------------------------- - -You can query encrypted fields using a `limited set of -query operators `_ -which must be specified in the field definition. For example, to query the ``ssn`` field for equality, you can use the -``EqualityQuery`` operator as shown in the example above. +You can query encrypted fields using a +:ref:`manual:qe-supported-query-operators` which must be specified in the +field definition. For example, to query the ``ssn`` field for equality, you +can use the ``EqualityQuery`` operator as shown in the example above. >>> Patient.objects.get(ssn="123-45-6789").ssn '123-45-6789' -If the ``ssn`` field provided in the query matches the encrypted value in the database, the query will succeed. +If the ``ssn`` field provided in the query matches the encrypted value in the +database, the query will succeed. + +Represented in BSON, from an encrypted client connection, the patient data +looks like this: + +.. code-block:: js + + { + _id: ObjectId('68825b066fac55353a8b2b41'), + ssn: '123-45-6789', + __safeContent__: [b'\xe0)NOFB\x9a,\x08\xd7\xdd\xb8\xa6\xba$…'] + } diff --git a/tests/encryption_/tests.py b/tests/encryption_/test_base.py similarity index 62% rename from tests/encryption_/tests.py rename to tests/encryption_/test_base.py index 409f844e7..f3cd6d46b 100644 --- a/tests/encryption_/tests.py +++ b/tests/encryption_/test_base.py @@ -1,13 +1,10 @@ import importlib from datetime import datetime, time -from io import StringIO from unittest.mock import patch import pymongo -from bson import json_util from bson.binary import Binary from django.conf import settings -from django.core.management import call_command from django.db import connections, models from django.test import TransactionTestCase, modify_settings, override_settings @@ -23,115 +20,6 @@ ) from .routers import TestEncryptedRouter -EXPECTED_ENCRYPTED_FIELDS_MAP = { - "encryption__billing": { - "fields": [ - { - "bsonType": "string", - "path": "cc_type", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b" \x901\x89\x1f\xafAX\x9b*\xb1\xc7\xc5\xfdl\xa4", 4), - }, - { - "bsonType": "long", - "path": "cc_number", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"\x97\xb4\x9d\xb8\xd5\xa6Ay\x85\xfe\x00\xc0\xd4{\xa2\xff", 4), - }, - { - "bsonType": "decimal", - "path": "account_balance", - "queries": {"queryType": "range"}, - # "keyId": Binary(b"\xcc\x01-s\xea\xd9B\x8d\x80\xd7\xf8!n\xc6\xf5U", 4), - }, - ] - }, - "encryption__patientrecord": { - "fields": [ - { - "bsonType": "string", - "path": "ssn", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), - }, - { - "bsonType": "date", - "path": "birth_date", - "queries": {"queryType": "range"}, - # "keyId": Binary(b"@\xdd\xb4\xd2%\xc2B\x94\xb5\x07\xbc(ER[s", 4), - }, - { - "bsonType": "binData", - "path": "profile_picture", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"Q\xa2\xebc!\xecD,\x8b\xe4$\xb6ul9\x9a", 4), - }, - { - "bsonType": "int", - "path": "patient_age", - "queries": {"queryType": "range"}, - # "keyId": Binary(b"\ro\x80\x1e\x8e1K\xde\xbc_\xc3bi\x95\xa6j", 4), - }, - { - "bsonType": "double", - "path": "weight", - "queries": {"queryType": "range"}, - # "keyId": Binary(b"\x9b\xfd:n\xe1\xd0N\xdd\xb3\xe7e)\x06\xea\x8a\x1d", 4), - }, - ] - }, - "encryption__patient": { - "fields": [ - { - "bsonType": "int", - "path": "patient_id", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"\x8ft\x16:\x8a\x91D\xc7\x8a\xdf\xe5O\n[\xfd\\", 4), - }, - { - "bsonType": "string", - "path": "patient_name", - # "keyId": Binary(b"<\x9b\xba\xeb:\xa4@m\x93\x0e\x0c\xcaN\x03\xfb\x05", 4), - }, - { - "bsonType": "string", - "path": "patient_notes", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"\x01\xe7\xd1isnB$\xa9(gwO\xca\x10\xbd", 4), - }, - { - "bsonType": "date", - "path": "registration_date", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"F\xfb\xae\x82\xd5\x9a@\xee\xbfJ\xaf#\x9c:-I", 4), - }, - { - "bsonType": "bool", - "path": "is_active", - "queries": {"queryType": "equality"}, - # "keyId": Binary(b"\xb2\xb5\xc4K53A\xda\xb9V\xa6\xa9\x97\x94\xea;", 4), - }, - {"bsonType": "string", "path": "email", "queries": {"queryType": "equality"}}, - ] - }, - "encryption__patientportaluser": { - "fields": [ - {"bsonType": "string", "path": "ip_address", "queries": {"queryType": "equality"}}, - {"bsonType": "string", "path": "url", "queries": {"queryType": "equality"}}, - ] - }, - "encryption__encryptednumbers": { - "fields": [ - {"bsonType": "int", "path": "pos_bigint", "queries": {"queryType": "equality"}}, - {"bsonType": "int", "path": "pos_smallint", "queries": {"queryType": "equality"}}, - {"bsonType": "int", "path": "smallint", "queries": {"queryType": "equality"}}, - ] - }, - "encryption__appointment": { - "fields": [{"bsonType": "date", "path": "time", "queries": {"queryType": "equality"}}] - }, -} - class EncryptedDurationField(EncryptedFieldMixin, models.DurationField): """ @@ -166,28 +54,26 @@ class EncryptedFieldTests(TransactionTestCase): available_apps = ["django_mongodb_backend", "encryption_"] def setUp(self): - self.appointment = Appointment(time="8:00") - self.appointment.save() + self.appointment = Appointment.objects.create(time="8:00") - self.billing = Billing(cc_type="Visa", cc_number=1234567890123456, account_balance=100.50) - self.billing.save() + self.billing = Billing.objects.create( + cc_type="Visa", cc_number=1234567890123456, account_balance=100.50 + ) - self.portal_user = PatientPortalUser( + self.portal_user = PatientPortalUser.objects.create( ip_address="127.0.0.1", url="https://example.com", ) - self.portal_user.save() - self.patientrecord = PatientRecord( + self.patientrecord = PatientRecord.objects.create( ssn="123-45-6789", birth_date="1970-01-01", profile_picture=b"image data", weight=175.5, patient_age=47, ) - self.patientrecord.save() - self.patient = Patient( + self.patient = Patient.objects.create( patient_id=1, patient_name="John Doe", patient_notes="patient notes " * 25, @@ -195,9 +81,9 @@ def setUp(self): is_active=True, email="john.doe@example.com", ) - self.patient.save() - # TODO: Embed billing and patient_record models in patient model then add tests + # TODO: Embed billing and patient_record models in patient model + # then add tests @classmethod def setUpClass(cls): @@ -284,25 +170,6 @@ def test_get_encrypted_fields_map(self): expected_encrypted_fields_map[db_table], ) - def test_show_schema_map(self): - self.maxDiff = None - out = StringIO() - call_command( - "showschemamap", - "--database", - "encrypted", - verbosity=0, - stdout=out, - ) - # Remove keyIds since they are different for each run. - output_json = json_util.loads(out.getvalue()) - for table in output_json: - for field in output_json[table]["fields"]: - del field["keyId"] - # TODO: probably we don't need to test the entire mapping, otherwise it - # requires updates every time a new model or field is added! - self.assertEqual(EXPECTED_ENCRYPTED_FIELDS_MAP, output_json) - def test_set_encrypted_fields_map_in_client(self): # TODO: Create new client with and without schema map provided then # sync database to ensure encrypted collections are created in both @@ -373,9 +240,10 @@ def test_patient(self): records = connections["encrypted"].database.encryption__patientrecord.find() self.assertTrue("__safeContent__" in records[0]) - -class EncryptedNumberFieldTests(EncryptedFieldTests): - def test_create_and_query(self): + def test_numeric_fields(self): + """ + Fields that have not been tested elsewhere. + """ EncryptedNumbers.objects.create( pos_bigint=1000000, # FIXME: pymongo.errors.EncryptionError: Cannot encrypt element of type int diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py new file mode 100644 index 000000000..cf0101180 --- /dev/null +++ b/tests/encryption_/test_management.py @@ -0,0 +1,144 @@ +from io import StringIO + +from bson import json_util +from django.core.management import call_command +from django.test import TransactionTestCase, modify_settings, override_settings + +from .routers import TestEncryptedRouter + +EXPECTED_ENCRYPTED_FIELDS_MAP = { + "encryption__billing": { + "fields": [ + { + "bsonType": "string", + "path": "cc_type", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b" \x901\x89\x1f\xafAX\x9b*\xb1\xc7\xc5\xfdl\xa4", 4), + }, + { + "bsonType": "long", + "path": "cc_number", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x97\xb4\x9d\xb8\xd5\xa6Ay\x85\xfe\x00\xc0\xd4{\xa2\xff", 4), + }, + { + "bsonType": "decimal", + "path": "account_balance", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\xcc\x01-s\xea\xd9B\x8d\x80\xd7\xf8!n\xc6\xf5U", 4), + }, + ] + }, + "encryption__patientrecord": { + "fields": [ + { + "bsonType": "string", + "path": "ssn", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), + }, + { + "bsonType": "date", + "path": "birth_date", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"@\xdd\xb4\xd2%\xc2B\x94\xb5\x07\xbc(ER[s", 4), + }, + { + "bsonType": "binData", + "path": "profile_picture", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"Q\xa2\xebc!\xecD,\x8b\xe4$\xb6ul9\x9a", 4), + }, + { + "bsonType": "int", + "path": "patient_age", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\ro\x80\x1e\x8e1K\xde\xbc_\xc3bi\x95\xa6j", 4), + }, + { + "bsonType": "double", + "path": "weight", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\x9b\xfd:n\xe1\xd0N\xdd\xb3\xe7e)\x06\xea\x8a\x1d", 4), + }, + ] + }, + "encryption__patient": { + "fields": [ + { + "bsonType": "int", + "path": "patient_id", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x8ft\x16:\x8a\x91D\xc7\x8a\xdf\xe5O\n[\xfd\\", 4), + }, + { + "bsonType": "string", + "path": "patient_name", + # "keyId": Binary(b"<\x9b\xba\xeb:\xa4@m\x93\x0e\x0c\xcaN\x03\xfb\x05", 4), + }, + { + "bsonType": "string", + "path": "patient_notes", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x01\xe7\xd1isnB$\xa9(gwO\xca\x10\xbd", 4), + }, + { + "bsonType": "date", + "path": "registration_date", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"F\xfb\xae\x82\xd5\x9a@\xee\xbfJ\xaf#\x9c:-I", 4), + }, + { + "bsonType": "bool", + "path": "is_active", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\xb2\xb5\xc4K53A\xda\xb9V\xa6\xa9\x97\x94\xea;", 4), + }, + {"bsonType": "string", "path": "email", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__patientportaluser": { + "fields": [ + {"bsonType": "string", "path": "ip_address", "queries": {"queryType": "equality"}}, + {"bsonType": "string", "path": "url", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__encryptednumbers": { + "fields": [ + {"bsonType": "int", "path": "pos_bigint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "pos_smallint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "smallint", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__appointment": { + "fields": [{"bsonType": "date", "path": "time", "queries": {"queryType": "equality"}}] + }, +} + + +@modify_settings( + INSTALLED_APPS={"prepend": "django_mongodb_backend"}, +) +@override_settings(DATABASE_ROUTERS=[TestEncryptedRouter()]) +class EncryptedFieldTests(TransactionTestCase): + databases = {"default", "encrypted"} + available_apps = ["django_mongodb_backend", "encryption_"] + + def test_show_schema_map(self): + self.maxDiff = None + out = StringIO() + call_command( + "showschemamap", + "--database", + "encrypted", + verbosity=0, + stdout=out, + ) + # Remove keyIds since they are different for each run. + output_json = json_util.loads(out.getvalue()) + for table in output_json: + for field in output_json[table]["fields"]: + del field["keyId"] + # TODO: probably we don't need to test the entire mapping, otherwise it + # requires updates every time a new model or field is added! + self.assertEqual(EXPECTED_ENCRYPTED_FIELDS_MAP, output_json) From 0e6e4691bbb6af2ef81f9e0f69dd55b5b80e2433 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Mon, 4 Aug 2025 22:15:27 -0400 Subject: [PATCH 5/5] Renama schema_map -> encrypted_fields_map - Client-side QE configuration mistakenly used `schema_map` to pass the encrypted fields map to Django's schema editor through `AutoEncryptionOpts`. Although confusing, and despite the error, client-side configuration still succeeded because the map given to `AutoEncryptionOpts` in `schema_map` was then correctly passed to `create_collection` via the `encryptedFields` arg. - Re-confirmed in local manual testing that client-side configuration works as expected and requires data keys. There is no code (as far as I can tell) to create data keys in PyMongo that is initiated by the existence of `encrypted_fields_map` alone. Rather, data keys appear to be created in `create_encrypted_collection` and only in `create_encrypted_collection`. - Renamed `showschemamap` -> `showfieldsmap` and updated tests and docs accordingly. --- .../commands/{showschemamap.py => showfieldsmap.py} | 8 ++++---- django_mongodb_backend/schema.py | 6 +++--- docs/source/faq.rst | 6 +++--- docs/source/ref/django-admin.rst | 10 +++++----- tests/encryption_/test_management.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) rename django_mongodb_backend/management/commands/{showschemamap.py => showfieldsmap.py} (88%) diff --git a/django_mongodb_backend/management/commands/showschemamap.py b/django_mongodb_backend/management/commands/showfieldsmap.py similarity index 88% rename from django_mongodb_backend/management/commands/showschemamap.py rename to django_mongodb_backend/management/commands/showfieldsmap.py index e7fcd4826..4aa103c33 100644 --- a/django_mongodb_backend/management/commands/showschemamap.py +++ b/django_mongodb_backend/management/commands/showfieldsmap.py @@ -8,7 +8,7 @@ class Command(BaseCommand): - help = "Generate a `schema_map` of encrypted fields for all encrypted" + help = "Generate an `encrypted_fields_map` of encrypted fields for all encrypted" " models in the database for use with `AutoEncryptionOpts` in" " client configuration." @@ -28,7 +28,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): db = options["database"] connection = connections[db] - schema_map = {} + encrypted_fields_map = {} for app_config in apps.get_app_configs(): for model in app_config.get_models(): if has_encrypted_fields(model): @@ -49,5 +49,5 @@ def handle(self, *args, **options): master_key=master_key, ) field["keyId"] = data_key - schema_map[model._meta.db_table] = fields - self.stdout.write(json_util.dumps(schema_map, indent=2)) + encrypted_fields_map[model._meta.db_table] = fields + self.stdout.write(json_util.dumps(encrypted_fields_map, indent=2)) diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 702aae241..1ea1553a4 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -424,7 +424,7 @@ def _field_should_have_unique(self, field): def _create_collection(self, model): """ Create a collection for the model with the encrypted fields. If - provided, use the `_schema_map` in the client's + provided, use the `_encrypted_fields_map` in the client's `auto_encryption_opts`. Otherwise, create the encrypted fields map with `_get_encrypted_fields_map`. """ @@ -434,8 +434,8 @@ def _create_collection(self, model): client = self.connection.connection options = getattr(client._options, "auto_encryption_opts", None) if options is not None: - if schema_map := getattr(options, "_schema_map", None): - db.create_collection(db_table, encryptedFields=schema_map[db_table]) + if encrypted_fields_map := getattr(options, "_encrypted_fields_map", None): + db.create_collection(db_table, encryptedFields=encrypted_fields_map[db_table]) else: ce = ClientEncryption( options._kms_providers, diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 3754b49b4..c60c63c77 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -75,15 +75,15 @@ In addition to the you will need to provide a ``schema_map`` to the ``AutoEncryptionOpts``. Fortunately, this is easy to do with Django MongoDB Backend. You can use -the ``showschemamap`` management command to generate the schema map +the ``showfieldsmap`` management command to generate the schema map for your encrypted fields, and then use the results in your settings. To generate the schema map, run the following command in your Django project: :: - python manage.py showschemamap + python manage.py showfieldsmap -.. note:: The ``showschemamap`` command is only available if you have the +.. note:: The ``showfieldsmap`` command is only available if you have the ``django_mongodb_backend`` app included in the :setting:`INSTALLED_APPS` setting. diff --git a/docs/source/ref/django-admin.rst b/docs/source/ref/django-admin.rst index 13927af06..7153f75b4 100644 --- a/docs/source/ref/django-admin.rst +++ b/docs/source/ref/django-admin.rst @@ -28,14 +28,14 @@ Available commands Defaults to ``default``. -``showschemamap`` +``showfieldsmap`` ---------------------------- -.. django-admin:: showschemamap +.. django-admin:: showfieldsmap - Creates a schema map for encrypted fields that can be used with - :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure - an encrypted client. + Creates an encrypted fields map that can be used with `encrypted_fields_map` + in :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure + client-side Queryable Encryption. .. django-admin-option:: --database DATABASE diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py index cf0101180..f5a706a82 100644 --- a/tests/encryption_/test_management.py +++ b/tests/encryption_/test_management.py @@ -128,7 +128,7 @@ def test_show_schema_map(self): self.maxDiff = None out = StringIO() call_command( - "showschemamap", + "showfieldsmap", "--database", "encrypted", verbosity=0,