Skip to content

INTPYTHON-527 Add Queryable Encryption support #329

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

INTPYTHON-527 Add Queryable Encryption support #329

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
82a3eb2
[temp] remove Django tests for faster CI
timgraham Nov 8, 2025
e3196e2
INTPYTHON-527 Add Queryable Encryption support
aclark4life Jun 25, 2025
f23c383
[wip] updates to field docs
timgraham Oct 30, 2025
7d63045
add QuerySet tests
timgraham Oct 30, 2025
bbf38be
use shared library instead of mongocryptd
timgraham Nov 5, 2025
755cb20
adjust message and add test for missing auto_encryption_opts error
timgraham Nov 7, 2025
51da0e9
polish howto
timgraham Nov 11, 2025
ac265ce
doc query limitations + docs polish + todos
timgraham Nov 12, 2025
175a638
edit "Dynamic library path configuration"
timgraham Nov 12, 2025
106e1c7
combine topic guide with howto
timgraham Nov 13, 2025
4284529
Remove QE from 5.2.0 beta 2 release notes
aclark4life Nov 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions .github/workflows/encrypted_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Settings for django_mongodb_backend/tests when encryption is supported.
import os

from mongodb_settings import * # noqa: F403
from pymongo.encryption import AutoEncryptionOpts

os.environ["LD_LIBRARY_PATH"] = os.environ["GITHUB_WORKSPACE"] + "/lib/"

DATABASES["encrypted"] = { # noqa: F405
"ENGINE": "django_mongodb_backend",
"NAME": "djangotests_encrypted",
"OPTIONS": {
"auto_encryption_opts": AutoEncryptionOpts(
key_vault_namespace="djangotests_encrypted.__keyVault",
kms_providers={"local": {"key": os.urandom(96)}},
crypt_shared_lib_path=os.environ["GITHUB_WORKSPACE"] + "/lib/mongo_crypt_v1.so",
Copy link
Collaborator Author

@aclark4life aclark4life Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest just using the POC:

      # 🧩 Install Mongo Crypt Shared Library
      - name: Install Mongo Crypt Shared Library
        run: |
          curl -sSL -o mongo_crypt_shared_v1.tgz https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
          tar -xzf mongo_crypt_shared_v1.tgz
          sudo mkdir -p /usr/local/lib/mongo_crypt_v1
          sudo cp lib/mongo_crypt_v1.so /usr/local/lib/mongo_crypt_v1/
          echo "✅ Installed Mongo Crypt Shared Library to /usr/local/lib/mongo_crypt_v1"

),
"directConnection": True,
},
"KMS_CREDENTIALS": {},
}


class EncryptedRouter:
def db_for_read(self, model, **hints):
if model._meta.app_label == "encryption_":
return "encrypted"
return None

db_for_write = db_for_read

def allow_migrate(self, db, app_label, model_name=None, **hints):
# The encryption_ app's models are only created in the encrypted
# database.
if app_label == "encryption_":
return db == "encrypted"
# Don't create other app's models in the encrypted database.
if db == "encrypted":
return False
return None


DATABASE_ROUTERS.append(EncryptedRouter()) # noqa: F405
3 changes: 2 additions & 1 deletion .github/workflows/mongodb_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Settings for django_mongodb_backend/tests.
# Settings for django_mongodb_backend/tests when encryption isn't supported.
from django_settings import * # noqa: F403

DATABASES["encrypted"] = {} # noqa: F405
DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]
145 changes: 0 additions & 145 deletions .github/workflows/runtests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,151 +6,6 @@
from django.core.exceptions import ImproperlyConfigured

test_apps = [
"admin_changelist",
"admin_checks",
"admin_custom_urls",
"admin_docs",
"admin_filters",
"admin_inlines",
"admin_ordering",
"admin_scripts",
"admin_utils",
"admin_views",
"admin_widgets",
"aggregation",
"aggregation_regress",
"annotations",
"apps",
"async",
"auth_tests",
"backends",
"basic",
"bulk_create",
"cache",
"check_framework",
"constraints",
"contenttypes_tests",
"context_processors",
"custom_columns",
"custom_lookups",
"custom_managers",
"custom_pk",
"datatypes",
"dates",
"datetimes",
"db_functions",
"defer",
"defer_regress",
"delete",
"delete_regress",
"empty",
"empty_models",
"expressions",
"expressions_case",
"field_defaults",
"file_storage",
"file_uploads",
"fixtures",
"fixtures_model_package",
"fixtures_regress",
"flatpages_tests",
"force_insert_update",
"foreign_object",
"forms_tests",
"from_db_value",
"generic_inline_admin",
"generic_relations",
"generic_relations_regress",
"generic_views",
"get_earliest_or_latest",
"get_object_or_404",
"get_or_create",
"i18n",
"indexes",
"inline_formsets",
"introspection",
"invalid_models_tests",
"known_related_objects",
"lookup",
"m2m_and_m2o",
"m2m_intermediary",
"m2m_multiple",
"m2m_recursive",
"m2m_regress",
"m2m_signals",
"m2m_through",
"m2m_through_regress",
"m2o_recursive",
"managers_regress",
"many_to_many",
"many_to_one",
"many_to_one_null",
"max_lengths",
"messages_tests",
"migrate_signals",
"migration_test_data_persistence",
"migrations",
"model_fields",
"model_forms",
"model_formsets",
"model_formsets_regress",
"model_indexes",
"model_inheritance",
"model_inheritance_regress",
"model_options",
"model_package",
"model_regress",
"model_utils",
"modeladmin",
"multiple_database",
"mutually_referential",
"nested_foreign_keys",
"null_fk",
"null_fk_ordering",
"null_queries",
"one_to_one",
"or_lookups",
"order_with_respect_to",
"ordering",
"pagination",
"prefetch_related",
"proxy_model_inheritance",
"proxy_models",
"queries",
"queryset_pickle",
"redirects_tests",
"reserved_names",
"reverse_lookup",
"save_delete_hooks",
"schema",
"select_for_update",
"select_related",
"select_related_onetoone",
"select_related_regress",
"serializers",
"servers",
"sessions_tests",
"shortcuts",
"signals",
"sitemaps_tests",
"sites_framework",
"sites_tests",
"string_lookup",
"swappable_models",
"syndication_tests",
"test_client",
"test_client_regress",
"test_runner",
"test_utils",
"timezones",
"transactions",
"unmanaged_models",
"update",
"update_only_fields",
"user_commands",
"validation",
"view_tests",
"xor_lookups",
# Add directories in django_mongodb_backend/tests
*sorted(
[
Expand Down
10 changes: 8 additions & 2 deletions .github/workflows/test-python-atlas.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: install django-mongodb-backend
run: |
pip3 install --upgrade pip
pip3 install -e .
pip3 install -e .[encryption]
- name: Checkout Django
uses: actions/checkout@v5
with:
Expand All @@ -51,8 +51,14 @@ jobs:
run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py
- name: Start local Atlas
working-directory: .
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15
Copy link
Collaborator

@timgraham timgraham Oct 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we should have a separate job for Atlas 8 so we still test with Atlas 7, although it's useful to keep it like this for now so we can see the modifications.

Copy link
Collaborator Author

@aclark4life aclark4life Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we care about 7 outside of QE? If not, let's just test 8. Still thinking about going the other direction and supporting query, queryPreview, etc. But in this one case I think it's reasonable to cling to non-preview and versions that support query.

Copy link
Collaborator

@timgraham timgraham Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. In fact, based on the driver policy, we have to support MongoDB 6 until July 2028. I've argued that since Django 5.2 is supported until April 2028, we could make Django 5.2 the last version to support MongoDB 6. This is similar to Django's version support for its built in databases. In any case, it would be nice to finalize and document a MongoDB version support policy.

Copy link
Collaborator Author

@aclark4life aclark4life Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK in that case to remove as much ambiguity as possible, and to support as much MongoDB as we can, how about if we go back to 7 for QE and document rangePreview. I haven't thought about the MongoDB support policy in the context of this project, but what you propose sounds fine. Maybe open a PR to the docs with that policy defined so we can discuss there and merge.

- name: Download crypt shared
run: |
wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so
- name: Run tests
run: python3 django_repo/tests/runtests_.py
permissions:
contents: read
env:
DJANGO_SETTINGS_MODULE: "encrypted_settings"
60 changes: 0 additions & 60 deletions .github/workflows/test-python-geo.yml
View file
Open in desktop

This file was deleted.

59 changes: 0 additions & 59 deletions .github/workflows/test-python.yml
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions django_mongodb_backend/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from .indexes import register_indexes # noqa: E402
from .lookups import register_lookups # noqa: E402
from .query import register_nodes # noqa: E402
from .routers import register_routers # noqa: E402

__all__ = ["parse_uri"]

Expand All @@ -25,3 +26,4 @@
register_indexes()
register_lookups()
register_nodes()
register_routers()
11 changes: 11 additions & 0 deletions django_mongodb_backend/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from django.utils.functional import cached_property
from pymongo.collection import Collection
from pymongo.driver_info import DriverInfo
from pymongo.encryption import ClientEncryption
from pymongo.mongo_client import MongoClient
from pymongo.uri_parser import parse_uri

Expand Down Expand Up @@ -241,6 +242,16 @@ def get_database(self):
return OperationDebugWrapper(self)
return self.database

@cached_property
def client_encryption(self):
auto_encryption_opts = self.connection._options.auto_encryption_opts
return ClientEncryption(
auto_encryption_opts._kms_providers,
auto_encryption_opts._key_vault_namespace,
self.connection,
self.connection.codec_options,
)

@cached_property
def database(self):
"""Connect to the database the first time it's accessed."""
Expand Down
Loading