INTPYTHON-527 Add Queryable Encryption support

.evergreen/config.yml

@@ -90,6 +90,28 @@ buildvariants:
     tasks:
       - name: run-tests
 
+  - name: tests-7-noauth-nossl
+    display_name: Run Tests 7.0 NoAuth NoSSL
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "7.0"
+      TOPOLOGY: server
+      AUTH: "noauth"
+      SSL: "nossl"
+    tasks:
+      - name: run-tests
+
+  - name: tests-7-auth-ssl
+    display_name: Run Tests 7.0 Auth SSL
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "7.0"
+      TOPOLOGY: server
+      AUTH: "auth"
+      SSL: "ssl"
+    tasks:
+      - name: run-tests
+
   - name: tests-8-noauth-nossl
     display_name: Run Tests 8.0 NoAuth NoSSL
     run_on: rhel87-small

.evergreen/run-tests.sh

@@ -6,6 +6,7 @@ set -eux
 /opt/python/3.10/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
+pip install ".[encryption]"
 pip install -e .
 
 # Install django and test dependencies

.github/workflows/mongodb_settings.py

@@ -1,5 +1,7 @@
 import os
 
+from pymongo.encryption_options import AutoEncryptionOpts
+
 from django_mongodb_backend import parse_uri
 
 if mongodb_uri := os.getenv("MONGODB_URI"):
@@ -27,6 +29,36 @@
         },
     }
 
+DATABASES["encrypted"] = {
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests-encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="my_encrypted_database.keyvault",
+            kms_providers={"local": {"key": os.urandom(96)}},
+        ),
+        "directConnection": True,
+    },
+    "KMS_PROVIDERS": {},
+    "KMS_CREDENTIALS": {},
+}
+
+
+class EncryptedRouter:
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        # The encryption_ app's models are only created in the encrypted database.
+        if app_label == "encryption_":
+            return db == "encrypted"
+        # Don't create other app's models in the encrypted database.
+        if db == "encrypted":
+            return False
+        return None
+
+    def kms_provider(self, model, **hints):
+        return "local"
+
+
+DATABASE_ROUTERS = [EncryptedRouter()]
 DEFAULT_AUTO_FIELD = "django_mongodb_backend.fields.ObjectIdAutoField"
 PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
 SECRET_KEY = "django_tests_secret_key"

.github/workflows/runtests.py

@@ -4,151 +4,6 @@
 import sys
 
 test_apps = [
-    "admin_changelist",
-    "admin_checks",
-    "admin_custom_urls",
-    "admin_docs",
-    "admin_filters",
-    "admin_inlines",
-    "admin_ordering",
-    "admin_scripts",
-    "admin_utils",
-    "admin_views",
-    "admin_widgets",
-    "aggregation",
-    "aggregation_regress",
-    "annotations",
-    "apps",
-    "async",
-    "auth_tests",
-    "backends",
-    "basic",
-    "bulk_create",
-    "cache",
-    "check_framework",
-    "constraints",
-    "contenttypes_tests",
-    "context_processors",
-    "custom_columns",
-    "custom_lookups",
-    "custom_managers",
-    "custom_pk",
-    "datatypes",
-    "dates",
-    "datetimes",
-    "db_functions",
-    "defer",
-    "defer_regress",
-    "delete",
-    "delete_regress",
-    "empty",
-    "empty_models",
-    "expressions",
-    "expressions_case",
-    "field_defaults",
-    "file_storage",
-    "file_uploads",
-    "fixtures",
-    "fixtures_model_package",
-    "fixtures_regress",
-    "flatpages_tests",
-    "force_insert_update",
-    "foreign_object",
-    "forms_tests",
-    "from_db_value",
-    "generic_inline_admin",
-    "generic_relations",
-    "generic_relations_regress",
-    "generic_views",
-    "get_earliest_or_latest",
-    "get_object_or_404",
-    "get_or_create",
-    "i18n",
-    "indexes",
-    "inline_formsets",
-    "introspection",
-    "invalid_models_tests",
-    "known_related_objects",
-    "lookup",
-    "m2m_and_m2o",
-    "m2m_intermediary",
-    "m2m_multiple",
-    "m2m_recursive",
-    "m2m_regress",
-    "m2m_signals",
-    "m2m_through",
-    "m2m_through_regress",
-    "m2o_recursive",
-    "managers_regress",
-    "many_to_many",
-    "many_to_one",
-    "many_to_one_null",
-    "max_lengths",
-    "messages_tests",
-    "migrate_signals",
-    "migration_test_data_persistence",
-    "migrations",
-    "model_fields",
-    "model_forms",
-    "model_formsets",
-    "model_formsets_regress",
-    "model_indexes",
-    "model_inheritance",
-    "model_inheritance_regress",
-    "model_options",
-    "model_package",
-    "model_regress",
-    "model_utils",
-    "modeladmin",
-    "multiple_database",
-    "mutually_referential",
-    "nested_foreign_keys",
-    "null_fk",
-    "null_fk_ordering",
-    "null_queries",
-    "one_to_one",
-    "or_lookups",
-    "order_with_respect_to",
-    "ordering",
-    "pagination",
-    "prefetch_related",
-    "proxy_model_inheritance",
-    "proxy_models",
-    "queries",
-    "queryset_pickle",
-    "redirects_tests",
-    "reserved_names",
-    "reverse_lookup",
-    "save_delete_hooks",
-    "schema",
-    "select_for_update",
-    "select_related",
-    "select_related_onetoone",
-    "select_related_regress",
-    "serializers",
-    "servers",
-    "sessions_tests",
-    "shortcuts",
-    "signals",
-    "sitemaps_tests",
-    "sites_framework",
-    "sites_tests",
-    "string_lookup",
-    "swappable_models",
-    "syndication_tests",
-    "test_client",
-    "test_client_regress",
-    "test_runner",
-    "test_utils",
-    "timezones",
-    "transactions",
-    "unmanaged_models",
-    "update",
-    "update_only_fields",
-    "user_commands",
-    "validation",
-    "view_tests",
-    "xor_lookups",
     # Add directories in django_mongodb_backend/tests
     *sorted(
         [

.github/workflows/test-python-atlas.yml

@@ -28,6 +28,7 @@ jobs:
       - name: install django-mongodb-backend
         run: |
           pip3 install --upgrade pip
+          pip3 install ".[encryption]"
           pip3 install -e .
       - name: Checkout Django
         uses: actions/checkout@v4

.github/workflows/test-python.yml

@@ -28,6 +28,7 @@ jobs:
       - name: install django-mongodb-backend
         run: |
           pip3 install --upgrade pip
+          pip3 install ".[encryption]"
           pip3 install -e .
       - name: Checkout Django
         uses: actions/checkout@v4

django_mongodb_backend/__init__.py

@@ -14,6 +14,7 @@
 from .indexes import register_indexes  # noqa: E402
 from .lookups import register_lookups  # noqa: E402
 from .query import register_nodes  # noqa: E402
+from .routers import register_routers  # noqa: E402
 
 __all__ = ["parse_uri"]
 
@@ -25,3 +26,4 @@
 register_indexes()
 register_lookups()
 register_nodes()
+register_routers()

django_mongodb_backend/base.py

@@ -229,4 +229,7 @@ def cursor(self):
 
     def get_database_version(self):
         """Return a tuple of the database's version."""
-        return tuple(self.connection.server_info()["versionArray"])
+        # Avoid using PyMongo to check the database version or require
+        # pymongocrypt>=1.14.2 which will contain a fix for the `buildInfo`
+        # command. https://jira.mongodb.org/browse/PYTHON-5429
+        return tuple(self.connection.admin.command("buildInfo")["versionArray"])

django_mongodb_backend/features.py

@@ -569,9 +569,17 @@ def django_test_expected_failures(self):
         },
     }
 
+    @cached_property
+    def mongodb_version(self):
+        return self.connection.get_database_version()  # e.g., (6, 3, 0)
+
     @cached_property
     def is_mongodb_6_3(self):
-        return self.connection.get_database_version() >= (6, 3)
+        return self.mongodb_version >= (6, 3)
+
+    @cached_property
+    def is_mongodb_7_0(self):
+        return self.mongodb_version >= (7, 0)
 
     @cached_property
     def supports_atlas_search(self):
@@ -601,3 +609,18 @@ def _supports_transactions(self):
         hello = client.command("hello")
         # a replica set or a sharded cluster
         return "setName" in hello or hello.get("msg") == "isdbgrid"
+
+    @cached_property
+    def supports_queryable_encryption(self):
+        """
+        Queryable Encryption requires a MongoDB 7.0 or later replica set or sharded
+        cluster, as well as MonogDB Atlas or Enterprise.
+        """
+        self.connection.ensure_connection()
+        build_info = self.connection.connection.admin.command("buildInfo")
+        is_enterprise = "enterprise" in build_info.get("modules")
+        return (
+            (is_enterprise or self.supports_atlas_search)
+            and self._supports_transactions
+            and self.is_mongodb_7_0
+        )

django_mongodb_backend/fields/__init__.py

@@ -3,6 +3,30 @@
 from .duration import register_duration_field
 from .embedded_model import EmbeddedModelField
 from .embedded_model_array import EmbeddedModelArrayField
+from .encryption import (
+    EncryptedBigIntegerField,
+    EncryptedBinaryField,
+    EncryptedBooleanField,
+    EncryptedCharField,
+    EncryptedDateField,
+    EncryptedDateTimeField,
+    EncryptedDecimalField,
+    EncryptedEmailField,
+    EncryptedFieldMixin,
+    EncryptedFloatField,
+    EncryptedGenericIPAddressField,
+    EncryptedIntegerField,
+    EncryptedPositiveBigIntegerField,
+    EncryptedPositiveIntegerField,
+    EncryptedPositiveSmallIntegerField,
+    EncryptedSmallIntegerField,
+    EncryptedTextField,
+    EncryptedTimeField,
+    EncryptedURLField,
+    EqualityQuery,
+    RangeQuery,
+    has_encrypted_fields,
+)
 from .json import register_json_field
 from .objectid import ObjectIdField
 from .polymorphic_embedded_model import PolymorphicEmbeddedModelField
@@ -12,10 +36,32 @@
     "ArrayField",
     "EmbeddedModelArrayField",
     "EmbeddedModelField",
+    "EncryptedBigIntegerField",
+    "EncryptedBinaryField",
+    "EncryptedBooleanField",
+    "EncryptedCharField",
+    "EncryptedDateField",
+    "EncryptedDateTimeField",
+    "EncryptedDecimalField",
+    "EncryptedEmailField",
+    "EncryptedFieldMixin",
+    "EncryptedFloatField",
+    "EncryptedGenericIPAddressField",
+    "EncryptedIntegerField",
+    "EncryptedPositiveBigIntegerField",
+    "EncryptedPositiveIntegerField",
+    "EncryptedPositiveSmallIntegerField",
+    "EncryptedSmallIntegerField",
+    "EncryptedTextField",
+    "EncryptedTimeField",
+    "EncryptedURLField",
+    "EqualityQuery",
     "ObjectIdAutoField",
     "ObjectIdField",
     "PolymorphicEmbeddedModelArrayField",
     "PolymorphicEmbeddedModelField",
+    "RangeQuery",
+    "has_encrypted_fields",
     "register_fields",
 ]