From fb1e120b8214771e96e7dcc6201d5478b9b166e6 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 25 Jun 2025 15:20:54 -0400 Subject: [PATCH 1/3] INTPYTHON-527 Add Queryable Encryption support --- .evergreen/config.yml | 22 + .evergreen/run-tests.sh | 1 + .github/workflows/mongodb_settings.py | 32 ++ .github/workflows/runtests.py | 145 ------- .github/workflows/test-python-atlas.yml | 1 + .github/workflows/test-python.yml | 1 + django_mongodb_backend/__init__.py | 2 + django_mongodb_backend/base.py | 5 +- django_mongodb_backend/features.py | 25 +- django_mongodb_backend/fields/__init__.py | 46 ++ django_mongodb_backend/fields/encryption.py | 122 ++++++ .../management/commands/showschemamap.py | 53 +++ django_mongodb_backend/routers.py | 26 +- django_mongodb_backend/schema.py | 67 ++- docs/source/faq.rst | 68 +++ docs/source/howto/index.rst | 1 + docs/source/howto/queryable-encryption.rst | 76 ++++ docs/source/index.rst | 1 + docs/source/ref/django-admin.rst | 17 + docs/source/ref/models/fields.rst | 89 ++++ docs/source/releases/5.2.x.rst | 1 + docs/source/topics/index.rst | 1 + docs/source/topics/known-issues.rst | 5 + docs/source/topics/queryable-encryption.rst | 67 +++ pyproject.toml | 3 +- tests/backend_/test_features.py | 80 ++++ tests/encryption_/__init__.py | 0 tests/encryption_/models.py | 87 ++++ tests/encryption_/routers.py | 24 ++ tests/encryption_/tests.py | 394 ++++++++++++++++++ 30 files changed, 1309 insertions(+), 153 deletions(-) create mode 100644 django_mongodb_backend/fields/encryption.py create mode 100644 django_mongodb_backend/management/commands/showschemamap.py create mode 100644 docs/source/howto/queryable-encryption.rst create mode 100644 docs/source/topics/queryable-encryption.rst create mode 100644 tests/encryption_/__init__.py create mode 100644 tests/encryption_/models.py create mode 100644 tests/encryption_/routers.py create mode 100644 tests/encryption_/tests.py diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d59bfe079..11695c462 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -90,6 +90,28 @@ buildvariants: tasks: - name: run-tests + - name: tests-7-noauth-nossl + display_name: Run Tests 7.0 NoAuth NoSSL + run_on: rhel87-small + expansions: + MONGODB_VERSION: "7.0" + TOPOLOGY: server + AUTH: "noauth" + SSL: "nossl" + tasks: + - name: run-tests + + - name: tests-7-auth-ssl + display_name: Run Tests 7.0 Auth SSL + run_on: rhel87-small + expansions: + MONGODB_VERSION: "7.0" + TOPOLOGY: server + AUTH: "auth" + SSL: "ssl" + tasks: + - name: run-tests + - name: tests-8-noauth-nossl display_name: Run Tests 8.0 NoAuth NoSSL run_on: rhel87-small diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f49a0e9a9..2170c6d5a 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -6,6 +6,7 @@ set -eux /opt/python/3.10/bin/python3 -m venv venv . venv/bin/activate python -m pip install -U pip +pip install ".[encryption]" pip install -e . # Install django and test dependencies diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 49d44a5fc..8b448ef14 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -1,5 +1,7 @@ import os +from pymongo.encryption_options import AutoEncryptionOpts + from django_mongodb_backend import parse_uri if mongodb_uri := os.getenv("MONGODB_URI"): @@ -27,6 +29,36 @@ }, } +DATABASES["encrypted"] = { + "ENGINE": "django_mongodb_backend", + "NAME": "djangotests-encrypted", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="my_encrypted_database.keyvault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_PROVIDERS": {}, + "KMS_CREDENTIALS": {}, +} + + +class EncryptedRouter: + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model, **hints): + return "local" + + +DATABASE_ROUTERS = [EncryptedRouter()] DEFAULT_AUTO_FIELD = "django_mongodb_backend.fields.ObjectIdAutoField" PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) SECRET_KEY = "django_tests_secret_key" diff --git a/.github/workflows/runtests.py b/.github/workflows/runtests.py index ebcc4876f..350ca0fc3 100755 --- a/.github/workflows/runtests.py +++ b/.github/workflows/runtests.py @@ -4,151 +4,6 @@ import sys test_apps = [ - "admin_changelist", - "admin_checks", - "admin_custom_urls", - "admin_docs", - "admin_filters", - "admin_inlines", - "admin_ordering", - "admin_scripts", - "admin_utils", - "admin_views", - "admin_widgets", - "aggregation", - "aggregation_regress", - "annotations", - "apps", - "async", - "auth_tests", - "backends", - "basic", - "bulk_create", - "cache", - "check_framework", - "constraints", - "contenttypes_tests", - "context_processors", - "custom_columns", - "custom_lookups", - "custom_managers", - "custom_pk", - "datatypes", - "dates", - "datetimes", - "db_functions", - "defer", - "defer_regress", - "delete", - "delete_regress", - "empty", - "empty_models", - "expressions", - "expressions_case", - "field_defaults", - "file_storage", - "file_uploads", - "fixtures", - "fixtures_model_package", - "fixtures_regress", - "flatpages_tests", - "force_insert_update", - "foreign_object", - "forms_tests", - "from_db_value", - "generic_inline_admin", - "generic_relations", - "generic_relations_regress", - "generic_views", - "get_earliest_or_latest", - "get_object_or_404", - "get_or_create", - "i18n", - "indexes", - "inline_formsets", - "introspection", - "invalid_models_tests", - "known_related_objects", - "lookup", - "m2m_and_m2o", - "m2m_intermediary", - "m2m_multiple", - "m2m_recursive", - "m2m_regress", - "m2m_signals", - "m2m_through", - "m2m_through_regress", - "m2o_recursive", - "managers_regress", - "many_to_many", - "many_to_one", - "many_to_one_null", - "max_lengths", - "messages_tests", - "migrate_signals", - "migration_test_data_persistence", - "migrations", - "model_fields", - "model_forms", - "model_formsets", - "model_formsets_regress", - "model_indexes", - "model_inheritance", - "model_inheritance_regress", - "model_options", - "model_package", - "model_regress", - "model_utils", - "modeladmin", - "multiple_database", - "mutually_referential", - "nested_foreign_keys", - "null_fk", - "null_fk_ordering", - "null_queries", - "one_to_one", - "or_lookups", - "order_with_respect_to", - "ordering", - "pagination", - "prefetch_related", - "proxy_model_inheritance", - "proxy_models", - "queries", - "queryset_pickle", - "redirects_tests", - "reserved_names", - "reverse_lookup", - "save_delete_hooks", - "schema", - "select_for_update", - "select_related", - "select_related_onetoone", - "select_related_regress", - "serializers", - "servers", - "sessions_tests", - "shortcuts", - "signals", - "sitemaps_tests", - "sites_framework", - "sites_tests", - "string_lookup", - "swappable_models", - "syndication_tests", - "test_client", - "test_client_regress", - "test_runner", - "test_utils", - "timezones", - "transactions", - "unmanaged_models", - "update", - "update_only_fields", - "user_commands", - "validation", - "view_tests", - "xor_lookups", # Add directories in django_mongodb_backend/tests *sorted( [ diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 6eab2b7e9..52f8b1aca 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -28,6 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip + pip3 install ".[encryption]" pip3 install -e . - name: Checkout Django uses: actions/checkout@v4 diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index ce7d300d0..93f0447f0 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -28,6 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip + pip3 install ".[encryption]" pip3 install -e . - name: Checkout Django uses: actions/checkout@v4 diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index 00700421a..25e431406 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -14,6 +14,7 @@ from .indexes import register_indexes # noqa: E402 from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 +from .routers import register_routers # noqa: E402 __all__ = ["parse_uri"] @@ -25,3 +26,4 @@ register_indexes() register_lookups() register_nodes() +register_routers() diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index 7f337cf82..180232c78 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -229,4 +229,7 @@ def cursor(self): def get_database_version(self): """Return a tuple of the database's version.""" - return tuple(self.connection.server_info()["versionArray"]) + # Avoid using PyMongo to check the database version or require + # pymongocrypt>=1.14.2 which will contain a fix for the `buildInfo` + # command. https://jira.mongodb.org/browse/PYTHON-5429 + return tuple(self.connection.admin.command("buildInfo")["versionArray"]) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 9f0245ec2..ec90bb4df 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -569,9 +569,17 @@ def django_test_expected_failures(self): }, } + @cached_property + def mongodb_version(self): + return self.connection.get_database_version() # e.g., (6, 3, 0) + @cached_property def is_mongodb_6_3(self): - return self.connection.get_database_version() >= (6, 3) + return self.mongodb_version >= (6, 3) + + @cached_property + def is_mongodb_7_0(self): + return self.mongodb_version >= (7, 0) @cached_property def supports_atlas_search(self): @@ -601,3 +609,18 @@ def _supports_transactions(self): hello = client.command("hello") # a replica set or a sharded cluster return "setName" in hello or hello.get("msg") == "isdbgrid" + + @cached_property + def supports_queryable_encryption(self): + """ + Queryable Encryption requires a MongoDB 7.0 or later replica set or sharded + cluster, as well as MonogDB Atlas or Enterprise. + """ + self.connection.ensure_connection() + build_info = self.connection.connection.admin.command("buildInfo") + is_enterprise = "enterprise" in build_info.get("modules") + return ( + (is_enterprise or self.supports_atlas_search) + and self._supports_transactions + and self.is_mongodb_7_0 + ) diff --git a/django_mongodb_backend/fields/__init__.py b/django_mongodb_backend/fields/__init__.py index 0c95afd69..2b4192098 100644 --- a/django_mongodb_backend/fields/__init__.py +++ b/django_mongodb_backend/fields/__init__.py @@ -3,6 +3,30 @@ from .duration import register_duration_field from .embedded_model import EmbeddedModelField from .embedded_model_array import EmbeddedModelArrayField +from .encryption import ( + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedEmailField, + EncryptedFieldMixin, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EqualityQuery, + RangeQuery, + has_encrypted_fields, +) from .json import register_json_field from .objectid import ObjectIdField from .polymorphic_embedded_model import PolymorphicEmbeddedModelField @@ -12,10 +36,32 @@ "ArrayField", "EmbeddedModelArrayField", "EmbeddedModelField", + "EncryptedBigIntegerField", + "EncryptedBinaryField", + "EncryptedBooleanField", + "EncryptedCharField", + "EncryptedDateField", + "EncryptedDateTimeField", + "EncryptedDecimalField", + "EncryptedEmailField", + "EncryptedFieldMixin", + "EncryptedFloatField", + "EncryptedGenericIPAddressField", + "EncryptedIntegerField", + "EncryptedPositiveBigIntegerField", + "EncryptedPositiveIntegerField", + "EncryptedPositiveSmallIntegerField", + "EncryptedSmallIntegerField", + "EncryptedTextField", + "EncryptedTimeField", + "EncryptedURLField", + "EqualityQuery", "ObjectIdAutoField", "ObjectIdField", "PolymorphicEmbeddedModelArrayField", "PolymorphicEmbeddedModelField", + "RangeQuery", + "has_encrypted_fields", "register_fields", ] diff --git a/django_mongodb_backend/fields/encryption.py b/django_mongodb_backend/fields/encryption.py new file mode 100644 index 000000000..29159dba7 --- /dev/null +++ b/django_mongodb_backend/fields/encryption.py @@ -0,0 +1,122 @@ +from django.db import models + + +def has_encrypted_fields(model): + return any(getattr(field, "encrypted", False) for field in model._meta.fields) + + +class EncryptedFieldMixin(models.Field): + encrypted = True + + def __init__(self, *args, queries=None, **kwargs): + self.queries = queries + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + + if self.queries is not None: + kwargs["queries"] = self.queries + + if path.startswith("django_mongodb_backend.fields.encrypted_model"): + path = path.replace( + "django_mongodb_backend.fields.encrypted_model", + "django_mongodb_backend.fields", + ) + + return name, path, args, kwargs + + +class EncryptedBigIntegerField(EncryptedFieldMixin, models.BigIntegerField): + pass + + +class EncryptedBinaryField(EncryptedFieldMixin, models.BinaryField): + pass + + +class EncryptedBooleanField(EncryptedFieldMixin, models.BooleanField): + pass + + +class EncryptedCharField(EncryptedFieldMixin, models.CharField): + pass + + +class EncryptedDateField(EncryptedFieldMixin, models.DateField): + pass + + +class EncryptedDateTimeField(EncryptedFieldMixin, models.DateTimeField): + pass + + +class EncryptedDecimalField(EncryptedFieldMixin, models.DecimalField): + pass + + +class EncryptedEmailField(EncryptedFieldMixin, models.EmailField): + pass + + +class EncryptedFloatField(EncryptedFieldMixin, models.FloatField): + pass + + +class EncryptedGenericIPAddressField(EncryptedFieldMixin, models.GenericIPAddressField): + pass + + +class EncryptedIntegerField(EncryptedFieldMixin, models.IntegerField): + pass + + +class EncryptedPositiveBigIntegerField(EncryptedFieldMixin, models.PositiveBigIntegerField): + pass + + +class EncryptedPositiveIntegerField(EncryptedFieldMixin, models.PositiveIntegerField): + pass + + +class EncryptedPositiveSmallIntegerField(EncryptedFieldMixin, models.PositiveSmallIntegerField): + pass + + +class EncryptedSmallIntegerField(EncryptedFieldMixin, models.SmallIntegerField): + pass + + +class EncryptedTimeField(EncryptedFieldMixin, models.TimeField): + pass + + +class EncryptedTextField(EncryptedFieldMixin, models.TextField): + pass + + +class EncryptedURLField(EncryptedFieldMixin, models.URLField): + pass + + +class EqualityQuery(dict): + def __init__(self, *, contention=None): + super().__init__(queryType="equality") + if contention is not None: + self["contention"] = contention + + +class RangeQuery(dict): + def __init__( + self, *, contention=None, max=None, min=None, precision=None, sparsity=None, trimFactor=None + ): + super().__init__(queryType="range") + options = { + "contention": contention, + "max": max, + "min": min, + "precision": precision, + "sparsity": sparsity, + "trimFactor": trimFactor, + } + self.update({k: v for k, v in options.items() if v is not None}) diff --git a/django_mongodb_backend/management/commands/showschemamap.py b/django_mongodb_backend/management/commands/showschemamap.py new file mode 100644 index 000000000..e7fcd4826 --- /dev/null +++ b/django_mongodb_backend/management/commands/showschemamap.py @@ -0,0 +1,53 @@ +from bson import json_util +from django.apps import apps +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS, connections, router +from pymongo.encryption import ClientEncryption + +from django_mongodb_backend.fields import has_encrypted_fields + + +class Command(BaseCommand): + help = "Generate a `schema_map` of encrypted fields for all encrypted" + " models in the database for use with `AutoEncryptionOpts` in" + " client configuration." + + def add_arguments(self, parser): + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + help="Specify the database to use for generating the encrypted" + "fields map. Defaults to the 'default' database.", + ) + parser.add_argument( + "--kms-provider", + default="local", + help="Specify the KMS provider to use for encryption. Defaults to 'local'.", + ) + + def handle(self, *args, **options): + db = options["database"] + connection = connections[db] + schema_map = {} + for app_config in apps.get_app_configs(): + for model in app_config.get_models(): + if has_encrypted_fields(model): + fields = connection.schema_editor()._get_encrypted_fields_map(model) + client = connection.connection + options = client._options.auto_encryption_opts + ce = ClientEncryption( + options._kms_providers, + options._key_vault_namespace, + client, + client.codec_options, + ) + kms_provider = router.kms_provider(model) + master_key = connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider) + for field in fields["fields"]: + data_key = ce.create_data_key( + kms_provider=kms_provider, + master_key=master_key, + ) + field["keyId"] = data_key + schema_map[model._meta.db_table] = fields + self.stdout.write(json_util.dumps(schema_map, indent=2)) diff --git a/django_mongodb_backend/routers.py b/django_mongodb_backend/routers.py index 60e54bbd8..15ed3b760 100644 --- a/django_mongodb_backend/routers.py +++ b/django_mongodb_backend/routers.py @@ -1,6 +1,8 @@ from django.apps import apps +from django.core.exceptions import ImproperlyConfigured +from django.db.utils import ConnectionRouter -from django_mongodb_backend.models import EmbeddedModel +from .fields import has_encrypted_fields class MongoRouter: @@ -9,10 +11,32 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): EmbeddedModels don't have their own collection and must be ignored by dumpdata. """ + from django_mongodb_backend.models import EmbeddedModel # noqa: PLC0415 + if not model_name: return None try: model = apps.get_model(app_label, model_name) except LookupError: return None + return False if issubclass(model, EmbeddedModel) else None + + +def kms_provider(self, model, *args, **kwargs): + for router in self.routers: + func = getattr(router, "kms_provider", None) + if func and callable(func): + result = func(model, *args, **kwargs) + if result is not None: + return result + if has_encrypted_fields(model): + raise ImproperlyConfigured("No kms_provider found in database router.") + return None + + +def register_routers(): + """ + Patch the ConnectionRouter to use the custom kms_provider method. + """ + ConnectionRouter.kms_provider = kms_provider diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index a12086a6e..702aae241 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -1,10 +1,12 @@ +from django.core.exceptions import ImproperlyConfigured +from django.db import router from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Index, UniqueConstraint +from pymongo.encryption import ClientEncryption from pymongo.operations import SearchIndexModel -from django_mongodb_backend.indexes import SearchIndex - -from .fields import EmbeddedModelField +from .fields import EmbeddedModelField, has_encrypted_fields +from .indexes import SearchIndex from .query import wrap_database_errors from .utils import OperationCollector @@ -41,7 +43,7 @@ def get_database(self): @wrap_database_errors @ignore_embedded_models def create_model(self, model): - self.get_database().create_collection(model._meta.db_table) + self._create_collection(model) self._create_model_indexes(model) # Make implicit M2M tables. for field in model._meta.local_many_to_many: @@ -418,3 +420,60 @@ def _field_should_have_unique(self, field): db_type = field.db_type(self.connection) # The _id column is automatically unique. return db_type and field.unique and field.column != "_id" + + def _create_collection(self, model): + """ + Create a collection for the model with the encrypted fields. If + provided, use the `_schema_map` in the client's + `auto_encryption_opts`. Otherwise, create the encrypted fields map + with `_get_encrypted_fields_map`. + """ + db = self.get_database() + db_table = model._meta.db_table + if has_encrypted_fields(model): + client = self.connection.connection + options = getattr(client._options, "auto_encryption_opts", None) + if options is not None: + if schema_map := getattr(options, "_schema_map", None): + db.create_collection(db_table, encryptedFields=schema_map[db_table]) + else: + ce = ClientEncryption( + options._kms_providers, + options._key_vault_namespace, + client, + client.codec_options, + ) + encrypted_fields_map = self._get_encrypted_fields_map(model) + provider = router.kms_provider(model) + credentials = self.connection.settings_dict.get("KMS_CREDENTIALS").get(provider) + ce.create_encrypted_collection( + db, + db_table, + encrypted_fields_map, + provider, + credentials, + ) + else: + raise ImproperlyConfigured( + "Encrypted fields found but the connection does not have " + "auto encryption options set. Please set `auto_encryption_opts` " + "in the connection settings." + ) + else: + db.create_collection(db_table) + + def _get_encrypted_fields_map(self, model): + connection = self.connection + fields = model._meta.fields + + return { + "fields": [ + { + "bsonType": field.db_type(connection), + "path": field.column, + **({"queries": field.queries} if getattr(field, "queries", None) else {}), + } + for field in fields + if getattr(field, "encrypted", False) + ] + } diff --git a/docs/source/faq.rst b/docs/source/faq.rst index bb52e1cde..3754b49b4 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -52,3 +52,71 @@ logging:: If running ``manage.py dumpdata`` results in ``CommandError: Unable to serialize database: 'EmbeddedModelManager' object has no attribute using'``, see :ref:`configuring-database-routers-setting`. + +.. _queryable-encryption: + +Queryable Encryption +==================== + +What about client side configuration? +------------------------------------- + +In the :doc:`Queryable Encryption how-to guide `, +server side Queryable Encryption configuration is covered. + +Client side Queryable Encryption configuration requires that the entire schema +for encrypted fields is known at the time of client connection. + +Schema Map +~~~~~~~~~~ + +In addition to the +:ref:`settings described in the how-to guide `, +you will need to provide a ``schema_map`` to the ``AutoEncryptionOpts``. + +Fortunately, this is easy to do with Django MongoDB Backend. You can use +the ``showschemamap`` management command to generate the schema map +for your encrypted fields, and then use the results in your settings. + +To generate the schema map, run the following command in your Django project: +:: + + python manage.py showschemamap + +.. note:: The ``showschemamap`` command is only available if you have the + ``django_mongodb_backend`` app included in the :setting:`INSTALLED_APPS` + setting. + +Settings +~~~~~~~~ + +Now include the generated schema map in your Django settings. + +:: + + … + DATABASES["encrypted"] = { + … + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + … + schema_map= { + "encryption__patientrecord": { + "fields": [ + { + "bsonType": "string", + "path": "ssn", + "queries": {"queryType": "equality"}, + "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), + }, + } + }, + # Add other models with encrypted fields here + }, + ), + … + }, + … + } + +You are now ready to use client side :doc:`Queryable Encryption ` in your Django project. diff --git a/docs/source/howto/index.rst b/docs/source/howto/index.rst index 95d7ef632..8451960ef 100644 --- a/docs/source/howto/index.rst +++ b/docs/source/howto/index.rst @@ -11,3 +11,4 @@ Project configuration :maxdepth: 1 contrib-apps + queryable-encryption diff --git a/docs/source/howto/queryable-encryption.rst b/docs/source/howto/queryable-encryption.rst new file mode 100644 index 000000000..94de44083 --- /dev/null +++ b/docs/source/howto/queryable-encryption.rst @@ -0,0 +1,76 @@ +================================ +Configuring Queryable Encryption +================================ + +Configuring Queryable Encryption in Django is similar to +`configuring Queryable Encryption in Python `_ +but with some additional steps required for Django. + +.. note:: This section describes how to configure server side Queryable Encryption in Django. + For configuration of client side Queryable Encryption, please refer to this :ref:`FAQ question `. + +Prerequisites +------------- + +In addition to :doc:`installing ` and +:doc:`configuring ` Django MongoDB Backend, +you will need to install PyMongo with Queryable Encryption support:: + + pip install django-mongodb-backend[encryption] + +.. note:: You can use Queryable Encryption on a MongoDB 7.0 or later replica + set or sharded cluster, but not a standalone instance. + `This table `_ + shows which MongoDB server products support which Queryable Encryption mechanisms. + +.. _server-side-queryable-encryption-settings: + +Settings +-------- + +Queryable Encryption in Django requires the use of an additional encrypted database +and Key Management Service (KMS) credentials as well as an encrypted database +router. Here's how to set it up in your Django settings. + +:: + + from django_mongodb_backend import parse_uri + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "default": parse_uri( + DATABASE_URL, + db_name="my_database", + ), + } + + DATABASES["encrypted"] = { + "ENGINE": "django_mongodb_backend", + "NAME": "my_encrypted_database", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="my_encrypted_database.keyvault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_PROVIDERS": {}, + "KMS_CREDENTIALS": {}, + } + + class EncryptedRouter: + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model, **hints): + return "local" + + DATABASE_ROUTERS = [EncryptedRouter()] + +You are now ready to use server side :doc:`Queryable Encryption ` in your Django project. diff --git a/docs/source/index.rst b/docs/source/index.rst index 9e0243487..2e584ad4b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -45,6 +45,7 @@ Models **Topic guides:** - :doc:`topics/embedded-models` +- :doc:`topics/queryable-encryption` Forms ===== diff --git a/docs/source/ref/django-admin.rst b/docs/source/ref/django-admin.rst index 93f90f9f6..4ee6d2b77 100644 --- a/docs/source/ref/django-admin.rst +++ b/docs/source/ref/django-admin.rst @@ -26,3 +26,20 @@ Available commands Specifies the database in which the cache collection(s) will be created. Defaults to ``default``. + + +``get_encrypted_fields_map`` +---------------------------- + +.. django-admin:: get_encrypted_fields_map + + Creates a schema map for encrypted fields that can be used with + :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure + an encrypted client. + + .. django-admin-option:: --database DATABASE + + Specifies the database to use. + Defaults to ``default``. + +.. TODO: Clarify how database specified could affect output. diff --git a/docs/source/ref/models/fields.rst b/docs/source/ref/models/fields.rst index 870d97061..796d3cc88 100644 --- a/docs/source/ref/models/fields.rst +++ b/docs/source/ref/models/fields.rst @@ -379,3 +379,92 @@ These indexes use 0-based indexing. .. admonition:: Forms are not supported ``PolymorphicEmbeddedModelArrayField``\s don't appear in model forms. + +.. _encrypted-fields: + +Encrypted fields +---------------- + +Encrypted fields are used to store sensitive data with MongoDB's Queryable +Encryption feature. They are subclasses of Django's built-in fields, and +they encrypt the data before storing it in the database. + ++----------------------------------------+------------------------------------------------------+ +| Encrypted Field | Django Field | ++========================================+======================================================+ +| ``EncryptedBigIntegerField`` | :class:`~django.db.models.BigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedBooleanField`` | :class:`~django.db.models.BooleanField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedCharField`` | :class:`~django.db.models.CharField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateField`` | :class:`~django.db.models.DateField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateTimeField`` | :class:`~django.db.models.DateTimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDecimalField`` | :class:`~django.db.models.DecimalField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedFloatField`` | :class:`~django.db.models.FloatField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedGenericIPAddressField`` | :class:`~django.db.models.GenericIPAddressField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedIntegerField`` | :class:`~django.db.models.IntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveBigIntegerField`` | :class:`~django.db.models.PositiveBigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveIntegerField`` | :class:`~django.db.models.PositiveIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveSmallIntegerField`` | :class:`~django.db.models.PositiveSmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedSmallIntegerField`` | :class:`~django.db.models.SmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTextField`` | :class:`~django.db.models.TextField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTimeField`` | :class:`~django.db.models.TimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedURLField`` | :class:`~django.db.models.URLField` | ++----------------------------------------+------------------------------------------------------+ + +.. _encrypted-fields-unsupported-fields: + +.. admonition:: Unsupported fields + + The following fields are supported by Django MongoDB Backend but are not supported by + Queryable Encryption. + + :class:`~django.db.models.SlugField` + +Query types +~~~~~~~~~~~ + +Django MongoDB Backend provides the following query type classes for use with +encrypted fields. + ++-------------------+----------------------------------------------------------------------------------------------+ +| ``EqualityQuery`` | This query type is used for equality checks. | ++-------------------+----------------------------------------------------------------------------------------------+ +| ``RangeQuery`` | This query type is used for range queries, such as greater than or less | +| | than checks. | ++-------------------+----------------------------------------------------------------------------------------------+ + +EncryptedFieldMixin +~~~~~~~~~~~~~~~~~~~ + +You can use the ``EncryptedFieldMixin`` to create your own encrypted fields. This mixin +supports the use of a ``queries`` argument in the field definition to specify query type +for the field:: + + from django.db import models + from django_mongodb_backend.fields import EncryptedFieldMixin, EqualityQuery + from .models import MyField + + + class MyEncryptedField(EncryptedFieldMixin, MyField): + pass + + + class MyModel(models.Model): + my_encrypted_field = MyEncryptedField( + queries=EqualityQuery(), + # Other field options... + ) diff --git a/docs/source/releases/5.2.x.rst b/docs/source/releases/5.2.x.rst index d7a86aca9..5ebe7461d 100644 --- a/docs/source/releases/5.2.x.rst +++ b/docs/source/releases/5.2.x.rst @@ -16,6 +16,7 @@ New features - Added :class:`~.fields.PolymorphicEmbeddedModelField` and :class:`~.fields.PolymorphicEmbeddedModelArrayField` for storing a model instance or list of model instances that may be of more than one model class. +- Added support for Queryable Encryption. Bug fixes --------- diff --git a/docs/source/topics/index.rst b/docs/source/topics/index.rst index 47e0c6dc0..f81c6630e 100644 --- a/docs/source/topics/index.rst +++ b/docs/source/topics/index.rst @@ -11,3 +11,4 @@ know: cache embedded-models known-issues + queryable-encryption diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 4b9edee70..3abab9ae2 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -102,3 +102,8 @@ Caching Secondly, you must use the :class:`django_mongodb_backend.cache.MongoDBCache` backend rather than Django's built-in database cache backend, ``django.core.cache.backends.db.DatabaseCache``. + +Queryable Encryption +==================== + +.. TODO: Add Django core limitations that affect Queryable Encryption. diff --git a/docs/source/topics/queryable-encryption.rst b/docs/source/topics/queryable-encryption.rst new file mode 100644 index 000000000..e9b3384c6 --- /dev/null +++ b/docs/source/topics/queryable-encryption.rst @@ -0,0 +1,67 @@ +Queryable Encryption +==================== + +Use :ref:`encrypted fields ` to store sensitive data in MongoDB +your data using `Queryable Encryption `_. + +.. _encrypted-field-example: + +The basics +---------- + +Let's consider this example:: + + from django.db import models + + from django_mongodb_backend.fields import EncryptedCharField, EqualityQuery + + + class Patient(models.Model): + ssn = EncryptedCharField(max_length=11, queries=EqualityQuery()) + + def __str__(self): + return self.ssn + +The API is similar to that of Django's relational fields, with some +security-related changes:: + + >>> bob = Patient(ssn="123-45-6789") + >>> bob.ssn + '123-45-6789' + +Represented in BSON, from an encrypted client connection, the patient data looks like this: + +.. code-block:: js + + { + _id: ObjectId('68825b066fac55353a8b2b41'), + ssn: '123-45-6789', + __safeContent__: [b'\xe0)NOFB\x9a,\x08\xd7\xdd\xb8\xa6\xba$…'] + } + +The ``ssn`` field is only visible from an encrypted client connection. From an unencrypted client connection, +the patient data looks like this: + +.. code-block:: js + + { + _id: ObjectId('6882566c586a440cd0649e8f'), + ssn: Binary.createFromBase64('DkrbD67ejkt2u…', 6), + } + +.. admonition:: List of encrypted fields + + See the full list of :ref:`encrypted fields ` in the :doc:`Model field reference `. + +Querying encrypted fields +------------------------- + +You can query encrypted fields using a `limited set of +query operators `_ +which must be specified in the field definition. For example, to query the ``ssn`` field for equality, you can use the +``EqualityQuery`` operator as shown in the example above. + + >>> Patient.objects.get(ssn="123-45-6789").ssn + '123-45-6789' + +If the ``ssn`` field provided in the query matches the encrypted value in the database, the query will succeed. diff --git a/pyproject.toml b/pyproject.toml index 7b412e1db..88e31897e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ classifiers = [ ] [project.optional-dependencies] -docs = [ "sphinx>=7"] +docs = ["sphinx>=7"] +encryption = ["pymongo[encryption]"] [project.urls] Homepage = "https://www.mongodb.org" diff --git a/tests/backend_/test_features.py b/tests/backend_/test_features.py index 05959fa70..dcdb93872 100644 --- a/tests/backend_/test_features.py +++ b/tests/backend_/test_features.py @@ -44,3 +44,83 @@ def mocked_command(command): with patch("pymongo.synchronous.database.Database.command", wraps=mocked_command): self.assertIs(connection.features._supports_transactions, False) + + +class SupportsQueryableEncryptionTests(TestCase): + def setUp(self): + # Clear the cached property. + connection.features.__dict__.pop("supports_queryable_encryption", None) + # Must initialize the feature before patching it. + connection.features._supports_transactions # noqa: B018 + + def tearDown(self): + del connection.features.supports_queryable_encryption + + @staticmethod + def enterprise_response(command): + if command == "buildInfo": + return {"modules": ["enterprise"]} + raise Exception("Unexpected command") + + @staticmethod + def non_enterprise_response(command): + if command == "buildInfo": + return {"modules": []} + raise Exception("Unexpected command") + + def test_supported_on_atlas(self): + """Supported on MongoDB 7.0+ Atlas replica set or sharded cluster.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", True), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_supported_on_enterprise(self): + """Supported on MongoDB 7.0+ Enterprise replica set or sharded cluster.""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_atlas_or_enterprise_required(self): + """Not supported on MongoDB Community Edition.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_transactions_required(self): + """ + Not supported if database isn't a replica set or sharded cluster + (i.e. DatabaseFeatures._supports_transactions = False). + """ + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", False), + patch("django.db.connection.features.is_mongodb_7_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_mongodb_7_0_required(self): + """Not supported on MongoDB < 7.0""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_7_0", False), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) diff --git a/tests/encryption_/__init__.py b/tests/encryption_/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/encryption_/models.py b/tests/encryption_/models.py new file mode 100644 index 000000000..35f627db5 --- /dev/null +++ b/tests/encryption_/models.py @@ -0,0 +1,87 @@ +from django.db import models + +from django_mongodb_backend.fields import ( + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedEmailField, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EqualityQuery, + RangeQuery, +) + + +class Appointment(models.Model): + time = EncryptedTimeField(queries=EqualityQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class Billing(models.Model): + cc_type = EncryptedCharField(max_length=20, queries=EqualityQuery()) + cc_number = EncryptedBigIntegerField(queries=EqualityQuery()) + account_balance = EncryptedDecimalField(max_digits=10, decimal_places=2, queries=RangeQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class PatientPortalUser(models.Model): + ip_address = EncryptedGenericIPAddressField(queries=EqualityQuery()) + url = EncryptedURLField(queries=EqualityQuery()) + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class PatientRecord(models.Model): + ssn = EncryptedCharField(max_length=11, queries=EqualityQuery()) + birth_date = EncryptedDateField(queries=RangeQuery()) + profile_picture = EncryptedBinaryField(queries=EqualityQuery()) + patient_age = EncryptedIntegerField("patient_age", queries=RangeQuery()) + weight = EncryptedFloatField(queries=RangeQuery()) + + # TODO: Embed Billing model + # billing = + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class Patient(models.Model): + patient_id = EncryptedIntegerField("patient_id", queries=EqualityQuery()) + patient_name = EncryptedCharField(max_length=100) + patient_notes = EncryptedTextField(queries=EqualityQuery()) + registration_date = EncryptedDateTimeField(queries=EqualityQuery()) + is_active = EncryptedBooleanField(queries=EqualityQuery()) + email = EncryptedEmailField(max_length=254, queries=EqualityQuery()) + + # TODO: Embed PatientRecord model + # patient_record = + + class Meta: + required_db_features = {"supports_queryable_encryption"} + + +class EncryptedNumbers(models.Model): + pos_bigint = EncryptedPositiveBigIntegerField(queries=EqualityQuery()) + + # FIXME: pymongo.errors.EncryptionError: Cannot encrypt element of type int + # because schema requires that type is one of: [ long ] + # pos_int = EncryptedPositiveIntegerField(queries=EqualityQuery()) + + pos_smallint = EncryptedPositiveSmallIntegerField(queries=EqualityQuery()) + smallint = EncryptedSmallIntegerField(queries=EqualityQuery()) diff --git a/tests/encryption_/routers.py b/tests/encryption_/routers.py new file mode 100644 index 000000000..941735091 --- /dev/null +++ b/tests/encryption_/routers.py @@ -0,0 +1,24 @@ +from django_mongodb_backend.fields import has_encrypted_fields + + +class TestEncryptedRouter: + """Router for testing encrypted models in Django. `kms_provider` + must be set on the global test router since table creation happens + at the start of the test suite, before @override_settings( + DATABASE_ROUTERS=[TestEncryptedRouter()]) takes effect. + """ + + def allow_migrate(self, db, app_label, model_name=None, model=None, **hints): + if model: + return db == ("encrypted" if has_encrypted_fields(model) else "default") + return db == "default" + + def db_for_read(self, model, **hints): + if has_encrypted_fields(model): + return "encrypted" + return "default" + + db_for_write = db_for_read + + def kms_provider(self, model, **hints): + return "local" diff --git a/tests/encryption_/tests.py b/tests/encryption_/tests.py new file mode 100644 index 000000000..463c93445 --- /dev/null +++ b/tests/encryption_/tests.py @@ -0,0 +1,394 @@ +import importlib +from datetime import datetime, time +from io import StringIO +from unittest.mock import patch + +import pymongo +from bson import json_util +from bson.binary import Binary +from django.core.management import call_command +from django.db import connections, models +from django.test import TransactionTestCase, modify_settings, override_settings + +from django_mongodb_backend.fields import EncryptedFieldMixin + +from .models import ( + Appointment, + Billing, + EncryptedNumbers, + Patient, + PatientPortalUser, + PatientRecord, +) +from .routers import TestEncryptedRouter + +EXPECTED_ENCRYPTED_FIELDS_MAP = { + "encryption__billing": { + "fields": [ + { + "bsonType": "string", + "path": "cc_type", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b" \x901\x89\x1f\xafAX\x9b*\xb1\xc7\xc5\xfdl\xa4", 4), + }, + { + "bsonType": "long", + "path": "cc_number", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x97\xb4\x9d\xb8\xd5\xa6Ay\x85\xfe\x00\xc0\xd4{\xa2\xff", 4), + }, + { + "bsonType": "decimal", + "path": "account_balance", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\xcc\x01-s\xea\xd9B\x8d\x80\xd7\xf8!n\xc6\xf5U", 4), + }, + ] + }, + "encryption__patientrecord": { + "fields": [ + { + "bsonType": "string", + "path": "ssn", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x14F\x89\xde\x8d\x04K7\xa9\x9a\xaf_\xca\x8a\xfb&", 4), + }, + { + "bsonType": "date", + "path": "birth_date", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"@\xdd\xb4\xd2%\xc2B\x94\xb5\x07\xbc(ER[s", 4), + }, + { + "bsonType": "binData", + "path": "profile_picture", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"Q\xa2\xebc!\xecD,\x8b\xe4$\xb6ul9\x9a", 4), + }, + { + "bsonType": "int", + "path": "patient_age", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\ro\x80\x1e\x8e1K\xde\xbc_\xc3bi\x95\xa6j", 4), + }, + { + "bsonType": "double", + "path": "weight", + "queries": {"queryType": "range"}, + # "keyId": Binary(b"\x9b\xfd:n\xe1\xd0N\xdd\xb3\xe7e)\x06\xea\x8a\x1d", 4), + }, + ] + }, + "encryption__patient": { + "fields": [ + { + "bsonType": "int", + "path": "patient_id", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x8ft\x16:\x8a\x91D\xc7\x8a\xdf\xe5O\n[\xfd\\", 4), + }, + { + "bsonType": "string", + "path": "patient_name", + # "keyId": Binary(b"<\x9b\xba\xeb:\xa4@m\x93\x0e\x0c\xcaN\x03\xfb\x05", 4), + }, + { + "bsonType": "string", + "path": "patient_notes", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\x01\xe7\xd1isnB$\xa9(gwO\xca\x10\xbd", 4), + }, + { + "bsonType": "date", + "path": "registration_date", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"F\xfb\xae\x82\xd5\x9a@\xee\xbfJ\xaf#\x9c:-I", 4), + }, + { + "bsonType": "bool", + "path": "is_active", + "queries": {"queryType": "equality"}, + # "keyId": Binary(b"\xb2\xb5\xc4K53A\xda\xb9V\xa6\xa9\x97\x94\xea;", 4), + }, + {"bsonType": "string", "path": "email", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__patientportaluser": { + "fields": [ + {"bsonType": "string", "path": "ip_address", "queries": {"queryType": "equality"}}, + {"bsonType": "string", "path": "url", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__encryptednumbers": { + "fields": [ + {"bsonType": "int", "path": "pos_bigint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "pos_smallint", "queries": {"queryType": "equality"}}, + {"bsonType": "int", "path": "smallint", "queries": {"queryType": "equality"}}, + ] + }, + "encryption__appointment": { + "fields": [{"bsonType": "date", "path": "time", "queries": {"queryType": "equality"}}] + }, +} + + +class EncryptedDurationField(EncryptedFieldMixin, models.DurationField): + """ + Unsupported by MongoDB when used with Queryable Encryption. + Included in tests until fix or wontfix. + """ + + +class EncryptedSlugField(EncryptedFieldMixin, models.SlugField): + """ + Unsupported by MongoDB when used with Queryable Encryption. + Included in tests until fix or wontfix. + """ + + +def reload_module(module): + """ + Reloads a module to ensure that any changes to environment variables + or other settings are applied without restarting the test runner. + """ + module = importlib.import_module(module) + importlib.reload(module) + return module + + +@modify_settings( + INSTALLED_APPS={"prepend": "django_mongodb_backend"}, +) +@override_settings(DATABASE_ROUTERS=[TestEncryptedRouter()]) +class EncryptedFieldTests(TransactionTestCase): + databases = {"default", "encrypted"} + available_apps = ["django_mongodb_backend", "encryption_"] + + def setUp(self): + self.appointment = Appointment(time="8:00") + self.appointment.save() + + self.billing = Billing(cc_type="Visa", cc_number=1234567890123456, account_balance=100.50) + self.billing.save() + + self.portal_user = PatientPortalUser( + ip_address="127.0.0.1", + url="https://example.com", + ) + self.portal_user.save() + + self.patientrecord = PatientRecord( + ssn="123-45-6789", + birth_date="1970-01-01", + profile_picture=b"image data", + weight=175.5, + patient_age=47, + ) + self.patientrecord.save() + + self.patient = Patient( + patient_id=1, + patient_name="John Doe", + patient_notes="patient notes " * 25, + registration_date=datetime(2023, 10, 1, 12, 0, 0), + is_active=True, + email="john.doe@example.com", + ) + self.patient.save() + + # TODO: Embed billing and patient_record models in patient model then add tests + + @classmethod + def setUpClass(cls): + super().setUpClass() + try: + from pymongo_auth_aws.auth import AwsCredential # noqa: PLC0415 + except ImportError: + cls.skipTest(cls, "pymongo_auth_aws not installed, skipping AWS credentials tests") + + cls.patch_aws = patch( + "pymongocrypt.synchronous.credentials.aws_temp_credentials", + return_value=AwsCredential(username="", password="", token=""), + ) + cls.patch_aws.start() + + cls.patch_azure = patch( + "pymongocrypt.synchronous.credentials._get_azure_credentials", return_value={} + ) + cls.patch_azure.start() + + cls.patch_gcp = patch( + "pymongocrypt.synchronous.credentials._get_gcp_credentials", return_value={} + ) + cls.patch_gcp.start() + + @classmethod + def tearDownClass(cls): + cls.patch_aws.stop() + cls.patch_azure.stop() + cls.patch_gcp.stop() + + def test_get_encrypted_fields_map(self): + """Test class method called by schema editor + and management command to get encrypted fields map for + `create_encrypted_collection` and `auto_encryption_opts` respectively. + There are no data keys in the results. + + Data keys for the schema editor are created by + `create_encrypted_collection` and data keys for the + management command are created by the management command + using code similar to the code in `create_encrypted_collection` + in Pymongo. + """ + expected_encrypted_fields_map = { + "encryption__patient": { + "fields": [ + { + "bsonType": "int", + "path": "patient_id", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "string", + "path": "patient_name", + }, + { + "bsonType": "string", + "path": "patient_notes", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "date", + "path": "registration_date", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "bool", + "path": "is_active", + "queries": {"queryType": "equality"}, + }, + { + "bsonType": "string", + "path": "email", + "queries": {"queryType": "equality"}, + }, + ] + }, + } + self.maxDiff = None + with connections["encrypted"].schema_editor() as editor: + db_table = self.patient._meta.db_table + self.assertEqual( + editor._get_encrypted_fields_map(self.patient), + expected_encrypted_fields_map[db_table], + ) + + def test_show_schema_map(self): + self.maxDiff = None + out = StringIO() + call_command( + "showschemamap", + "--database", + "encrypted", + verbosity=0, + stdout=out, + ) + # Remove keyIds since they are different for each run. + output_json = json_util.loads(out.getvalue()) + for table in output_json: + for field in output_json[table]["fields"]: + del field["keyId"] + # TODO: probably we don't need to test the entire mapping, otherwise it + # requires updates every time a new model or field is added! + self.assertEqual(EXPECTED_ENCRYPTED_FIELDS_MAP, output_json) + + def test_set_encrypted_fields_map_in_client(self): + # TODO: Create new client with and without schema map provided then + # sync database to ensure encrypted collections are created in both + pass + + def test_appointment(self): + self.assertEqual(Appointment.objects.get(time="8:00").time, time(8, 0)) + + def test_billing(self): + self.assertEqual( + Billing.objects.get(cc_number=1234567890123456).cc_number, 1234567890123456 + ) + self.assertEqual(Billing.objects.get(cc_type="Visa").cc_type, "Visa") + self.assertTrue(Billing.objects.filter(account_balance__gte=100.0).exists()) + + def test_patientportaluser(self): + self.assertEqual( + PatientPortalUser.objects.get(ip_address="127.0.0.1").ip_address, "127.0.0.1" + ) + + def test_patientrecord(self): + self.assertEqual(PatientRecord.objects.get(ssn="123-45-6789").ssn, "123-45-6789") + with self.assertRaises(PatientRecord.DoesNotExist): + PatientRecord.objects.get(ssn="000-00-0000") + self.assertTrue(PatientRecord.objects.filter(birth_date__gte="1969-01-01").exists()) + self.assertEqual( + PatientRecord.objects.get(ssn="123-45-6789").profile_picture, b"image data" + ) + with self.assertRaises(AssertionError): + self.assertEqual( + PatientRecord.objects.get(ssn="123-45-6789").profile_picture, b"bad image data" + ) + self.assertTrue(PatientRecord.objects.filter(patient_age__gte=40).exists()) + self.assertFalse(PatientRecord.objects.filter(patient_age__gte=200).exists()) + self.assertTrue(PatientRecord.objects.filter(weight__gte=175.0).exists()) + + # Test encrypted patient record in unencrypted database. + conn_params = connections["encrypted"].get_connection_params() + if conn_params.pop("auto_encryption_opts", False): + # Call MongoClient instead of get_new_connection because + # get_new_connection will return the encrypted connection + # from the connection pool. + with pymongo.MongoClient(**conn_params) as new_connection: + patientrecords = new_connection["test_encrypted"].encryption__patientrecord.find() + ssn = patientrecords[0]["ssn"] + self.assertTrue(isinstance(ssn, Binary)) + + def test_patient(self): + self.assertEqual( + Patient.objects.get(patient_notes="patient notes " * 25).patient_notes, + "patient notes " * 25, + ) + self.assertEqual( + Patient.objects.get( + registration_date=datetime(2023, 10, 1, 12, 0, 0) + ).registration_date, + datetime(2023, 10, 1, 12, 0, 0), + ) + self.assertTrue(Patient.objects.get(patient_id=1).is_active) + self.assertEqual( + Patient.objects.get(email="john.doe@example.com").email, "john.doe@example.com" + ) + + # Test decrypted patient record in encrypted database. + patients = connections["encrypted"].database.encryption__patient.find() + self.assertEqual(len(list(patients)), 1) + records = connections["encrypted"].database.encryption__patientrecord.find() + self.assertTrue("__safeContent__" in records[0]) + + +class EncryptedNumberFieldTests(EncryptedFieldTests): + def test_create_and_query(self): + EncryptedNumbers.objects.create( + pos_bigint=1000000, + # FIXME: pymongo.errors.EncryptionError: Cannot encrypt element of type int + # because schema requires that type is one of: [ long ] + # pos_int=1, + pos_smallint=12345, + smallint=-12345, + ) + + obj = EncryptedNumbers.objects.get(pos_bigint=1000000) + # obj = EncryptedNumbers.objects.get(pos_int=1) + obj = EncryptedNumbers.objects.get(pos_smallint=12345) + obj = EncryptedNumbers.objects.get(smallint=-12345) + + self.assertEqual(obj.pos_bigint, 1000000) + # self.assertEqual(obj.pos_int, 1) + self.assertEqual(obj.pos_smallint, 12345) + self.assertEqual(obj.smallint, -12345) From c7424292ca842e5101a216c51bd30054cbbc4e58 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Mon, 4 Aug 2025 07:37:30 -0400 Subject: [PATCH 2/3] Rename get_encrypted_fields_map -> showschemamap --- docs/source/ref/django-admin.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/ref/django-admin.rst b/docs/source/ref/django-admin.rst index 4ee6d2b77..13927af06 100644 --- a/docs/source/ref/django-admin.rst +++ b/docs/source/ref/django-admin.rst @@ -28,10 +28,10 @@ Available commands Defaults to ``default``. -``get_encrypted_fields_map`` +``showschemamap`` ---------------------------- -.. django-admin:: get_encrypted_fields_map +.. django-admin:: showschemamap Creates a schema map for encrypted fields that can be used with :class:`~pymongo.encryption_options.AutoEncryptionOpts` to configure @@ -41,5 +41,3 @@ Available commands Specifies the database to use. Defaults to ``default``. - -.. TODO: Clarify how database specified could affect output. From 5e0818dd73d86b4c79674a45060c2190f65ef2e5 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 4 Aug 2025 09:21:57 -0400 Subject: [PATCH 3/3] remove hardcoded db name --- tests/encryption_/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/encryption_/tests.py b/tests/encryption_/tests.py index 463c93445..409f844e7 100644 --- a/tests/encryption_/tests.py +++ b/tests/encryption_/tests.py @@ -6,6 +6,7 @@ import pymongo from bson import json_util from bson.binary import Binary +from django.conf import settings from django.core.management import call_command from django.db import connections, models from django.test import TransactionTestCase, modify_settings, override_settings @@ -340,12 +341,13 @@ def test_patientrecord(self): # Test encrypted patient record in unencrypted database. conn_params = connections["encrypted"].get_connection_params() + db_name = settings.DATABASES["encrypted"]["NAME"] if conn_params.pop("auto_encryption_opts", False): # Call MongoClient instead of get_new_connection because # get_new_connection will return the encrypted connection # from the connection pool. with pymongo.MongoClient(**conn_params) as new_connection: - patientrecords = new_connection["test_encrypted"].encryption__patientrecord.find() + patientrecords = new_connection[db_name].encryption__patientrecord.find() ssn = patientrecords[0]["ssn"] self.assertTrue(isinstance(ssn, Binary))