From 9306a9ea48cb758dbaf44b67a6f8cbe171c2bf69 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Mon, 17 Mar 2025 09:29:07 -0700 Subject: [PATCH 01/20] REVERT ME: modify pymongocrypt_source to test branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 993b3e5aee..824c862f88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" ] pymongocrypt_source = [ - "pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python" + "pymongocrypt@git+https://github.com/sleepyStick/libmongocrypt@PYTHON-5162#subdirectory=bindings/python" ] perf = ["simplejson"] typing = [ From c69c3bdab48be30993f0964773e61d7faf4ea082 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Mon, 17 Mar 2025 09:37:45 -0700 Subject: [PATCH 02/20] resync specs --- .../etc/data/lookup/key-doc.json | 30 +++++++++++++++++++ .../etc/data/lookup/schema-csfle.json | 19 ++++++++++++ .../etc/data/lookup/schema-csfle2.json | 19 ++++++++++++ .../etc/data/lookup/schema-qe.json | 20 +++++++++++++ .../etc/data/lookup/schema-qe2.json | 20 +++++++++++++ 5 files changed, 108 insertions(+) create mode 100644 test/client-side-encryption/etc/data/lookup/key-doc.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-csfle.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-csfle2.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-qe.json create mode 100644 test/client-side-encryption/etc/data/lookup/schema-qe2.json diff --git a/test/client-side-encryption/etc/data/lookup/key-doc.json b/test/client-side-encryption/etc/data/lookup/key-doc.json new file mode 100644 index 0000000000..566b56c354 --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/key-doc.json @@ -0,0 +1,30 @@ +{ + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-csfle.json b/test/client-side-encryption/etc/data/lookup/schema-csfle.json new file mode 100644 index 0000000000..29ac9ad5da --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-csfle.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-csfle2.json b/test/client-side-encryption/etc/data/lookup/schema-csfle2.json new file mode 100644 index 0000000000..3f1c02781c --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-csfle2.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle2": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-qe.json b/test/client-side-encryption/etc/data/lookup/schema-qe.json new file mode 100644 index 0000000000..9428ea1b45 --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-qe.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe.esc", + "ecocCollection": "enxcol_.qe.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} diff --git a/test/client-side-encryption/etc/data/lookup/schema-qe2.json b/test/client-side-encryption/etc/data/lookup/schema-qe2.json new file mode 100644 index 0000000000..77d5bd37cb --- /dev/null +++ b/test/client-side-encryption/etc/data/lookup/schema-qe2.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe2.esc", + "ecocCollection": "enxcol_.qe2.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe2", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} From 1fb56b0cd7bd8baa5812d771273cfc2286f4f94a Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 10:23:49 -0700 Subject: [PATCH 03/20] add tests draft --- pymongo/asynchronous/encryption.py | 9 +- pymongo/synchronous/encryption.py | 9 +- test/asynchronous/test_encryption.py | 328 ++++++++++++++++++++++++++- test/test_encryption.py | 328 ++++++++++++++++++++++++++- 4 files changed, 666 insertions(+), 8 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index ef8d817b2c..43a1ac7f4f 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -242,7 +242,7 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: ) raise exc from final_err - async def collection_info(self, database: str, filter: bytes) -> Optional[bytes]: + async def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]: """Get the collection info for a namespace. The returned collection info is passed to libmongocrypt which reads @@ -256,8 +256,11 @@ async def collection_info(self, database: str, filter: bytes) -> Optional[bytes] async with await self.client_ref()[database].list_collections( filter=RawBSONDocument(filter) ) as cursor: + lst = [] async for doc in cursor: - return _dict_to_bson(doc, False, _DATA_KEY_OPTS) + lst.append(_dict_to_bson(doc, False, _DATA_KEY_OPTS)) + if len(lst) > 0: + return lst return None def spawn(self) -> None: @@ -551,7 +554,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. if kwargs.get("key_expiration_ms") is None: kwargs.pop("key_expiration_ms", None) - return MongoCryptOptions(**kwargs) + return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True) class AsyncClientEncryption(Generic[_DocumentType]): diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index a97534ed41..9899548330 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -241,7 +241,7 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: ) raise exc from final_err - def collection_info(self, database: str, filter: bytes) -> Optional[bytes]: + def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]]: """Get the collection info for a namespace. The returned collection info is passed to libmongocrypt which reads @@ -253,8 +253,11 @@ def collection_info(self, database: str, filter: bytes) -> Optional[bytes]: :return: The first document from the listCollections command response as BSON. """ with self.client_ref()[database].list_collections(filter=RawBSONDocument(filter)) as cursor: + lst = [] for doc in cursor: - return _dict_to_bson(doc, False, _DATA_KEY_OPTS) + lst.append(_dict_to_bson(doc, False, _DATA_KEY_OPTS)) + if len(lst) > 0: + return lst return None def spawn(self) -> None: @@ -548,7 +551,7 @@ def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions: # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms. if kwargs.get("key_expiration_ms") is None: kwargs.pop("key_expiration_ms", None) - return MongoCryptOptions(**kwargs) + return MongoCryptOptions(**kwargs, enable_multiple_collinfo=True) class ClientEncryption(Generic[_DocumentType]): diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 728a6f9139..7bb754724c 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -73,7 +73,7 @@ is_greenthread_patched, ) -from bson import DatetimeMS, Decimal128, encode, json_util +from bson import BSON, DatetimeMS, Decimal128, encode, json_util from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation from bson.codec_options import CodecOptions from bson.errors import BSONError @@ -94,6 +94,7 @@ EncryptionError, InvalidOperation, OperationFailure, + PyMongoError, ServerSelectionTimeoutError, WriteError, ) @@ -2419,6 +2420,331 @@ async def test_05_roundtrip_encrypted_unindexed(self): self.assertEqual(decrypted, val) +# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup +class TestLookupProse(AsyncEncryptionIntegrationTest): + async def asyncSetUp(self): + await super().asyncSetUp() + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + await self.encrypted_client.db.drop_collection("keyvault") + + keyvault = json_data("etc", "data", "lookup", "key-doc.json") + await create_key_vault(self.encrypted_client.keyvault, keyvault) + + await self.encrypted_client.db.drop("csfle") + await self.encrypted_client.db.create_collection( + "csfle", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, + ) + + await self.encrypted_client.db.drop("csfle2") + await self.encrypted_client.db.create_collection( + "csfle2", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, + ) + + await self.encrypted_client.db.drop("qe") + await self.encrypted_client.db.create_collection( + "qe", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe.json")} + ) + + await self.encrypted_client.db.drop("qe2") + await self.encrypted_client.db.create_collection( + "qe2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe2.json")} + ) + + await self.encrypted_client.db.drop("no_schema") + await self.encrypted_client.db.create_collection("no-schema") + + await self.encrypted_client.db.drop("no_schema2") + await self.encrypted_client.db.create_collection("no_schema2") + + self.unencrypted_client = await self.async_rs_or_single_client() + + self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = self.unencrypted_client.db.csfle.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = self.unencrypted_client.db.csfle2.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = self.unencrypted_client.db.qe.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = self.unencrypted_client.db.qe2.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_1(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_2(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_3(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "csfle", + "as": "matched", + "pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_4(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [ + {"$match": {"qe": "qe"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_5(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "csfle2", + "as": "matched", + "pipeline": [ + {"$match": {"csfle2": "csfle2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_6(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "qe2", + "as": "matched", + "pipeline": [ + {"$match": {"qe2": "qe2"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_7(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = anext( + await self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @async_client_context.require_version_min(8, 1, -1) + async def test_8(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + # not sure if this is the right error! + with self.assertRaises(PyMongoError) as exc: + _ = anext( + await self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + # check that the exc contains the substring not supported + self.assertTrue("not supported" in exc.msg) + + # Test requires mongocryptd/crypt_shared <8.1. + @async_client_context.require_version_max(8, 1, -1) + async def test_9(self): + self.encrypted_client = await self.async_rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + # not sure if this is the right error! + with self.assertRaises(PyMongoError) as exc: + _ = anext( + await self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + # check that the exc contains the substring Upgrade + self.assertTrue("Upgrade" in exc.msg) + + # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap class TestRewrapWithSeparateClientEncryption(AsyncEncryptionIntegrationTest): MASTER_KEYS: Mapping[str, Mapping[str, Any]] = { diff --git a/test/test_encryption.py b/test/test_encryption.py index 36c0ab0e24..33136d0f3b 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -73,7 +73,7 @@ ) from test.utils_spec_runner import SpecRunner -from bson import DatetimeMS, Decimal128, encode, json_util +from bson import BSON, DatetimeMS, Decimal128, encode, json_util from bson.binary import UUID_SUBTYPE, Binary, UuidRepresentation from bson.codec_options import CodecOptions from bson.errors import BSONError @@ -91,6 +91,7 @@ EncryptionError, InvalidOperation, OperationFailure, + PyMongoError, ServerSelectionTimeoutError, WriteError, ) @@ -2403,6 +2404,331 @@ def test_05_roundtrip_encrypted_unindexed(self): self.assertEqual(decrypted, val) +# https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup +class TestLookupProse(EncryptionIntegrationTest): + def setUp(self): + super().setUp() + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + self.encrypted_client.db.drop_collection("keyvault") + + keyvault = json_data("etc", "data", "lookup", "key-doc.json") + create_key_vault(self.encrypted_client.keyvault, keyvault) + + self.encrypted_client.db.drop("csfle") + self.encrypted_client.db.create_collection( + "csfle", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, + ) + + self.encrypted_client.db.drop("csfle2") + self.encrypted_client.db.create_collection( + "csfle2", + validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, + ) + + self.encrypted_client.db.drop("qe") + self.encrypted_client.db.create_collection( + "qe", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe.json")} + ) + + self.encrypted_client.db.drop("qe2") + self.encrypted_client.db.create_collection( + "qe2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe2.json")} + ) + + self.encrypted_client.db.drop("no_schema") + self.encrypted_client.db.create_collection("no-schema") + + self.encrypted_client.db.drop("no_schema2") + self.encrypted_client.db.create_collection("no_schema2") + + self.unencrypted_client = self.rs_or_single_client() + + self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = self.unencrypted_client.db.csfle.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = self.unencrypted_client.db.csfle2.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = self.unencrypted_client.db.qe.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = self.unencrypted_client.db.qe2.find_one() + self.assertTrue(isinstance(doc, BSON)) + self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_1(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_2(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_3(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "csfle", + "as": "matched", + "pipeline": [{"$match": {"csfle": "csfle"}}, {"$project": {"_id": 0}}], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_4(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "qe", + "as": "matched", + "pipeline": [ + {"$match": {"qe": "qe"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_5(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "csfle2", + "as": "matched", + "pipeline": [ + {"$match": {"csfle2": "csfle2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_6(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.qe.aggregate( + [ + {"$match": {"qe": "qe"}}, + { + "$lookup": { + "from": "qe2", + "as": "matched", + "pipeline": [ + {"$match": {"qe2": "qe2"}}, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ], + } + }, + {"$project": {"_id": 0, "__safeContent__": 0}}, + ] + ) + ) + self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_7(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + doc = next( + self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + + # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + @client_context.require_version_min(8, 1, -1) + def test_8(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + # not sure if this is the right error! + with self.assertRaises(PyMongoError) as exc: + _ = next( + self.encrypted_client.db.no_schema.aggregate( + [ + {"$match": {"no_schema": "no_schema"}}, + { + "$lookup": { + "from": "no_schema2", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema2": "no_schema2"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + # check that the exc contains the substring not supported + self.assertTrue("not supported" in exc.msg) + + # Test requires mongocryptd/crypt_shared <8.1. + @client_context.require_version_max(8, 1, -1) + def test_9(self): + self.encrypted_client = self.rs_or_single_client( + auto_encryption_opts=AutoEncryptionOpts( + key_vault_namespace="db.keyvault", + kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, + ) + ) + # not sure if this is the right error! + with self.assertRaises(PyMongoError) as exc: + _ = next( + self.encrypted_client.db.csfle.aggregate( + [ + {"$match": {"csfle": "csfle"}}, + { + "$lookup": { + "from": "no_schema", + "as": "matched", + "pipeline": [ + {"$match": {"no_schema": "no_schema"}}, + {"$project": {"_id": 0}}, + ], + } + }, + {"$project": {"_id": 0}}, + ] + ) + ) + # check that the exc contains the substring Upgrade + self.assertTrue("Upgrade" in exc.msg) + + # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap class TestRewrapWithSeparateClientEncryption(EncryptionIntegrationTest): MASTER_KEYS: Mapping[str, Mapping[str, Any]] = { From ff300a99725099606ca2fc06ff70aa658ac9dd37 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 11:20:13 -0700 Subject: [PATCH 04/20] fix typing --- test/asynchronous/test_encryption.py | 4 ++-- test/test_encryption.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 7bb754724c..5f76081c4a 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2710,7 +2710,7 @@ async def test_8(self): ) ) # check that the exc contains the substring not supported - self.assertTrue("not supported" in exc.msg) + self.assertTrue("not supported" in str(exc)) # Test requires mongocryptd/crypt_shared <8.1. @async_client_context.require_version_max(8, 1, -1) @@ -2742,7 +2742,7 @@ async def test_9(self): ) ) # check that the exc contains the substring Upgrade - self.assertTrue("Upgrade" in exc.msg) + self.assertTrue("Upgrade" in str(exc)) # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap diff --git a/test/test_encryption.py b/test/test_encryption.py index 33136d0f3b..df2659050a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2694,7 +2694,7 @@ def test_8(self): ) ) # check that the exc contains the substring not supported - self.assertTrue("not supported" in exc.msg) + self.assertTrue("not supported" in str(exc)) # Test requires mongocryptd/crypt_shared <8.1. @client_context.require_version_max(8, 1, -1) @@ -2726,7 +2726,7 @@ def test_9(self): ) ) # check that the exc contains the substring Upgrade - self.assertTrue("Upgrade" in exc.msg) + self.assertTrue("Upgrade" in str(exc)) # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap From ea56a2e5b0a245bcd6dab3f6d0b46e55c6b7be31 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 11:20:26 -0700 Subject: [PATCH 05/20] revert this in the future! --- .evergreen/scripts/configure-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fa37b8fb08..e6e6b59a89 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -76,8 +76,8 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs +BRANCH=PYTHON-5046 +ORG=sleepyStick git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env From 3ea9a259c18f0199247dc98004ffddbcab724954 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 11:55:43 -0700 Subject: [PATCH 06/20] undo changes --- .evergreen/scripts/configure-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index e6e6b59a89..fa37b8fb08 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -76,8 +76,8 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS -BRANCH=PYTHON-5046 -ORG=sleepyStick +BRANCH=master +ORG=mongodb-labs git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env From d967ffa0234209ddcc04b0e79feeeb5b476435f1 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 12:04:40 -0700 Subject: [PATCH 07/20] modify uv.lock --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 8b5d592dc0..23f4b242e7 100644 --- a/uv.lock +++ b/uv.lock @@ -1109,7 +1109,7 @@ eventlet = [{ name = "eventlet" }] gevent = [{ name = "gevent" }] mockupdb = [{ name = "mockupdb", git = "https://github.com/mongodb-labs/mongo-mockup-db?rev=master" }] perf = [{ name = "simplejson" }] -pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] +pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162" }] typing = [ { name = "mypy", specifier = "==1.14.1" }, { name = "pip" }, @@ -1133,7 +1133,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1e96c283162aa7789cf01f99f211e0ace8e6d49f" } +source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#caacf80d5d67a9667e0c72450c5a194cd0a00f18" } dependencies = [ { name = "cffi" }, { name = "cryptography" }, From c9a4bd03232e7e40eaf67aacd8f59d2c567d92f9 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 12:12:30 -0700 Subject: [PATCH 08/20] change uv.lock commit hash? --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 23f4b242e7..726f942636 100644 --- a/uv.lock +++ b/uv.lock @@ -1133,7 +1133,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#caacf80d5d67a9667e0c72450c5a194cd0a00f18" } +source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#039e85e16156727176b2193730ecb0038c9f48c1" } dependencies = [ { name = "cffi" }, { name = "cryptography" }, From 590e45551d862f872a616f8e66e6b0cf6d1077bf Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 16:55:57 -0700 Subject: [PATCH 09/20] still not done, but test setup runs now --- test/asynchronous/test_encryption.py | 54 +++++++++++++++------------- test/test_encryption.py | 34 ++++++++++-------- uv.lock | 2 +- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 5f76081c4a..4f7b72e36a 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2422,6 +2422,9 @@ async def test_05_roundtrip_encrypted_unindexed(self): # https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup class TestLookupProse(AsyncEncryptionIntegrationTest): + # check libmongocrypt version? + @async_client_context.require_no_standalone + @async_client_context.require_version_min(7, 0, -1) async def asyncSetUp(self): await super().asyncSetUp() self.encrypted_client = await self.async_rs_or_single_client( @@ -2432,53 +2435,54 @@ async def asyncSetUp(self): ) await self.encrypted_client.db.drop_collection("keyvault") - keyvault = json_data("etc", "data", "lookup", "key-doc.json") - await create_key_vault(self.encrypted_client.keyvault, keyvault) + key_doc = json_data("etc", "data", "lookup", "key-doc.json") + key_vault = await create_key_vault(self.encrypted_client.db.keyvault, key_doc) + self.addCleanup(key_vault.drop) - await self.encrypted_client.db.drop("csfle") + await self.encrypted_client.db.drop_collection("csfle") await self.encrypted_client.db.create_collection( "csfle", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, ) - await self.encrypted_client.db.drop("csfle2") + await self.encrypted_client.db.drop_collection("csfle2") await self.encrypted_client.db.create_collection( "csfle2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, ) - await self.encrypted_client.db.drop("qe") + await self.encrypted_client.db.drop_collection("qe") await self.encrypted_client.db.create_collection( - "qe", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe.json")} + "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") ) - await self.encrypted_client.db.drop("qe2") + await self.encrypted_client.db.drop_collection("qe2") await self.encrypted_client.db.create_collection( - "qe2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe2.json")} + "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") ) - await self.encrypted_client.db.drop("no_schema") - await self.encrypted_client.db.create_collection("no-schema") + await self.encrypted_client.db.drop_collection("no_schema") + await self.encrypted_client.db.create_collection("no_schema") - await self.encrypted_client.db.drop("no_schema2") + await self.encrypted_client.db.drop_collection("no_schema2") await self.encrypted_client.db.create_collection("no_schema2") self.unencrypted_client = await self.async_rs_or_single_client() - self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) - doc = self.unencrypted_client.db.csfle.find_one() - self.assertTrue(isinstance(doc, BSON)) - self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) - doc = self.unencrypted_client.db.csfle2.find_one() - self.assertTrue(isinstance(doc, BSON)) - self.encrypted_client.db.qe.insert_one({"qe": "qe"}) - doc = self.unencrypted_client.db.qe.find_one() - self.assertTrue(isinstance(doc, BSON)) - self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) - doc = self.unencrypted_client.db.qe2.find_one() - self.assertTrue(isinstance(doc, BSON)) - self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) - self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + await self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = await self.unencrypted_client.db.csfle.find_one() + self.assertTrue(isinstance(doc["csfle"], Binary)) + await self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = await self.unencrypted_client.db.csfle2.find_one() + self.assertTrue(isinstance(doc["csfle2"], Binary)) + await self.encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = await self.unencrypted_client.db.qe.find_one() + self.assertTrue(isinstance(doc["qe"], Binary)) + await self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = await self.unencrypted_client.db.qe2.find_one() + self.assertTrue(isinstance(doc["qe2"], Binary)) + await self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + await self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) diff --git a/test/test_encryption.py b/test/test_encryption.py index df2659050a..be94148bd4 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2406,6 +2406,9 @@ def test_05_roundtrip_encrypted_unindexed(self): # https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup class TestLookupProse(EncryptionIntegrationTest): + # check libmongocrypt version? + @client_context.require_no_standalone + @client_context.require_version_min(7, 0, -1) def setUp(self): super().setUp() self.encrypted_client = self.rs_or_single_client( @@ -2416,51 +2419,52 @@ def setUp(self): ) self.encrypted_client.db.drop_collection("keyvault") - keyvault = json_data("etc", "data", "lookup", "key-doc.json") - create_key_vault(self.encrypted_client.keyvault, keyvault) + key_doc = json_data("etc", "data", "lookup", "key-doc.json") + key_vault = create_key_vault(self.encrypted_client.db.keyvault, key_doc) + self.addCleanup(key_vault.drop) - self.encrypted_client.db.drop("csfle") + self.encrypted_client.db.drop_collection("csfle") self.encrypted_client.db.create_collection( "csfle", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, ) - self.encrypted_client.db.drop("csfle2") + self.encrypted_client.db.drop_collection("csfle2") self.encrypted_client.db.create_collection( "csfle2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, ) - self.encrypted_client.db.drop("qe") + self.encrypted_client.db.drop_collection("qe") self.encrypted_client.db.create_collection( - "qe", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe.json")} + "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") ) - self.encrypted_client.db.drop("qe2") + self.encrypted_client.db.drop_collection("qe2") self.encrypted_client.db.create_collection( - "qe2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-qe2.json")} + "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") ) - self.encrypted_client.db.drop("no_schema") - self.encrypted_client.db.create_collection("no-schema") + self.encrypted_client.db.drop_collection("no_schema") + self.encrypted_client.db.create_collection("no_schema") - self.encrypted_client.db.drop("no_schema2") + self.encrypted_client.db.drop_collection("no_schema2") self.encrypted_client.db.create_collection("no_schema2") self.unencrypted_client = self.rs_or_single_client() self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) doc = self.unencrypted_client.db.csfle.find_one() - self.assertTrue(isinstance(doc, BSON)) + self.assertTrue(isinstance(doc["csfle"], Binary)) self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) doc = self.unencrypted_client.db.csfle2.find_one() - self.assertTrue(isinstance(doc, BSON)) + self.assertTrue(isinstance(doc["csfle2"], Binary)) self.encrypted_client.db.qe.insert_one({"qe": "qe"}) doc = self.unencrypted_client.db.qe.find_one() - self.assertTrue(isinstance(doc, BSON)) + self.assertTrue(isinstance(doc["qe"], Binary)) self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) doc = self.unencrypted_client.db.qe2.find_one() - self.assertTrue(isinstance(doc, BSON)) + self.assertTrue(isinstance(doc["qe2"], Binary)) self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) diff --git a/uv.lock b/uv.lock index 726f942636..634927f406 100644 --- a/uv.lock +++ b/uv.lock @@ -1133,7 +1133,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#039e85e16156727176b2193730ecb0038c9f48c1" } +source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#5be264a1d523d0d410f69327efa3c284503fc1ca" } dependencies = [ { name = "cffi" }, { name = "cryptography" }, From 6e33d29f9d0138305c993f0d84d7b013745ff0c4 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 17:42:52 -0700 Subject: [PATCH 10/20] add missing await --- test/asynchronous/test_encryption.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 4f7b72e36a..457ebe7a14 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2493,7 +2493,7 @@ async def test_1(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, @@ -2522,7 +2522,7 @@ async def test_2(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, @@ -2551,7 +2551,7 @@ async def test_3(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, @@ -2577,7 +2577,7 @@ async def test_4(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, @@ -2606,7 +2606,7 @@ async def test_5(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, @@ -2635,7 +2635,7 @@ async def test_6(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, @@ -2664,7 +2664,7 @@ async def test_7(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - doc = anext( + doc = await anext( await self.encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, @@ -2695,7 +2695,7 @@ async def test_8(self): ) # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: - _ = anext( + _ = await anext( await self.encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, @@ -2727,7 +2727,7 @@ async def test_9(self): ) # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: - _ = anext( + _ = await anext( await self.encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, From 74fac795f8cad9ce1578fbfe0c86efd489415282 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Tue, 18 Mar 2025 19:07:03 -0700 Subject: [PATCH 11/20] fix test 8 --- test/asynchronous/test_encryption.py | 13 ++++--------- test/test_encryption.py | 13 ++++--------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 457ebe7a14..1f4417cd15 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2696,24 +2696,20 @@ async def test_8(self): # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = await anext( - await self.encrypted_client.db.no_schema.aggregate( + await self.encrypted_client.db.csfle.aggregate( [ - {"$match": {"no_schema": "no_schema"}}, + {"$match": {"csfle": "qe"}}, { "$lookup": { - "from": "no_schema2", + "from": "qe", "as": "matched", - "pipeline": [ - {"$match": {"no_schema2": "no_schema2"}}, - {"$project": {"_id": 0}}, - ], + "pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}], } }, {"$project": {"_id": 0}}, ] ) ) - # check that the exc contains the substring not supported self.assertTrue("not supported" in str(exc)) # Test requires mongocryptd/crypt_shared <8.1. @@ -2745,7 +2741,6 @@ async def test_9(self): ] ) ) - # check that the exc contains the substring Upgrade self.assertTrue("Upgrade" in str(exc)) diff --git a/test/test_encryption.py b/test/test_encryption.py index be94148bd4..b638768a37 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2680,24 +2680,20 @@ def test_8(self): # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = next( - self.encrypted_client.db.no_schema.aggregate( + self.encrypted_client.db.csfle.aggregate( [ - {"$match": {"no_schema": "no_schema"}}, + {"$match": {"csfle": "qe"}}, { "$lookup": { - "from": "no_schema2", + "from": "qe", "as": "matched", - "pipeline": [ - {"$match": {"no_schema2": "no_schema2"}}, - {"$project": {"_id": 0}}, - ], + "pipeline": [{"$match": {"qe": "qe"}}, {"$project": {"_id": 0}}], } }, {"$project": {"_id": 0}}, ] ) ) - # check that the exc contains the substring not supported self.assertTrue("not supported" in str(exc)) # Test requires mongocryptd/crypt_shared <8.1. @@ -2729,7 +2725,6 @@ def test_9(self): ] ) ) - # check that the exc contains the substring Upgrade self.assertTrue("Upgrade" in str(exc)) From daa420deee9f76ca24ec7bf1dbe7b429cee2cc1e Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 11:39:34 -0700 Subject: [PATCH 12/20] remove notes to self --- test/asynchronous/test_encryption.py | 12 ------------ test/test_encryption.py | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 1f4417cd15..e8a86b13d1 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2422,7 +2422,6 @@ async def test_05_roundtrip_encrypted_unindexed(self): # https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup class TestLookupProse(AsyncEncryptionIntegrationTest): - # check libmongocrypt version? @async_client_context.require_no_standalone @async_client_context.require_version_min(7, 0, -1) async def asyncSetUp(self): @@ -2484,7 +2483,6 @@ async def asyncSetUp(self): await self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) await self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_1(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2513,7 +2511,6 @@ async def test_1(self): ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_2(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2542,7 +2539,6 @@ async def test_2(self): ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_3(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2568,7 +2564,6 @@ async def test_3(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_4(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2597,7 +2592,6 @@ async def test_4(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_5(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2626,7 +2620,6 @@ async def test_5(self): ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_6(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2655,7 +2648,6 @@ async def test_6(self): ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_7(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2684,7 +2676,6 @@ async def test_7(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @async_client_context.require_version_min(8, 1, -1) async def test_8(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2693,7 +2684,6 @@ async def test_8(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = await anext( await self.encrypted_client.db.csfle.aggregate( @@ -2712,7 +2702,6 @@ async def test_8(self): ) self.assertTrue("not supported" in str(exc)) - # Test requires mongocryptd/crypt_shared <8.1. @async_client_context.require_version_max(8, 1, -1) async def test_9(self): self.encrypted_client = await self.async_rs_or_single_client( @@ -2721,7 +2710,6 @@ async def test_9(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = await anext( await self.encrypted_client.db.csfle.aggregate( diff --git a/test/test_encryption.py b/test/test_encryption.py index b638768a37..a52a67ff47 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2406,7 +2406,6 @@ def test_05_roundtrip_encrypted_unindexed(self): # https://github.com/mongodb/specifications/blob/527e22d5090ec48bf1e144c45fc831de0f1935f6/source/client-side-encryption/tests/README.md#25-test-lookup class TestLookupProse(EncryptionIntegrationTest): - # check libmongocrypt version? @client_context.require_no_standalone @client_context.require_version_min(7, 0, -1) def setUp(self): @@ -2468,7 +2467,6 @@ def setUp(self): self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_1(self): self.encrypted_client = self.rs_or_single_client( @@ -2497,7 +2495,6 @@ def test_1(self): ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_2(self): self.encrypted_client = self.rs_or_single_client( @@ -2526,7 +2523,6 @@ def test_2(self): ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_3(self): self.encrypted_client = self.rs_or_single_client( @@ -2552,7 +2548,6 @@ def test_3(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_4(self): self.encrypted_client = self.rs_or_single_client( @@ -2581,7 +2576,6 @@ def test_4(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_5(self): self.encrypted_client = self.rs_or_single_client( @@ -2610,7 +2604,6 @@ def test_5(self): ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_6(self): self.encrypted_client = self.rs_or_single_client( @@ -2639,7 +2632,6 @@ def test_6(self): ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_7(self): self.encrypted_client = self.rs_or_single_client( @@ -2668,7 +2660,6 @@ def test_7(self): ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) - # Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. @client_context.require_version_min(8, 1, -1) def test_8(self): self.encrypted_client = self.rs_or_single_client( @@ -2677,7 +2668,6 @@ def test_8(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = next( self.encrypted_client.db.csfle.aggregate( @@ -2696,7 +2686,6 @@ def test_8(self): ) self.assertTrue("not supported" in str(exc)) - # Test requires mongocryptd/crypt_shared <8.1. @client_context.require_version_max(8, 1, -1) def test_9(self): self.encrypted_client = self.rs_or_single_client( @@ -2705,7 +2694,6 @@ def test_9(self): kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - # not sure if this is the right error! with self.assertRaises(PyMongoError) as exc: _ = next( self.encrypted_client.db.csfle.aggregate( From 2c46edbfbddbf6848a9ec58324a691341fc0b1ef Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 12:59:10 -0700 Subject: [PATCH 13/20] Revert "REVERT ME: modify pymongocrypt_source to test branch" This reverts commit 9306a9ea48cb758dbaf44b67a6f8cbe171c2bf69. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 824c862f88..993b3e5aee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" ] pymongocrypt_source = [ - "pymongocrypt@git+https://github.com/sleepyStick/libmongocrypt@PYTHON-5162#subdirectory=bindings/python" + "pymongocrypt@git+https://github.com/mongodb/libmongocrypt@master#subdirectory=bindings/python" ] perf = ["simplejson"] typing = [ From 8770445ef3db0336fe6aa913bf73014b740fb139 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 13:01:03 -0700 Subject: [PATCH 14/20] Update uv.lock --- uv.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 634927f406..fbf1f06702 100644 --- a/uv.lock +++ b/uv.lock @@ -997,7 +997,6 @@ sdist = { url = "https://files.pythonhosted.org/packages/07/e9/ae44ea7d7605df9e5 [[package]] name = "pymongo" -version = "4.12.0.dev0" source = { editable = "." } dependencies = [ { name = "dnspython" }, @@ -1109,7 +1108,7 @@ eventlet = [{ name = "eventlet" }] gevent = [{ name = "gevent" }] mockupdb = [{ name = "mockupdb", git = "https://github.com/mongodb-labs/mongo-mockup-db?rev=master" }] perf = [{ name = "simplejson" }] -pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162" }] +pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ { name = "mypy", specifier = "==1.14.1" }, { name = "pip" }, @@ -1133,7 +1132,7 @@ wheels = [ [[package]] name = "pymongocrypt" version = "1.13.0.dev0" -source = { git = "https://github.com/sleepyStick/libmongocrypt?subdirectory=bindings%2Fpython&rev=PYTHON-5162#5be264a1d523d0d410f69327efa3c284503fc1ca" } +source = { git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master#1cad4ad1c4cd6c11c6a4710da2127dab6a374471" } dependencies = [ { name = "cffi" }, { name = "cryptography" }, From 8c45daa6b640acbf8139cbeabb71665bd72de05c Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 15:28:55 -0700 Subject: [PATCH 15/20] address comments --- doc/changelog.rst | 1 + pymongo/asynchronous/encryption.py | 9 +-- pymongo/synchronous/encryption.py | 9 +-- test/asynchronous/test_encryption.py | 105 ++++++++++++--------------- test/test_encryption.py | 105 ++++++++++++--------------- uv.lock | 1 + 6 files changed, 100 insertions(+), 130 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 21e86953c6..257cf1b747 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ PyMongo 4.12 brings a number of changes including: - Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to :class:`~pymongo.encryption_options.AutoEncryptionOpts`. +- Support for $lookup in CSFLE and QE.pr Issues Resolved ............... diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 43a1ac7f4f..6d7cea718f 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -251,17 +251,12 @@ async def collection_info(self, database: str, filter: bytes) -> Optional[list[b :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The first document from the listCollections command response as BSON. + :return: The all documents from the listCollections command response as BSON. """ async with await self.client_ref()[database].list_collections( filter=RawBSONDocument(filter) ) as cursor: - lst = [] - async for doc in cursor: - lst.append(_dict_to_bson(doc, False, _DATA_KEY_OPTS)) - if len(lst) > 0: - return lst - return None + return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) async for doc in cursor] def spawn(self) -> None: """Spawn mongocryptd. diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 9899548330..052f4c5a07 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -250,15 +250,10 @@ def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]] :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The first document from the listCollections command response as BSON. + :return: The all documents from the listCollections command response as BSON. """ with self.client_ref()[database].list_collections(filter=RawBSONDocument(filter)) as cursor: - lst = [] - for doc in cursor: - lst.append(_dict_to_bson(doc, False, _DATA_KEY_OPTS)) - if len(lst) > 0: - return lst - return None + return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) for doc in cursor] def spawn(self) -> None: """Spawn mongocryptd. diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index e8a86b13d1..06ccb28c8f 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2426,73 +2426,62 @@ class TestLookupProse(AsyncEncryptionIntegrationTest): @async_client_context.require_version_min(7, 0, -1) async def asyncSetUp(self): await super().asyncSetUp() - self.encrypted_client = await self.async_rs_or_single_client( + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - await self.encrypted_client.db.drop_collection("keyvault") + await encrypted_client.drop_database("db") key_doc = json_data("etc", "data", "lookup", "key-doc.json") - key_vault = await create_key_vault(self.encrypted_client.db.keyvault, key_doc) + key_vault = await create_key_vault(encrypted_client.db.keyvault, key_doc) self.addCleanup(key_vault.drop) - await self.encrypted_client.db.drop_collection("csfle") - await self.encrypted_client.db.create_collection( + await encrypted_client.db.create_collection( "csfle", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, ) - - await self.encrypted_client.db.drop_collection("csfle2") - await self.encrypted_client.db.create_collection( + await encrypted_client.db.create_collection( "csfle2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, ) - - await self.encrypted_client.db.drop_collection("qe") - await self.encrypted_client.db.create_collection( + await encrypted_client.db.create_collection( "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") ) - - await self.encrypted_client.db.drop_collection("qe2") - await self.encrypted_client.db.create_collection( + await encrypted_client.db.create_collection( "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") ) + await encrypted_client.db.create_collection("no_schema") + await encrypted_client.db.create_collection("no_schema2") - await self.encrypted_client.db.drop_collection("no_schema") - await self.encrypted_client.db.create_collection("no_schema") + unencrypted_client = await self.async_rs_or_single_client() - await self.encrypted_client.db.drop_collection("no_schema2") - await self.encrypted_client.db.create_collection("no_schema2") - - self.unencrypted_client = await self.async_rs_or_single_client() - - await self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) - doc = await self.unencrypted_client.db.csfle.find_one() + await encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = await unencrypted_client.db.csfle.find_one() self.assertTrue(isinstance(doc["csfle"], Binary)) - await self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) - doc = await self.unencrypted_client.db.csfle2.find_one() + await encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = await unencrypted_client.db.csfle2.find_one() self.assertTrue(isinstance(doc["csfle2"], Binary)) - await self.encrypted_client.db.qe.insert_one({"qe": "qe"}) - doc = await self.unencrypted_client.db.qe.find_one() + await encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = await unencrypted_client.db.qe.find_one() self.assertTrue(isinstance(doc["qe"], Binary)) - await self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) - doc = await self.unencrypted_client.db.qe2.find_one() + await encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = await unencrypted_client.db.qe2.find_one() self.assertTrue(isinstance(doc["qe2"], Binary)) - await self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) - await self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + await encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + await encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) @async_client_context.require_version_min(8, 1, -1) - async def test_1(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_1_csfle_joins_no_schema(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.csfle.aggregate( + await encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { @@ -2512,15 +2501,15 @@ async def test_1(self): self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_2(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_2_qe_joins_no_schema(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.qe.aggregate( + await encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, { @@ -2540,15 +2529,15 @@ async def test_2(self): self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_3(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_3_no_schema_joins_csfle(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.no_schema.aggregate( + await encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2565,15 +2554,15 @@ async def test_3(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_4(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_4_no_schema_joins_qe(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.no_schema.aggregate( + await encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2593,15 +2582,15 @@ async def test_4(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_5(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_5_csfle_joins_csfle2(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.csfle.aggregate( + await encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { @@ -2621,15 +2610,15 @@ async def test_5(self): self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_6(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_6_qe_joins_qe2(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.qe.aggregate( + await encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, { @@ -2649,15 +2638,15 @@ async def test_6(self): self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_7(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_7_no_schema_joins_no_schema2(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = await anext( - await self.encrypted_client.db.no_schema.aggregate( + await encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2677,8 +2666,8 @@ async def test_7(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) @async_client_context.require_version_min(8, 1, -1) - async def test_8(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_8_csfle_joins_qe(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, @@ -2686,7 +2675,7 @@ async def test_8(self): ) with self.assertRaises(PyMongoError) as exc: _ = await anext( - await self.encrypted_client.db.csfle.aggregate( + await encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "qe"}}, { @@ -2703,8 +2692,8 @@ async def test_8(self): self.assertTrue("not supported" in str(exc)) @async_client_context.require_version_max(8, 1, -1) - async def test_9(self): - self.encrypted_client = await self.async_rs_or_single_client( + async def test_9_error(self): + encrypted_client = await self.async_rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, @@ -2712,7 +2701,7 @@ async def test_9(self): ) with self.assertRaises(PyMongoError) as exc: _ = await anext( - await self.encrypted_client.db.csfle.aggregate( + await encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { diff --git a/test/test_encryption.py b/test/test_encryption.py index a52a67ff47..938b276b7f 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2410,73 +2410,62 @@ class TestLookupProse(EncryptionIntegrationTest): @client_context.require_version_min(7, 0, -1) def setUp(self): super().setUp() - self.encrypted_client = self.rs_or_single_client( + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) - self.encrypted_client.db.drop_collection("keyvault") + encrypted_client.drop_database("db") key_doc = json_data("etc", "data", "lookup", "key-doc.json") - key_vault = create_key_vault(self.encrypted_client.db.keyvault, key_doc) + key_vault = create_key_vault(encrypted_client.db.keyvault, key_doc) self.addCleanup(key_vault.drop) - self.encrypted_client.db.drop_collection("csfle") - self.encrypted_client.db.create_collection( + encrypted_client.db.create_collection( "csfle", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle.json")}, ) - - self.encrypted_client.db.drop_collection("csfle2") - self.encrypted_client.db.create_collection( + encrypted_client.db.create_collection( "csfle2", validator={"$jsonSchema": json_data("etc", "data", "lookup", "schema-csfle2.json")}, ) - - self.encrypted_client.db.drop_collection("qe") - self.encrypted_client.db.create_collection( + encrypted_client.db.create_collection( "qe", encryptedFields=json_data("etc", "data", "lookup", "schema-qe.json") ) - - self.encrypted_client.db.drop_collection("qe2") - self.encrypted_client.db.create_collection( + encrypted_client.db.create_collection( "qe2", encryptedFields=json_data("etc", "data", "lookup", "schema-qe2.json") ) + encrypted_client.db.create_collection("no_schema") + encrypted_client.db.create_collection("no_schema2") - self.encrypted_client.db.drop_collection("no_schema") - self.encrypted_client.db.create_collection("no_schema") + unencrypted_client = self.rs_or_single_client() - self.encrypted_client.db.drop_collection("no_schema2") - self.encrypted_client.db.create_collection("no_schema2") - - self.unencrypted_client = self.rs_or_single_client() - - self.encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) - doc = self.unencrypted_client.db.csfle.find_one() + encrypted_client.db.csfle.insert_one({"csfle": "csfle"}) + doc = unencrypted_client.db.csfle.find_one() self.assertTrue(isinstance(doc["csfle"], Binary)) - self.encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) - doc = self.unencrypted_client.db.csfle2.find_one() + encrypted_client.db.csfle2.insert_one({"csfle2": "csfle2"}) + doc = unencrypted_client.db.csfle2.find_one() self.assertTrue(isinstance(doc["csfle2"], Binary)) - self.encrypted_client.db.qe.insert_one({"qe": "qe"}) - doc = self.unencrypted_client.db.qe.find_one() + encrypted_client.db.qe.insert_one({"qe": "qe"}) + doc = unencrypted_client.db.qe.find_one() self.assertTrue(isinstance(doc["qe"], Binary)) - self.encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) - doc = self.unencrypted_client.db.qe2.find_one() + encrypted_client.db.qe2.insert_one({"qe2": "qe2"}) + doc = unencrypted_client.db.qe2.find_one() self.assertTrue(isinstance(doc["qe2"], Binary)) - self.encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) - self.encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) + encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) @client_context.require_version_min(8, 1, -1) - def test_1(self): - self.encrypted_client = self.rs_or_single_client( + def test_1_csfle_joins_no_schema(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.csfle.aggregate( + encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { @@ -2496,15 +2485,15 @@ def test_1(self): self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) @client_context.require_version_min(8, 1, -1) - def test_2(self): - self.encrypted_client = self.rs_or_single_client( + def test_2_qe_joins_no_schema(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.qe.aggregate( + encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, { @@ -2524,15 +2513,15 @@ def test_2(self): self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) @client_context.require_version_min(8, 1, -1) - def test_3(self): - self.encrypted_client = self.rs_or_single_client( + def test_3_no_schema_joins_csfle(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.no_schema.aggregate( + encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2549,15 +2538,15 @@ def test_3(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) @client_context.require_version_min(8, 1, -1) - def test_4(self): - self.encrypted_client = self.rs_or_single_client( + def test_4_no_schema_joins_qe(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.no_schema.aggregate( + encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2577,15 +2566,15 @@ def test_4(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) @client_context.require_version_min(8, 1, -1) - def test_5(self): - self.encrypted_client = self.rs_or_single_client( + def test_5_csfle_joins_csfle2(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.csfle.aggregate( + encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { @@ -2605,15 +2594,15 @@ def test_5(self): self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) @client_context.require_version_min(8, 1, -1) - def test_6(self): - self.encrypted_client = self.rs_or_single_client( + def test_6_qe_joins_qe2(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.qe.aggregate( + encrypted_client.db.qe.aggregate( [ {"$match": {"qe": "qe"}}, { @@ -2633,15 +2622,15 @@ def test_6(self): self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) @client_context.require_version_min(8, 1, -1) - def test_7(self): - self.encrypted_client = self.rs_or_single_client( + def test_7_no_schema_joins_no_schema2(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, ) ) doc = next( - self.encrypted_client.db.no_schema.aggregate( + encrypted_client.db.no_schema.aggregate( [ {"$match": {"no_schema": "no_schema"}}, { @@ -2661,8 +2650,8 @@ def test_7(self): self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) @client_context.require_version_min(8, 1, -1) - def test_8(self): - self.encrypted_client = self.rs_or_single_client( + def test_8_csfle_joins_qe(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, @@ -2670,7 +2659,7 @@ def test_8(self): ) with self.assertRaises(PyMongoError) as exc: _ = next( - self.encrypted_client.db.csfle.aggregate( + encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "qe"}}, { @@ -2687,8 +2676,8 @@ def test_8(self): self.assertTrue("not supported" in str(exc)) @client_context.require_version_max(8, 1, -1) - def test_9(self): - self.encrypted_client = self.rs_or_single_client( + def test_9_error(self): + encrypted_client = self.rs_or_single_client( auto_encryption_opts=AutoEncryptionOpts( key_vault_namespace="db.keyvault", kms_providers={"local": {"key": LOCAL_MASTER_KEY}}, @@ -2696,7 +2685,7 @@ def test_9(self): ) with self.assertRaises(PyMongoError) as exc: _ = next( - self.encrypted_client.db.csfle.aggregate( + encrypted_client.db.csfle.aggregate( [ {"$match": {"csfle": "csfle"}}, { diff --git a/uv.lock b/uv.lock index fbf1f06702..2dc9d46f12 100644 --- a/uv.lock +++ b/uv.lock @@ -997,6 +997,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/07/e9/ae44ea7d7605df9e5 [[package]] name = "pymongo" +version = "4.12.0.dev0" source = { editable = "." } dependencies = [ { name = "dnspython" }, From 53da94dae2a48b43ea5b3f98c75eec32c0f7f30b Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 16:50:56 -0700 Subject: [PATCH 16/20] close clients and fix changelog --- doc/changelog.rst | 2 +- test/asynchronous/test_encryption.py | 12 ++++++++++++ test/test_encryption.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 257cf1b747..6d70bb5979 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,7 +8,7 @@ PyMongo 4.12 brings a number of changes including: - Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to :class:`~pymongo.encryption_options.AutoEncryptionOpts`. -- Support for $lookup in CSFLE and QE.pr +- Support for $lookup in CSFLE and QE supported on MongoDB 8.1+. Issues Resolved ............... diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 06ccb28c8f..44aa1c6b02 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2472,6 +2472,9 @@ async def asyncSetUp(self): await encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) await encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + await encrypted_client.close() + await unencrypted_client.close() + @async_client_context.require_version_min(8, 1, -1) async def test_1_csfle_joins_no_schema(self): encrypted_client = await self.async_rs_or_single_client( @@ -2499,6 +2502,7 @@ async def test_1_csfle_joins_no_schema(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_2_qe_joins_no_schema(self): @@ -2527,6 +2531,7 @@ async def test_2_qe_joins_no_schema(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_3_no_schema_joins_csfle(self): @@ -2552,6 +2557,7 @@ async def test_3_no_schema_joins_csfle(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_4_no_schema_joins_qe(self): @@ -2580,6 +2586,7 @@ async def test_4_no_schema_joins_qe(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_5_csfle_joins_csfle2(self): @@ -2608,6 +2615,7 @@ async def test_5_csfle_joins_csfle2(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_6_qe_joins_qe2(self): @@ -2636,6 +2644,7 @@ async def test_6_qe_joins_qe2(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_7_no_schema_joins_no_schema2(self): @@ -2664,6 +2673,7 @@ async def test_7_no_schema_joins_no_schema2(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_8_csfle_joins_qe(self): @@ -2690,6 +2700,7 @@ async def test_8_csfle_joins_qe(self): ) ) self.assertTrue("not supported" in str(exc)) + await encrypted_client.close() @async_client_context.require_version_max(8, 1, -1) async def test_9_error(self): @@ -2719,6 +2730,7 @@ async def test_9_error(self): ) ) self.assertTrue("Upgrade" in str(exc)) + await encrypted_client.close() # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap diff --git a/test/test_encryption.py b/test/test_encryption.py index 938b276b7f..2bf02e200a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2456,6 +2456,9 @@ def setUp(self): encrypted_client.db.no_schema.insert_one({"no_schema": "no_schema"}) encrypted_client.db.no_schema2.insert_one({"no_schema2": "no_schema2"}) + encrypted_client.close() + unencrypted_client.close() + @client_context.require_version_min(8, 1, -1) def test_1_csfle_joins_no_schema(self): encrypted_client = self.rs_or_single_client( @@ -2483,6 +2486,7 @@ def test_1_csfle_joins_no_schema(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_2_qe_joins_no_schema(self): @@ -2511,6 +2515,7 @@ def test_2_qe_joins_no_schema(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_3_no_schema_joins_csfle(self): @@ -2536,6 +2541,7 @@ def test_3_no_schema_joins_csfle(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_4_no_schema_joins_qe(self): @@ -2564,6 +2570,7 @@ def test_4_no_schema_joins_qe(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_5_csfle_joins_csfle2(self): @@ -2592,6 +2599,7 @@ def test_5_csfle_joins_csfle2(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_6_qe_joins_qe2(self): @@ -2620,6 +2628,7 @@ def test_6_qe_joins_qe2(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_7_no_schema_joins_no_schema2(self): @@ -2648,6 +2657,7 @@ def test_7_no_schema_joins_no_schema2(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) + encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_8_csfle_joins_qe(self): @@ -2674,6 +2684,7 @@ def test_8_csfle_joins_qe(self): ) ) self.assertTrue("not supported" in str(exc)) + encrypted_client.close() @client_context.require_version_max(8, 1, -1) def test_9_error(self): @@ -2703,6 +2714,7 @@ def test_9_error(self): ) ) self.assertTrue("Upgrade" in str(exc)) + encrypted_client.close() # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap From c85a856d48fa2f699eaf8aa2a11b399442748caf Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 17:19:29 -0700 Subject: [PATCH 17/20] delete unnecessary client.close --- test/asynchronous/test_encryption.py | 9 --------- test/test_encryption.py | 9 --------- 2 files changed, 18 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 44aa1c6b02..3ea747dfda 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2502,7 +2502,6 @@ async def test_1_csfle_joins_no_schema(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_2_qe_joins_no_schema(self): @@ -2531,7 +2530,6 @@ async def test_2_qe_joins_no_schema(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_3_no_schema_joins_csfle(self): @@ -2557,7 +2555,6 @@ async def test_3_no_schema_joins_csfle(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_4_no_schema_joins_qe(self): @@ -2586,7 +2583,6 @@ async def test_4_no_schema_joins_qe(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_5_csfle_joins_csfle2(self): @@ -2615,7 +2611,6 @@ async def test_5_csfle_joins_csfle2(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_6_qe_joins_qe2(self): @@ -2644,7 +2639,6 @@ async def test_6_qe_joins_qe2(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_7_no_schema_joins_no_schema2(self): @@ -2673,7 +2667,6 @@ async def test_7_no_schema_joins_no_schema2(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) - await encrypted_client.close() @async_client_context.require_version_min(8, 1, -1) async def test_8_csfle_joins_qe(self): @@ -2700,7 +2693,6 @@ async def test_8_csfle_joins_qe(self): ) ) self.assertTrue("not supported" in str(exc)) - await encrypted_client.close() @async_client_context.require_version_max(8, 1, -1) async def test_9_error(self): @@ -2730,7 +2722,6 @@ async def test_9_error(self): ) ) self.assertTrue("Upgrade" in str(exc)) - await encrypted_client.close() # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap diff --git a/test/test_encryption.py b/test/test_encryption.py index 2bf02e200a..57623b5b71 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2486,7 +2486,6 @@ def test_1_csfle_joins_no_schema(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"no_schema": "no_schema"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_2_qe_joins_no_schema(self): @@ -2515,7 +2514,6 @@ def test_2_qe_joins_no_schema(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"no_schema": "no_schema"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_3_no_schema_joins_csfle(self): @@ -2541,7 +2539,6 @@ def test_3_no_schema_joins_csfle(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"csfle": "csfle"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_4_no_schema_joins_qe(self): @@ -2570,7 +2567,6 @@ def test_4_no_schema_joins_qe(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"qe": "qe"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_5_csfle_joins_csfle2(self): @@ -2599,7 +2595,6 @@ def test_5_csfle_joins_csfle2(self): ) ) self.assertEqual(doc, {"csfle": "csfle", "matched": [{"csfle2": "csfle2"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_6_qe_joins_qe2(self): @@ -2628,7 +2623,6 @@ def test_6_qe_joins_qe2(self): ) ) self.assertEqual(doc, {"qe": "qe", "matched": [{"qe2": "qe2"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_7_no_schema_joins_no_schema2(self): @@ -2657,7 +2651,6 @@ def test_7_no_schema_joins_no_schema2(self): ) ) self.assertEqual(doc, {"no_schema": "no_schema", "matched": [{"no_schema2": "no_schema2"}]}) - encrypted_client.close() @client_context.require_version_min(8, 1, -1) def test_8_csfle_joins_qe(self): @@ -2684,7 +2677,6 @@ def test_8_csfle_joins_qe(self): ) ) self.assertTrue("not supported" in str(exc)) - encrypted_client.close() @client_context.require_version_max(8, 1, -1) def test_9_error(self): @@ -2714,7 +2706,6 @@ def test_9_error(self): ) ) self.assertTrue("Upgrade" in str(exc)) - encrypted_client.close() # https://github.com/mongodb/specifications/blob/072601/source/client-side-encryption/tests/README.md#rewrap From 09a82da6e0af358fdb023a505bfc3d4432b16bce Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Wed, 19 Mar 2025 18:04:10 -0700 Subject: [PATCH 18/20] drop db instead of just keyvault --- test/asynchronous/test_encryption.py | 4 ++-- test/test_encryption.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 3ea747dfda..c4f789e651 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2435,8 +2435,8 @@ async def asyncSetUp(self): await encrypted_client.drop_database("db") key_doc = json_data("etc", "data", "lookup", "key-doc.json") - key_vault = await create_key_vault(encrypted_client.db.keyvault, key_doc) - self.addCleanup(key_vault.drop) + await create_key_vault(encrypted_client.db.keyvault, key_doc) + self.addAsyncCleanup(async_client_context.client.drop_database("db")) await encrypted_client.db.create_collection( "csfle", diff --git a/test/test_encryption.py b/test/test_encryption.py index 57623b5b71..fb66f3851e 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2419,8 +2419,8 @@ def setUp(self): encrypted_client.drop_database("db") key_doc = json_data("etc", "data", "lookup", "key-doc.json") - key_vault = create_key_vault(encrypted_client.db.keyvault, key_doc) - self.addCleanup(key_vault.drop) + create_key_vault(encrypted_client.db.keyvault, key_doc) + self.addCleanup(client_context.client.drop_database("db")) encrypted_client.db.create_collection( "csfle", From 74b338b6285ce3a6c34d977221f3def9d170a53e Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Thu, 20 Mar 2025 09:30:19 -0700 Subject: [PATCH 19/20] fix asynccleanup call --- test/asynchronous/test_encryption.py | 2 +- test/test_encryption.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index c4f789e651..3b9096ef6a 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2436,7 +2436,7 @@ async def asyncSetUp(self): key_doc = json_data("etc", "data", "lookup", "key-doc.json") await create_key_vault(encrypted_client.db.keyvault, key_doc) - self.addAsyncCleanup(async_client_context.client.drop_database("db")) + self.addAsyncCleanup(async_client_context.client.drop_database, "db") await encrypted_client.db.create_collection( "csfle", diff --git a/test/test_encryption.py b/test/test_encryption.py index fb66f3851e..6d669a538d 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2420,7 +2420,7 @@ def setUp(self): key_doc = json_data("etc", "data", "lookup", "key-doc.json") create_key_vault(encrypted_client.db.keyvault, key_doc) - self.addCleanup(client_context.client.drop_database("db")) + self.addCleanup(client_context.client.drop_database, "db") encrypted_client.db.create_collection( "csfle", From 7dcf8bd5d25c2f3e9881c9f77349fe291567b1e6 Mon Sep 17 00:00:00 2001 From: Iris Ho Date: Thu, 20 Mar 2025 11:20:20 -0700 Subject: [PATCH 20/20] fix docstring --- pymongo/asynchronous/encryption.py | 2 +- pymongo/synchronous/encryption.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 6d7cea718f..262d964a4e 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -251,7 +251,7 @@ async def collection_info(self, database: str, filter: bytes) -> Optional[list[b :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The all documents from the listCollections command response as BSON. + :return: All documents from the listCollections command response as BSON. """ async with await self.client_ref()[database].list_collections( filter=RawBSONDocument(filter) diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 052f4c5a07..d6aa5c1ec1 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -250,7 +250,7 @@ def collection_info(self, database: str, filter: bytes) -> Optional[list[bytes]] :param database: The database on which to run listCollections. :param filter: The filter to pass to listCollections. - :return: The all documents from the listCollections command response as BSON. + :return: All documents from the listCollections command response as BSON. """ with self.client_ref()[database].list_collections(filter=RawBSONDocument(filter)) as cursor: return [_dict_to_bson(doc, False, _DATA_KEY_OPTS) for doc in cursor]