From ecaf7a8238e07a2a35272c86ce0cc772e8816cbf Mon Sep 17 00:00:00 2001 From: Carlos Cruz Date: Tue, 16 Jun 2026 13:00:28 -0600 Subject: [PATCH 1/2] Add virtual decrypted types to support top-level list fields --- ming/encryption.py | 92 ++++++++++++++++++++++++++++++++++- ming/odm/__init__.py | 4 +- ming/odm/property.py | 3 ++ ming/odm/property.pyi | 5 +- ming/tests/test_encryption.py | 84 +++++++++++++++++++++++++++++++- 5 files changed, 181 insertions(+), 7 deletions(-) diff --git a/ming/encryption.py b/ming/encryption.py index 18ca9d1..1bc67fd 100644 --- a/ming/encryption.py +++ b/ming/encryption.py @@ -654,6 +654,87 @@ def __set__(self, instance: EncryptedMixin, value: T): setattr(instance, self.encrypted_field, instance.encr(value)) +class DecryptedListField: + """Virtual Document field for lists stored in a sibling encrypted field. + + ``DecryptedListField('emails_encrypted')`` exposes plaintext list + operations while storing encrypted values in ``emails_encrypted``. + """ + + def __init__(self, encrypted_field: str): + self.encrypted_field = encrypted_field + + def _encrypt_list(self, encr_func, value): + if value is None: + return [] + return _encrypt_list_recursive(value, [S.Binary], encr_func, self.encrypted_field, force_encrypt=True) + + def __get__(self, instance: EncryptedMixin, owner): + if instance is None: + return self + + doc = instance.get(self.encrypted_field) + if doc is None: + doc = [] + instance[self.encrypted_field] = doc + return EncryptedListWrapper( + doc=doc, + tracker=None, + item_schema=S.Binary, + instance=instance, + items_encrypted=True, + ) + + def __set__(self, instance: EncryptedMixin, value): + instance[self.encrypted_field] = self._encrypt_list(instance.encr, value) + + def __delete__(self, instance): + del instance[self.encrypted_field] + + +class DecryptedListProperty: + """Virtual ODM property for lists stored in a sibling encrypted field. + + ``DecryptedListProperty('emails_encrypted')`` exposes plaintext list + operations while storing encrypted values in ``emails_encrypted``. + """ + + def __init__(self, encrypted_field: str): + self.encrypted_field = encrypted_field + + def _encrypt_list(self, encr_func, value): + if value is None: + return [] + return _encrypt_list_recursive(value, [S.Binary], encr_func, self.encrypted_field, force_encrypt=True) + + def __get__(self, instance: EncryptedMixin, owner): + if instance is None: + return self + + from ming.odm.base import state + + st = state(instance) + doc = st.document.get(self.encrypted_field) + if doc is None: + doc = [] + st.document[self.encrypted_field] = doc + return EncryptedListWrapper( + doc=doc, + tracker=st.tracker, + item_schema=S.Binary, + instance=instance, + items_encrypted=True, + ) + + def __set__(self, instance: EncryptedMixin, value): + setattr(instance, self.encrypted_field, self._encrypt_list(instance.encr, value)) + + def __delete__(self, instance): + from ming.odm.base import state + + state(instance).delete(self.encrypted_field) + + class EncryptedMixin: """A mixin intended to be used with :class:`~ming.declarative.Document` or :class:`~ming.odm.declarative.MappedClass` to provide encryption. @@ -763,7 +844,11 @@ def encrypt_some_fields(cls, data: dict) -> dict: for fld in cls.decrypted_field_names(): if fld in encrypted_data: val = encrypted_data.pop(fld) - encrypted_data[f'{fld}_encrypted'] = cls.encr(val) + prop = getattr(cls, fld, None) + if isinstance(prop, (DecryptedListField, DecryptedListProperty)): + encrypted_data[prop.encrypted_field] = prop._encrypt_list(cls.encr, val) + else: + encrypted_data[f'{fld}_encrypted'] = cls.encr(val) # Handle nested encrypted field/property instances. for field_name, field in cls._encrypted_field_index().items(): @@ -787,7 +872,10 @@ def decrypt_some_fields(self) -> dict: for k in self._field_names: if k.endswith('_encrypted'): k_decrypted = k.replace('_encrypted', '') - decrypted_data[k_decrypted] = getattr(self, k_decrypted) + value = getattr(self, k_decrypted) + if isinstance(getattr(type(self), k_decrypted, None), (DecryptedListField, DecryptedListProperty)): + value = list(value) + decrypted_data[k_decrypted] = value else: decrypted_data[k] = getattr(self, k) return decrypted_data diff --git a/ming/odm/__init__.py b/ming/odm/__init__.py index cb86a12..85ec089 100644 --- a/ming/odm/__init__.py +++ b/ming/odm/__init__.py @@ -2,7 +2,7 @@ from ming.odm.mapper import mapper, Mapper, MapperExtension from ming.odm.property import RelationProperty, ForeignIdProperty -from ming.odm.property import FieldProperty, FieldPropertyWithMissingNone, DecryptedProperty +from ming.odm.property import FieldProperty, FieldPropertyWithMissingNone, DecryptedProperty, DecryptedListProperty from ming.odm.odmsession import ODMSession, ThreadLocalODMSession, SessionExtension from ming.odm.odmsession import ContextualODMSession @@ -15,5 +15,5 @@ __all__ = ('state', 'session', 'mapper', 'Mapper', 'MapperExtension', 'RelationProperty', 'ForeignIdProperty', 'FieldProperty', 'DecryptedProperty', - 'FieldPropertyWithMissingNone', 'ODMSession', 'ThreadLocalODMSession', + 'DecryptedListProperty', 'FieldPropertyWithMissingNone', 'ODMSession', 'ThreadLocalODMSession', 'SessionExtension', 'MappedClass', 'ContextualODMSession') diff --git a/ming/odm/property.py b/ming/odm/property.py index 941a186..8fbf027 100644 --- a/ming/odm/property.py +++ b/ming/odm/property.py @@ -32,6 +32,9 @@ def __repr__(self): class DecryptedProperty(ming.encryption.DecryptedField): pass +class DecryptedListProperty(ming.encryption.DecryptedListProperty): + pass + class FieldProperty(ORMProperty): """Declares property for a value stored in a MongoDB Document. diff --git a/ming/odm/property.pyi b/ming/odm/property.pyi index ee8dc04..11af0b7 100644 --- a/ming/odm/property.pyi +++ b/ming/odm/property.pyi @@ -25,5 +25,8 @@ class RelationProperty: @overload def __new__(self, related: Type[MC], via: str=None, fetch=True) -> Iterable[MC]:... +class DecryptedListProperty: + def __init__(self, encrypted_field: str) -> None: ... -def __getattr__(name) -> Any: ... # marks file as incomplete \ No newline at end of file + +def __getattr__(name) -> Any: ... # marks file as incomplete diff --git a/ming/tests/test_encryption.py b/ming/tests/test_encryption.py index fa28e21..9639b12 100644 --- a/ming/tests/test_encryption.py +++ b/ming/tests/test_encryption.py @@ -4,8 +4,11 @@ import ming from ming import create_datastore, Document, Field, schema as S -from ming.odm import state, session, ODMSession, Mapper, MappedClass, FieldProperty, DecryptedProperty -from ming.encryption import DecryptedField, NestedEncryptedField, NestedEncryptedProperty +from ming.odm import ( + state, session, ODMSession, Mapper, MappedClass, FieldProperty, DecryptedProperty, + DecryptedListProperty, +) +from ming.encryption import DecryptedField, DecryptedListField, NestedEncryptedField, NestedEncryptedProperty from ming.odm.odmsession import ThreadLocalODMSession from . import make_encryption_key @@ -769,6 +772,43 @@ class __mongometa__: self.assertEqual(doc['author'], {'username_encrypted': None}) self.assertIsNone(doc.author.username) + def test_decrypted_list_field_uses_sibling_encrypted_field(self): + class TestDoc(Document): + class __mongometa__: + name = 'test_decrypted_list_field' + session = ming.Session.by_name('test_db') + + _id = Field(S.Anything) + emails = DecryptedListField('emails_encrypted') + emails_encrypted = Field([S.Binary]) + + doc = TestDoc.make_encr(dict( + _id=1, + emails=['first@example.com', None], + )) + doc.m.save() + + self.assertNotIn('emails', doc) + self.assertEqual(doc['emails_encrypted'][0], TestDoc.encr('first@example.com')) + self.assertIsNone(doc['emails_encrypted'][1]) + self.assertEqual(list(doc.emails), ['first@example.com', None]) + self.assertEqual(doc.emails[0], 'first@example.com') + + doc.emails.append('second@example.com') + doc.m.save() + self.assertEqual(list(doc.emails), ['first@example.com', None, 'second@example.com']) + self.assertEqual(doc['emails_encrypted'][2], TestDoc.encr('second@example.com')) + + doc.emails[0] = 'updated@example.com' + del doc.emails[1] + doc.m.save() + self.assertEqual(list(doc.emails), ['updated@example.com', 'second@example.com']) + self.assertEqual(doc['emails_encrypted'][0], TestDoc.encr('updated@example.com')) + + doc.emails = ['reset@example.com'] + self.assertEqual(doc['emails_encrypted'], [TestDoc.encr('reset@example.com')]) + self.assertEqual(doc.decrypt_some_fields(), {'_id': 1, 'emails': ['reset@example.com']}) + class TestNestedEncryptedPropertyMapped(TestCase): DATASTORE = 'mim:///test_db' @@ -990,3 +1030,43 @@ class __mongometa__: self.assertEqual(state(obj).status, state(obj).dirty) self.session.flush() self.assertEqual(list(obj.secrets), ['m', 'm', 'm']) + + def test_decrypted_list_property_uses_sibling_encrypted_field(self): + class TestMappedEmails(MappedClass): + class __mongometa__: + name = 'test_decrypted_list_property' + session = self.session + + _id = FieldProperty(S.ObjectId) + emails = DecryptedListProperty('emails_encrypted') + emails_encrypted = FieldProperty([S.Binary]) + + obj = TestMappedEmails(_id=None, emails=['first@example.com', None]) + self.session.flush() + + raw_doc = state(obj).document + self.assertNotIn('emails', raw_doc) + self.assertEqual(raw_doc['emails_encrypted'][0], TestMappedEmails.encr('first@example.com')) + self.assertIsNone(raw_doc['emails_encrypted'][1]) + self.assertEqual(list(obj.emails), ['first@example.com', None]) + self.assertEqual(obj.emails[0], 'first@example.com') + + obj.emails.append('second@example.com') + self.assertEqual(state(obj).status, state(obj).dirty) + self.session.flush() + self.assertEqual(list(obj.emails), ['first@example.com', None, 'second@example.com']) + self.assertEqual(raw_doc['emails_encrypted'][2], TestMappedEmails.encr('second@example.com')) + + obj.emails[0] = 'updated@example.com' + del obj.emails[1] + self.session.flush() + self.assertEqual(list(obj.emails), ['updated@example.com', 'second@example.com']) + self.assertEqual(raw_doc['emails_encrypted'][0], TestMappedEmails.encr('updated@example.com')) + + encrypted = TestMappedEmails.encrypt_some_fields({ + 'emails': ['one@example.com', 'two@example.com'], + }) + self.assertNotIn('emails', encrypted) + self.assertEqual(encrypted['emails_encrypted'][0], TestMappedEmails.encr('one@example.com')) + + self.assertEqual(obj.decrypt_some_fields()['emails'], ['updated@example.com', 'second@example.com']) From 5101e7ab53fda9da6e7f1a3cd8f3518fc5a9cf42 Mon Sep 17 00:00:00 2001 From: Carlos Cruz Date: Fri, 19 Jun 2026 17:44:31 +0000 Subject: [PATCH 2/2] Add encrypted list wrapper list behavior --- ming/encryption.py | 74 +++++++++++++++++++++++++++++++---- ming/tests/test_encryption.py | 62 ++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/ming/encryption.py b/ming/encryption.py index 1bc67fd..600fbbe 100644 --- a/ming/encryption.py +++ b/ming/encryption.py @@ -226,20 +226,30 @@ def _wrap_item(self, item): def _encrypt_item(self, item): """Encrypt or process a single item for writing.""" - encrypted_iem = _encrypt_value_recursive( + encrypted_item = _encrypt_value_recursive( item, self._item_schema, self._instance.encr, field_name=ENCRYPTED_SUFFIX if self._items_encrypted else None, force_encrypt=self._items_encrypted, ) - return encrypted_iem + return encrypted_item def _mark_dirty(self): """Mark the list as modified for dirty tracking.""" if self._tracker is not None: self._tracker.added_item(self._doc) + def _plain_list(self): + return list(self) + + def _coerce_list_comparison(self, other): + if isinstance(other, EncryptedListWrapper): + return other._plain_list() + if isinstance(other, list): + return other + return NotImplemented + def __getitem__(self, index): if isinstance(index, slice): return [self._wrap_item(item) for item in self._doc[index]] @@ -283,6 +293,42 @@ def __imul__(self, n): self.extend(current) return self + def __eq__(self, other): + other_list = self._coerce_list_comparison(other) + if other_list is NotImplemented: + return False + return self._plain_list() == other_list + + def __ne__(self, other): + return not self == other + + def __lt__(self, other): + other_list = self._coerce_list_comparison(other) + if other_list is NotImplemented: + return NotImplemented + return self._plain_list() < other_list + + def __le__(self, other): + other_list = self._coerce_list_comparison(other) + if other_list is NotImplemented: + return NotImplemented + return self._plain_list() <= other_list + + def __gt__(self, other): + other_list = self._coerce_list_comparison(other) + if other_list is NotImplemented: + return NotImplemented + return self._plain_list() > other_list + + def __ge__(self, other): + other_list = self._coerce_list_comparison(other) + if other_list is NotImplemented: + return NotImplemented + return self._plain_list() >= other_list + + def __reversed__(self): + return reversed(self._plain_list()) + def append(self, value): self._doc.append(self._encrypt_item(value)) self._mark_dirty() @@ -302,14 +348,27 @@ def pop(self, index=-1): return self._wrap_item(item) def remove(self, value): - # Need to find and remove the encrypted version - encrypted_value = self._encrypt_item(value) - self._doc.remove(encrypted_value) + del self._doc[self.index(value)] self._mark_dirty() def index(self, value, *args): - encrypted_value = self._encrypt_item(value) - return self._doc.index(encrypted_value, *args) + return self._plain_list().index(value, *args) + + def count(self, value): + return self._plain_list().count(value) + + def copy(self): + return self._plain_list().copy() + + def reverse(self): + self._doc.reverse() + self._mark_dirty() + + def sort(self, *, key=None, reverse=False): + values = self._plain_list() + values.sort(key=key, reverse=reverse) + self._doc[:] = [self._encrypt_item(value) for value in values] + self._mark_dirty() def replace(self, values): self[:] = values @@ -334,7 +393,6 @@ def __contains__(self, value): def __repr__(self): return f"EncryptedListWrapper({list(self)})" - class EncryptedDictWrapper: """Generic dict wrapper that transparently encrypts/decrypts specified fields. diff --git a/ming/tests/test_encryption.py b/ming/tests/test_encryption.py index 9639b12..bc539d2 100644 --- a/ming/tests/test_encryption.py +++ b/ming/tests/test_encryption.py @@ -8,7 +8,10 @@ state, session, ODMSession, Mapper, MappedClass, FieldProperty, DecryptedProperty, DecryptedListProperty, ) -from ming.encryption import DecryptedField, DecryptedListField, NestedEncryptedField, NestedEncryptedProperty +from ming.encryption import ( + DecryptedField, DecryptedListField, EncryptedListWrapper, + NestedEncryptedField, NestedEncryptedProperty, +) from ming.odm.odmsession import ThreadLocalODMSession from . import make_encryption_key @@ -1031,6 +1034,63 @@ class __mongometa__: self.session.flush() self.assertEqual(list(obj.secrets), ['m', 'm', 'm']) + def test_mapped_list_wrapper_matches_list_behavior(self): + class TestMappedListBehavior(MappedClass): + class __mongometa__: + name = 'test_nested_encrypted_property_list_behavior' + session = self.session + + _id = FieldProperty(S.ObjectId) + secrets = NestedEncryptedProperty([S.Binary]) + + obj = TestMappedListBehavior(_id=None, secrets=['b', 'a', 'b']) + self.session.flush() + self.assertEqual(state(obj).status, state(obj).clean) + + self.assertEqual(obj.secrets, ['b', 'a', 'b']) + self.assertNotEqual(obj.secrets, ['a', 'b']) + self.assertLess(obj.secrets, ['c']) + self.assertGreater(obj.secrets, ['a']) + self.assertEqual(list(reversed(obj.secrets)), ['b', 'a', 'b']) + self.assertEqual(obj.secrets.count('b'), 2) + self.assertEqual(obj.secrets.copy(), ['b', 'a', 'b']) + + obj.secrets.reverse() + self.assertEqual(state(obj).status, state(obj).dirty) + self.session.flush() + self.assertEqual(list(obj.secrets), ['b', 'a', 'b']) + + obj.secrets.sort() + self.assertEqual(state(obj).status, state(obj).dirty) + self.session.flush() + self.assertEqual(list(obj.secrets), ['a', 'b', 'b']) + + def test_encrypted_list_wrapper_index_and_remove_compare_plaintext_values(self): + class NonDeterministicEncryption: + def __init__(self): + self.calls = 0 + + def encr(self, value): + self.calls += 1 + return f'{value}:{self.calls}'.encode() + + def decr(self, value): + return value.decode().split(':', 1)[0] + + instance = NonDeterministicEncryption() + doc = [instance.encr('a'), instance.encr('b')] + wrapped = EncryptedListWrapper( + doc=doc, + tracker=None, + item_schema=S.Binary, + instance=instance, + items_encrypted=True, + ) + + self.assertEqual(wrapped.index('a'), 0) + wrapped.remove('b') + self.assertEqual(list(wrapped), ['a']) + def test_decrypted_list_property_uses_sibling_encrypted_field(self): class TestMappedEmails(MappedClass): class __mongometa__: