Add support for collections in RealmValue (#1469)

* Initial stab at collections in mixed

* regenerate ffi bindings

* Add more tests

* Add notification tests

* Return the types

* Address some PR feedback

* Add RealmValueType

* Fix some tests

* Add query tests

* Add tests for indexOf/contains; add tests for sets

* Create a symlink for test data inside tests

* Add a few more tests

* Correct Core issue link

* enable notifications tests

* Enable some tests

* Re-generate ffi

* Fix test

* Added a changelog entry

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## vNext-next (TBD)
 
+### Breaking Changes
+* `RealmValue.type` is now an enum of type `RealmValueType` rather than `Type`. If you need the runtime type of the value wrapped in `RealmValue`, use `RealmValue.value.runtimeType`. (Issue [#1505](https://github.com/realm/realm-dart/issues/1505))
+* Renamed `RealmValue.uint8List` constructor to `RealmValue.binary`. (PR [#1469](https://github.com/realm/realm-dart/pull/1469))
+
 ### Enhancements
 * Added `isCollectionDeleted` to `RealmListChanges`, `RealmSetChanges`, and `RealmMapChanges` which will be `true` if the parent object, containing the collection has been deleted. (Core 14.0.0)
 * Added `isCleared` to `RealmMapChanges` which will be `true` if the map has been cleared. (Core 14.0.0)
@@ -14,6 +18,39 @@
   realm.query<Owner>('dogs[LAST].age = 5'); // Query all owners whose last dog is 5 years old
   realm.query<Owner>('dogs[SIZE] = 10'); // Query all owners who have 10 dogs
   ```
+* Added support for storing lists and maps inside a `RealmValue` property. (Issue [#1504](https://github.com/realm/realm-dart/issues/1504))
+  ```dart
+  class _Container {
+    late RealmValue anything;
+  }
+
+  realm.write(() {
+    realm.add(Container(anything: RealmValue.from([1, 'foo', 3.14])));
+  });
+
+  final container = realm.all<Container>().first;
+
+  final list = container.anything.asList(); // will throw if cast is invalid
+  for (final item in containerValue) {
+    switch (item.type) {
+      case RealmValueType.int:
+        print('Integer: ${item.value as int}');
+        break;
+      case RealmValueType.string:
+        print('String: ${item.value as String}');
+        break;
+      case RealmValueType.double:
+        print('Double: ${item.value as double}');
+        break;
+    }
+  }
+
+  final subscription = list.changes.listen((event) {
+    // The list changed
+  });
+  ```
+* Added `RealmValueType` enum that contains all the possible types that can be wrapped by a `RealmValue`. (PR [#1469](https://github.com/realm/realm-dart/pull/1469))
+
 
 ### Fixed
 * If you have more than 8388606 links pointing to one specific object, the program will crash. (Core 14.0.0)

common/lib/src/realm_types.dart

@@ -16,7 +16,6 @@
 //
 ////////////////////////////////////////////////////////////////////////////////
 
-import 'dart:ffi';
 import 'dart:math';
 import 'dart:typed_data';
 import 'package:objectid/objectid.dart';
@@ -157,6 +156,51 @@ abstract class EmbeddedObjectMarker implements RealmObjectBaseMarker {}
 /// @nodoc
 abstract class AsymmetricObjectMarker implements RealmObjectBaseMarker {}
 
+/// An enum describing the possible types that can be wrapped inside [RealmValue]
+enum RealmValueType {
+  /// The [RealmValue] represents `null`
+  nullValue,
+
+  /// The [RealmValue] represents a [boolean] value
+  boolean,
+
+  /// The [RealmValue] represents a [String] value
+  string,
+
+  /// The [RealmValue] represents an [int] value
+  int,
+
+  /// The [RealmValue] represents a [double] value
+  double,
+
+  /// The [RealmValue] represents a `RealmObject` instance value
+  object,
+
+  /// The [RealmValue] represents an [ObjectId] value
+  objectId,
+
+  /// The [RealmValue] represents a [DateTime] value
+  dateTime,
+
+  /// The [RealmValue] represents a [Decimal128] value
+  decimal,
+
+  /// The [RealmValue] represents an [Uuid] value
+  uuid,
+
+  /// The [RealmValue] represents a binary ([Uint8List]) value
+  binary,
+
+  /// The [RealmValue] represents a `List<RealmValue>`
+  list,
+
+  /// The [RealmValue] represents a `Map<String, RealmValue>`
+  map;
+
+  /// Returns `true` if the enum value represents a collection - i.e. it's [list] or [map].
+  bool get isCollection => this == RealmValueType.list || this == RealmValueType.map;
+}
+
 /// A type that can represent any valid realm data type, except collections and embedded objects.
 ///
 /// You can use [RealmValue] to declare fields on realm models, in which case it must be non-nullable,
@@ -188,60 +232,77 @@ abstract class AsymmetricObjectMarker implements RealmObjectBaseMarker {}
 /// ```
 class RealmValue {
   final Object? value;
-  Type get type => value.runtimeType;
+
+  final RealmValueType type;
+
+  /// Casts [value] to [T]. An exception will be thrown if the value is not convertible to [T].
   T as<T>() => value as T; // better for code completion
 
   // This is private, so user cannot accidentally construct an invalid instance
-  const RealmValue._(this.value);
+  const RealmValue._(this.value, this.type);
 
-  const RealmValue.nullValue() : this._(null);
-  const RealmValue.bool(bool b) : this._(b);
-  const RealmValue.string(String text) : this._(text);
-  const RealmValue.int(int i) : this._(i);
-  const RealmValue.double(double d) : this._(d);
+  const RealmValue.nullValue() : this._(null, RealmValueType.nullValue);
+  const RealmValue.bool(bool b) : this._(b, RealmValueType.boolean);
+  const RealmValue.string(String text) : this._(text, RealmValueType.string);
+  const RealmValue.int(int i) : this._(i, RealmValueType.int);
+  const RealmValue.double(double d) : this._(d, RealmValueType.double);
   // TODO: RealmObjectMarker introduced to avoid dependency inversion. It would be better if we could use RealmObject directly. https://github.com/realm/realm-dart/issues/701
-  const RealmValue.realmObject(RealmObjectMarker o) : this._(o);
-  const RealmValue.dateTime(DateTime timestamp) : this._(timestamp);
-  const RealmValue.objectId(ObjectId id) : this._(id);
-  const RealmValue.decimal128(Decimal128 decimal) : this._(decimal);
-  const RealmValue.uuid(Uuid uuid) : this._(uuid);
-  const RealmValue.uint8List(Uint8List binary) : this._(binary);
-
-  /// Will throw [ArgumentError]
+  const RealmValue.realmObject(RealmObjectMarker o) : this._(o, RealmValueType.object);
+  const RealmValue.dateTime(DateTime timestamp) : this._(timestamp, RealmValueType.dateTime);
+  const RealmValue.objectId(ObjectId id) : this._(id, RealmValueType.objectId);
+  const RealmValue.decimal128(Decimal128 decimal) : this._(decimal, RealmValueType.decimal);
+  const RealmValue.uuid(Uuid uuid) : this._(uuid, RealmValueType.uuid);
+  const RealmValue.binary(Uint8List binary) : this._(binary, RealmValueType.binary);
+  const RealmValue.list(List<RealmValue> list) : this._(list, RealmValueType.list);
+  const RealmValue.map(Map<String, RealmValue> map) : this._(map, RealmValueType.map);
+
+  /// Constructs a RealmValue from an arbitrary object. Collections will be converted recursively as long
+  /// as all their values are compatible.
+  ///
+  /// Throws [ArgumentError] if any of the values inside the graph cannot be stored in a [RealmValue].
   factory RealmValue.from(Object? object) {
-    if (object == null ||
-        object is bool ||
-        object is String ||
-        object is int ||
-        object is Float ||
-        object is double ||
-        object is RealmObjectMarker ||
-        object is DateTime ||
-        object is ObjectId ||
-        object is Decimal128 ||
-        object is Uuid ||
-        object is Uint8List) {
-      return RealmValue._(object);
-    } else {
-      throw ArgumentError.value(object, 'object', 'Unsupported type');
-    }
+    return switch (object) {
+      null => RealmValue.nullValue(),
+      bool b => RealmValue.bool(b),
+      String text => RealmValue.string(text),
+      int i => RealmValue.int(i),
+      double d => RealmValue.double(d),
+      RealmObjectMarker o => RealmValue.realmObject(o),
+      DateTime d => RealmValue.dateTime(d),
+      ObjectId id => RealmValue.objectId(id),
+      Decimal128 decimal => RealmValue.decimal128(decimal),
+      Uuid uuid => RealmValue.uuid(uuid),
+      Uint8List binary => RealmValue.binary(binary),
+      Map<String, RealmValue> d => RealmValue.map(d),
+      Map<String, dynamic> d => RealmValue.map(d.map((key, value) => MapEntry(key, RealmValue.from(value)))),
+      List<RealmValue> l => RealmValue.list(l),
+      List<dynamic> l => RealmValue.list(l.map((o) => RealmValue.from(o)).toList()),
+      Iterable<RealmValue> i => RealmValue.list(i.toList()),
+      Iterable<dynamic> i => RealmValue.list(i.map((o) => RealmValue.from(o)).toList()),
+      _ => throw ArgumentError.value(object.runtimeType, 'object', 'Unsupported type'),
+    };
   }
 
   @override
   operator ==(Object? other) {
+    // We always return false when comparing two RealmValue collections.
+    if (type.isCollection) {
+      return false;
+    }
+
     if (other is RealmValue) {
       if (value is Uint8List && other.value is Uint8List) {
         return ListEquality().equals(value as Uint8List, other.value as Uint8List);
       }
 
-      return value == other.value;
+      return type == other.type && value == other.value;
     }
 
     return value == other;
   }
 
   @override
-  int get hashCode => value.hashCode;
+  int get hashCode => Object.hash(type, value);
 
   @override
   String toString() => 'RealmValue($value)';

flutter/realm_flutter/tests/data

@@ -0,0 +1 @@
+../../../test/data

lib/src/list.dart

@@ -97,6 +97,7 @@ class ManagedRealmList<T extends Object?> with RealmEntity, ListMixin<T> impleme
     if (element is RealmObjectBase && !element.isManaged) {
       throw RealmStateError('Cannot call remove on a managed list with an element that is an unmanaged object');
     }
+
     final index = indexOf(element);
     if (index < 0) {
       return false;
@@ -173,6 +174,17 @@ class ManagedRealmList<T extends Object?> with RealmEntity, ListMixin<T> impleme
     if (element is RealmObjectBase && !element.isManaged) {
       throw RealmStateError('Cannot call indexOf on a managed list with an element that is an unmanaged object');
     }
+
+    if (element is RealmValue) {
+      if (element.type.isCollection) {
+        return -1;
+      }
+
+      if (element.value is RealmObjectBase && !(element.value as RealmObjectBase).isManaged) {
+        return -1;
+      }
+    }
+
     if (start < 0) start = 0;
     final index = realmCore.listFind(this, element);
     return index < start ? -1 : index; // to align with dart list semantics
@@ -205,7 +217,13 @@ class ManagedRealmList<T extends Object?> with RealmEntity, ListMixin<T> impleme
 }
 
 class UnmanagedRealmList<T extends Object?> extends collection.DelegatingList<T> with RealmEntity implements RealmList<T> {
-  UnmanagedRealmList([Iterable<T>? items]) : super(List<T>.from(items ?? <T>[]));
+  final List<T> _base;
+
+  UnmanagedRealmList([Iterable<T>? items]) : this._(List<T>.from(items ?? <T>[]));
+
+  UnmanagedRealmList._(List<T> items)
+      : _base = items,
+        super(items);
 
   @override
   RealmObjectMetadata? get _metadata => throw RealmException("Unmanaged lists don't have metadata associated with them.");
@@ -224,6 +242,14 @@ class UnmanagedRealmList<T extends Object?> extends collection.DelegatingList<T>
 
   @override
   Stream<RealmListChanges<T>> get changes => throw RealmStateError("Unmanaged lists don't support changes");
+
+  @override
+  bool operator ==(Object? other) {
+    return _base == other;
+  }
+
+  @override
+  int get hashCode => _base.hashCode;
 }
 
 // The query operations on lists, only work for list of objects (core restriction),
@@ -264,6 +290,10 @@ extension RealmListInternal<T extends Object?> on RealmList<T> {
 
   RealmObjectMetadata? get metadata => asManaged()._metadata;
 
+  static RealmList<T> createFromList<T>(List<T> items) {
+    return UnmanagedRealmList._(items);
+  }
+
   static RealmList<T> create<T extends Object?>(RealmListHandle handle, Realm realm, RealmObjectMetadata? metadata) => RealmList<T>._(handle, realm, metadata);
 
   static void setValue(RealmListHandle handle, Realm realm, int index, Object? value, {bool update = false, bool insert = false}) {
@@ -288,19 +318,14 @@ extension RealmListInternal<T extends Object?> on RealmList<T> {
         return;
       }
 
-      if (value is RealmValue) {
-        value = value.value;
+      if (value is RealmValue && value.type.isCollection) {
+        realmCore.listAddCollectionAt(handle, realm, index, value, insert || index >= length);
+        return;
       }
 
-      if (value is RealmObject && !value.isManaged) {
-        realm.add<RealmObject>(value, update: update);
-      }
+      realm.addUnmanagedRealmObjectFromValue(value, update);
 
-      if (insert || index >= length) {
-        realmCore.listInsertElementAt(handle, index, value);
-      } else {
-        realmCore.listSetElementAt(handle, index, value);
-      }
+      realmCore.listAddElementAt(handle, index, value, insert || index >= length);
     } on Exception catch (e) {
       throw RealmException("Error setting value at index $index. Error: $e");
     }

lib/src/map.dart

@@ -49,7 +49,13 @@ abstract class RealmMap<T extends Object?> with RealmEntity implements MapBase<S
 }
 
 class UnmanagedRealmMap<T extends Object?> extends collection.DelegatingMap<String, T> with RealmEntity implements RealmMap<T> {
-  UnmanagedRealmMap([Map<String, T>? items]) : super(Map<String, T>.from(items ?? <String, T>{}));
+  final Map<String, T> _base;
+
+  UnmanagedRealmMap([Map<String, T>? items]) : this._(Map<String, T>.from(items ?? <String, T>{}));
+
+  UnmanagedRealmMap._(Map<String, T> items)
+      : _base = items,
+        super(items);
 
   @override
   bool get isValid => true;
@@ -59,6 +65,14 @@ class UnmanagedRealmMap<T extends Object?> extends collection.DelegatingMap<Stri
 
   @override
   Stream<RealmMapChanges<T>> get changes => throw RealmStateError("Unmanaged maps don't support changes");
+
+  @override
+  bool operator ==(Object? other) {
+    return _base == other;
+  }
+
+  @override
+  int get hashCode => _base.hashCode;
 }
 
 class ManagedRealmMap<T extends Object?> with RealmEntity, MapMixin<String, T> implements RealmMap<T> {
@@ -172,8 +186,14 @@ class ManagedRealmMap<T extends Object?> with RealmEntity, MapMixin<String, T> i
       return false;
     }
 
-    if (value is RealmValue && value.value is RealmObjectBase && !(value.value as RealmObjectBase).isManaged) {
-      return false;
+    if (value is RealmValue) {
+      if (value.value is RealmObjectBase && !(value.value as RealmObjectBase).isManaged) {
+        return false;
+      }
+
+      if (value.type.isCollection) {
+        return false;
+      }
     }
 
     return realmCore.mapContainsValue(this, value);
@@ -246,6 +266,8 @@ extension RealmMapInternal<T extends Object?> on RealmMap<T> {
 
   RealmObjectMetadata? get metadata => asManaged()._metadata;
 
+  static RealmMap<T> createFromMap<T>(Map<String, T> map) => UnmanagedRealmMap._(map);
+
   static RealmMap<T> create<T extends Object?>(RealmMapHandle handle, Realm realm, RealmObjectMetadata? metadata) =>
       ManagedRealmMap<T>._(handle, realm, metadata);
 
@@ -261,13 +283,12 @@ extension RealmMapInternal<T extends Object?> on RealmMap<T> {
         return;
       }
 
-      if (value is RealmValue) {
-        value = value.value;
+      if (value is RealmValue && value.type.isCollection) {
+        realmCore.mapInsertCollection(handle, realm, key, value);
+        return;
       }
 
-      if (value is RealmObject && !value.isManaged) {
-        realm.add<RealmObject>(value, update: update);
-      }
+      realm.addUnmanagedRealmObjectFromValue(value, update);
 
       realmCore.mapInsertValue(handle, key, value);
     } on Exception catch (e) {