realm compact

CHANGELOG.md

@@ -12,6 +12,7 @@
 * Support results of primitives, ie. `RealmResult<int>`. ([#162](https://github.com/realm/realm-dart/issues/162))
 * Support notifications on all managed realm lists, including list of primitives, ie. `RealmList<int>.changes` is supported. ([#893](https://github.com/realm/realm-dart/pull/893))
 * Support named backlinks on realm models. You can now add and annotate a realm object iterator field with `@Backlink(#fieldName)`. ([#996](https://github.com/realm/realm-dart/pull/996))
+* Added Realm file compaction support. ([#1005](https://github.com/realm/realm-dart/pull/1005))
 * Allow `@Indexed` attribute on all indexable type, and ensure appropriate indexes are created in the realm. ([#797](https://github.com/realm/realm-dart/issues/797))
 * Add `parent` getter on embedded objects. ([#979](https://github.com/realm/realm-dart/pull/979))
 * Support [Client Resets](https://www.mongodb.com/docs/atlas/app-services/sync/error-handling/client-resets/). Atlas App Services automatically detects the need for client resets and the realm client automatically performs it according to the configured callbacks for the type of client reset handlers set on `FlexibleSyncConfiguration`. A parameter `clientResetHandler` is added to `Configuration.flexibleSync`. Supported client reset handlers are `ManualRecoveryHandler`, `DiscardUnsyncedChangesHandler`, `RecoverUnsyncedChangesHandler` and `RecoverOrDiscardUnsyncedChangesHandler`. `RecoverOrDiscardUnsyncedChangesHandler` is the default strategy. ([#925](https://github.com/realm/realm-dart/pull/925)) An example usage of the default `clientResetHandler` is as follows:

lib/src/native/realm_core.dart

@@ -2249,6 +2249,14 @@ class _RealmCore {
     });
   }
 
+  bool compact(Realm realm) {
+    return using((arena) {
+      final out_did_compact = arena<Bool>();
+      _realmLib.invokeGetBool(() => _realmLib.realm_compact(realm.handle._pointer, out_did_compact));
+      return out_did_compact.value;
+    });
+  }
+
   void immediatelyRunFileActions(App app, String realmPath) {
     using((arena) {
       _realmLib.invokeGetBool(() => _realmLib.realm_sync_immediately_run_file_actions(app.handle._pointer, realmPath.toCharPtr(arena)),

lib/src/realm_class.dart

@@ -497,6 +497,44 @@ class Realm implements Finalizable {
       throw RealmError('Starting a write transaction on a frozen Realm is not allowed.');
     }
   }
+
+  /// Compacts a Realm file. A Realm file usually contains free/unused space.
+  ///
+  /// This method removes this free space and the file size is thereby reduced.
+  /// Objects within the Realm file are untouched.
+  /// Note: The file system should have free space for at least a copy of the Realm file. This method must not be called inside a transaction.
+  /// The Realm file is left untouched if any file operation fails.
+  static bool compact(Configuration config) {
+    if (config is InMemoryConfiguration) {
+      throw RealmException("Can't compact an in-memory Realm");
+    }
+
+    late Configuration compactConfig;
+
+    if (!File(config.path).existsSync()) {
+      return false;
+    }
+
+    if (config is LocalConfiguration) {
+      // `compact` opens the realm file so it can triger schema version upgrade, file format upgrade, migration and initial data callbacks etc.
+      // We must allow that to happen so use the local config as is.
+      compactConfig = config;
+    } else if (config is DisconnectedSyncConfiguration) {
+      compactConfig = config;
+    } else if (config is FlexibleSyncConfiguration) {
+      compactConfig = Configuration.disconnectedSync(config.schemaObjects.toList(),
+          fifoFilesFallbackPath: config.fifoFilesFallbackPath, path: config.path, encryptionKey: config.encryptionKey);
+    } else {
+      throw RealmError("Unsupported realm configuration type ${config.runtimeType}");
+    }
+
+    final realm = Realm(compactConfig);
+    try {
+      return realmCore.compact(realm);
+    } finally {
+      realm.close();
+    }
+  }
 }
 
 /// Provides a scope to safely write data to a [Realm]. Can be created using [Realm.beginWrite] or

test/realm_test.dart

@@ -20,6 +20,8 @@
 
 import 'dart:convert';
 import 'dart:io';
+import 'dart:isolate';
+import 'dart:math';
 import 'package:test/test.dart' hide test, throws;
 import 'package:timezone/timezone.dart' as tz;
 import 'package:timezone/data/latest.dart' as tz;
@@ -1394,6 +1396,173 @@ Future<void> main([List<String>? args]) async {
     expect(await realmIsCancelled, isTrue);
     expect(progressReturned, isFalse);
   });
+
+  const compactTest = "compact_test";
+  void addDataForCompact(Realm realm) {
+    realm.write(() {
+      for (var i = 0; i < 2500; i++) {
+        realm.add(Product(ObjectId(), compactTest));
+      }
+    });
+
+    realm.write(() => realm.deleteMany(realm.query<Product>("stringQueryField CONTAINS '$compactTest'")));
+
+    realm.write(() {
+      for (var i = 0; i < 10; i++) {
+        realm.add(Product(ObjectId(), compactTest));
+      }
+    });
+  }
+
+  Future<int> createRealmForCompact(Configuration config) async {
+    var realm = getRealm(config);
+
+    if (config is FlexibleSyncConfiguration) {
+      realm.subscriptions.update((mutableSubscriptions) {
+        mutableSubscriptions.add(realm.query<Product>("stringQueryField CONTAINS '$compactTest'"));
+      });
+      await realm.subscriptions.waitForSynchronization();
+    }
+
+    addDataForCompact(realm);
+
+    if (config is FlexibleSyncConfiguration) {
+      await realm.syncSession.waitForDownload();
+      await realm.syncSession.waitForUpload();
+    }
+
+    final beforeSize = await File(config.path).stat().then((value) => value.size);
+
+    realm.close();
+    return beforeSize;
+  }
+
+  void validateCompact(bool compacted, String realmPath, int beforeCompactSizeSize) async {
+    expect(compacted, true);
+    final afterCompactSize  = await File(realmPath).stat().then((value) => value.size);
+    expect(beforeCompactSizeSize, greaterThan(afterCompactSize));
+  }
+
+  test('Realm - local realm can be compacted', () async {
+    var config = Configuration.local([Product.schema], path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+    final beforeCompactSizeSize = await createRealmForCompact(config);
+
+    final compacted = Realm.compact(config);
+
+    validateCompact(compacted, config.path, beforeCompactSizeSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(config);
+  });
+
+   test('Realm - non existing realm can not be compacted', () async {
+    var config = Configuration.local([Product.schema], path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+    final compacted = Realm.compact(config);
+    expect(compacted, false);
+  });
+
+  test('Realm - local realm can be compacted in worker isolate', () async {
+    var config = Configuration.local([Product.schema], path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+    final beforeCompactSizeSize = await createRealmForCompact(config);
+
+    final receivePort = ReceivePort();
+    await Isolate.spawn((List<Object> args) async {
+      SendPort sendPort = args[0] as SendPort;
+      final path = args[1] as String;
+      var config = Configuration.local([Product.schema], path: path);
+      final compacted = Realm.compact(config);
+      Isolate.exit(sendPort, compacted);
+    }, [receivePort.sendPort, config.path]);
+
+    final compacted = await receivePort.first as bool;
+
+    validateCompact(compacted, config.path, beforeCompactSizeSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(config);
+  });
+
+  test('Realm - local encrypted realm can be compacted', () async {
+    final config = Configuration.local([Product.schema],
+        encryptionKey: generateEncryptionKey(), path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+
+    final beforeCompactSizeSize = await createRealmForCompact(config);
+
+    final compacted = Realm.compact(config);
+
+    validateCompact(compacted, config.path, beforeCompactSizeSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(config);
+  });
+
+  test('Realm - in-memory realm can not be compacted', () async {
+    var config = Configuration.inMemory([Product.schema], path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+    expect(() => Realm.compact(config), throws<RealmException>("Can't compact an in-memory Realm"));
+  });
+
+  test('Realm - readonly realm can not be compacted', () async {
+    var path = p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm");
+    var config = Configuration.local([Product.schema], path: path);
+    final beforeCompactSize = await createRealmForCompact(config);
+
+    config = Configuration.local([Product.schema], isReadOnly: true, path: path);
+    expect(() => Realm.compact(config), throws<RealmException>("Can't compact a read-only Realm"));
+
+    //test the realm can be opened.
+    final realm = getRealm(config);
+  });
+
+  test('Realm - disconnected sync realm can be compacted', () async {
+    var config = Configuration.disconnectedSync([Product.schema], path: p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm"));
+
+    final beforeCompactSize = await createRealmForCompact(config);
+
+    final compacted = Realm.compact(config);
+
+    validateCompact(compacted, config.path, beforeCompactSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(config);
+  });
+
+  baasTest('Realm - synced realm can be compacted', (appConfiguration) async {
+    final app = App(appConfiguration);
+    final credentials = Credentials.anonymous();
+    final user = await app.logIn(credentials);
+    final path = p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm");
+    final config = Configuration.flexibleSync(user, [Product.schema], path: path);
+    final beforeCompactSize = await createRealmForCompact(config);
+    user.logOut();
+    Future<void>.delayed(Duration(seconds: 5));
+
+    final compacted = Realm.compact(config);
+
+    validateCompact(compacted, config.path, beforeCompactSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(Configuration.disconnectedSync([Product.schema], path: path));
+  });
+
+  baasTest('Realm - synced encrypted realm can be compacted', (appConfiguration) async {
+    final app = App(appConfiguration);
+    final credentials = Credentials.anonymous();
+    final path = p.join(Configuration.defaultStoragePath, "${generateRandomString(8)}.realm");
+    final user = await app.logIn(credentials);
+    List<int> key = List<int>.generate(encryptionKeySize, (i) => random.nextInt(256));
+    final config =
+        Configuration.flexibleSync(user, [Product.schema], encryptionKey: key, path: path);
+    final beforeCompactSize = await createRealmForCompact(config);
+    user.logOut();
+    Future<void>.delayed(Duration(seconds: 5));
+
+    final compacted = Realm.compact(config);
+
+    validateCompact(compacted, config.path, beforeCompactSize);
+
+    //test the realm can be opened.
+    final realm = getRealm(Configuration.disconnectedSync([Product.schema], path: path, encryptionKey: key));
+  });
 }
 
 List<int> generateEncryptionKey() {

test/test.dart

@@ -398,8 +398,8 @@ String generateRandomRealmPath() {
 
 final random = Random();
 String generateRandomString(int len) {
-  const _chars = 'abcdefghjklmnopqrstuvwxuz';
-  return List.generate(len, (index) => _chars[random.nextInt(_chars.length)]).join();
+  const chars = 'abcdefghjklmnopqrstuvwxuz';
+  return List.generate(len, (index) => chars[random.nextInt(chars.length)]).join();
 }
 
 Realm getRealm(Configuration config) {
@@ -467,13 +467,21 @@ Future<void> tryDeleteRealm(String path) async {
     return;
   }
 
+  final dummy = File("");
+  const duration = Duration(milliseconds: 100);
   for (var i = 0; i < 5; i++) {
     try {
       Realm.deleteRealm(path);
-      await File('$path.lock').delete();
+
+      //delete lock file
+      await File('$path.lock').delete().onError((error, stackTrace) => dummy);
+
+      //Bug in Core https://github.com/realm/realm-core/issues/5997. Remove when fixed
+      //delete compaction space file
+      await File('$path.tmp_compaction_space').delete().onError((error, stackTrace) => dummy);
+
       return;
     } catch (e) {
-      const duration = Duration(milliseconds: 100);
       print('Failed to delete realm at path $path. Trying again in ${duration.inMilliseconds}ms');
       await Future<void>.delayed(duration);
     }