improve query performance and commit performance in some cases

src/Crdt.Core/CommitBase.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics.CodeAnalysis;
+using System.ComponentModel.DataAnnotations;
+using System.Diagnostics.CodeAnalysis;
 using System.IO.Hashing;
 using System.Text.Json.Serialization;
 
@@ -12,37 +13,23 @@ public abstract class CommitBase
 {
     public const string NullParentHash = "0000";
     [JsonConstructor]
-    protected internal CommitBase(Guid id, string hash, string parentHash, HybridDateTime hybridDateTime)
+    protected internal CommitBase(Guid id, HybridDateTime hybridDateTime)
     {
         Id = id;
-        Hash = hash;
-        ParentHash = parentHash;
         HybridDateTime = hybridDateTime;
     }
 
     internal CommitBase(Guid id)
     {
         Id = id;
-        Hash = GenerateHash(NullParentHash);
-        ParentHash = NullParentHash;
     }
 
     public (DateTimeOffset, long, Guid) CompareKey => (HybridDateTime.DateTime, HybridDateTime.Counter, Id);
     public Guid Id { get; }
     public required HybridDateTime HybridDateTime { get; init; }
     public DateTimeOffset DateTime => HybridDateTime.DateTime;
-    [JsonIgnore]
-    public string Hash { get; private set; }
-
-    [JsonIgnore]
-    public string ParentHash { get; private set; }
     public CommitMetadata Metadata { get; init; } = new();
 
-    public void SetParentHash(string parentHash)
-    {
-        Hash = GenerateHash(parentHash);
-        ParentHash = parentHash;
-    }
 
     public string GenerateHash(string parentHash)
     {
@@ -65,7 +52,7 @@ public override string ToString()
 /// <inheritdoc cref="CommitBase"/>
 public abstract class CommitBase<TChange> : CommitBase
 {
-    internal CommitBase(Guid id, string hash, string parentHash, HybridDateTime hybridDateTime) : base(id, hash, parentHash, hybridDateTime)
+    internal CommitBase(Guid id, HybridDateTime hybridDateTime) : base(id, hybridDateTime)
     {
     }
 

src/Crdt.Core/QueryHelpers.cs

@@ -32,9 +32,11 @@ public static async Task<ChangesResult<TCommit>> GetChanges<TCommit, TChange>(th
                 var otherDt = DateTimeOffset.FromUnixTimeMilliseconds(otherTimestamp);
                 //todo even slower we want to also filter out changes that are already in the other history
                 //client has newer history than the other history
-                newHistory.AddRange(await commits.Include(c => c.ChangeEntities).DefaultOrder()
+                newHistory.AddRange((await commits.Include(c => c.ChangeEntities).DefaultOrder()
                     .Where(c => c.ClientId == clientId && c.HybridDateTime.DateTime > otherDt)
-                    .ToArrayAsync());
+                    .ToArrayAsync())
+                    //fixes an issue where the query would include commits that are already in the other history
+                    .Where(c => c.DateTime.ToUnixTimeMilliseconds() > otherTimestamp));
             }
         }
 

src/Crdt.Core/ServerCommit.cs

@@ -7,9 +7,7 @@ namespace Crdt.Core;
 public class ServerCommit : CommitBase<ServerJsonChange>
 {
     [JsonConstructor]
-    protected ServerCommit(Guid id, string hash, string parentHash, HybridDateTime hybridDateTime) : base(id,
-        hash,
-        parentHash,
+    protected ServerCommit(Guid id, HybridDateTime hybridDateTime) : base(id,
         hybridDateTime)
     {
     }

src/Crdt.Tests/RepositoryTests.cs

@@ -238,4 +238,23 @@ await _repository.AddSnapshots([
         _crdtDbContext.Snapshots.Should().ContainSingle()
             .Which.CommitId.Should().Be(ids[0]);
     }
+
+    [Fact]
+    public async Task GetChanges_HandlesExactDateFilters()
+    {
+        var tmpTime = Time(2, 0);
+        //by adding a tick we cause an error and commit 2 will be returned by the query
+        var commit2Time = tmpTime with { DateTime = tmpTime.DateTime.AddTicks(1) };
+        await _repository.AddCommits([
+            Commit(Guid.NewGuid(), Time(1, 0)),
+            Commit(Guid.NewGuid(), commit2Time),
+            Commit(Guid.NewGuid(), Time(3, 0)),
+        ]);
+
+        var changes = await _repository.GetChanges(new SyncState(new()
+        {
+            { Guid.Empty, commit2Time.DateTime.ToUnixTimeMilliseconds() }
+        }));
+        changes.MissingFromClient.Select(c => c.DateTime.ToUnixTimeMilliseconds()).Should().ContainSingle("because {0} is only before the last commit", commit2Time.DateTime.ToUnixTimeMilliseconds());
+    }
 }

src/Crdt/Commit.cs

@@ -9,21 +9,34 @@ public class Commit : CommitBase<IChange>
 {
     [JsonConstructor]
     protected Commit(Guid id, string hash, string parentHash, HybridDateTime hybridDateTime) : base(id,
-        hash,
-        parentHash,
         hybridDateTime)
     {
+        Hash = hash;
+        ParentHash = parentHash;
     }
 
     internal Commit(Guid id) : base(id)
     {
+        Hash = GenerateHash(NullParentHash);
+        ParentHash = NullParentHash;
     }
 
+    public void SetParentHash(string parentHash)
+    {
+        Hash = GenerateHash(parentHash);
+        ParentHash = parentHash;
+    }
     internal Commit() : this(Guid.NewGuid())
     {
 
     }
 
     [JsonIgnore]
     public List<ObjectSnapshot> Snapshots { get; init; } = [];
+
+    [JsonIgnore]
+    public string Hash { get; private set; }
+
+    [JsonIgnore]
+    public string ParentHash { get; private set; }
 }

src/Crdt/CrdtConfig.cs

@@ -9,7 +9,11 @@ namespace Crdt;
 
 public class CrdtConfig
 {
-    public bool EnableProjectedTables { get; set; } = false;
+    /// <summary>
+    /// recommended to increase query performance, as getting objects can just query the table for that object.
+    /// it does however increase database size as now objects are stored both in snapshots and in their projected tables
+    /// </summary>
+    public bool EnableProjectedTables { get; set; } = true;
     public ChangeTypeListBuilder ChangeTypeListBuilder { get; } = new();
     public ObjectTypeListBuilder ObjectTypeListBuilder { get; } = new();
 

src/Crdt/DataModel.cs

@@ -9,7 +9,7 @@ namespace Crdt;
 
 public record SyncResults(Commit[] MissingFromLocal, Commit[] MissingFromRemote, bool IsSynced);
 
-public class DataModel : ISyncable
+public class DataModel : ISyncable, IAsyncDisposable
 {
     /// <summary>
     /// after adding any commit validate the commit history, not great for performance but good for testing.
@@ -61,7 +61,8 @@ public async Task<Commit> AddChanges(
         Guid clientId,
         IEnumerable<IChange> changes,
         Guid commitId = default,
-        CommitMetadata? commitMetadata = null)
+        CommitMetadata? commitMetadata = null,
+        bool deferCommit = false)
     {
         commitId = commitId == default ? Guid.NewGuid() : commitId;
         var commit = new Commit(commitId)
@@ -71,21 +72,47 @@ public async Task<Commit> AddChanges(
             ChangeEntities = [..changes.Select(ToChangeEntity)],
             Metadata = commitMetadata ?? new()
         };
-        await Add(commit);
+        await Add(commit, deferCommit);
         return commit;
-    }
+    } 
 
-    private async Task Add(Commit commit)
+    private List<Commit> _deferredCommits = [];
+    private async Task Add(Commit commit, bool deferSnapshotUpdates)
     {
         if (await _crdtRepository.HasCommit(commit.Id)) return;
 
         await using var transaction = await _crdtRepository.BeginTransactionAsync();
         await _crdtRepository.AddCommit(commit);
-        await UpdateSnapshots(commit, [commit]);
-        if (_autoValidate) await ValidateCommits();
+        if (!deferSnapshotUpdates)
+        {
+            //if there are deferred commits, update snapshots with them first
+            if (_deferredCommits is not []) await UpdateSnapshotsByDeferredCommits();
+            await UpdateSnapshots(commit, [commit]);
+            if (_autoValidate) await ValidateCommits();
+        }
+        else
+        {
+            _deferredCommits.Add(commit);
+        }
         await transaction.CommitAsync();
     }
 
+    public async ValueTask DisposeAsync()
+    {
+        if (_deferredCommits is []) return;
+        await UpdateSnapshotsByDeferredCommits();
+    }
+
+    private async Task UpdateSnapshotsByDeferredCommits()
+    {
+        var commits = Interlocked.Exchange(ref _deferredCommits, []);
+        var oldestChange = commits.MinBy(c => c.CompareKey);
+        if (oldestChange is null) return;
+        await UpdateSnapshots(oldestChange, commits.ToArray());
+        if (_autoValidate) await ValidateCommits();
+    }
+
+
     private static ChangeEntity<IChange> ToChangeEntity(IChange change, int index)
     {
         return new ChangeEntity<IChange>()
@@ -103,6 +130,8 @@ async Task ISyncable.AddRangeFromSync(IEnumerable<Commit> commits)
         if (oldestChange is null || newCommits is []) return;
 
         await using var transaction = await _crdtRepository.BeginTransactionAsync();
+        //if there are deferred commits, update snapshots with them first
+        if (_deferredCommits is not []) await UpdateSnapshotsByDeferredCommits();
         //don't save since UpdateSnapshots will also modify newCommits with hashes, so changes will be saved once that's done
         await _crdtRepository.AddCommits(newCommits, false);
         await UpdateSnapshots(oldestChange, newCommits);

src/Crdt/Db/CrdtRepository.cs

@@ -137,15 +137,14 @@ public async Task<IObjectBase> GetObjectBySnapshotId(Guid snapshotId)
         return snapshot?.Entity.Is<T>();
     }
 
-    public IQueryable<T> GetCurrentObjects<T>(Expression<Func<ObjectSnapshot, bool>>? predicate = null) where T : class, IObjectBase
+    public IQueryable<T> GetCurrentObjects<T>() where T : class, IObjectBase
     {
-        if (crdtConfig.Value.EnableProjectedTables && predicate is null)
+        if (crdtConfig.Value.EnableProjectedTables)
         {
-            return _dbContext.Set<T>().Where(e => CurrentSnapshotIds().Contains(EF.Property<Guid>(e, ObjectSnapshot.ShadowRefName)));
+            return _dbContext.Set<T>();
         }
         var typeName = DerivedTypeHelper.GetEntityDiscriminator<T>();
         var queryable = CurrentSnapshots().Where(s => s.TypeName == typeName && !s.EntityIsDeleted);
-        if (predicate is not null) queryable = queryable.Where(predicate);
         return queryable.Select(s => (T)s.Entity);
     }