From 280a94ebe9937874d6e5025fffd37ccfe31fcd79 Mon Sep 17 00:00:00 2001 From: "andrew.john.hill@gmail.com" Date: Fri, 25 Aug 2023 09:51:13 +1000 Subject: [PATCH 1/4] When there are continuations to the container scan, the later pages should not advance the timestamp past when the first page of the scan occurs. Without this change an update to a file on the first page that occurs while the later API calls are occurring might be missed. --- .../Blobs/Listeners/ContainerScanInfo.cs | 1 + .../Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs index f9008f957..932e5187e 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs @@ -17,5 +17,6 @@ internal class ContainerScanInfo public DateTime CurrentSweepCycleLatestModified { get; set; } public BlobContinuationToken ContinuationToken { get; set; } + public DateTime CurrentScanBeginTime { get; set; } } } diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index 81a9992f0..3da53bf54 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -214,6 +214,7 @@ public async Task> PollNewBlobsAsync( // if starting the cycle, reset the sweep time if (continuationToken == null) { + containerScanInfo.CurrentScanBeginTime = DateTime.UtcNow; containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue; } @@ -259,7 +260,7 @@ public async Task> PollNewBlobsAsync( var properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; - if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified) + if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && (continuationToken is null || lastModifiedTimestamp< containerScanInfo.CurrentScanBeginTime)) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } From 2fa027f8e9e39935c092f1bbd3aa51822921e9ca Mon Sep 17 00:00:00 2001 From: "andrew.john.hill@gmail.com" Date: Fri, 25 Aug 2023 10:07:07 +1000 Subject: [PATCH 2/4] codestyle and comment. --- .../Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index 3da53bf54..e15009149 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -260,7 +260,8 @@ public async Task> PollNewBlobsAsync( var properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; - if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && (continuationToken is null || lastModifiedTimestamp< containerScanInfo.CurrentScanBeginTime)) + // if we are not the first page of this scan (ie, we have a continuation token), then we can not advance past the time the first page was called, otherwise we might miss updates to those blobs + if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && (continuationToken == null || lastModifiedTimestamp < containerScanInfo.CurrentScanBeginTime)) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } From 786fcaf814aac6781d784ae9532b5b59976fad80 Mon Sep 17 00:00:00 2001 From: "andrew.john.hill@gmail.com" Date: Fri, 25 Aug 2023 10:40:32 +1000 Subject: [PATCH 3/4] init and var rename. --- .../Blobs/Listeners/ContainerScanInfo.cs | 2 +- .../Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs index 932e5187e..ef3531531 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ContainerScanInfo.cs @@ -17,6 +17,6 @@ internal class ContainerScanInfo public DateTime CurrentSweepCycleLatestModified { get; set; } public BlobContinuationToken ContinuationToken { get; set; } - public DateTime CurrentScanBeginTime { get; set; } + public DateTime CurrentSweepCycleStartTime { get; set; } } } diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index e15009149..fbe6f0dd0 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -76,7 +76,8 @@ public async Task RegisterAsync(CloudBlobContainer container, ITriggerExecutor>(), LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue, - CurrentSweepCycleLatestModified = DateTime.MinValue, + CurrentSweepCycleStartTime = DateTime.MinValue, + CurrentSweepCycleLatestModified = DateTime.MinValue, // value not used as it is set on first use when the continuationToken is null. ContinuationToken = null }; @@ -214,7 +215,7 @@ public async Task> PollNewBlobsAsync( // if starting the cycle, reset the sweep time if (continuationToken == null) { - containerScanInfo.CurrentScanBeginTime = DateTime.UtcNow; + containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow; containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue; } @@ -261,7 +262,7 @@ public async Task> PollNewBlobsAsync( DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; // if we are not the first page of this scan (ie, we have a continuation token), then we can not advance past the time the first page was called, otherwise we might miss updates to those blobs - if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && (continuationToken == null || lastModifiedTimestamp < containerScanInfo.CurrentScanBeginTime)) + if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified && (continuationToken == null || lastModifiedTimestamp < containerScanInfo.CurrentSweepCycleStartTime)) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } From 1615f07bde4c56d695dfd0d416b28788e914172f Mon Sep 17 00:00:00 2001 From: "andrew.john.hill@gmail.com" Date: Fri, 25 Aug 2023 10:41:20 +1000 Subject: [PATCH 4/4] oops --- .../Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs index fbe6f0dd0..1ac17df88 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs +++ b/src/Microsoft.Azure.WebJobs.Extensions.Storage/Blobs/Listeners/ScanBlobScanLogHybridPollingStrategy.cs @@ -76,8 +76,8 @@ public async Task RegisterAsync(CloudBlobContainer container, ITriggerExecutor>(), LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue, - CurrentSweepCycleStartTime = DateTime.MinValue, - CurrentSweepCycleLatestModified = DateTime.MinValue, // value not used as it is set on first use when the continuationToken is null. + CurrentSweepCycleStartTime = DateTime.MinValue, // value not used as it is set on first use when the continuationToken is null. + CurrentSweepCycleLatestModified = DateTime.MinValue, ContinuationToken = null };