From 746536f0b94142483ba54c2e3c2b90afcf87b69b Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 13 Oct 2023 11:53:44 -0700 Subject: [PATCH 1/2] More fixes for Pause/Resume --- .../src/BlobsStorageResourceProvider.cs | 15 ++++++++---- .../src/CheckpointerExtensions.cs | 17 ++++++++------ .../src/DataTransferProperties.cs | 6 +++-- .../src/Shared/DataMovementConstants.cs | 3 ++- .../src/Shared/DataMovementExtensions.cs | 16 ++++--------- .../src/Shared/JobPlan/JobPlanHeader.cs | 15 ++++++++++++ .../src/Shared/LocalTransferCheckpointer.cs | 2 ++ .../tests/GetTransfersTests.cs | 22 ++++++++++++++---- .../tests/JobPlanHeaderTests.cs | 11 +++++++-- .../tests/LocalTransferCheckpointerFactory.cs | 2 ++ .../tests/PauseResumeTransferTests.cs | 20 ++++++++-------- .../tests/Resources/SampleJobPlanFile.b1.ndm | Bin 107 -> 108 bytes .../tests/Shared/CheckpointerTesting.cs | 2 ++ 13 files changed, 89 insertions(+), 42 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs index cb204c7fc6dc2..35796dffd72ac 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs @@ -729,13 +729,20 @@ public async Task RehydrateAsync( #endregion private static ResourceType GetType(string typeId, bool isContainer) - => typeId switch + { + if (isContainer) { - "BlockBlob" => isContainer ? ResourceType.BlobContainer : ResourceType.BlockBlob, - "PageBlob" => isContainer ? ResourceType.BlobContainer : ResourceType.PageBlob, - "AppendBlob" => isContainer ? ResourceType.BlobContainer : ResourceType.AppendBlob, + return ResourceType.BlobContainer; + } + + return typeId switch + { + "BlockBlob" => ResourceType.BlockBlob, + "PageBlob" => ResourceType.PageBlob, + "AppendBlob" => ResourceType.AppendBlob, _ => ResourceType.Unknown }; + } private static ArgumentException BadResourceTypeException(ResourceType resourceType) => new ArgumentException( diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs index 8082e027d88fd..c5ffedb841ef5 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs @@ -54,23 +54,26 @@ internal static async Task GetDataTransferPropertiesAsyn header = JobPlanHeader.Deserialize(stream); } - (string sourceResourceId, string destResourceId) = await checkpointer.GetResourceIdsAsync( + string sourceTypeId = default; + string destinationTypeId = default; + // Only need to get type ids for single transfers + if (!header.IsContainer) + { + (sourceTypeId, destinationTypeId) = await checkpointer.GetResourceIdsAsync( transferId, cancellationToken).ConfigureAwait(false); - - bool isContainer = - (await checkpointer.CurrentJobPartCountAsync(transferId, cancellationToken).ConfigureAwait(false)) > 1; + } return new DataTransferProperties { TransferId = transferId, - SourceTypeId = sourceResourceId, + SourceTypeId = sourceTypeId, SourceUri = new Uri(header.ParentSourcePath), SourceProviderId = header.SourceProviderId, - DestinationTypeId = destResourceId, + DestinationTypeId = destinationTypeId, DestinationUri = new Uri(header.ParentDestinationPath), DestinationProviderId = header.DestinationProviderId, - IsContainer = isContainer, + IsContainer = header.IsContainer, }; } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs index 4a82a90abd499..f3b1605da236a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperties.cs @@ -21,7 +21,8 @@ public class DataTransferProperties public virtual string TransferId { get; internal set; } /// - /// Contains the Source Scheme of the Storage Resource to rehydrate the StorageResource from. + /// Contains the type id for the source resource to use during rehydration. + /// Will be null if is true. /// public virtual string SourceTypeId { get; internal set; } @@ -36,7 +37,8 @@ public class DataTransferProperties public virtual string SourceProviderId { get; internal set; } /// - /// Contains the Source Scheme of the Storage Resource to rehydrate the StorageResource from. + /// Contains the type id for the destination resource to use during rehydration. + /// Will be null if is true. /// public virtual string DestinationTypeId { get; internal set; } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs index d1f08f199e3c4..c71a94ae80e9c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs @@ -81,7 +81,8 @@ internal static class JobPlanFile internal const int OperationTypeIndex = CrateTimeIndex + LongSizeInBytes; internal const int SourceProviderIdIndex = OperationTypeIndex + OneByte; internal const int DestinationProviderIdIndex = SourceProviderIdIndex + ProviderIdNumBytes; - internal const int EnumerationCompleteIndex = DestinationProviderIdIndex + ProviderIdNumBytes; + internal const int IsContainerIndex = DestinationProviderIdIndex + ProviderIdNumBytes; + internal const int EnumerationCompleteIndex = IsContainerIndex + OneByte; internal const int JobStatusIndex = EnumerationCompleteIndex + OneByte; internal const int ParentSourcePathOffsetIndex = JobStatusIndex + IntSizeInBytes; internal const int ParentSourcePathLengthIndex = ParentSourcePathOffsetIndex + IntSizeInBytes; diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementExtensions.cs index 7321b30c574b2..de00ca6dbce4c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementExtensions.cs @@ -105,7 +105,7 @@ public static async Task ToJobPartAsync( // Apply credentials to the saved transfer job path string childSourcePath = header.SourcePath; - string childSourceName = childSourcePath.Substring(sourceResource.Uri.GetPath().Length + 1); + string childSourceName = childSourcePath.Substring(sourceResource.Uri.AbsoluteUri.Length + 1); string childDestinationPath = header.DestinationPath; string childDestinationName = childDestinationPath.Substring(destinationResource.Uri.AbsoluteUri.Length + 1); DataTransferStatus jobPartStatus = header.AtomicJobStatus; @@ -163,7 +163,7 @@ public static async Task ToJobPartAsync( string childSourcePath = header.SourcePath; string childSourceName = childSourcePath.Substring(sourceResource.Uri.AbsoluteUri.Length + 1); string childDestinationPath = header.DestinationPath; - string childDestinationName = childDestinationPath.Substring(destinationResource.Uri.GetPath().Length + 1); + string childDestinationName = childDestinationPath.Substring(destinationResource.Uri.AbsoluteUri.Length + 1); DataTransferStatus jobPartStatus = header.AtomicJobStatus; UriToStreamJobPart jobPart = await UriToStreamJobPart.CreateJobPartAsync( job: baseJob, @@ -257,11 +257,8 @@ internal static void VerifyJobPartPlanHeader(this JobPartInternal jobPart, JobPa throw Errors.MismatchTransferId(jobPart._dataTransfer.Id, header.TransferId); } - // Check source path' - // Remove any query or SAS that could be attach to the Uri - UriBuilder sourceUriBuilder = new UriBuilder(jobPart._sourceResource.Uri.AbsoluteUri); - sourceUriBuilder.Query = ""; - string passedSourcePath = sourceUriBuilder.Uri.AbsoluteUri; + // Check source path + string passedSourcePath = jobPart._sourceResource.Uri.ToSanitizedString(); // We only check if it starts with the path because if we're passed a container // then we only need to check if the prefix matches @@ -271,10 +268,7 @@ internal static void VerifyJobPartPlanHeader(this JobPartInternal jobPart, JobPa } // Check destination path - // Remove any query or SAS that could be attach to the Uri - UriBuilder destinationUriBuilder = new UriBuilder(jobPart._destinationResource.Uri.AbsoluteUri); - destinationUriBuilder.Query = ""; - string passedDestinationPath = destinationUriBuilder.Uri.AbsoluteUri; + string passedDestinationPath = jobPart._destinationResource.Uri.ToSanitizedString(); // We only check if it starts with the path because if we're passed a container // then we only need to check if the prefix matches diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs index e87e6f5456521..2226b249558e8 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPlanHeader.cs @@ -40,6 +40,11 @@ internal class JobPlanHeader /// public string DestinationProviderId; + /// + /// Whether the transfer is of a container or not. + /// + public bool IsContainer; + /// /// Whether or not the enumeration of the parent container has completed. /// @@ -67,6 +72,7 @@ public JobPlanHeader( JobPlanOperation operationType, string sourceProviderId, string destinationProviderId, + bool isContainer, bool enumerationComplete, DataTransferStatus jobStatus, string parentSourcePath, @@ -95,6 +101,7 @@ public JobPlanHeader( OperationType = operationType; SourceProviderId = sourceProviderId; DestinationProviderId = destinationProviderId; + IsContainer = isContainer; EnumerationComplete = enumerationComplete; JobStatus = jobStatus; ParentSourcePath = parentSourcePath; @@ -127,6 +134,9 @@ public void Serialize(Stream stream) // DestinationProviderId WritePaddedString(writer, DestinationProviderId, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + // IsContainer + writer.Write(Convert.ToByte(IsContainer)); + // EnumerationComplete writer.Write(Convert.ToByte(EnumerationComplete)); @@ -180,6 +190,10 @@ public static JobPlanHeader Deserialize(Stream stream) // DestinationProviderId string destProviderId = ReadPaddedString(reader, DataMovementConstants.JobPlanFile.ProviderIdNumBytes); + // IsContainer + byte isContainerByte = reader.ReadByte(); + bool isContainer = Convert.ToBoolean(isContainerByte); + // EnumerationComplete byte enumerationCompleteByte = reader.ReadByte(); bool enumerationComplete = Convert.ToBoolean(enumerationCompleteByte); @@ -224,6 +238,7 @@ public static JobPlanHeader Deserialize(Stream stream) operationType, sourceProviderId, destProviderId, + isContainer, enumerationComplete, jobPlanStatus.ToDataTransferStatus(), parentSourcePath, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs index cb80bace0d7ea..17160e35b1f4e 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/LocalTransferCheckpointer.cs @@ -67,6 +67,7 @@ public override async Task AddNewJobAsync( throw Errors.CollisionTransferIdCheckpointer(transferId); } + bool isContainer = source is StorageResourceContainer; JobPlanHeader header = new( DataMovementConstants.JobPlanFile.SchemaVersion, transferId, @@ -74,6 +75,7 @@ public override async Task AddNewJobAsync( GetOperationType(source, destination), source.ProviderId, destination.ProviderId, + isContainer, false, /* enumerationComplete */ new DataTransferStatusInternal(), source.Uri.ToSanitizedString(), diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs index 744c100c7ac91..eccbc82438ff1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/GetTransfersTests.cs @@ -246,8 +246,8 @@ public async Task GetResumableTransfers_LocalCheckpointer() new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = "LocalFile", SourceUri = new Uri(parentLocalUri1, "file1"), DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationUri = new Uri(parentRemoteUri, "file1"), IsContainer = false }, new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourceUri = new Uri(parentRemoteUri, "file2/"), DestinationProviderId = "local", DestinationTypeId = "LocalFile", DestinationUri = new Uri(parentLocalUri1, "file2/"), IsContainer = false }, new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourceUri = new Uri(parentRemoteUri, "file3"), DestinationProviderId = "blob", DestinationTypeId = "BlockBlob", DestinationUri = new Uri(parentRemoteUri, "file3"), IsContainer = false }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = "BlockBlob", SourceUri = parentRemoteUri, DestinationProviderId = "local", DestinationTypeId = "LocalFile", DestinationUri = parentLocalUri1, IsContainer = true }, - new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = "LocalFile", SourceUri = parentLocalUri2, DestinationProviderId = "blob", DestinationTypeId = "AppendBlob", DestinationUri = parentRemoteUri, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "blob", SourceTypeId = default, SourceUri = parentRemoteUri, DestinationProviderId = "local", DestinationTypeId = default, DestinationUri = parentLocalUri1, IsContainer = true }, + new DataTransferProperties { TransferId = Guid.NewGuid().ToString(), SourceProviderId = "local", SourceTypeId = default, SourceUri = parentLocalUri2, DestinationProviderId = "blob", DestinationTypeId = default, DestinationUri = parentRemoteUri, IsContainer = true }, }; // Add a transfer for each expected result @@ -322,7 +322,8 @@ private void AddTransferFromDataTransferProperties( parentSourcePath: properties.SourceUri.AbsoluteUri, parentDestinationPath: properties.DestinationUri.AbsoluteUri, sourceProviderId: properties.SourceProviderId, - destinationProviderId: properties.DestinationProviderId); + destinationProviderId: properties.DestinationProviderId, + isContainer: properties.IsContainer); if (properties.IsContainer) { @@ -343,6 +344,9 @@ private void AddTransferFromDataTransferProperties( destinationPaths.Add(properties.DestinationUri + $"file{i}"); } + // Because type ID is null on container transfers, derive a type from provider id + string sourceTypeId = GetTypeIdForProvider(properties.SourceProviderId); + string destinationTypeId = GetTypeIdForProvider(properties.DestinationProviderId); factory.CreateStubJobPartPlanFilesAsync( checkpointerPath, properties.TransferId, @@ -350,8 +354,8 @@ private void AddTransferFromDataTransferProperties( InProgressStatus, sourcePaths, destinationPaths, - sourceResourceId: properties.SourceTypeId, - destinationResourceId: properties.DestinationTypeId); + sourceResourceId: sourceTypeId, + destinationResourceId: destinationTypeId); } else { @@ -378,5 +382,13 @@ private void AssertTransferProperties(DataTransferProperties expected, DataTrans Assert.AreEqual(expected.DestinationUri.AbsoluteUri.TrimEnd('\\', '/'), actual.DestinationUri.AbsoluteUri.TrimEnd('\\', '/')); Assert.AreEqual(expected.IsContainer, actual.IsContainer); } + + private string GetTypeIdForProvider(string providerId) + => providerId switch + { + "blob" => "BlockBlob", + "local" => "LocalFile", + _ => "Unknown" + }; } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs index 156a3879b2ad7..f9c93668899bb 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs @@ -25,7 +25,8 @@ public void Ctor() Assert.AreEqual(DefaultJobPlanOperation, header.OperationType); Assert.AreEqual(DefaultSourceProviderId, header.SourceProviderId); Assert.AreEqual(DefaultDestinationProviderId, header.DestinationProviderId); - Assert.AreEqual(false, header.EnumerationComplete); + Assert.IsFalse(header.IsContainer); + Assert.IsFalse(header.EnumerationComplete); Assert.AreEqual(DefaultJobStatus, header.JobStatus); Assert.AreEqual(DefaultSourcePath, header.ParentSourcePath); Assert.AreEqual(DefaultDestinationPath, header.ParentDestinationPath); @@ -48,6 +49,11 @@ public void Serialize() CollectionAssert.AreEqual(expected, actual); } + + //using (FileStream fileStream = File.OpenWrite("D:\\azure-sdk-for-net\\sdk\\storage\\Azure.Storage.DataMovement\\tests\\Resources\\SampleJobPlanFile.b1.ndm")) + //{ + // header.Serialize(fileStream); + //} } [Test] @@ -81,7 +87,8 @@ private void DeserializeAndVerify(Stream stream, string version) Assert.AreEqual(DefaultJobPlanOperation, deserialized.OperationType); Assert.AreEqual(DefaultSourceProviderId, deserialized.SourceProviderId); Assert.AreEqual(DefaultDestinationProviderId, deserialized.DestinationProviderId); - Assert.AreEqual(false, deserialized.EnumerationComplete); + Assert.IsFalse(deserialized.IsContainer); + Assert.IsFalse(deserialized.EnumerationComplete); Assert.AreEqual(DefaultJobStatus, deserialized.JobStatus); Assert.AreEqual(DefaultSourcePath, deserialized.ParentSourcePath); Assert.AreEqual(DefaultDestinationPath, deserialized.ParentDestinationPath); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs index 73e7533bf5a54..4508a7e1aa937 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalTransferCheckpointerFactory.cs @@ -152,6 +152,7 @@ internal void CreateStubJobPlanFile( string parentDestinationPath = _testDestinationPath, string sourceProviderId = _testSourceProviderId, string destinationProviderId = _testDestinationProviderId, + bool isContainer = false, DataTransferStatus status = default) { status ??= new DataTransferStatus(); @@ -162,6 +163,7 @@ internal void CreateStubJobPlanFile( JobPlanOperation.ServiceToService, sourceProviderId, destinationProviderId, + isContainer, false, /* enumerationComplete */ status, parentSourcePath, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs index 5ac73d8e39fbf..5e5116989fe5a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs @@ -227,7 +227,7 @@ private async Task CreateSingleLongTransferAsync( return await manager.StartTransferAsync(sourceResource, destinationResource, transferOptions); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -281,7 +281,7 @@ public async Task TryPauseTransferAsync_Id(TransferDirection transferType) Assert.IsTrue(File.Exists(fileName.FullPath)); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -351,7 +351,7 @@ public void TryPauseTransferAsync_Error() Assert.CatchAsync(async () => await transferManager.PauseTransferIfRunningAsync("bad transfer Id")); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -410,7 +410,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused(TransferDirection transfer Assert.IsTrue(File.Exists(fileName.FullPath)); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -485,7 +485,7 @@ await AssertSourceAndDestinationAsync( destinationContainer: destinationContainer.Container); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -707,7 +707,7 @@ private async Task CreateDirectoryLongTransferAsync( return await manager.StartTransferAsync(sourceResource, destinationResource, transferOptions); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -758,7 +758,7 @@ public async Task TryPauseTransferAsync_Id_Directory(TransferDirection transferT Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -809,7 +809,7 @@ public async Task TryPauseTransferAsync_DataTransfer_Directory(TransferDirection Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -866,7 +866,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -945,7 +945,7 @@ await AssertDirectorySourceAndDestinationAsync( destinationContainer: destinationContainer.Container); } - [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm b/sdk/storage/Azure.Storage.DataMovement/tests/Resources/SampleJobPlanFile.b1.ndm index 07cf481e1571d9b8ddbaeac5a13bc00857867334..8966811c3de500711141fbfe67361133dc827c8d 100644 GIT binary patch delta 22 acmd1KnP9}nz{tSB;10wBKpZ~NUIzdvvjbBA delta 21 Zcmd1Fo?yhw$iTqh2E+nD95&HG2LLDS15p3~ diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs index 2235d0d96a4c1..186e48337b531 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/CheckpointerTesting.cs @@ -185,6 +185,7 @@ internal static JobPlanHeader CreateDefaultJobHeader( JobPlanOperation operationType = DefaultJobPlanOperation, string sourceProviderId = DefaultSourceProviderId, string destinationProviderId = DefaultDestinationProviderId, + bool isContainer = false, bool enumerationComplete = false, DataTransferStatus jobStatus = default, string parentSourcePath = DefaultSourcePath, @@ -203,6 +204,7 @@ internal static JobPlanHeader CreateDefaultJobHeader( operationType, sourceProviderId, destinationProviderId, + isContainer, enumerationComplete, jobStatus, parentSourcePath, From 6bb39e12f0409963b45667228828bae3abe4c2b4 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 13 Oct 2023 11:56:23 -0700 Subject: [PATCH 2/2] Fix tests --- .../tests/JobPlanHeaderTests.cs | 5 ----- .../tests/PauseResumeTransferTests.cs | 20 +++++++++---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs index f9c93668899bb..73fd0a9b10bac 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/JobPlanHeaderTests.cs @@ -49,11 +49,6 @@ public void Serialize() CollectionAssert.AreEqual(expected, actual); } - - //using (FileStream fileStream = File.OpenWrite("D:\\azure-sdk-for-net\\sdk\\storage\\Azure.Storage.DataMovement\\tests\\Resources\\SampleJobPlanFile.b1.ndm")) - //{ - // header.Serialize(fileStream); - //} } [Test] diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs index 5e5116989fe5a..5ac73d8e39fbf 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs @@ -227,7 +227,7 @@ private async Task CreateSingleLongTransferAsync( return await manager.StartTransferAsync(sourceResource, destinationResource, transferOptions); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -281,7 +281,7 @@ public async Task TryPauseTransferAsync_Id(TransferDirection transferType) Assert.IsTrue(File.Exists(fileName.FullPath)); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -351,7 +351,7 @@ public void TryPauseTransferAsync_Error() Assert.CatchAsync(async () => await transferManager.PauseTransferIfRunningAsync("bad transfer Id")); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -410,7 +410,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused(TransferDirection transfer Assert.IsTrue(File.Exists(fileName.FullPath)); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -485,7 +485,7 @@ await AssertSourceAndDestinationAsync( destinationContainer: destinationContainer.Container); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -707,7 +707,7 @@ private async Task CreateDirectoryLongTransferAsync( return await manager.StartTransferAsync(sourceResource, destinationResource, transferOptions); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -758,7 +758,7 @@ public async Task TryPauseTransferAsync_Id_Directory(TransferDirection transferT Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -809,7 +809,7 @@ public async Task TryPauseTransferAsync_DataTransfer_Directory(TransferDirection Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -866,7 +866,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio Assert.AreEqual(DataTransferState.Paused, transfer.TransferStatus.State); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)] @@ -945,7 +945,7 @@ await AssertDirectorySourceAndDestinationAsync( destinationContainer: destinationContainer.Container); } - //[Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/35439")] [RecordedTest] [TestCase(TransferDirection.Upload)] [TestCase(TransferDirection.Download)]