[Storage][DataMovement] Add ProviderId to resources and job plan file (…

…Azure#39163)
matthohn-msft · Oct 27, 2023 · 444001c · 444001c
1 parent fdc77b4
commit 444001c
Show file tree

Hide file tree

Showing 37 changed files with 189 additions and 104 deletions.
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs
@@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag
         public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
         public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
         public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
-        protected override string TypeId { get { throw null; } }
+        protected override string ProviderId { get { throw null; } }
         public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; }

diff --git a/...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs b/...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs
@@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag
         public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
         public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
         public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
-        protected override string TypeId { get { throw null; } }
+        protected override string ProviderId { get { throw null; } }
         public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; }

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs
@@ -20,16 +20,12 @@ internal class AppendBlobStorageResource : StorageResourceItemInternal
         internal long? _length;
         internal ETag? _etagDownloadLock = default;
 
-        /// <summary>
-        /// The identifier for the type of storage resource.
-        /// </summary>
         protected override string ResourceId => "AppendBlob";
 
-        /// <summary>
-        /// Gets the Uri of the Storage Resource
-        /// </summary>
         public override Uri Uri => BlobClient.Uri;
 
+        public override string ProviderId => "blob";
+
         /// <summary>
         /// Defines the recommended Transfer Type for the storage resource.
         /// </summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs
@@ -29,6 +29,8 @@ internal class BlobStorageResourceContainer : StorageResourceContainerInternal
         /// </summary>
         public override Uri Uri => _uri;
 
+        public override string ProviderId => "blob";
+
         /// <summary>
         /// The constructor to create an instance of the BlobStorageResourceContainer.
         /// </summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs
@@ -66,7 +66,7 @@ private enum CredentialType
         }
 
         /// <inheritdoc/>
-        protected override string TypeId => "blob";
+        protected override string ProviderId => "blob";
 
         private readonly CredentialType _credentialType;
         private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential;

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs
@@ -30,16 +30,12 @@ internal class BlockBlobStorageResource : StorageResourceItemInternal
         /// </summary>
         private ConcurrentDictionary<long, string> _blocks;
 
-        /// <summary>
-        /// The identifier for the type of storage resource.
-        /// </summary>
         protected override string ResourceId => "BlockBlob";
 
-        /// <summary>
-        /// Gets the Uri of the StorageResource
-        /// </summary>
         public override Uri Uri => BlobClient.Uri;
 
+        public override string ProviderId => "blob";
+
         /// <summary>
         /// Defines the recommended Transfer Type of the storage resource.
         /// </summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs
@@ -20,16 +20,12 @@ internal class PageBlobStorageResource : StorageResourceItemInternal
         internal long? _length;
         internal ETag? _etagDownloadLock = default;
 
-        /// <summary>
-        /// The identifier for the type of storage resource.
-        /// </summary>
         protected override string ResourceId => "PageBlob";
 
-        /// <summary>
-        /// Gets the Uri of the Storage Resource
-        /// </summary>
         public override Uri Uri => BlobClient.Uri;
 
+        public override string ProviderId => "blob";
+
         /// <summary>
         /// Defines the recommended Transfer Type for the storage resource.
         /// </summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs
@@ -41,13 +41,27 @@ private static string ToResourceId(StorageResourceType type)
             };
         }
 
+        private static string ToProviderId(StorageResourceType type)
+        {
+            return type switch
+            {
+                StorageResourceType.BlockBlob => "blob",
+                StorageResourceType.PageBlob => "blob",
+                StorageResourceType.AppendBlob => "blob",
+                StorageResourceType.Local => "local",
+                _ => throw new NotImplementedException(),
+            };
+        }
+
         private static Mock<DataTransferProperties> GetProperties(
             string checkpointerPath,
             string transferId,
             string sourcePath,
             string destinationPath,
             string sourceResourceId,
             string destinationResourceId,
+            string sourceProviderId,
+            string destinationProviderId,
             bool isContainer)
         {
             var mock = new Mock<DataTransferProperties>(MockBehavior.Strict);
@@ -57,6 +71,8 @@ private static Mock<DataTransferProperties> GetProperties(
             mock.Setup(p => p.DestinationPath).Returns(destinationPath);
             mock.Setup(p => p.SourceTypeId).Returns(sourceResourceId);
             mock.Setup(p => p.DestinationTypeId).Returns(destinationResourceId);
+            mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId);
+            mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId);
             mock.Setup(p => p.IsContainer).Returns(isContainer);
             return mock;
         }
@@ -112,8 +128,10 @@ private async Task AddJobPartToCheckpointer(
             // Use mock resources that don't correspond to correct paths
             var sourceMock = new Mock<StorageResource>();
             sourceMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebSourcePath));
+            sourceMock.Setup(s => s.ProviderId).Returns(ToProviderId(sourceType));
             var destMock = new Mock<StorageResource>();
             destMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebDestinationPath));
+            destMock.Setup(s => s.ProviderId).Returns(ToProviderId(destinationType));
             await checkpointer.AddNewJobAsync(transferId, sourceMock.Object, destMock.Object);
 
             for (int currentPart = 0; currentPart < partCount; currentPart++)
@@ -159,6 +177,8 @@ public async Task RehydrateBlockBlob(
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             await AddJobPartToCheckpointer(
@@ -196,6 +216,8 @@ public async Task RehydrateBlockBlob_Options()
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             IDictionary<string, string> metadata = DataProvider.BuildMetadata();
@@ -250,6 +272,8 @@ public async Task RehydratePageBlob(
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             await AddJobPartToCheckpointer(
@@ -287,6 +311,8 @@ public async Task RehydratePageBlob_Options()
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             IDictionary<string, string> metadata = DataProvider.BuildMetadata();
@@ -341,6 +367,8 @@ public async Task RehydrateAppendBlob(
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             await AddJobPartToCheckpointer(
@@ -378,6 +406,8 @@ public async Task RehydrateAppendBlob_Options()
                 destinationPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: false).Object;
 
             IDictionary<string, string> metadata = DataProvider.BuildMetadata();
@@ -441,6 +471,8 @@ public async Task RehydrateBlobContainer(
                 destinationParentPath,
                 ToResourceId(sourceType),
                 ToResourceId(destinationType),
+                ToProviderId(sourceType),
+                ToProviderId(destinationType),
                 isContainer: true).Object;
 
             await AddJobPartToCheckpointer(

diff --git a/...e.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs b/...e.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs
@@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares
         public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
         public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
         public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
-        protected override string TypeId { get { throw null; } }
+        protected override string ProviderId { get { throw null; } }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
         protected override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }

diff --git a/...e.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs b/...e.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs
@@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares
         public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
         public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
         public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
-        protected override string TypeId { get { throw null; } }
+        protected override string ProviderId { get { throw null; } }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
         public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
         protected override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }

diff --git a/...age/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs b/...age/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs
@@ -20,6 +20,8 @@ internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer
 
         public override Uri Uri => ShareDirectoryClient.Uri;
 
+        public override string ProviderId => "share";
+
         internal ShareDirectoryStorageResourceContainer(ShareDirectoryClient shareDirectoryClient, ShareFileStorageResourceOptions options)
         {
             ShareDirectoryClient = shareDirectoryClient;

diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs
@@ -22,6 +22,8 @@ internal class ShareFileStorageResource : StorageResourceItemInternal
 
         public override Uri Uri => ShareFileClient.Uri;
 
+        public override string ProviderId => "share";
+
         protected override string ResourceId => "ShareFile";
 
         protected override DataTransferOrder TransferType => DataTransferOrder.Sequential;

diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs
@@ -64,7 +64,7 @@ private enum CredentialType
         }
 
         /// <inheritdoc/>
-        protected override string TypeId => "share";
+        protected override string ProviderId => "share";
 
         private readonly CredentialType _credentialType;
         private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential;

diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs
@@ -64,9 +64,11 @@ public partial class DataTransferProperties
         protected internal DataTransferProperties() { }
         public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } }
         public virtual string DestinationPath { get { throw null; } }
+        public virtual string DestinationProviderId { get { throw null; } }
         public virtual string DestinationTypeId { get { throw null; } }
         public virtual bool IsContainer { get { throw null; } }
         public virtual string SourcePath { get { throw null; } }
+        public virtual string SourceProviderId { get { throw null; } }
         public virtual string SourceTypeId { get { throw null; } }
         public virtual string TransferId { get { throw null; } }
     }
@@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta
     public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider
     {
         public LocalFilesStorageResourceProvider() { }
-        protected internal override string TypeId { get { throw null; } }
+        protected internal override string ProviderId { get { throw null; } }
         protected internal override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
         public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; }
         public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; }
@@ -113,6 +115,7 @@ public abstract partial class StorageResource
     {
         protected StorageResource() { }
         protected internal abstract bool IsContainer { get; }
+        public abstract string ProviderId { get; }
         public abstract System.Uri Uri { get; }
     }
     public abstract partial class StorageResourceCheckpointData
@@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date
     public abstract partial class StorageResourceProvider
     {
         protected StorageResourceProvider() { }
-        protected internal abstract string TypeId { get; }
+        protected internal abstract string ProviderId { get; }
         protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
         protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
     }

diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs
@@ -64,9 +64,11 @@ public partial class DataTransferProperties
         protected internal DataTransferProperties() { }
         public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } }
         public virtual string DestinationPath { get { throw null; } }
+        public virtual string DestinationProviderId { get { throw null; } }
         public virtual string DestinationTypeId { get { throw null; } }
         public virtual bool IsContainer { get { throw null; } }
         public virtual string SourcePath { get { throw null; } }
+        public virtual string SourceProviderId { get { throw null; } }
         public virtual string SourceTypeId { get { throw null; } }
         public virtual string TransferId { get { throw null; } }
     }
@@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta
     public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider
     {
         public LocalFilesStorageResourceProvider() { }
-        protected internal override string TypeId { get { throw null; } }
+        protected internal override string ProviderId { get { throw null; } }
         protected internal override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
         public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; }
         public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; }
@@ -113,6 +115,7 @@ public abstract partial class StorageResource
     {
         protected StorageResource() { }
         protected internal abstract bool IsContainer { get; }
+        public abstract string ProviderId { get; }
         public abstract System.Uri Uri { get; }
     }
     public abstract partial class StorageResourceCheckpointData
@@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date
     public abstract partial class StorageResourceProvider
     {
         protected StorageResourceProvider() { }
-        protected internal abstract string TypeId { get; }
+        protected internal abstract string ProviderId { get; }
         protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
         protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
     }