From 3cc1d0d05856309a26b7f00e2c85143bb56c4d1f Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 12:47:26 +0200 Subject: [PATCH 01/39] r/storage_blob: moving the legacy sdk functions into their own file to aid refactoring --- azurerm/resource_arm_storage_blob.go | 387 ------------------- azurerm/resource_arm_storage_blob_legacy.go | 395 ++++++++++++++++++++ 2 files changed, 395 insertions(+), 387 deletions(-) create mode 100644 azurerm/resource_arm_storage_blob_legacy.go diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 7a661dac56d0..1a3d880d7ead 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -1,17 +1,9 @@ package azurerm import ( - "bytes" - "crypto/rand" - "encoding/base64" "fmt" - "io" "log" - "net/url" - "os" - "runtime" "strings" - "sync" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf" @@ -20,10 +12,7 @@ import ( "github.com/hashicorp/terraform/helper/validation" - "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" - "github.com/Azure/azure-sdk-for-go/storage" - azauto "github.com/Azure/go-autorest/autorest/azure" "github.com/hashicorp/terraform/helper/schema" ) @@ -223,326 +212,6 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro return resourceArmStorageBlobRead(d, meta) } -type resourceArmStorageBlobPage struct { - offset int64 - section *io.SectionReader -} - -func resourceArmStorageBlobPageUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { - workerCount := parallelism * runtime.NumCPU() - - file, err := os.Open(source) - if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", source, err) - } - defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) - - blobSize, pageList, err := resourceArmStorageBlobPageSplit(file) - if err != nil { - return fmt.Errorf("Error splitting source file %q into pages: %s", source, err) - } - - options := &storage.PutBlobOptions{} - containerRef := client.GetContainerReference(container) - blob := containerRef.GetBlobReference(name) - blob.Properties.ContentLength = blobSize - blob.Properties.ContentType = contentType - err = blob.PutPageBlob(options) - if err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - - pages := make(chan resourceArmStorageBlobPage, len(pageList)) - errors := make(chan error, len(pageList)) - wg := &sync.WaitGroup{} - wg.Add(len(pageList)) - - total := int64(0) - for _, page := range pageList { - total += page.section.Size() - pages <- page - } - close(pages) - - for i := 0; i < workerCount; i++ { - go resourceArmStorageBlobPageUploadWorker(resourceArmStorageBlobPageUploadContext{ - container: container, - name: name, - source: source, - blobSize: blobSize, - client: client, - pages: pages, - errors: errors, - wg: wg, - attempts: attempts, - }) - } - - wg.Wait() - - if len(errors) > 0 { - return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) - } - - return nil -} - -func resourceArmStorageBlobPageSplit(file *os.File) (int64, []resourceArmStorageBlobPage, error) { - const ( - minPageSize int64 = 4 * 1024 - maxPageSize int64 = 4 * 1024 * 1024 - ) - - info, err := file.Stat() - if err != nil { - return int64(0), nil, fmt.Errorf("Could not stat file %q: %s", file.Name(), err) - } - - blobSize := info.Size() - if info.Size()%minPageSize != 0 { - blobSize = info.Size() + (minPageSize - (info.Size() % minPageSize)) - } - - emptyPage := make([]byte, minPageSize) - - type byteRange struct { - offset int64 - length int64 - } - - var nonEmptyRanges []byteRange - var currentRange byteRange - for i := int64(0); i < blobSize; i += minPageSize { - pageBuf := make([]byte, minPageSize) - _, err = file.ReadAt(pageBuf, i) - if err != nil && err != io.EOF { - return int64(0), nil, fmt.Errorf("Could not read chunk at %d: %s", i, err) - } - - if bytes.Equal(pageBuf, emptyPage) { - if currentRange.length != 0 { - nonEmptyRanges = append(nonEmptyRanges, currentRange) - } - currentRange = byteRange{ - offset: i + minPageSize, - } - } else { - currentRange.length += minPageSize - if currentRange.length == maxPageSize || (currentRange.offset+currentRange.length == blobSize) { - nonEmptyRanges = append(nonEmptyRanges, currentRange) - currentRange = byteRange{ - offset: i + minPageSize, - } - } - } - } - - var pages []resourceArmStorageBlobPage - for _, nonEmptyRange := range nonEmptyRanges { - pages = append(pages, resourceArmStorageBlobPage{ - offset: nonEmptyRange.offset, - section: io.NewSectionReader(file, nonEmptyRange.offset, nonEmptyRange.length), - }) - } - - return info.Size(), pages, nil -} - -type resourceArmStorageBlobPageUploadContext struct { - container string - name string - source string - blobSize int64 - client *storage.BlobStorageClient - pages chan resourceArmStorageBlobPage - errors chan error - wg *sync.WaitGroup - attempts int -} - -func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUploadContext) { - for page := range ctx.pages { - start := page.offset - end := page.offset + page.section.Size() - 1 - if end > ctx.blobSize-1 { - end = ctx.blobSize - 1 - } - size := end - start + 1 - - chunk := make([]byte, size) - _, err := page.section.Read(chunk) - if err != nil && err != io.EOF { - ctx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", ctx.source, page.offset, err) - ctx.wg.Done() - continue - } - - for x := 0; x < ctx.attempts; x++ { - container := ctx.client.GetContainerReference(ctx.container) - blob := container.GetBlobReference(ctx.name) - blobRange := storage.BlobRange{ - Start: uint64(start), - End: uint64(end), - } - options := &storage.PutPageOptions{} - reader := bytes.NewReader(chunk) - err = blob.WriteRange(blobRange, reader, options) - if err == nil { - break - } - } - if err != nil { - ctx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, ctx.source, err) - ctx.wg.Done() - continue - } - - ctx.wg.Done() - } -} - -type resourceArmStorageBlobBlock struct { - section *io.SectionReader - id string -} - -func resourceArmStorageBlobBlockUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { - workerCount := parallelism * runtime.NumCPU() - - file, err := os.Open(source) - if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", source, err) - } - defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) - - blockList, parts, err := resourceArmStorageBlobBlockSplit(file) - if err != nil { - return fmt.Errorf("Error reading and splitting source file for upload %q: %s", source, err) - } - - wg := &sync.WaitGroup{} - blocks := make(chan resourceArmStorageBlobBlock, len(parts)) - errors := make(chan error, len(parts)) - - wg.Add(len(parts)) - for _, p := range parts { - blocks <- p - } - close(blocks) - - for i := 0; i < workerCount; i++ { - go resourceArmStorageBlobBlockUploadWorker(resourceArmStorageBlobBlockUploadContext{ - client: client, - source: source, - container: container, - name: name, - blocks: blocks, - errors: errors, - wg: wg, - attempts: attempts, - }) - } - - wg.Wait() - - if len(errors) > 0 { - return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) - } - - containerReference := client.GetContainerReference(container) - blobReference := containerReference.GetBlobReference(name) - blobReference.Properties.ContentType = contentType - options := &storage.PutBlockListOptions{} - err = blobReference.PutBlockList(blockList, options) - if err != nil { - return fmt.Errorf("Error updating block list for source file %q: %s", source, err) - } - - return nil -} - -func resourceArmStorageBlobBlockSplit(file *os.File) ([]storage.Block, []resourceArmStorageBlobBlock, error) { - const ( - idSize = 64 - blockSize int64 = 4 * 1024 * 1024 - ) - var parts []resourceArmStorageBlobBlock - var blockList []storage.Block - - info, err := file.Stat() - if err != nil { - return nil, nil, fmt.Errorf("Error stating source file %q: %s", file.Name(), err) - } - - for i := int64(0); i < info.Size(); i = i + blockSize { - entropy := make([]byte, idSize) - _, err = rand.Read(entropy) - if err != nil { - return nil, nil, fmt.Errorf("Error generating a random block ID for source file %q: %s", file.Name(), err) - } - - sectionSize := blockSize - remainder := info.Size() - i - if remainder < blockSize { - sectionSize = remainder - } - - block := storage.Block{ - ID: base64.StdEncoding.EncodeToString(entropy), - Status: storage.BlockStatusUncommitted, - } - - blockList = append(blockList, block) - - parts = append(parts, resourceArmStorageBlobBlock{ - id: block.ID, - section: io.NewSectionReader(file, i, sectionSize), - }) - } - - return blockList, parts, nil -} - -type resourceArmStorageBlobBlockUploadContext struct { - client *storage.BlobStorageClient - container string - name string - source string - attempts int - blocks chan resourceArmStorageBlobBlock - errors chan error - wg *sync.WaitGroup -} - -func resourceArmStorageBlobBlockUploadWorker(ctx resourceArmStorageBlobBlockUploadContext) { - for block := range ctx.blocks { - buffer := make([]byte, block.section.Size()) - - _, err := block.section.Read(buffer) - if err != nil { - ctx.errors <- fmt.Errorf("Error reading source file %q: %s", ctx.source, err) - ctx.wg.Done() - continue - } - - for i := 0; i < ctx.attempts; i++ { - container := ctx.client.GetContainerReference(ctx.container) - blob := container.GetBlobReference(ctx.name) - options := &storage.PutBlockOptions{} - if err = blob.PutBlock(block.id, buffer, options); err == nil { - break - } - } - if err != nil { - ctx.errors <- fmt.Errorf("Error uploading block %q for source file %q: %s", block.id, ctx.source, err) - ctx.wg.Done() - continue - } - - ctx.wg.Done() - } -} - func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) error { armClient := meta.(*ArmClient) ctx := armClient.StopContext @@ -709,62 +378,6 @@ func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) erro return nil } -type storageBlobId struct { - storageAccountName string - containerName string - blobName string -} - -func parseStorageBlobID(input string, environment azauto.Environment) (*storageBlobId, error) { - uri, err := url.Parse(input) - if err != nil { - return nil, fmt.Errorf("Error parsing %q as URI: %+v", input, err) - } - - // trim the leading `/` - segments := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/") - if len(segments) < 2 { - return nil, fmt.Errorf("Expected number of segments in the path to be < 2 but got %d", len(segments)) - } - - storageAccountName := strings.Replace(uri.Host, fmt.Sprintf(".blob.%s", environment.StorageEndpointSuffix), "", 1) - containerName := segments[0] - blobName := strings.TrimPrefix(uri.Path, fmt.Sprintf("/%s/", containerName)) - - id := storageBlobId{ - storageAccountName: storageAccountName, - containerName: containerName, - blobName: blobName, - } - return &id, nil -} - -func determineResourceGroupForStorageAccount(accountName string, client *ArmClient) (*string, error) { - storageClient := client.storage.AccountsClient - ctx := client.StopContext - - // first locate which resource group the storage account is in - groupsResp, err := storageClient.List(ctx) - if err != nil { - return nil, fmt.Errorf("Error loading the Resource Groups for Storage Account %q: %+v", accountName, err) - } - - if groups := groupsResp.Value; groups != nil { - for _, group := range *groups { - if group.Name != nil && *group.Name == accountName { - groupId, err := azure.ParseAzureResourceID(*group.ID) - if err != nil { - return nil, err - } - - return &groupId.ResourceGroup, nil - } - } - } - - return nil, nil -} - func expandStorageAccountBlobMetadata(d *schema.ResourceData) storage.BlobMetadata { blobMetadata := make(map[string]string) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go new file mode 100644 index 000000000000..23c3d1374229 --- /dev/null +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -0,0 +1,395 @@ +package azurerm + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "os" + "runtime" + "strings" + "sync" + + "github.com/Azure/azure-sdk-for-go/storage" + azauto "github.com/Azure/go-autorest/autorest/azure" + "github.com/hashicorp/go-getter/helper/url" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" +) + +type storageBlobId struct { + storageAccountName string + containerName string + blobName string +} + +func parseStorageBlobID(input string, environment azauto.Environment) (*storageBlobId, error) { + uri, err := url.Parse(input) + if err != nil { + return nil, fmt.Errorf("Error parsing %q as URI: %+v", input, err) + } + + // trim the leading `/` + segments := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/") + if len(segments) < 2 { + return nil, fmt.Errorf("Expected number of segments in the path to be < 2 but got %d", len(segments)) + } + + storageAccountName := strings.Replace(uri.Host, fmt.Sprintf(".blob.%s", environment.StorageEndpointSuffix), "", 1) + containerName := segments[0] + blobName := strings.TrimPrefix(uri.Path, fmt.Sprintf("/%s/", containerName)) + + id := storageBlobId{ + storageAccountName: storageAccountName, + containerName: containerName, + blobName: blobName, + } + return &id, nil +} + +func determineResourceGroupForStorageAccount(accountName string, client *ArmClient) (*string, error) { + storageClient := client.storage.AccountsClient + ctx := client.StopContext + + // first locate which resource group the storage account is in + groupsResp, err := storageClient.List(ctx) + if err != nil { + return nil, fmt.Errorf("Error loading the Resource Groups for Storage Account %q: %+v", accountName, err) + } + + if groups := groupsResp.Value; groups != nil { + for _, group := range *groups { + if group.Name != nil && *group.Name == accountName { + groupId, err := azure.ParseAzureResourceID(*group.ID) + if err != nil { + return nil, err + } + + return &groupId.ResourceGroup, nil + } + } + } + + return nil, nil +} + +type resourceArmStorageBlobPage struct { + offset int64 + section *io.SectionReader +} + +func resourceArmStorageBlobPageUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { + workerCount := parallelism * runtime.NumCPU() + + file, err := os.Open(source) + if err != nil { + return fmt.Errorf("Error opening source file for upload %q: %s", source, err) + } + defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) + + blobSize, pageList, err := resourceArmStorageBlobPageSplit(file) + if err != nil { + return fmt.Errorf("Error splitting source file %q into pages: %s", source, err) + } + + options := &storage.PutBlobOptions{} + containerRef := client.GetContainerReference(container) + blob := containerRef.GetBlobReference(name) + blob.Properties.ContentLength = blobSize + blob.Properties.ContentType = contentType + err = blob.PutPageBlob(options) + if err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + + pages := make(chan resourceArmStorageBlobPage, len(pageList)) + errors := make(chan error, len(pageList)) + wg := &sync.WaitGroup{} + wg.Add(len(pageList)) + + total := int64(0) + for _, page := range pageList { + total += page.section.Size() + pages <- page + } + close(pages) + + for i := 0; i < workerCount; i++ { + go resourceArmStorageBlobPageUploadWorker(resourceArmStorageBlobPageUploadContext{ + container: container, + name: name, + source: source, + blobSize: blobSize, + client: client, + pages: pages, + errors: errors, + wg: wg, + attempts: attempts, + }) + } + + wg.Wait() + + if len(errors) > 0 { + return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) + } + + return nil +} + +func resourceArmStorageBlobPageSplit(file *os.File) (int64, []resourceArmStorageBlobPage, error) { + const ( + minPageSize int64 = 4 * 1024 + maxPageSize int64 = 4 * 1024 * 1024 + ) + + info, err := file.Stat() + if err != nil { + return int64(0), nil, fmt.Errorf("Could not stat file %q: %s", file.Name(), err) + } + + blobSize := info.Size() + if info.Size()%minPageSize != 0 { + blobSize = info.Size() + (minPageSize - (info.Size() % minPageSize)) + } + + emptyPage := make([]byte, minPageSize) + + type byteRange struct { + offset int64 + length int64 + } + + var nonEmptyRanges []byteRange + var currentRange byteRange + for i := int64(0); i < blobSize; i += minPageSize { + pageBuf := make([]byte, minPageSize) + _, err = file.ReadAt(pageBuf, i) + if err != nil && err != io.EOF { + return int64(0), nil, fmt.Errorf("Could not read chunk at %d: %s", i, err) + } + + if bytes.Equal(pageBuf, emptyPage) { + if currentRange.length != 0 { + nonEmptyRanges = append(nonEmptyRanges, currentRange) + } + currentRange = byteRange{ + offset: i + minPageSize, + } + } else { + currentRange.length += minPageSize + if currentRange.length == maxPageSize || (currentRange.offset+currentRange.length == blobSize) { + nonEmptyRanges = append(nonEmptyRanges, currentRange) + currentRange = byteRange{ + offset: i + minPageSize, + } + } + } + } + + var pages []resourceArmStorageBlobPage + for _, nonEmptyRange := range nonEmptyRanges { + pages = append(pages, resourceArmStorageBlobPage{ + offset: nonEmptyRange.offset, + section: io.NewSectionReader(file, nonEmptyRange.offset, nonEmptyRange.length), + }) + } + + return info.Size(), pages, nil +} + +type resourceArmStorageBlobPageUploadContext struct { + container string + name string + source string + blobSize int64 + client *storage.BlobStorageClient + pages chan resourceArmStorageBlobPage + errors chan error + wg *sync.WaitGroup + attempts int +} + +func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUploadContext) { + for page := range ctx.pages { + start := page.offset + end := page.offset + page.section.Size() - 1 + if end > ctx.blobSize-1 { + end = ctx.blobSize - 1 + } + size := end - start + 1 + + chunk := make([]byte, size) + _, err := page.section.Read(chunk) + if err != nil && err != io.EOF { + ctx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", ctx.source, page.offset, err) + ctx.wg.Done() + continue + } + + for x := 0; x < ctx.attempts; x++ { + container := ctx.client.GetContainerReference(ctx.container) + blob := container.GetBlobReference(ctx.name) + blobRange := storage.BlobRange{ + Start: uint64(start), + End: uint64(end), + } + options := &storage.PutPageOptions{} + reader := bytes.NewReader(chunk) + err = blob.WriteRange(blobRange, reader, options) + if err == nil { + break + } + } + if err != nil { + ctx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, ctx.source, err) + ctx.wg.Done() + continue + } + + ctx.wg.Done() + } +} + +type resourceArmStorageBlobBlock struct { + section *io.SectionReader + id string +} + +func resourceArmStorageBlobBlockUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { + workerCount := parallelism * runtime.NumCPU() + + file, err := os.Open(source) + if err != nil { + return fmt.Errorf("Error opening source file for upload %q: %s", source, err) + } + defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) + + blockList, parts, err := resourceArmStorageBlobBlockSplit(file) + if err != nil { + return fmt.Errorf("Error reading and splitting source file for upload %q: %s", source, err) + } + + wg := &sync.WaitGroup{} + blocks := make(chan resourceArmStorageBlobBlock, len(parts)) + errors := make(chan error, len(parts)) + + wg.Add(len(parts)) + for _, p := range parts { + blocks <- p + } + close(blocks) + + for i := 0; i < workerCount; i++ { + go resourceArmStorageBlobBlockUploadWorker(resourceArmStorageBlobBlockUploadContext{ + client: client, + source: source, + container: container, + name: name, + blocks: blocks, + errors: errors, + wg: wg, + attempts: attempts, + }) + } + + wg.Wait() + + if len(errors) > 0 { + return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) + } + + containerReference := client.GetContainerReference(container) + blobReference := containerReference.GetBlobReference(name) + blobReference.Properties.ContentType = contentType + options := &storage.PutBlockListOptions{} + err = blobReference.PutBlockList(blockList, options) + if err != nil { + return fmt.Errorf("Error updating block list for source file %q: %s", source, err) + } + + return nil +} + +func resourceArmStorageBlobBlockSplit(file *os.File) ([]storage.Block, []resourceArmStorageBlobBlock, error) { + const ( + idSize = 64 + blockSize int64 = 4 * 1024 * 1024 + ) + var parts []resourceArmStorageBlobBlock + var blockList []storage.Block + + info, err := file.Stat() + if err != nil { + return nil, nil, fmt.Errorf("Error stating source file %q: %s", file.Name(), err) + } + + for i := int64(0); i < info.Size(); i = i + blockSize { + entropy := make([]byte, idSize) + _, err = rand.Read(entropy) + if err != nil { + return nil, nil, fmt.Errorf("Error generating a random block ID for source file %q: %s", file.Name(), err) + } + + sectionSize := blockSize + remainder := info.Size() - i + if remainder < blockSize { + sectionSize = remainder + } + + block := storage.Block{ + ID: base64.StdEncoding.EncodeToString(entropy), + Status: storage.BlockStatusUncommitted, + } + + blockList = append(blockList, block) + + parts = append(parts, resourceArmStorageBlobBlock{ + id: block.ID, + section: io.NewSectionReader(file, i, sectionSize), + }) + } + + return blockList, parts, nil +} + +type resourceArmStorageBlobBlockUploadContext struct { + client *storage.BlobStorageClient + container string + name string + source string + attempts int + blocks chan resourceArmStorageBlobBlock + errors chan error + wg *sync.WaitGroup +} + +func resourceArmStorageBlobBlockUploadWorker(ctx resourceArmStorageBlobBlockUploadContext) { + for block := range ctx.blocks { + buffer := make([]byte, block.section.Size()) + + _, err := block.section.Read(buffer) + if err != nil { + ctx.errors <- fmt.Errorf("Error reading source file %q: %s", ctx.source, err) + ctx.wg.Done() + continue + } + + for i := 0; i < ctx.attempts; i++ { + container := ctx.client.GetContainerReference(ctx.container) + blob := container.GetBlobReference(ctx.name) + options := &storage.PutBlockOptions{} + if err = blob.PutBlock(block.id, buffer, options); err == nil { + break + } + } + if err != nil { + ctx.errors <- fmt.Errorf("Error uploading block %q for source file %q: %s", block.id, ctx.source, err) + ctx.wg.Done() + continue + } + + ctx.wg.Done() + } +} From 3eb5c67879db4abfcdf29fc2dbe0048f464a911e Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 12:52:59 +0200 Subject: [PATCH 02/39] r/storage_blob: switching to using the new metadata object --- azurerm/resource_arm_storage_blob.go | 72 ++++++++++------------------ 1 file changed, 26 insertions(+), 46 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 1a3d880d7ead..4e670fb0ded0 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -5,15 +5,14 @@ import ( "log" "strings" + legacy "github.com/Azure/azure-sdk-for-go/storage" + "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/helper/validation" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/validate" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/features" - - "github.com/hashicorp/terraform/helper/validation" - - "github.com/Azure/azure-sdk-for-go/storage" - "github.com/hashicorp/terraform/helper/schema" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/storage" ) func resourceArmStorageBlob() *schema.Resource { @@ -33,20 +32,24 @@ func resourceArmStorageBlob() *schema.Resource { Type: schema.TypeString, Required: true, ForceNew: true, + // TODO: add validation }, + // TODO: this can be deprecated with the new sdk? "resource_group_name": azure.SchemaResourceGroupName(), "storage_account_name": { Type: schema.TypeString, Required: true, ForceNew: true, + // TODO: add validation }, "storage_container_name": { Type: schema.TypeString, Required: true, ForceNew: true, + // TODO: add validation }, "type": { @@ -105,15 +108,7 @@ func resourceArmStorageBlob() *schema.Resource { ValidateFunc: validation.IntAtLeast(1), }, - "metadata": { - Type: schema.TypeMap, - Optional: true, - Computed: true, - Elem: &schema.Schema{ - Type: schema.TypeString, - ValidateFunc: validate.NoEmptyStrings, - }, - }, + "metadata": storage.MetaDataSchema(), }, } } @@ -158,14 +153,14 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } if sourceUri != "" { - options := &storage.CopyOptions{} + options := &legacy.CopyOptions{} if err := blob.Copy(sourceUri, options); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } else { switch strings.ToLower(blobType) { case "block": - options := &storage.PutBlobOptions{} + options := &legacy.PutBlobOptions{} if err := blob.CreateBlockBlob(options); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } @@ -190,7 +185,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } else { size := int64(d.Get("size").(int)) - options := &storage.PutBlobOptions{} + options := &legacy.PutBlobOptions{} blob.Properties.ContentLength = size blob.Properties.ContentType = contentType @@ -201,9 +196,10 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } - blob.Metadata = expandStorageAccountBlobMetadata(d) + metaDataRaw := d.Get("metadata").(map[string]interface{}) + blob.Metadata = storage.ExpandMetaData(metaDataRaw) - opts := &storage.SetBlobMetadataOptions{} + opts := &legacy.SetBlobMetadataOptions{} if err := blob.SetMetadata(opts); err != nil { return fmt.Errorf("Error setting metadata for storage blob on Azure: %s", err) } @@ -245,16 +241,17 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro blob.Properties.ContentType = d.Get("content_type").(string) } - options := &storage.SetBlobPropertiesOptions{} + options := &legacy.SetBlobPropertiesOptions{} err = blob.SetProperties(options) if err != nil { return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) } if d.HasChange("metadata") { - blob.Metadata = expandStorageAccountBlobMetadata(d) + metaDataRaw := d.Get("metadata").(map[string]interface{}) + blob.Metadata = storage.ExpandMetaData(metaDataRaw) - opts := &storage.SetBlobMetadataOptions{} + opts := &legacy.SetBlobMetadataOptions{} if err := blob.SetMetadata(opts); err != nil { return fmt.Errorf("Error setting metadata for storage blob on Azure: %s", err) } @@ -305,13 +302,13 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error return nil } - options := &storage.GetBlobPropertiesOptions{} + options := &legacy.GetBlobPropertiesOptions{} err = blob.GetProperties(options) if err != nil { return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) } - metadataOptions := &storage.GetBlobMetadataOptions{} + metadataOptions := &legacy.GetBlobMetadataOptions{} err = blob.GetMetadata(metadataOptions) if err != nil { return fmt.Errorf("Error getting metadata of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) @@ -334,7 +331,10 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error log.Printf("[INFO] URL for %q is empty", id.blobName) } d.Set("url", u) - d.Set("metadata", flattenStorageAccountBlobMetadata(blob.Metadata)) + + if err := d.Set("metadata", storage.FlattenMetaData(blob.Metadata)); err != nil { + return fmt.Errorf("Error setting `metadata`: %+v", err) + } return nil } @@ -367,7 +367,7 @@ func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) erro } log.Printf("[INFO] Deleting storage blob %q", id.blobName) - options := &storage.DeleteBlobOptions{} + options := &legacy.DeleteBlobOptions{} container := blobClient.GetContainerReference(id.containerName) blob := container.GetBlobReference(id.blobName) _, err = blob.DeleteIfExists(options) @@ -377,23 +377,3 @@ func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) erro return nil } - -func expandStorageAccountBlobMetadata(d *schema.ResourceData) storage.BlobMetadata { - blobMetadata := make(map[string]string) - - blobMetadataRaw := d.Get("metadata").(map[string]interface{}) - for key, value := range blobMetadataRaw { - blobMetadata[key] = value.(string) - } - return blobMetadata -} - -func flattenStorageAccountBlobMetadata(in storage.BlobMetadata) map[string]interface{} { - blobMetadata := make(map[string]interface{}) - - for key, value := range in { - blobMetadata[key] = value - } - - return blobMetadata -} From 42e93f505e820273293bdcda6eef4855397dfdc6 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 18:45:20 +0200 Subject: [PATCH 03/39] r/storage_blob: switching delete over to the new sdk --- azurerm/resource_arm_storage_blob.go | 37 ++++++++++++---------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 4e670fb0ded0..04ba0e572cd4 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -13,6 +13,7 @@ import ( "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/validate" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/features" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/storage" + "github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs" ) func resourceArmStorageBlob() *schema.Resource { @@ -340,39 +341,33 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error } func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) error { - armClient := meta.(*ArmClient) - ctx := armClient.StopContext + storageClient := meta.(*ArmClient).storage + ctx := meta.(*ArmClient).StopContext - id, err := parseStorageBlobID(d.Id(), armClient.environment) + id, err := blobs.ParseResourceID(d.Id()) if err != nil { - return err + return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient) + resourceGroup, err := storageClient.FindResourceGroup(ctx, id.AccountName) if err != nil { - return fmt.Errorf("Unable to determine Resource Group for Storage Account %q: %+v", id.storageAccountName, err) + return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", id.AccountName, err) } if resourceGroup == nil { - log.Printf("[INFO] Resource Group doesn't exist so the blob won't exist") - return nil + return fmt.Errorf("Unable to locate Resource Group for Storage Account %q (Disk %q)!", id.AccountName, uri) } - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.storageAccountName) + blobsClient, err := storageClient.BlobsClient(ctx, *resourceGroup, id.AccountName) if err != nil { - return err - } - if !accountExists { - log.Printf("[INFO] Storage Account %q doesn't exist so the blob won't exist", id.storageAccountName) - return nil + return fmt.Errorf("Error building Blobs Client: %s", err) } - log.Printf("[INFO] Deleting storage blob %q", id.blobName) - options := &legacy.DeleteBlobOptions{} - container := blobClient.GetContainerReference(id.containerName) - blob := container.GetBlobReference(id.blobName) - _, err = blob.DeleteIfExists(options) - if err != nil { - return fmt.Errorf("Error deleting storage blob %q: %s", id.blobName, err) + log.Printf("[INFO] Deleting Blob %q from Container %q / Storage Account %q", id.BlobName, id.ContainerName, id.AccountName) + input := blobs.DeleteInput{ + DeleteSnapshots: true, + } + if _, err := blobsClient.Delete(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { + return fmt.Errorf("Error deleting Blob %q (Container %q / Account %q): %s", id.BlobName, id.ContainerName, id.AccountName, err) } return nil From 83622e37785d93368aff75166b5d103c53c00fae Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 18:50:24 +0200 Subject: [PATCH 04/39] r/storage_blob: switching id parsing over to the new sdk --- azurerm/resource_arm_storage_blob.go | 54 ++++++++++----------- azurerm/resource_arm_storage_blob_legacy.go | 33 ------------- 2 files changed, 27 insertions(+), 60 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 04ba0e572cd4..d3e4b321e027 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -213,30 +213,30 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro armClient := meta.(*ArmClient) ctx := armClient.StopContext - id, err := parseStorageBlobID(d.Id(), armClient.environment) + id, err := blobs.ParseResourceID(d.Id()) if err != nil { - return err + return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient) + resourceGroup, err := determineResourceGroupForStorageAccount(id.AccountName, armClient) if err != nil { return err } if resourceGroup == nil { - return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.storageAccountName) + return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.AccountName) } - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.storageAccountName) + blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.AccountName) if err != nil { - return fmt.Errorf("Error getting storage account %s: %+v", id.storageAccountName, err) + return fmt.Errorf("Error getting storage account %s: %+v", id.AccountName, err) } if !accountExists { - return fmt.Errorf("Storage account %s not found in resource group %s", id.storageAccountName, *resourceGroup) + return fmt.Errorf("Storage account %s not found in resource group %s", id.AccountName, *resourceGroup) } - container := blobClient.GetContainerReference(id.containerName) - blob := container.GetBlobReference(id.blobName) + container := blobClient.GetContainerReference(id.ContainerName) + blob := container.GetBlobReference(id.BlobName) if d.HasChange("content_type") { blob.Properties.ContentType = d.Get("content_type").(string) @@ -245,7 +245,7 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro options := &legacy.SetBlobPropertiesOptions{} err = blob.SetProperties(options) if err != nil { - return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) + return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) } if d.HasChange("metadata") { @@ -265,40 +265,40 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error armClient := meta.(*ArmClient) ctx := armClient.StopContext - id, err := parseStorageBlobID(d.Id(), armClient.environment) + id, err := blobs.ParseResourceID(d.Id()) if err != nil { - return err + return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient) + resourceGroup, err := determineResourceGroupForStorageAccount(id.AccountName, armClient) if err != nil { return err } if resourceGroup == nil { - return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.storageAccountName) + return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.AccountName) } - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.storageAccountName) + blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.AccountName) if err != nil { return err } if !accountExists { - log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", id.storageAccountName, d.Id()) + log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", id.AccountName, d.Id()) d.SetId("") return nil } - log.Printf("[INFO] Checking for existence of storage blob %q in container %q.", id.blobName, id.containerName) - container := blobClient.GetContainerReference(id.containerName) - blob := container.GetBlobReference(id.blobName) + log.Printf("[INFO] Checking for existence of storage blob %q in container %q.", id.BlobName, id.ContainerName) + container := blobClient.GetContainerReference(id.ContainerName) + blob := container.GetBlobReference(id.BlobName) exists, err := blob.Exists() if err != nil { - return fmt.Errorf("error checking for existence of storage blob %q: %s", id.blobName, err) + return fmt.Errorf("error checking for existence of storage blob %q: %s", id.BlobName, err) } if !exists { - log.Printf("[INFO] Storage blob %q no longer exists, removing from state...", id.blobName) + log.Printf("[INFO] Storage blob %q no longer exists, removing from state...", id.BlobName) d.SetId("") return nil } @@ -306,18 +306,18 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error options := &legacy.GetBlobPropertiesOptions{} err = blob.GetProperties(options) if err != nil { - return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) + return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) } metadataOptions := &legacy.GetBlobMetadataOptions{} err = blob.GetMetadata(metadataOptions) if err != nil { - return fmt.Errorf("Error getting metadata of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err) + return fmt.Errorf("Error getting metadata of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) } - d.Set("name", id.blobName) - d.Set("storage_container_name", id.containerName) - d.Set("storage_account_name", id.storageAccountName) + d.Set("name", id.BlobName) + d.Set("storage_container_name", id.ContainerName) + d.Set("storage_account_name", id.AccountName) d.Set("resource_group_name", resourceGroup) d.Set("content_type", blob.Properties.ContentType) @@ -329,7 +329,7 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error u := blob.GetURL() if u == "" { - log.Printf("[INFO] URL for %q is empty", id.blobName) + log.Printf("[INFO] URL for %q is empty", id.BlobName) } d.Set("url", u) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 23c3d1374229..7f6ef598a4fe 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -8,46 +8,13 @@ import ( "io" "os" "runtime" - "strings" "sync" "github.com/Azure/azure-sdk-for-go/storage" - azauto "github.com/Azure/go-autorest/autorest/azure" - "github.com/hashicorp/go-getter/helper/url" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" ) -type storageBlobId struct { - storageAccountName string - containerName string - blobName string -} - -func parseStorageBlobID(input string, environment azauto.Environment) (*storageBlobId, error) { - uri, err := url.Parse(input) - if err != nil { - return nil, fmt.Errorf("Error parsing %q as URI: %+v", input, err) - } - - // trim the leading `/` - segments := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/") - if len(segments) < 2 { - return nil, fmt.Errorf("Expected number of segments in the path to be < 2 but got %d", len(segments)) - } - - storageAccountName := strings.Replace(uri.Host, fmt.Sprintf(".blob.%s", environment.StorageEndpointSuffix), "", 1) - containerName := segments[0] - blobName := strings.TrimPrefix(uri.Path, fmt.Sprintf("/%s/", containerName)) - - id := storageBlobId{ - storageAccountName: storageAccountName, - containerName: containerName, - blobName: blobName, - } - return &id, nil -} - func determineResourceGroupForStorageAccount(accountName string, client *ArmClient) (*string, error) { storageClient := client.storage.AccountsClient ctx := client.StopContext From 6c950bc15cd1570ff11146330303baec79531b85 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 18:57:54 +0200 Subject: [PATCH 05/39] r/storage_blob: reusing the 'find resource group' method --- azurerm/resource_arm_storage_blob.go | 6 ++--- azurerm/resource_arm_storage_blob_legacy.go | 27 --------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index d3e4b321e027..fc89e994e974 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -218,7 +218,7 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := determineResourceGroupForStorageAccount(id.AccountName, armClient) + resourceGroup, err := armClient.storage.FindResourceGroup(ctx, id.AccountName) if err != nil { return err } @@ -270,7 +270,7 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := determineResourceGroupForStorageAccount(id.AccountName, armClient) + resourceGroup, err := armClient.storage.FindResourceGroup(ctx, id.AccountName) if err != nil { return err } @@ -354,7 +354,7 @@ func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) erro return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", id.AccountName, err) } if resourceGroup == nil { - return fmt.Errorf("Unable to locate Resource Group for Storage Account %q (Disk %q)!", id.AccountName, uri) + return fmt.Errorf("Unable to locate Resource Group for Storage Account %q!", id.AccountName) } blobsClient, err := storageClient.BlobsClient(ctx, *resourceGroup, id.AccountName) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 7f6ef598a4fe..929535e6e5d5 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -11,36 +11,9 @@ import ( "sync" "github.com/Azure/azure-sdk-for-go/storage" - "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" ) -func determineResourceGroupForStorageAccount(accountName string, client *ArmClient) (*string, error) { - storageClient := client.storage.AccountsClient - ctx := client.StopContext - - // first locate which resource group the storage account is in - groupsResp, err := storageClient.List(ctx) - if err != nil { - return nil, fmt.Errorf("Error loading the Resource Groups for Storage Account %q: %+v", accountName, err) - } - - if groups := groupsResp.Value; groups != nil { - for _, group := range *groups { - if group.Name != nil && *group.Name == accountName { - groupId, err := azure.ParseAzureResourceID(*group.ID) - if err != nil { - return nil, err - } - - return &groupId.ResourceGroup, nil - } - } - } - - return nil, nil -} - type resourceArmStorageBlobPage struct { offset int64 section *io.SectionReader From 517976f9464d5df0569564003d0c147dc2a028f3 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 19:15:31 +0200 Subject: [PATCH 06/39] r/storage_blob: switching the Read method over to the new sdk --- azurerm/resource_arm_storage_blob.go | 65 ++++++++++------------------ 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index fc89e994e974..0e64968a3104 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -13,6 +13,7 @@ import ( "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/validate" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/features" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/storage" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" "github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs" ) @@ -262,57 +263,40 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro } func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error { - armClient := meta.(*ArmClient) - ctx := armClient.StopContext + storageClient := meta.(*ArmClient).storage + ctx := meta.(*ArmClient).StopContext id, err := blobs.ParseResourceID(d.Id()) if err != nil { return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := armClient.storage.FindResourceGroup(ctx, id.AccountName) + resourceGroup, err := storageClient.FindResourceGroup(ctx, id.AccountName) if err != nil { - return err + return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", id.AccountName, err) } - if resourceGroup == nil { - return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.AccountName) - } - - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.AccountName) - if err != nil { - return err - } - if !accountExists { - log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", id.AccountName, d.Id()) + log.Printf("[DEBUG] Unable to locate Storage Account for Blob %q (Container %q / Account %q) - assuming removed & removing from state!", id.BlobName, id.ContainerName, id.AccountName) d.SetId("") return nil } - log.Printf("[INFO] Checking for existence of storage blob %q in container %q.", id.BlobName, id.ContainerName) - container := blobClient.GetContainerReference(id.ContainerName) - blob := container.GetBlobReference(id.BlobName) - exists, err := blob.Exists() + blobsClient, err := storageClient.BlobsClient(ctx, *resourceGroup, id.AccountName) if err != nil { - return fmt.Errorf("error checking for existence of storage blob %q: %s", id.BlobName, err) - } - - if !exists { - log.Printf("[INFO] Storage blob %q no longer exists, removing from state...", id.BlobName) - d.SetId("") - return nil + return fmt.Errorf("Error building Blobs Client: %s", err) } - options := &legacy.GetBlobPropertiesOptions{} - err = blob.GetProperties(options) + log.Printf("[INFO] Retrieving Storage Blob %q (Container %q / Account %q).", id.BlobName, id.ContainerName, id.AccountName) + input := blobs.GetPropertiesInput{} + props, err := blobsClient.GetProperties(ctx, id.AccountName, id.ContainerName, id.BlobName, input) if err != nil { - return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) - } + if utils.ResponseWasNotFound(props.Response) { + log.Printf("[INFO] Blob %q was not found in Container %q / Account %q - assuming removed & removing from state...", id.BlobName, id.ContainerName, id.AccountName) + d.SetId("") + return nil + } - metadataOptions := &legacy.GetBlobMetadataOptions{} - err = blob.GetMetadata(metadataOptions) - if err != nil { - return fmt.Errorf("Error getting metadata of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) + return fmt.Errorf("Error retrieving properties for Blob %q (Container %q / Account %q): %s", id.BlobName, id.ContainerName, id.AccountName, err) } d.Set("name", id.BlobName) @@ -320,20 +304,15 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error d.Set("storage_account_name", id.AccountName) d.Set("resource_group_name", resourceGroup) - d.Set("content_type", blob.Properties.ContentType) - - d.Set("source_uri", blob.Properties.CopySource) + d.Set("content_type", props.ContentType) + d.Set("source_uri", props.CopySource) - blobType := strings.ToLower(strings.Replace(string(blob.Properties.BlobType), "Blob", "", 1)) + blobType := strings.ToLower(strings.Replace(string(props.BlobType), "Blob", "", 1)) d.Set("type", blobType) - u := blob.GetURL() - if u == "" { - log.Printf("[INFO] URL for %q is empty", id.BlobName) - } - d.Set("url", u) + d.Set("url", d.Id()) - if err := d.Set("metadata", storage.FlattenMetaData(blob.Metadata)); err != nil { + if err := d.Set("metadata", storage.FlattenMetaData(props.MetaData)); err != nil { return fmt.Errorf("Error setting `metadata`: %+v", err) } From 2839bd46ebde3eaddaa01ad50bb0dc7f1d6f152b Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 22 Aug 2019 19:49:38 +0200 Subject: [PATCH 07/39] r/storage_blob: switching update over to the new sdk --- azurerm/resource_arm_storage_blob.go | 50 +++++++++++++--------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 0e64968a3104..f160c5b29f03 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -211,52 +211,48 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) error { - armClient := meta.(*ArmClient) - ctx := armClient.StopContext + storageClient := meta.(*ArmClient).storage + ctx := meta.(*ArmClient).StopContext id, err := blobs.ParseResourceID(d.Id()) if err != nil { return fmt.Errorf("Error parsing %q: %s", d.Id(), err) } - resourceGroup, err := armClient.storage.FindResourceGroup(ctx, id.AccountName) + resourceGroup, err := storageClient.FindResourceGroup(ctx, id.AccountName) if err != nil { - return err + return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", id.AccountName, err) } - if resourceGroup == nil { - return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.AccountName) + return fmt.Errorf("Unable to locate Resource Group for Blob %q (Container %q / Account %q)", id.BlobName, id.ContainerName, id.AccountName) } - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, *resourceGroup, id.AccountName) + blobsClient, err := storageClient.BlobsClient(ctx, *resourceGroup, id.AccountName) if err != nil { - return fmt.Errorf("Error getting storage account %s: %+v", id.AccountName, err) - } - if !accountExists { - return fmt.Errorf("Storage account %s not found in resource group %s", id.AccountName, *resourceGroup) + return fmt.Errorf("Error building Blobs Client: %s", err) } - container := blobClient.GetContainerReference(id.ContainerName) - blob := container.GetBlobReference(id.BlobName) - if d.HasChange("content_type") { - blob.Properties.ContentType = d.Get("content_type").(string) - } - - options := &legacy.SetBlobPropertiesOptions{} - err = blob.SetProperties(options) - if err != nil { - return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", id.BlobName, id.ContainerName, id.AccountName, err) + log.Printf("[DEBUG] Updating Properties for Blob %q (Container %q / Account %q)...", id.BlobName, id.ContainerName, id.AccountName) + input := blobs.SetPropertiesInput{ + ContentType: utils.String(d.Get("content_type").(string)), + } + if _, err := blobsClient.SetProperties(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { + return fmt.Errorf("Error updating Properties for Blob %q (Container %q / Account %q): %s", id.BlobName, id.ContainerName, id.AccountName, err) + } + log.Printf("[DEBUG] Updated Properties for Blob %q (Container %q / Account %q).", id.BlobName, id.ContainerName, id.AccountName) } if d.HasChange("metadata") { + log.Printf("[DEBUG] Updating MetaData for Blob %q (Container %q / Account %q)...", id.BlobName, id.ContainerName, id.AccountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) - blob.Metadata = storage.ExpandMetaData(metaDataRaw) - - opts := &legacy.SetBlobMetadataOptions{} - if err := blob.SetMetadata(opts); err != nil { - return fmt.Errorf("Error setting metadata for storage blob on Azure: %s", err) + input := blobs.SetMetaDataInput{ + MetaData: storage.ExpandMetaData(metaDataRaw), + } + if _, err := blobsClient.SetMetaData(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { + return fmt.Errorf("Error updating MetaData for Blob %q (Container %q / Account %q): %s", id.BlobName, id.ContainerName, id.AccountName, err) } + log.Printf("[DEBUG] Updated MetaData for Blob %q (Container %q / Account %q).", id.BlobName, id.ContainerName, id.AccountName) } return nil @@ -276,7 +272,7 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", id.AccountName, err) } if resourceGroup == nil { - log.Printf("[DEBUG] Unable to locate Storage Account for Blob %q (Container %q / Account %q) - assuming removed & removing from state!", id.BlobName, id.ContainerName, id.AccountName) + log.Printf("[DEBUG] Unable to locate Resource Group for Blob %q (Container %q / Account %q) - assuming removed & removing from state!", id.BlobName, id.ContainerName, id.AccountName) d.SetId("") return nil } From 47b95c61beecfd5ad8ded1e4d06ce77241f893be Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 07:53:16 +0200 Subject: [PATCH 08/39] r/storage_blob: switching to use the new sdk for some calls in create --- azurerm/resource_arm_storage_blob.go | 63 ++++++++++++++++------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index f160c5b29f03..dc71fb288a01 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -116,37 +116,41 @@ func resourceArmStorageBlob() *schema.Resource { } func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) error { - armClient := meta.(*ArmClient) - ctx := armClient.StopContext - env := armClient.environment + storageClient := meta.(*ArmClient).storage + ctx := meta.(*ArmClient).StopContext - resourceGroupName := d.Get("resource_group_name").(string) - storageAccountName := d.Get("storage_account_name").(string) + accountName := d.Get("storage_account_name").(string) + containerName := d.Get("storage_container_name").(string) + name := d.Get("name").(string) - blobClient, accountExists, err := armClient.storage.LegacyBlobClient(ctx, resourceGroupName, storageAccountName) + resourceGroup, err := storageClient.FindResourceGroup(ctx, accountName) if err != nil { - return err + return fmt.Errorf("Error locating Resource Group for Storage Account %q: %s", accountName, err) } - if !accountExists { - return fmt.Errorf("Storage Account %q Not Found", storageAccountName) + if resourceGroup == nil { + return fmt.Errorf("Unable to locate Resource Group for Blob %q (Container %q / Account %q)", name, containerName, accountName) } - name := d.Get("name").(string) - blobType := d.Get("type").(string) - containerName := d.Get("storage_container_name").(string) - sourceUri := d.Get("source_uri").(string) - contentType := d.Get("content_type").(string) + blobsClient, err := storageClient.BlobsClient(ctx, *resourceGroup, accountName) + if err != nil { + return fmt.Errorf("Error building Blobs Client: %s", err) + } - log.Printf("[INFO] Creating blob %q in container %q within storage account %q", name, containerName, storageAccountName) - container := blobClient.GetContainerReference(containerName) + legacyBlobsClient, accountExists, err := storageClient.LegacyBlobClient(ctx, *resourceGroup, accountName) + if err != nil { + return err + } + if !accountExists { + return fmt.Errorf("Storage Account %q Not Found", accountName) + } + container := legacyBlobsClient.GetContainerReference(containerName) blob := container.GetBlobReference(name) - // gives us https://example.blob.core.windows.net/container/file.vhd - id := fmt.Sprintf("https://%s.blob.%s/%s/%s", storageAccountName, env.StorageEndpointSuffix, containerName, name) + id := blobsClient.GetResourceID(accountName, containerName, name) if features.ShouldResourcesBeImported() && d.IsNewResource() { exists, err := blob.Exists() if err != nil { - return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, storageAccountName, resourceGroupName, err) + return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, accountName, resourceGroup, err) } if exists { @@ -154,6 +158,11 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } + log.Printf("[INFO] Creating blob %q in container %q within storage account %q", name, containerName, accountName) + blobType := d.Get("type").(string) + sourceUri := d.Get("source_uri").(string) + contentType := d.Get("content_type").(string) + if sourceUri != "" { options := &legacy.CopyOptions{} if err := blob.Copy(sourceUri, options); err != nil { @@ -172,7 +181,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro parallelism := d.Get("parallelism").(int) attempts := d.Get("attempts").(int) - if err := resourceArmStorageBlobBlockUploadFromSource(containerName, name, source, contentType, blobClient, parallelism, attempts); err != nil { + if err := resourceArmStorageBlobBlockUploadFromSource(containerName, name, source, contentType, legacyBlobsClient, parallelism, attempts); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } @@ -182,7 +191,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro parallelism := d.Get("parallelism").(int) attempts := d.Get("attempts").(int) - if err := resourceArmStorageBlobPageUploadFromSource(containerName, name, source, contentType, blobClient, parallelism, attempts); err != nil { + if err := resourceArmStorageBlobPageUploadFromSource(containerName, name, source, contentType, legacyBlobsClient, parallelism, attempts); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } else { @@ -198,13 +207,15 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } + log.Printf("[DEBUG] Setting the MetaData for Blob %q (Container %q / Account %q)...", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) - blob.Metadata = storage.ExpandMetaData(metaDataRaw) - - opts := &legacy.SetBlobMetadataOptions{} - if err := blob.SetMetadata(opts); err != nil { - return fmt.Errorf("Error setting metadata for storage blob on Azure: %s", err) + input := blobs.SetMetaDataInput{ + MetaData: storage.ExpandMetaData(metaDataRaw), + } + if _, err := blobsClient.SetMetaData(ctx, accountName, containerName, name, input); err != nil { + return fmt.Errorf("Error setting MetaData for Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) } + log.Printf("[DEBUG] Set the MetaData for Blob %q (Container %q / Account %q).", name, containerName, accountName) d.SetId(id) return resourceArmStorageBlobRead(d, meta) From 1eb08482b61c4218d60e30345d1765b0e135145a Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 08:18:19 +0200 Subject: [PATCH 09/39] r/storage_blob: refactoring the legacy functionality into a helper --- azurerm/resource_arm_storage_blob.go | 66 ++++++--------------- azurerm/resource_arm_storage_blob_legacy.go | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+), 48 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index dc71fb288a01..9f04a5629ba8 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -5,7 +5,6 @@ import ( "log" "strings" - legacy "github.com/Azure/azure-sdk-for-go/storage" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/helper/validation" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure" @@ -146,6 +145,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro container := legacyBlobsClient.GetContainerReference(containerName) blob := container.GetBlobReference(name) + // TODO: switch to the new sdk id := blobsClient.GetResourceID(accountName, containerName, name) if features.ShouldResourcesBeImported() && d.IsNewResource() { exists, err := blob.Exists() @@ -158,54 +158,24 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } - log.Printf("[INFO] Creating blob %q in container %q within storage account %q", name, containerName, accountName) - blobType := d.Get("type").(string) - sourceUri := d.Get("source_uri").(string) - contentType := d.Get("content_type").(string) - - if sourceUri != "" { - options := &legacy.CopyOptions{} - if err := blob.Copy(sourceUri, options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } else { - switch strings.ToLower(blobType) { - case "block": - options := &legacy.PutBlobOptions{} - if err := blob.CreateBlockBlob(options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - - source := d.Get("source").(string) - if source != "" { - parallelism := d.Get("parallelism").(int) - attempts := d.Get("attempts").(int) - - if err := resourceArmStorageBlobBlockUploadFromSource(containerName, name, source, contentType, legacyBlobsClient, parallelism, attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } - case "page": - source := d.Get("source").(string) - if source != "" { - parallelism := d.Get("parallelism").(int) - attempts := d.Get("attempts").(int) - - if err := resourceArmStorageBlobPageUploadFromSource(containerName, name, source, contentType, legacyBlobsClient, parallelism, attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } else { - size := int64(d.Get("size").(int)) - options := &legacy.PutBlobOptions{} - - blob.Properties.ContentLength = size - blob.Properties.ContentType = contentType - if err := blob.PutPageBlob(options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } - } + log.Printf("[INFO] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) + blobInput := StorageBlobUpload{ + accountName: accountName, + containerName: containerName, + blobName: name, + legacyClient: legacyBlobsClient, + attempts: d.Get("attempts").(int), + contentType: d.Get("content_type").(string), + parallelism: d.Get("parallelism").(int), + size: d.Get("size").(int), + source: d.Get("source").(string), + sourceUri: d.Get("source_uri").(string), + blobType: d.Get("type").(string), + } + if err := blobInput.Create(ctx); err != nil { + return fmt.Errorf("Error creating Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) } + log.Printf("[INFO] Created Blob %q in Container %q within Storage Account %q.", name, containerName, accountName) log.Printf("[DEBUG] Setting the MetaData for Blob %q (Container %q / Account %q)...", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 929535e6e5d5..670e6ed61c58 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -2,18 +2,79 @@ package azurerm import ( "bytes" + "context" "crypto/rand" "encoding/base64" "fmt" "io" "os" "runtime" + "strings" "sync" "github.com/Azure/azure-sdk-for-go/storage" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" ) +type StorageBlobUpload struct { + accountName string + containerName string + blobName string + + attempts int + blobType string + contentType string + parallelism int + size int + source string + sourceUri string + + legacyClient *storage.BlobStorageClient +} + +func (sbu StorageBlobUpload) Create(ctx context.Context) error { + container := sbu.legacyClient.GetContainerReference(sbu.containerName) + blob := container.GetBlobReference(sbu.blobName) + + if sbu.sourceUri != "" { + options := &storage.CopyOptions{} + if err := blob.Copy(sbu.sourceUri, options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + } else { + switch strings.ToLower(sbu.blobType) { + case "block": + options := &storage.PutBlobOptions{} + if err := blob.CreateBlockBlob(options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + + if sbu.source != "" { + if err := resourceArmStorageBlobBlockUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + } + case "page": + if sbu.source != "" { + if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + } else { + size := int64(sbu.size) + options := &storage.PutBlobOptions{} + + blob.Properties.ContentLength = size + blob.Properties.ContentType = sbu.contentType + if err := blob.PutPageBlob(options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + } + } + } + + return nil +} + type resourceArmStorageBlobPage struct { offset int64 section *io.SectionReader From b6d8e943b8cde6ba961d97cf2da1e9d82c3508e4 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 08:25:11 +0200 Subject: [PATCH 10/39] r/storage_blob: switching exists over to the new sdk --- azurerm/resource_arm_storage_blob.go | 31 +++++++++++++--------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 9f04a5629ba8..43eec166e306 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -135,30 +135,27 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro return fmt.Errorf("Error building Blobs Client: %s", err) } - legacyBlobsClient, accountExists, err := storageClient.LegacyBlobClient(ctx, *resourceGroup, accountName) - if err != nil { - return err - } - if !accountExists { - return fmt.Errorf("Storage Account %q Not Found", accountName) - } - container := legacyBlobsClient.GetContainerReference(containerName) - blob := container.GetBlobReference(name) - - // TODO: switch to the new sdk id := blobsClient.GetResourceID(accountName, containerName, name) if features.ShouldResourcesBeImported() && d.IsNewResource() { - exists, err := blob.Exists() + input := blobs.GetPropertiesInput{} + props, err := blobsClient.GetProperties(ctx, accountName, containerName, name, input) if err != nil { - return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, accountName, resourceGroup, err) + if !utils.ResponseWasNotFound(props.Response) { + return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, accountName, resourceGroup, err) + } } - - if exists { + if !utils.ResponseWasNotFound(props.Response) { return tf.ImportAsExistsError("azurerm_storage_blob", id) } } - log.Printf("[INFO] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) + // TODO: remove me + legacyBlobsClient, _, err := storageClient.LegacyBlobClient(ctx, *resourceGroup, accountName) + if err != nil { + return err + } + + log.Printf("[DEBUG] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) blobInput := StorageBlobUpload{ accountName: accountName, containerName: containerName, @@ -175,7 +172,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro if err := blobInput.Create(ctx); err != nil { return fmt.Errorf("Error creating Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) } - log.Printf("[INFO] Created Blob %q in Container %q within Storage Account %q.", name, containerName, accountName) + log.Printf("[DEBUG] Created Blob %q in Container %q within Storage Account %q.", name, containerName, accountName) log.Printf("[DEBUG] Setting the MetaData for Blob %q (Container %q / Account %q)...", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) From 967c1bcc4c367f847c3e03d7619eee668a10b866 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 08:35:27 +0200 Subject: [PATCH 11/39] r/storage_blob: refactoring --- azurerm/resource_arm_storage_blob_legacy.go | 51 +++++++++++---------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 670e6ed61c58..1b362c3836a3 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -41,33 +41,36 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { if err := blob.Copy(sbu.sourceUri, options); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } - } else { - switch strings.ToLower(sbu.blobType) { - case "block": - options := &storage.PutBlobOptions{} - if err := blob.CreateBlockBlob(options); err != nil { + + return nil + } + + switch strings.ToLower(sbu.blobType) { + // TODO: new feature for 'append' blobs? + case "block": + options := &storage.PutBlobOptions{} + if err := blob.CreateBlockBlob(options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + + if sbu.source != "" { + if err := resourceArmStorageBlobBlockUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } - - if sbu.source != "" { - if err := resourceArmStorageBlobBlockUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } + } + case "page": + if sbu.source != "" { + if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) } - case "page": - if sbu.source != "" { - if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } else { - size := int64(sbu.size) - options := &storage.PutBlobOptions{} - - blob.Properties.ContentLength = size - blob.Properties.ContentType = sbu.contentType - if err := blob.PutPageBlob(options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } + } else { + size := int64(sbu.size) + options := &storage.PutBlobOptions{} + + blob.Properties.ContentLength = size + blob.Properties.ContentType = sbu.contentType + if err := blob.PutPageBlob(options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } } From 06d474c3f6370831127537413b9cc1db60f522d6 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 08:36:19 +0200 Subject: [PATCH 12/39] r/storage_blob: fixing the log message --- azurerm/resource_arm_storage_blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 43eec166e306..f7ea46674f82 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -141,7 +141,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro props, err := blobsClient.GetProperties(ctx, accountName, containerName, name, input) if err != nil { if !utils.ResponseWasNotFound(props.Response) { - return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, accountName, resourceGroup, err) + return fmt.Errorf("Error checking if Blob %q exists (Container %q / Account %q / Resource Group %q): %s", name, containerName, accountName, *resourceGroup, err) } } if !utils.ResponseWasNotFound(props.Response) { From 6824b0b3fc104a00ae6ec04f9d6dea215e138773 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 08:51:21 +0200 Subject: [PATCH 13/39] r/storage_blob: switching copies over to the new sdk --- azurerm/resource_arm_storage_blob.go | 34 ++++++------ azurerm/resource_arm_storage_blob_legacy.go | 61 +++++++++++++-------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index f7ea46674f82..46d848f19d2d 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -156,34 +156,31 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } log.Printf("[DEBUG] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) + metaDataRaw := d.Get("metadata").(map[string]interface{}) blobInput := StorageBlobUpload{ accountName: accountName, containerName: containerName, blobName: name, - legacyClient: legacyBlobsClient, - attempts: d.Get("attempts").(int), - contentType: d.Get("content_type").(string), - parallelism: d.Get("parallelism").(int), - size: d.Get("size").(int), - source: d.Get("source").(string), - sourceUri: d.Get("source_uri").(string), - blobType: d.Get("type").(string), + + client: blobsClient, + legacyClient: legacyBlobsClient, + + blobType: d.Get("type").(string), + contentType: d.Get("content_type").(string), + metaData: storage.ExpandMetaData(metaDataRaw), + size: d.Get("size").(int), + source: d.Get("source").(string), + sourceUri: d.Get("source_uri").(string), + + // TODO: deprecate these + attempts: d.Get("attempts").(int), + parallelism: d.Get("parallelism").(int), } if err := blobInput.Create(ctx); err != nil { return fmt.Errorf("Error creating Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) } log.Printf("[DEBUG] Created Blob %q in Container %q within Storage Account %q.", name, containerName, accountName) - log.Printf("[DEBUG] Setting the MetaData for Blob %q (Container %q / Account %q)...", name, containerName, accountName) - metaDataRaw := d.Get("metadata").(map[string]interface{}) - input := blobs.SetMetaDataInput{ - MetaData: storage.ExpandMetaData(metaDataRaw), - } - if _, err := blobsClient.SetMetaData(ctx, accountName, containerName, name, input); err != nil { - return fmt.Errorf("Error setting MetaData for Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) - } - log.Printf("[DEBUG] Set the MetaData for Blob %q (Container %q / Account %q).", name, containerName, accountName) - d.SetId(id) return resourceArmStorageBlobRead(d, meta) } @@ -213,6 +210,7 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro if d.HasChange("content_type") { log.Printf("[DEBUG] Updating Properties for Blob %q (Container %q / Account %q)...", id.BlobName, id.ContainerName, id.AccountName) input := blobs.SetPropertiesInput{ + // TODO: other properties? ContentType: utils.String(d.Get("content_type").(string)), } if _, err := blobsClient.SetProperties(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 1b362c3836a3..e2791bb48688 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -11,11 +11,15 @@ import ( "runtime" "strings" "sync" + "time" - "github.com/Azure/azure-sdk-for-go/storage" + legacy "github.com/Azure/azure-sdk-for-go/storage" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" + "github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs" ) +const pollingInterval = time.Second * 15 + type StorageBlobUpload struct { accountName string containerName string @@ -24,12 +28,14 @@ type StorageBlobUpload struct { attempts int blobType string contentType string + metaData map[string]string parallelism int size int source string sourceUri string - legacyClient *storage.BlobStorageClient + client *blobs.Client + legacyClient *legacy.BlobStorageClient } func (sbu StorageBlobUpload) Create(ctx context.Context) error { @@ -37,18 +43,13 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { blob := container.GetBlobReference(sbu.blobName) if sbu.sourceUri != "" { - options := &storage.CopyOptions{} - if err := blob.Copy(sbu.sourceUri, options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - - return nil + return sbu.copy(ctx) } switch strings.ToLower(sbu.blobType) { // TODO: new feature for 'append' blobs? case "block": - options := &storage.PutBlobOptions{} + options := &legacy.PutBlobOptions{} if err := blob.CreateBlockBlob(options); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } @@ -65,7 +66,7 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { } } else { size := int64(sbu.size) - options := &storage.PutBlobOptions{} + options := &legacy.PutBlobOptions{} blob.Properties.ContentLength = size blob.Properties.ContentType = sbu.contentType @@ -78,12 +79,26 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { return nil } +func (sbu StorageBlobUpload) copy(ctx context.Context) error { + input := blobs.CopyInput{ + CopySource: sbu.sourceUri, + MetaData: sbu.metaData, + } + if err := sbu.client.CopyAndWait(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input, pollingInterval); err != nil { + return fmt.Errorf("Error copy/waiting: %s", err) + } + + return nil +} + +// TODO: remove below here + type resourceArmStorageBlobPage struct { offset int64 section *io.SectionReader } -func resourceArmStorageBlobPageUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { +func resourceArmStorageBlobPageUploadFromSource(container, name, source, contentType string, client *legacy.BlobStorageClient, parallelism, attempts int) error { workerCount := parallelism * runtime.NumCPU() file, err := os.Open(source) @@ -97,7 +112,7 @@ func resourceArmStorageBlobPageUploadFromSource(container, name, source, content return fmt.Errorf("Error splitting source file %q into pages: %s", source, err) } - options := &storage.PutBlobOptions{} + options := &legacy.PutBlobOptions{} containerRef := client.GetContainerReference(container) blob := containerRef.GetBlobReference(name) blob.Properties.ContentLength = blobSize @@ -208,7 +223,7 @@ type resourceArmStorageBlobPageUploadContext struct { name string source string blobSize int64 - client *storage.BlobStorageClient + client *legacy.BlobStorageClient pages chan resourceArmStorageBlobPage errors chan error wg *sync.WaitGroup @@ -235,11 +250,11 @@ func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUpload for x := 0; x < ctx.attempts; x++ { container := ctx.client.GetContainerReference(ctx.container) blob := container.GetBlobReference(ctx.name) - blobRange := storage.BlobRange{ + blobRange := legacy.BlobRange{ Start: uint64(start), End: uint64(end), } - options := &storage.PutPageOptions{} + options := &legacy.PutPageOptions{} reader := bytes.NewReader(chunk) err = blob.WriteRange(blobRange, reader, options) if err == nil { @@ -261,7 +276,7 @@ type resourceArmStorageBlobBlock struct { id string } -func resourceArmStorageBlobBlockUploadFromSource(container, name, source, contentType string, client *storage.BlobStorageClient, parallelism, attempts int) error { +func resourceArmStorageBlobBlockUploadFromSource(container, name, source, contentType string, client *legacy.BlobStorageClient, parallelism, attempts int) error { workerCount := parallelism * runtime.NumCPU() file, err := os.Open(source) @@ -307,7 +322,7 @@ func resourceArmStorageBlobBlockUploadFromSource(container, name, source, conten containerReference := client.GetContainerReference(container) blobReference := containerReference.GetBlobReference(name) blobReference.Properties.ContentType = contentType - options := &storage.PutBlockListOptions{} + options := &legacy.PutBlockListOptions{} err = blobReference.PutBlockList(blockList, options) if err != nil { return fmt.Errorf("Error updating block list for source file %q: %s", source, err) @@ -316,13 +331,13 @@ func resourceArmStorageBlobBlockUploadFromSource(container, name, source, conten return nil } -func resourceArmStorageBlobBlockSplit(file *os.File) ([]storage.Block, []resourceArmStorageBlobBlock, error) { +func resourceArmStorageBlobBlockSplit(file *os.File) ([]legacy.Block, []resourceArmStorageBlobBlock, error) { const ( idSize = 64 blockSize int64 = 4 * 1024 * 1024 ) var parts []resourceArmStorageBlobBlock - var blockList []storage.Block + var blockList []legacy.Block info, err := file.Stat() if err != nil { @@ -342,9 +357,9 @@ func resourceArmStorageBlobBlockSplit(file *os.File) ([]storage.Block, []resourc sectionSize = remainder } - block := storage.Block{ + block := legacy.Block{ ID: base64.StdEncoding.EncodeToString(entropy), - Status: storage.BlockStatusUncommitted, + Status: legacy.BlockStatusUncommitted, } blockList = append(blockList, block) @@ -359,7 +374,7 @@ func resourceArmStorageBlobBlockSplit(file *os.File) ([]storage.Block, []resourc } type resourceArmStorageBlobBlockUploadContext struct { - client *storage.BlobStorageClient + client *legacy.BlobStorageClient container string name string source string @@ -383,7 +398,7 @@ func resourceArmStorageBlobBlockUploadWorker(ctx resourceArmStorageBlobBlockUplo for i := 0; i < ctx.attempts; i++ { container := ctx.client.GetContainerReference(ctx.container) blob := container.GetBlobReference(ctx.name) - options := &storage.PutBlockOptions{} + options := &legacy.PutBlockOptions{} if err = blob.PutBlock(block.id, buffer, options); err == nil { break } From 5a4fb8d51df50f1086c103b1c6446a719eefab78 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:04:17 +0200 Subject: [PATCH 14/39] r/storage_blob: switching block blob to the new sdk --- azurerm/resource_arm_storage_blob_legacy.go | 95 ++++++--------------- 1 file changed, 27 insertions(+), 68 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index e2791bb48688..4d000e5681a1 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -39,9 +39,6 @@ type StorageBlobUpload struct { } func (sbu StorageBlobUpload) Create(ctx context.Context) error { - container := sbu.legacyClient.GetContainerReference(sbu.containerName) - blob := container.GetBlobReference(sbu.blobName) - if sbu.sourceUri != "" { return sbu.copy(ctx) } @@ -49,22 +46,16 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { switch strings.ToLower(sbu.blobType) { // TODO: new feature for 'append' blobs? case "block": - options := &legacy.PutBlobOptions{} - if err := blob.CreateBlockBlob(options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - - if sbu.source != "" { - if err := resourceArmStorageBlobBlockUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } + return sbu.uploadBlockBlob(ctx) case "page": if sbu.source != "" { if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } else { + container := sbu.legacyClient.GetContainerReference(sbu.containerName) + blob := container.GetBlobReference(sbu.blobName) + size := int64(sbu.size) options := &legacy.PutBlobOptions{} @@ -91,6 +82,29 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { return nil } +func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { + if sbu.source == "" { + return fmt.Errorf("A `source` is required when uploading a Block Blob") + } + + file, err := os.Open(sbu.source) + if err != nil { + return fmt.Errorf("Error opening: %s", err) + } + defer file.Close() + + input := blobs.PutBlockBlobInput{ + ContentType: utils.String(sbu.contentType), + MetaData: sbu.metaData, + } + + if err := sbu.client.PutBlockBlobFromFile(ctx, sbu.accountName, sbu.containerName, sbu.blobName, file, input); err != nil { + return fmt.Errorf("Error PutBlockBlobFromFile: %s", err) + } + + return nil +} + // TODO: remove below here type resourceArmStorageBlobPage struct { @@ -276,61 +290,6 @@ type resourceArmStorageBlobBlock struct { id string } -func resourceArmStorageBlobBlockUploadFromSource(container, name, source, contentType string, client *legacy.BlobStorageClient, parallelism, attempts int) error { - workerCount := parallelism * runtime.NumCPU() - - file, err := os.Open(source) - if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", source, err) - } - defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) - - blockList, parts, err := resourceArmStorageBlobBlockSplit(file) - if err != nil { - return fmt.Errorf("Error reading and splitting source file for upload %q: %s", source, err) - } - - wg := &sync.WaitGroup{} - blocks := make(chan resourceArmStorageBlobBlock, len(parts)) - errors := make(chan error, len(parts)) - - wg.Add(len(parts)) - for _, p := range parts { - blocks <- p - } - close(blocks) - - for i := 0; i < workerCount; i++ { - go resourceArmStorageBlobBlockUploadWorker(resourceArmStorageBlobBlockUploadContext{ - client: client, - source: source, - container: container, - name: name, - blocks: blocks, - errors: errors, - wg: wg, - attempts: attempts, - }) - } - - wg.Wait() - - if len(errors) > 0 { - return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) - } - - containerReference := client.GetContainerReference(container) - blobReference := containerReference.GetBlobReference(name) - blobReference.Properties.ContentType = contentType - options := &legacy.PutBlockListOptions{} - err = blobReference.PutBlockList(blockList, options) - if err != nil { - return fmt.Errorf("Error updating block list for source file %q: %s", source, err) - } - - return nil -} - func resourceArmStorageBlobBlockSplit(file *os.File) ([]legacy.Block, []resourceArmStorageBlobBlock, error) { const ( idSize = 64 From 21e5cc2c7d837fb52bafb53e536fba5220b9bf7b Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:07:06 +0200 Subject: [PATCH 15/39] r/storage_blob: removing unused code --- azurerm/resource_arm_storage_blob_legacy.go | 89 --------------------- 1 file changed, 89 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 4d000e5681a1..96f452ed5d55 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -3,8 +3,6 @@ package azurerm import ( "bytes" "context" - "crypto/rand" - "encoding/base64" "fmt" "io" "os" @@ -284,90 +282,3 @@ func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUpload ctx.wg.Done() } } - -type resourceArmStorageBlobBlock struct { - section *io.SectionReader - id string -} - -func resourceArmStorageBlobBlockSplit(file *os.File) ([]legacy.Block, []resourceArmStorageBlobBlock, error) { - const ( - idSize = 64 - blockSize int64 = 4 * 1024 * 1024 - ) - var parts []resourceArmStorageBlobBlock - var blockList []legacy.Block - - info, err := file.Stat() - if err != nil { - return nil, nil, fmt.Errorf("Error stating source file %q: %s", file.Name(), err) - } - - for i := int64(0); i < info.Size(); i = i + blockSize { - entropy := make([]byte, idSize) - _, err = rand.Read(entropy) - if err != nil { - return nil, nil, fmt.Errorf("Error generating a random block ID for source file %q: %s", file.Name(), err) - } - - sectionSize := blockSize - remainder := info.Size() - i - if remainder < blockSize { - sectionSize = remainder - } - - block := legacy.Block{ - ID: base64.StdEncoding.EncodeToString(entropy), - Status: legacy.BlockStatusUncommitted, - } - - blockList = append(blockList, block) - - parts = append(parts, resourceArmStorageBlobBlock{ - id: block.ID, - section: io.NewSectionReader(file, i, sectionSize), - }) - } - - return blockList, parts, nil -} - -type resourceArmStorageBlobBlockUploadContext struct { - client *legacy.BlobStorageClient - container string - name string - source string - attempts int - blocks chan resourceArmStorageBlobBlock - errors chan error - wg *sync.WaitGroup -} - -func resourceArmStorageBlobBlockUploadWorker(ctx resourceArmStorageBlobBlockUploadContext) { - for block := range ctx.blocks { - buffer := make([]byte, block.section.Size()) - - _, err := block.section.Read(buffer) - if err != nil { - ctx.errors <- fmt.Errorf("Error reading source file %q: %s", ctx.source, err) - ctx.wg.Done() - continue - } - - for i := 0; i < ctx.attempts; i++ { - container := ctx.client.GetContainerReference(ctx.container) - blob := container.GetBlobReference(ctx.name) - options := &legacy.PutBlockOptions{} - if err = blob.PutBlock(block.id, buffer, options); err == nil { - break - } - } - if err != nil { - ctx.errors <- fmt.Errorf("Error uploading block %q for source file %q: %s", block.id, ctx.source, err) - ctx.wg.Done() - continue - } - - ctx.wg.Done() - } -} From 75a70aa7c6f70bd5a743f2f683c8233d8680075b Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:14:35 +0200 Subject: [PATCH 16/39] r/storage_blob: bugfix setting the metadata for page blobs --- azurerm/resource_arm_storage_blob_legacy.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 96f452ed5d55..fb5e5057b694 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -41,10 +41,15 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { return sbu.copy(ctx) } - switch strings.ToLower(sbu.blobType) { + blobType := strings.ToLower(sbu.blobType) + // TODO: new feature for 'append' blobs? - case "block": + + if blobType == "block" { return sbu.uploadBlockBlob(ctx) + } + + switch strings.ToLower(sbu.blobType) { case "page": if sbu.source != "" { if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { @@ -63,6 +68,13 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } } + + input := blobs.SetMetaDataInput{ + MetaData: sbu.metaData, + } + if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error setting MetaData: %s", err) + } } return nil From 4844259b3ac7cec6505950f79dff96f14ffbdcfc Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:42:44 +0200 Subject: [PATCH 17/39] r/storage_blob: removing unsupported tests --- azurerm/resource_arm_storage_blob_test.go | 120 +++++++++------------- 1 file changed, 49 insertions(+), 71 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_test.go b/azurerm/resource_arm_storage_blob_test.go index cd2a801e0157..654e3e9ee1d6 100644 --- a/azurerm/resource_arm_storage_blob_test.go +++ b/azurerm/resource_arm_storage_blob_test.go @@ -21,6 +21,15 @@ import ( var supportsNewStorageFeatures = false func TestAccAzureRMStorageBlob_disappears(t *testing.T) { + sourceBlob, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("Failed to create local source blob file") + } + + if err := testAccAzureRMStorageBlob_populateTempFile(sourceBlob); err != nil { + t.Fatalf("Error populating temp file: %s", err) + } + resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -32,7 +41,7 @@ func TestAccAzureRMStorageBlob_disappears(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockEmpty(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), testCheckAzureRMStorageBlobDisappears(resourceName), @@ -105,7 +114,7 @@ func TestAccAzureRMStorageBlob_appendEmptyMetaData(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockEmpty(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -117,7 +126,7 @@ func TestAccAzureRMStorageBlob_blockEmpty(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockEmpty(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromPublicBlob(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -132,7 +141,7 @@ func TestAccAzureRMStorageBlob_blockEmpty(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockEmptyMetaData(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -144,7 +153,7 @@ func TestAccAzureRMStorageBlob_blockEmptyMetaData(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockEmptyMetaData(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromPublicFile(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -159,7 +168,7 @@ func TestAccAzureRMStorageBlob_blockEmptyMetaData(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -171,7 +180,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromPublicBlob(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromExistingBlob(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -186,34 +195,16 @@ func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { - resourceName := "azurerm_storage_blob.test" - ri := tf.AccRandTimeInt() - rs := strings.ToLower(acctest.RandString(11)) - location := testLocation() +func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { + sourceBlob, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("Failed to create local source blob file") + } - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testCheckAzureRMStorageBlobDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAzureRMStorageBlob_blockFromPublicFile(ri, rs, location), - Check: resource.ComposeTestCheckFunc( - testCheckAzureRMStorageBlobExists(resourceName), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, - }, - }, - }) -} + if err := testAccAzureRMStorageBlob_populateTempFile(sourceBlob); err != nil { + t.Fatalf("Error populating temp file: %s", err) + } -func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -225,22 +216,23 @@ func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromExistingBlob(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), + testCheckAzureRMStorageBlobMatchesFile(resourceName, storage.BlobTypeBlock, sourceBlob.Name()), ), }, { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source", "type"}, }, }, }) } -func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromLocalFileMetaData(t *testing.T) { sourceBlob, err := ioutil.TempFile("", "") if err != nil { t.Fatalf("Failed to create local source blob file") @@ -261,7 +253,7 @@ func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), + Config: testAccAzureRMStorageBlob_blockFromLocalBlobMetaData(ri, rs, location, sourceBlob.Name()), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), testCheckAzureRMStorageBlobMatchesFile(resourceName, storage.BlobTypeBlock, sourceBlob.Name()), @@ -711,40 +703,6 @@ resource "azurerm_storage_blob" "test" { `, template) } -func testAccAzureRMStorageBlob_blockEmpty(rInt int, rString string, location string) string { - template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") - return fmt.Sprintf(` -%s - -resource "azurerm_storage_blob" "test" { - name = "example.vhd" - resource_group_name = "${azurerm_resource_group.test.name}" - storage_account_name = "${azurerm_storage_account.test.name}" - storage_container_name = "${azurerm_storage_container.test.name}" - type = "block" -} -`, template) -} - -func testAccAzureRMStorageBlob_blockEmptyMetaData(rInt int, rString string, location string) string { - template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") - return fmt.Sprintf(` -%s - -resource "azurerm_storage_blob" "test" { - name = "example.vhd" - resource_group_name = "${azurerm_resource_group.test.name}" - storage_account_name = "${azurerm_storage_account.test.name}" - storage_container_name = "${azurerm_storage_container.test.name}" - type = "block" - - metadata = { - hello = "world" - } -} -`, template) -} - func testAccAzureRMStorageBlob_blockFromPublicBlob(rInt int, rString, location string) string { template := testAccAzureRMStorageBlob_template(rInt, rString, location, "blob") return fmt.Sprintf(` @@ -839,6 +797,26 @@ resource "azurerm_storage_blob" "test" { `, template, fileName) } +func testAccAzureRMStorageBlob_blockFromLocalBlobMetaData(rInt int, rString, location, fileName string) string { + template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") + return fmt.Sprintf(` +%s + +resource "azurerm_storage_blob" "test" { + name = "example.vhd" + resource_group_name = "${azurerm_resource_group.test.name}" + storage_account_name = "${azurerm_storage_account.test.name}" + storage_container_name = "${azurerm_storage_container.test.name}" + type = "block" + source = "%s" + + metadata = { + hello = "world" + } +} +`, template, fileName) +} + func testAccAzureRMStorageBlob_contentType(rInt int, rString, location string) string { template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") return fmt.Sprintf(` From 92f2816ab7554ea22a3e28b8fe9f183815d32ce5 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:49:04 +0200 Subject: [PATCH 18/39] r/storage_blob: refactoring into methods --- azurerm/resource_arm_storage_blob_legacy.go | 72 ++++++++++++--------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index fb5e5057b694..1296e1093894 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -46,38 +46,14 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { // TODO: new feature for 'append' blobs? if blobType == "block" { - return sbu.uploadBlockBlob(ctx) + return sbu.createBlockBlob(ctx) } - switch strings.ToLower(sbu.blobType) { - case "page": - if sbu.source != "" { - if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } else { - container := sbu.legacyClient.GetContainerReference(sbu.containerName) - blob := container.GetBlobReference(sbu.blobName) - - size := int64(sbu.size) - options := &legacy.PutBlobOptions{} - - blob.Properties.ContentLength = size - blob.Properties.ContentType = sbu.contentType - if err := blob.PutPageBlob(options); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - } - - input := blobs.SetMetaDataInput{ - MetaData: sbu.metaData, - } - if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { - return fmt.Errorf("Error setting MetaData: %s", err) - } + if blobType == "page" { + return sbu.createPageBlob(ctx) } - return nil + return fmt.Errorf("Unsupported Blob Type: %q", blobType) } func (sbu StorageBlobUpload) copy(ctx context.Context) error { @@ -92,7 +68,7 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { +func (sbu StorageBlobUpload) createBlockBlob(ctx context.Context) error { if sbu.source == "" { return fmt.Errorf("A `source` is required when uploading a Block Blob") } @@ -115,6 +91,44 @@ func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { return nil } +func (sbu StorageBlobUpload) createPageBlob(ctx context.Context) error { + if sbu.source != "" { + if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + + input := blobs.SetMetaDataInput{ + MetaData: sbu.metaData, + } + if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error setting MetaData: %s", err) + } + + return nil + } + + container := sbu.legacyClient.GetContainerReference(sbu.containerName) + blob := container.GetBlobReference(sbu.blobName) + + size := int64(sbu.size) + options := &legacy.PutBlobOptions{} + + blob.Properties.ContentLength = size + blob.Properties.ContentType = sbu.contentType + if err := blob.PutPageBlob(options); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) + } + + input := blobs.SetMetaDataInput{ + MetaData: sbu.metaData, + } + if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error setting MetaData: %s", err) + } + + return nil +} + // TODO: remove below here type resourceArmStorageBlobPage struct { From 283b72d5d14b1142a8ddd4f8f6fc02db263b9e56 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:54:24 +0200 Subject: [PATCH 19/39] r/storage_blob: switching empty page blobs to the new sdk --- azurerm/resource_arm_storage_blob_legacy.go | 47 ++++++++++----------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 1296e1093894..12145e06d6f5 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -46,11 +46,15 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { // TODO: new feature for 'append' blobs? if blobType == "block" { - return sbu.createBlockBlob(ctx) + return sbu.uploadBlockBlob(ctx) } if blobType == "page" { - return sbu.createPageBlob(ctx) + if sbu.source != "" { + return sbu.uploadPageBlob(ctx) + } + + return sbu.createEmptyPageBlob(ctx) } return fmt.Errorf("Unsupported Blob Type: %q", blobType) @@ -68,7 +72,7 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) createBlockBlob(ctx context.Context) error { +func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { if sbu.source == "" { return fmt.Errorf("A `source` is required when uploading a Block Blob") } @@ -91,31 +95,26 @@ func (sbu StorageBlobUpload) createBlockBlob(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) createPageBlob(ctx context.Context) error { - if sbu.source != "" { - if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - - input := blobs.SetMetaDataInput{ - MetaData: sbu.metaData, - } - if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { - return fmt.Errorf("Error setting MetaData: %s", err) - } - - return nil +func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { + if sbu.size == 0 { + return fmt.Errorf("`size` cannot be zero for a page blob") } - container := sbu.legacyClient.GetContainerReference(sbu.containerName) - blob := container.GetBlobReference(sbu.blobName) + input := blobs.PutPageBlobInput{ + BlobContentLengthBytes: int64(sbu.size), + ContentType: utils.String(sbu.contentType), + MetaData: sbu.metaData, + } + // TODO: access tiers? + if _, err := sbu.client.PutPageBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error PutPageBlob: %s", err) + } - size := int64(sbu.size) - options := &legacy.PutBlobOptions{} + return nil +} - blob.Properties.ContentLength = size - blob.Properties.ContentType = sbu.contentType - if err := blob.PutPageBlob(options); err != nil { +func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { + if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } From b4a6fa8417898de3957537bf53c9cd73b105fd2c Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 09:56:16 +0200 Subject: [PATCH 20/39] r/storage_blob: fixing the tests --- azurerm/resource_arm_storage_blob_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_test.go b/azurerm/resource_arm_storage_blob_test.go index 654e3e9ee1d6..df8017678439 100644 --- a/azurerm/resource_arm_storage_blob_test.go +++ b/azurerm/resource_arm_storage_blob_test.go @@ -443,7 +443,7 @@ func TestAccAzureRMStorageBlob_requiresImport(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockEmpty(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromPublicBlob(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -963,7 +963,7 @@ func testAccAzureRMStorageBlob_populateTempFile(input *os.File) error { } func testAccAzureRMStorageBlob_requiresImport(rInt int, rString, location string) string { - template := testAccAzureRMStorageBlob_blockEmpty(rInt, rString, location) + template := testAccAzureRMStorageBlob_blockFromPublicBlob(rInt, rString, location) return fmt.Sprintf(` %s From 32784dfe162234359d9a0edf58efe6c8bdc8828a Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 10:41:33 +0100 Subject: [PATCH 21/39] Revert "r/storage_blob: removing unsupported tests" This reverts commit 9971cceb623029ffd323e25a738688046b8c69fa. --- azurerm/resource_arm_storage_blob_test.go | 120 +++++++++++++--------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_test.go b/azurerm/resource_arm_storage_blob_test.go index df8017678439..377413223730 100644 --- a/azurerm/resource_arm_storage_blob_test.go +++ b/azurerm/resource_arm_storage_blob_test.go @@ -21,15 +21,6 @@ import ( var supportsNewStorageFeatures = false func TestAccAzureRMStorageBlob_disappears(t *testing.T) { - sourceBlob, err := ioutil.TempFile("", "") - if err != nil { - t.Fatalf("Failed to create local source blob file") - } - - if err := testAccAzureRMStorageBlob_populateTempFile(sourceBlob); err != nil { - t.Fatalf("Error populating temp file: %s", err) - } - resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -41,7 +32,7 @@ func TestAccAzureRMStorageBlob_disappears(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), + Config: testAccAzureRMStorageBlob_blockEmpty(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), testCheckAzureRMStorageBlobDisappears(resourceName), @@ -114,7 +105,7 @@ func TestAccAzureRMStorageBlob_appendEmptyMetaData(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { +func TestAccAzureRMStorageBlob_blockEmpty(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -126,7 +117,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromPublicBlob(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockEmpty(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -141,7 +132,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { +func TestAccAzureRMStorageBlob_blockEmptyMetaData(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -153,7 +144,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromPublicFile(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockEmptyMetaData(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -168,7 +159,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -180,7 +171,7 @@ func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromExistingBlob(ri, rs, location), + Config: testAccAzureRMStorageBlob_blockFromPublicBlob(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), ), @@ -195,16 +186,34 @@ func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { }) } -func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { - sourceBlob, err := ioutil.TempFile("", "") - if err != nil { - t.Fatalf("Failed to create local source blob file") - } +func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { + resourceName := "azurerm_storage_blob.test" + ri := tf.AccRandTimeInt() + rs := strings.ToLower(acctest.RandString(11)) + location := testLocation() - if err := testAccAzureRMStorageBlob_populateTempFile(sourceBlob); err != nil { - t.Fatalf("Error populating temp file: %s", err) - } + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testCheckAzureRMStorageBlobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAzureRMStorageBlob_blockFromPublicFile(ri, rs, location), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMStorageBlobExists(resourceName), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + }, + }, + }) +} +func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { resourceName := "azurerm_storage_blob.test" ri := tf.AccRandTimeInt() rs := strings.ToLower(acctest.RandString(11)) @@ -216,23 +225,22 @@ func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), + Config: testAccAzureRMStorageBlob_blockFromExistingBlob(ri, rs, location), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), - testCheckAzureRMStorageBlobMatchesFile(resourceName, storage.BlobTypeBlock, sourceBlob.Name()), ), }, { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, }, }, }) } -func TestAccAzureRMStorageBlob_blockFromLocalFileMetaData(t *testing.T) { +func TestAccAzureRMStorageBlob_blockFromLocalFile(t *testing.T) { sourceBlob, err := ioutil.TempFile("", "") if err != nil { t.Fatalf("Failed to create local source blob file") @@ -253,7 +261,7 @@ func TestAccAzureRMStorageBlob_blockFromLocalFileMetaData(t *testing.T) { CheckDestroy: testCheckAzureRMStorageBlobDestroy, Steps: []resource.TestStep{ { - Config: testAccAzureRMStorageBlob_blockFromLocalBlobMetaData(ri, rs, location, sourceBlob.Name()), + Config: testAccAzureRMStorageBlob_blockFromLocalBlob(ri, rs, location, sourceBlob.Name()), Check: resource.ComposeTestCheckFunc( testCheckAzureRMStorageBlobExists(resourceName), testCheckAzureRMStorageBlobMatchesFile(resourceName, storage.BlobTypeBlock, sourceBlob.Name()), @@ -703,6 +711,40 @@ resource "azurerm_storage_blob" "test" { `, template) } +func testAccAzureRMStorageBlob_blockEmpty(rInt int, rString string, location string) string { + template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") + return fmt.Sprintf(` +%s + +resource "azurerm_storage_blob" "test" { + name = "example.vhd" + resource_group_name = "${azurerm_resource_group.test.name}" + storage_account_name = "${azurerm_storage_account.test.name}" + storage_container_name = "${azurerm_storage_container.test.name}" + type = "block" +} +`, template) +} + +func testAccAzureRMStorageBlob_blockEmptyMetaData(rInt int, rString string, location string) string { + template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") + return fmt.Sprintf(` +%s + +resource "azurerm_storage_blob" "test" { + name = "example.vhd" + resource_group_name = "${azurerm_resource_group.test.name}" + storage_account_name = "${azurerm_storage_account.test.name}" + storage_container_name = "${azurerm_storage_container.test.name}" + type = "block" + + metadata = { + hello = "world" + } +} +`, template) +} + func testAccAzureRMStorageBlob_blockFromPublicBlob(rInt int, rString, location string) string { template := testAccAzureRMStorageBlob_template(rInt, rString, location, "blob") return fmt.Sprintf(` @@ -797,26 +839,6 @@ resource "azurerm_storage_blob" "test" { `, template, fileName) } -func testAccAzureRMStorageBlob_blockFromLocalBlobMetaData(rInt int, rString, location, fileName string) string { - template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") - return fmt.Sprintf(` -%s - -resource "azurerm_storage_blob" "test" { - name = "example.vhd" - resource_group_name = "${azurerm_resource_group.test.name}" - storage_account_name = "${azurerm_storage_account.test.name}" - storage_container_name = "${azurerm_storage_container.test.name}" - type = "block" - source = "%s" - - metadata = { - hello = "world" - } -} -`, template, fileName) -} - func testAccAzureRMStorageBlob_contentType(rInt int, rString, location string) string { template := testAccAzureRMStorageBlob_template(rInt, rString, location, "private") return fmt.Sprintf(` From 712d77f493eef2521dd609d940c0dff63a49eb27 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Fri, 23 Aug 2019 11:02:18 +0100 Subject: [PATCH 22/39] WIP: empty block blobs --- azurerm/resource_arm_storage_blob_legacy.go | 20 +++++++++++++++---- go.mod | 2 +- go.sum | 2 ++ .../2018-11-09/blob/blobs/append_block.go | 15 ++++++++++---- .../2018-11-09/blob/blobs/properties_get.go | 2 +- .../2018-11-09/blob/blobs/put_block_blob.go | 16 ++++++++++----- .../blob/blobs/put_block_blob_file.go | 2 +- .../2018-11-09/blob/blobs/put_page_blob.go | 2 +- vendor/modules.txt | 4 ++-- 9 files changed, 46 insertions(+), 19 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 12145e06d6f5..8cbbbafbc2bc 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -46,7 +46,11 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { // TODO: new feature for 'append' blobs? if blobType == "block" { - return sbu.uploadBlockBlob(ctx) + if sbu.source != "" { + return sbu.uploadBlockBlob(ctx) + } + + return sbu.createEmptyBlockBlob(ctx) } if blobType == "page" { @@ -72,11 +76,19 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { - if sbu.source == "" { - return fmt.Errorf("A `source` is required when uploading a Block Blob") +func (sbu StorageBlobUpload) createEmptyBlockBlob(ctx context.Context) error { + input := blobs.PutBlockBlobInput{ + ContentType: utils.String(sbu.contentType), + MetaData: sbu.metaData, + } + if _, err := sbu.client.PutBlockBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error PutBlockBlob: %s", err) } + return nil +} + +func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { file, err := os.Open(sbu.source) if err != nil { return fmt.Errorf("Error opening: %s", err) diff --git a/go.mod b/go.mod index 102cd29caa49..c78c95e4c5a9 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/satori/uuid v0.0.0-20160927100844-b061729afc07 github.com/sirupsen/logrus v1.2.0 // indirect github.com/terraform-providers/terraform-provider-azuread v0.6.0 - github.com/tombuildsstuff/giovanni v0.3.2 + github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190502183928-7f726cade0ab gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 69aa7f5cb018..3fae28098d8c 100644 --- a/go.sum +++ b/go.sum @@ -488,6 +488,8 @@ github.com/terraform-providers/terraform-provider-openstack v1.15.0/go.mod h1:2a github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tombuildsstuff/giovanni v0.3.2 h1:uKt7irq74daF0lvaA0X9bEuFKSDhceVy4DyWVPDRYeU= github.com/tombuildsstuff/giovanni v0.3.2/go.mod h1:3UHcoRZCvWTjXMWCJ0epD1VROlPMwZeeof3vfP6NiWM= +github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 h1:M5CRmx8+0gcgEG6qpPml39RS3Squ1zbFD0xVZ1owpzM= +github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5/go.mod h1:3UHcoRZCvWTjXMWCJ0epD1VROlPMwZeeof3vfP6NiWM= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5 h1:cMjKdf4PxEBN9K5HaD9UMW8gkTbM0kMzkTa9SJe0WNQ= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/append_block.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/append_block.go index 7fed86a319b0..80d9d8f74aaa 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/append_block.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/append_block.go @@ -28,7 +28,8 @@ type AppendBlockInput struct { BlobConditionMaxSize *int64 // The Bytes which should be appended to the end of this Append Blob. - Content []byte + // This can either be nil, which creates an empty blob, or a byte array + Content *[]byte // An MD5 hash of the block content. // This hash is used to verify the integrity of the block during transport. @@ -68,7 +69,7 @@ func (client Client) AppendBlock(ctx context.Context, accountName, containerName if blobName == "" { return result, validation.NewError("blobs.Client", "AppendBlock", "`blobName` cannot be an empty string.") } - if len(input.Content) > (4 * 1024 * 1024) { + if input.Content != nil && len(*input.Content) > (4*1024*1024) { return result, validation.NewError("files.Client", "PutByteRange", "`input.Content` must be at most 4MB.") } @@ -122,13 +123,19 @@ func (client Client) AppendBlockPreparer(ctx context.Context, accountName, conta headers["x-ms-lease-id"] = *input.LeaseID } - preparer := autorest.CreatePreparer( + decorators := []autorest.PrepareDecorator{ autorest.AsPut(), autorest.WithBaseURL(endpoints.GetBlobEndpoint(client.BaseURI, accountName)), autorest.WithPathParameters("/{containerName}/{blobName}", pathParameters), autorest.WithQueryParameters(queryParameters), autorest.WithHeaders(headers), - autorest.WithBytes(&input.Content)) + } + + if input.Content != nil { + decorators = append(decorators, autorest.WithBytes(input.Content)) + } + + preparer := autorest.CreatePreparer(decorators...) return preparer.Prepare((&http.Request{}).WithContext(ctx)) } diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/properties_get.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/properties_get.go index de7c5fced3b5..5640687d6d92 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/properties_get.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/properties_get.go @@ -248,7 +248,7 @@ func (client Client) GetPropertiesResponder(resp *http.Response) (result GetProp result.CopyCompletionTime = resp.Header.Get("x-ms-copy-completion-time") result.CopyDestinationSnapshot = resp.Header.Get("x-ms-copy-destination-snapshot") result.CopyID = resp.Header.Get("x-ms-copy-id") - result.CopyProgress = resp.Header.Get(" x-ms-copy-progress") + result.CopyProgress = resp.Header.Get("x-ms-copy-progress") result.CopySource = resp.Header.Get("x-ms-copy-source") result.CopyStatus = CopyStatus(resp.Header.Get("x-ms-copy-status")) result.CopyStatusDescription = resp.Header.Get("x-ms-copy-status-description") diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob.go index fa29dd38011e..972a39da2ffa 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob.go @@ -15,7 +15,7 @@ import ( type PutBlockBlobInput struct { CacheControl *string - Content []byte + Content *[]byte ContentDisposition *string ContentEncoding *string ContentLanguage *string @@ -40,8 +40,8 @@ func (client Client) PutBlockBlob(ctx context.Context, accountName, containerNam if blobName == "" { return result, validation.NewError("blobs.Client", "PutBlockBlob", "`blobName` cannot be an empty string.") } - if len(input.Content) == 0 { - return result, validation.NewError("blobs.Client", "PutBlockBlob", "`input.Content` cannot be empty.") + if input.Content != nil && len(*input.Content) == 0 { + return result, validation.NewError("blobs.Client", "PutBlockBlob", "`input.Content` must either be nil or not empty.") } if err := metadata.Validate(input.MetaData); err != nil { return result, validation.NewError("blobs.Client", "PutBlockBlob", fmt.Sprintf("`input.MetaData` is not valid: %s.", err)) @@ -105,12 +105,18 @@ func (client Client) PutBlockBlobPreparer(ctx context.Context, accountName, cont headers = metadata.SetIntoHeaders(headers, input.MetaData) - preparer := autorest.CreatePreparer( + decorators := []autorest.PrepareDecorator{ autorest.AsPut(), autorest.WithBaseURL(endpoints.GetBlobEndpoint(client.BaseURI, accountName)), autorest.WithPathParameters("/{containerName}/{blobName}", pathParameters), autorest.WithHeaders(headers), - autorest.WithBytes(&input.Content)) + } + + if input.Content != nil { + decorators = append(decorators, autorest.WithBytes(input.Content)) + } + + preparer := autorest.CreatePreparer(decorators...) return preparer.Prepare((&http.Request{}).WithContext(ctx)) } diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob_file.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob_file.go index 7232e5e4e456..b7499498aeca 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob_file.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_block_blob_file.go @@ -24,7 +24,7 @@ func (client Client) PutBlockBlobFromFile(ctx context.Context, accountName, cont } } - input.Content = bytes + input.Content = &bytes if _, err = client.PutBlockBlob(ctx, accountName, containerName, blobName, input); err != nil { return fmt.Errorf("Error putting bytes: %s", err) diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_page_blob.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_page_blob.go index ad3c8788c862..4cfba95ae37b 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_page_blob.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs/put_page_blob.go @@ -43,7 +43,7 @@ func (client Client) PutPageBlob(ctx context.Context, accountName, containerName return result, validation.NewError("blobs.Client", "PutPageBlob", "`blobName` cannot be an empty string.") } if input.BlobContentLengthBytes == 0 || input.BlobContentLengthBytes%512 != 0 { - return result, validation.NewError("blobs.Client", "PutPageBlob", "`blobName` must be aligned to a 512-byte boundary.") + return result, validation.NewError("blobs.Client", "PutPageBlob", "`input.BlobContentLengthBytes` must be aligned to a 512-byte boundary.") } req, err := client.PutPageBlobPreparer(ctx, accountName, containerName, blobName, input) diff --git a/vendor/modules.txt b/vendor/modules.txt index 380505428004..834729a112ac 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -234,9 +234,9 @@ github.com/hashicorp/terraform/helper/resource github.com/hashicorp/terraform/helper/schema github.com/hashicorp/terraform/helper/structure github.com/hashicorp/terraform/helper/validation -github.com/hashicorp/terraform/httpclient github.com/hashicorp/terraform/terraform github.com/hashicorp/terraform/helper/acctest +github.com/hashicorp/terraform/httpclient github.com/hashicorp/terraform/helper/mutexkv github.com/hashicorp/terraform/configs/configschema github.com/hashicorp/terraform/helper/plugin @@ -328,7 +328,7 @@ github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/slices github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/tf github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/validate github.com/terraform-providers/terraform-provider-azuread/version -# github.com/tombuildsstuff/giovanni v0.3.2 +# github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers github.com/tombuildsstuff/giovanni/storage/2018-11-09/file/directories From b70b7f207ba48cb57ae6fe720598c3884edc4541 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 12:35:35 +0100 Subject: [PATCH 23/39] r/storage_blob: refactoring --- azurerm/resource_arm_storage_blob_legacy.go | 43 ++++++++++++--------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 8cbbbafbc2bc..10ff40f53740 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -126,7 +126,11 @@ func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { } func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { - if err := resourceArmStorageBlobPageUploadFromSource(sbu.containerName, sbu.blobName, sbu.source, sbu.contentType, sbu.legacyClient, sbu.parallelism, sbu.attempts); err != nil { + if sbu.size != 0 { + // the user shouldn't need to specify this since we infer it + } + + if err := sbu.resourceArmStorageBlobPageUploadFromSource(); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } @@ -147,30 +151,33 @@ type resourceArmStorageBlobPage struct { section *io.SectionReader } -func resourceArmStorageBlobPageUploadFromSource(container, name, source, contentType string, client *legacy.BlobStorageClient, parallelism, attempts int) error { - workerCount := parallelism * runtime.NumCPU() +func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource() error { + workerCount := sbu.parallelism * runtime.NumCPU() - file, err := os.Open(source) + file, err := os.Open(sbu.source) if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", source, err) + return fmt.Errorf("Error opening source file for upload %q: %s", sbu.source, err) } - defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", name, source)) + defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", sbu.blobName, sbu.source)) - blobSize, pageList, err := resourceArmStorageBlobPageSplit(file) + // first we chunk the file and assign them to 'pages' + blobSize, pageList, err := sbu.resourceArmStorageBlobPageSplit(file) if err != nil { - return fmt.Errorf("Error splitting source file %q into pages: %s", source, err) + return fmt.Errorf("Error splitting source file %q into pages: %s", sbu.source, err) } + // then we create an empty file with this size options := &legacy.PutBlobOptions{} - containerRef := client.GetContainerReference(container) - blob := containerRef.GetBlobReference(name) + containerRef := sbu.legacyClient.GetContainerReference(sbu.containerName) + blob := containerRef.GetBlobReference(sbu.blobName) blob.Properties.ContentLength = blobSize - blob.Properties.ContentType = contentType + blob.Properties.ContentType = sbu.contentType err = blob.PutPageBlob(options) if err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } + // finally we upload the contents of said file pages := make(chan resourceArmStorageBlobPage, len(pageList)) errors := make(chan error, len(pageList)) wg := &sync.WaitGroup{} @@ -185,28 +192,28 @@ func resourceArmStorageBlobPageUploadFromSource(container, name, source, content for i := 0; i < workerCount; i++ { go resourceArmStorageBlobPageUploadWorker(resourceArmStorageBlobPageUploadContext{ - container: container, - name: name, - source: source, + container: sbu.containerName, + name: sbu.blobName, + source: sbu.source, blobSize: blobSize, - client: client, + client: sbu.legacyClient, pages: pages, errors: errors, wg: wg, - attempts: attempts, + attempts: sbu.attempts, }) } wg.Wait() if len(errors) > 0 { - return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) + return fmt.Errorf("Error while uploading source file %q: %s", sbu.source, <-errors) } return nil } -func resourceArmStorageBlobPageSplit(file *os.File) (int64, []resourceArmStorageBlobPage, error) { +func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File) (int64, []resourceArmStorageBlobPage, error) { const ( minPageSize int64 = 4 * 1024 maxPageSize int64 = 4 * 1024 * 1024 From d1b132d4c74ac6b5c7dfcb14426fbdb9bd7127f3 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 12:58:27 +0100 Subject: [PATCH 24/39] r/storage_blob: refactoring away the old sdk --- azurerm/resource_arm_storage_blob_legacy.go | 83 ++++++++++----------- azurerm/utils/io.go | 2 + 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 10ff40f53740..260700cd6218 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -130,15 +130,35 @@ func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { // the user shouldn't need to specify this since we infer it } - if err := sbu.resourceArmStorageBlobPageUploadFromSource(); err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) + // determine the details about the file + file, err := os.Open(sbu.source) + if err != nil { + return fmt.Errorf("Error opening source file for upload %q: %s", sbu.source, err) } + defer file.Close() - input := blobs.SetMetaDataInput{ - MetaData: sbu.metaData, + // TODO: all of this ultimately can be moved into Giovanni + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("Could not stat file %q: %s", file.Name(), err) + } + + fileSize := info.Size() + + // first let's create a file of the specified file size + input := blobs.PutPageBlobInput{ + BlobContentLengthBytes: int64(fileSize), + ContentType: utils.String(sbu.contentType), + MetaData: sbu.metaData, + } + // TODO: access tiers? + if _, err := sbu.client.PutPageBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + return fmt.Errorf("Error PutPageBlob: %s", err) } - if _, err := sbu.client.SetMetaData(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { - return fmt.Errorf("Error setting MetaData: %s", err) + + if err := sbu.resourceArmStorageBlobPageUploadFromSource(ctx, file, fileSize); err != nil { + return fmt.Errorf("Error creating storage blob on Azure: %s", err) } return nil @@ -151,32 +171,15 @@ type resourceArmStorageBlobPage struct { section *io.SectionReader } -func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource() error { +func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { workerCount := sbu.parallelism * runtime.NumCPU() - file, err := os.Open(sbu.source) - if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", sbu.source, err) - } - defer utils.IoCloseAndLogError(file, fmt.Sprintf("Error closing Storage Blob `%s` file `%s` after upload", sbu.blobName, sbu.source)) - // first we chunk the file and assign them to 'pages' - blobSize, pageList, err := sbu.resourceArmStorageBlobPageSplit(file) + pageList, err := sbu.resourceArmStorageBlobPageSplit(file, fileSize) if err != nil { return fmt.Errorf("Error splitting source file %q into pages: %s", sbu.source, err) } - // then we create an empty file with this size - options := &legacy.PutBlobOptions{} - containerRef := sbu.legacyClient.GetContainerReference(sbu.containerName) - blob := containerRef.GetBlobReference(sbu.blobName) - blob.Properties.ContentLength = blobSize - blob.Properties.ContentType = sbu.contentType - err = blob.PutPageBlob(options) - if err != nil { - return fmt.Errorf("Error creating storage blob on Azure: %s", err) - } - // finally we upload the contents of said file pages := make(chan resourceArmStorageBlobPage, len(pageList)) errors := make(chan error, len(pageList)) @@ -195,7 +198,7 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource() error container: sbu.containerName, name: sbu.blobName, source: sbu.source, - blobSize: blobSize, + blobSize: fileSize, client: sbu.legacyClient, pages: pages, errors: errors, @@ -213,20 +216,16 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource() error return nil } -func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File) (int64, []resourceArmStorageBlobPage, error) { - const ( - minPageSize int64 = 4 * 1024 - maxPageSize int64 = 4 * 1024 * 1024 - ) - - info, err := file.Stat() - if err != nil { - return int64(0), nil, fmt.Errorf("Could not stat file %q: %s", file.Name(), err) - } +const ( + minPageSize int64 = 4 * 1024 + maxPageSize int64 = 4 * 1024 * 1024 +) - blobSize := info.Size() - if info.Size()%minPageSize != 0 { - blobSize = info.Size() + (minPageSize - (info.Size() % minPageSize)) +func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, fileSize int64) ([]resourceArmStorageBlobPage, error) { + // whilst the file size can be any arbitary size, it must be uploaded in fixed-size pages + blobSize := fileSize + if fileSize%minPageSize != 0 { + blobSize = fileSize + (minPageSize - (fileSize % minPageSize)) } emptyPage := make([]byte, minPageSize) @@ -240,9 +239,9 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File) (int var currentRange byteRange for i := int64(0); i < blobSize; i += minPageSize { pageBuf := make([]byte, minPageSize) - _, err = file.ReadAt(pageBuf, i) + _, err := file.ReadAt(pageBuf, i) if err != nil && err != io.EOF { - return int64(0), nil, fmt.Errorf("Could not read chunk at %d: %s", i, err) + return nil, fmt.Errorf("Could not read chunk at %d: %s", i, err) } if bytes.Equal(pageBuf, emptyPage) { @@ -271,7 +270,7 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File) (int }) } - return info.Size(), pages, nil + return pages, nil } type resourceArmStorageBlobPageUploadContext struct { diff --git a/azurerm/utils/io.go b/azurerm/utils/io.go index be6eaecbc352..90c0b4fa5c30 100644 --- a/azurerm/utils/io.go +++ b/azurerm/utils/io.go @@ -5,6 +5,8 @@ import ( "log" ) +// TODO: deprecate / remove me + func IoCloseAndLogError(c io.Closer, message string) { if err := c.Close(); err != nil { log.Printf("%s: %v", message, err) From c20ff4e095b302249d79931725b296a439f2e32c Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 13:06:32 +0100 Subject: [PATCH 25/39] r/storage_blob: refactoring/renaming the page upload functions --- azurerm/resource_arm_storage_blob_legacy.go | 56 +++++++++------------ 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 260700cd6218..df21db173a1c 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -157,7 +157,7 @@ func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { return fmt.Errorf("Error PutPageBlob: %s", err) } - if err := sbu.resourceArmStorageBlobPageUploadFromSource(ctx, file, fileSize); err != nil { + if err := sbu.pageUploadFromSource(ctx, file, fileSize); err != nil { return fmt.Errorf("Error creating storage blob on Azure: %s", err) } @@ -166,12 +166,12 @@ func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { // TODO: remove below here -type resourceArmStorageBlobPage struct { +type storageBlobPage struct { offset int64 section *io.SectionReader } -func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { +func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { workerCount := sbu.parallelism * runtime.NumCPU() // first we chunk the file and assign them to 'pages' @@ -181,7 +181,7 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource(ctx cont } // finally we upload the contents of said file - pages := make(chan resourceArmStorageBlobPage, len(pageList)) + pages := make(chan storageBlobPage, len(pageList)) errors := make(chan error, len(pageList)) wg := &sync.WaitGroup{} wg.Add(len(pageList)) @@ -194,16 +194,12 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageUploadFromSource(ctx cont close(pages) for i := 0; i < workerCount; i++ { - go resourceArmStorageBlobPageUploadWorker(resourceArmStorageBlobPageUploadContext{ - container: sbu.containerName, - name: sbu.blobName, - source: sbu.source, - blobSize: fileSize, - client: sbu.legacyClient, - pages: pages, - errors: errors, - wg: wg, - attempts: sbu.attempts, + go sbu.blobPageUploadWorker(blobPageUploadContext{ + blobSize: fileSize, + pages: pages, + errors: errors, + wg: wg, + attempts: sbu.attempts, }) } @@ -221,7 +217,7 @@ const ( maxPageSize int64 = 4 * 1024 * 1024 ) -func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, fileSize int64) ([]resourceArmStorageBlobPage, error) { +func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { // whilst the file size can be any arbitary size, it must be uploaded in fixed-size pages blobSize := fileSize if fileSize%minPageSize != 0 { @@ -262,9 +258,9 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, file } } - var pages []resourceArmStorageBlobPage + var pages []storageBlobPage for _, nonEmptyRange := range nonEmptyRanges { - pages = append(pages, resourceArmStorageBlobPage{ + pages = append(pages, storageBlobPage{ offset: nonEmptyRange.offset, section: io.NewSectionReader(file, nonEmptyRange.offset, nonEmptyRange.length), }) @@ -273,19 +269,15 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, file return pages, nil } -type resourceArmStorageBlobPageUploadContext struct { - container string - name string - source string - blobSize int64 - client *legacy.BlobStorageClient - pages chan resourceArmStorageBlobPage - errors chan error - wg *sync.WaitGroup - attempts int +type blobPageUploadContext struct { + blobSize int64 + pages chan storageBlobPage + errors chan error + wg *sync.WaitGroup + attempts int } -func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUploadContext) { +func (sbu StorageBlobUpload) blobPageUploadWorker(ctx blobPageUploadContext) { for page := range ctx.pages { start := page.offset end := page.offset + page.section.Size() - 1 @@ -297,14 +289,14 @@ func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUpload chunk := make([]byte, size) _, err := page.section.Read(chunk) if err != nil && err != io.EOF { - ctx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", ctx.source, page.offset, err) + ctx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.source, page.offset, err) ctx.wg.Done() continue } for x := 0; x < ctx.attempts; x++ { - container := ctx.client.GetContainerReference(ctx.container) - blob := container.GetBlobReference(ctx.name) + container := sbu.legacyClient.GetContainerReference(sbu.containerName) + blob := container.GetBlobReference(sbu.blobName) blobRange := legacy.BlobRange{ Start: uint64(start), End: uint64(end), @@ -317,7 +309,7 @@ func resourceArmStorageBlobPageUploadWorker(ctx resourceArmStorageBlobPageUpload } } if err != nil { - ctx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, ctx.source, err) + ctx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.source, err) ctx.wg.Done() continue } From 65c9ea3f7d0fc442ec5f6c6d7d4aa54d1d6ec53c Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 13:20:21 +0100 Subject: [PATCH 26/39] r/storage_blob: switching to use the new client for uploading pages --- azurerm/resource_arm_storage_blob.go | 10 +---- azurerm/resource_arm_storage_blob_legacy.go | 50 ++++++++++----------- 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 46d848f19d2d..2a4462b34a93 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -149,21 +149,13 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro } } - // TODO: remove me - legacyBlobsClient, _, err := storageClient.LegacyBlobClient(ctx, *resourceGroup, accountName) - if err != nil { - return err - } - log.Printf("[DEBUG] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) blobInput := StorageBlobUpload{ accountName: accountName, containerName: containerName, blobName: name, - - client: blobsClient, - legacyClient: legacyBlobsClient, + client: blobsClient, blobType: d.Get("type").(string), contentType: d.Get("content_type").(string), diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index df21db173a1c..40ad502016ec 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -11,7 +11,6 @@ import ( "sync" "time" - legacy "github.com/Azure/azure-sdk-for-go/storage" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" "github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs" ) @@ -19,21 +18,22 @@ import ( const pollingInterval = time.Second * 15 type StorageBlobUpload struct { + client *blobs.Client + accountName string containerName string blobName string - attempts int blobType string contentType string metaData map[string]string - parallelism int size int source string sourceUri string - client *blobs.Client - legacyClient *legacy.BlobStorageClient + // TODO: deprecate/remove + attempts int + parallelism int } func (sbu StorageBlobUpload) Create(ctx context.Context) error { @@ -194,7 +194,7 @@ func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os. close(pages) for i := 0; i < workerCount; i++ { - go sbu.blobPageUploadWorker(blobPageUploadContext{ + go sbu.blobPageUploadWorker(ctx, blobPageUploadContext{ blobSize: fileSize, pages: pages, errors: errors, @@ -214,6 +214,8 @@ func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os. const ( minPageSize int64 = 4 * 1024 + + // TODO: investigate whether this can be bumped to 100MB with the new API maxPageSize int64 = 4 * 1024 * 1024 ) @@ -235,8 +237,7 @@ func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, file var currentRange byteRange for i := int64(0); i < blobSize; i += minPageSize { pageBuf := make([]byte, minPageSize) - _, err := file.ReadAt(pageBuf, i) - if err != nil && err != io.EOF { + if _, err := file.ReadAt(pageBuf, i); err != nil && err != io.EOF { return nil, fmt.Errorf("Could not read chunk at %d: %s", i, err) } @@ -277,43 +278,40 @@ type blobPageUploadContext struct { attempts int } -func (sbu StorageBlobUpload) blobPageUploadWorker(ctx blobPageUploadContext) { - for page := range ctx.pages { +func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPageUploadContext) { + for page := range uploadCtx.pages { start := page.offset end := page.offset + page.section.Size() - 1 - if end > ctx.blobSize-1 { - end = ctx.blobSize - 1 + if end > uploadCtx.blobSize-1 { + end = uploadCtx.blobSize - 1 } size := end - start + 1 chunk := make([]byte, size) _, err := page.section.Read(chunk) if err != nil && err != io.EOF { - ctx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.source, page.offset, err) - ctx.wg.Done() + uploadCtx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.source, page.offset, err) + uploadCtx.wg.Done() continue } - for x := 0; x < ctx.attempts; x++ { - container := sbu.legacyClient.GetContainerReference(sbu.containerName) - blob := container.GetBlobReference(sbu.blobName) - blobRange := legacy.BlobRange{ - Start: uint64(start), - End: uint64(end), + for x := 0; x < uploadCtx.attempts; x++ { + input := blobs.PutPageUpdateInput{ + StartByte: start, + EndByte: end, + Content: chunk, } - options := &legacy.PutPageOptions{} - reader := bytes.NewReader(chunk) - err = blob.WriteRange(blobRange, reader, options) + _, err = sbu.client.PutPageUpdate(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input) if err == nil { break } } if err != nil { - ctx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.source, err) - ctx.wg.Done() + uploadCtx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.source, err) + uploadCtx.wg.Done() continue } - ctx.wg.Done() + uploadCtx.wg.Done() } } From a96d90dab08f3fdc70f6c70865c13e85cef8f406 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 13:35:35 +0100 Subject: [PATCH 27/39] r/storage_blob: removing support for 'attempts' since go-autorest should do this for us --- azurerm/resource_arm_storage_blob_legacy.go | 23 +++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 40ad502016ec..2f4b072109c4 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -175,7 +175,7 @@ func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os. workerCount := sbu.parallelism * runtime.NumCPU() // first we chunk the file and assign them to 'pages' - pageList, err := sbu.resourceArmStorageBlobPageSplit(file, fileSize) + pageList, err := sbu.storageBlobPageSplit(file, fileSize) if err != nil { return fmt.Errorf("Error splitting source file %q into pages: %s", sbu.source, err) } @@ -199,7 +199,6 @@ func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os. pages: pages, errors: errors, wg: wg, - attempts: sbu.attempts, }) } @@ -219,7 +218,7 @@ const ( maxPageSize int64 = 4 * 1024 * 1024 ) -func (sbu StorageBlobUpload) resourceArmStorageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { +func (sbu StorageBlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { // whilst the file size can be any arbitary size, it must be uploaded in fixed-size pages blobSize := fileSize if fileSize%minPageSize != 0 { @@ -275,7 +274,6 @@ type blobPageUploadContext struct { pages chan storageBlobPage errors chan error wg *sync.WaitGroup - attempts int } func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPageUploadContext) { @@ -295,18 +293,13 @@ func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx continue } - for x := 0; x < uploadCtx.attempts; x++ { - input := blobs.PutPageUpdateInput{ - StartByte: start, - EndByte: end, - Content: chunk, - } - _, err = sbu.client.PutPageUpdate(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input) - if err == nil { - break - } + input := blobs.PutPageUpdateInput{ + StartByte: start, + EndByte: end, + Content: chunk, } - if err != nil { + + if _, err = sbu.client.PutPageUpdate(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { uploadCtx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.source, err) uploadCtx.wg.Done() continue From 805c3a458dcc9f7eb6365cf00a9b91fa96eb5f07 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 26 Aug 2019 13:41:57 +0100 Subject: [PATCH 28/39] r/storage_blob: deprecating 'attempts' --- azurerm/resource_arm_storage_blob.go | 15 ++++++++------- azurerm/resource_arm_storage_blob_legacy.go | 10 +++------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 2a4462b34a93..612514d2386c 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -94,6 +94,7 @@ func resourceArmStorageBlob() *schema.Resource { }, "parallelism": { + // TODO: @tombuildsstuff - a note this only works for Page blobs Type: schema.TypeInt, Optional: true, Default: 8, @@ -101,15 +102,18 @@ func resourceArmStorageBlob() *schema.Resource { ValidateFunc: validation.IntAtLeast(1), }, + "metadata": storage.MetaDataSchema(), + + // Deprecated fields "attempts": { + // TODO: @tombuildsstuff - add to the deprecation guide Type: schema.TypeInt, Optional: true, Default: 1, ForceNew: true, + Deprecated: "Retries are now handled by the Azure SDK as such this field is no longer necessary and will be removed in v2.0 of the Azure Provider", ValidateFunc: validation.IntAtLeast(1), }, - - "metadata": storage.MetaDataSchema(), }, } } @@ -160,13 +164,10 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro blobType: d.Get("type").(string), contentType: d.Get("content_type").(string), metaData: storage.ExpandMetaData(metaDataRaw), + parallelism: d.Get("parallelism").(int), size: d.Get("size").(int), source: d.Get("source").(string), sourceUri: d.Get("source_uri").(string), - - // TODO: deprecate these - attempts: d.Get("attempts").(int), - parallelism: d.Get("parallelism").(int), } if err := blobInput.Create(ctx); err != nil { return fmt.Errorf("Error creating Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) @@ -202,7 +203,7 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro if d.HasChange("content_type") { log.Printf("[DEBUG] Updating Properties for Blob %q (Container %q / Account %q)...", id.BlobName, id.ContainerName, id.AccountName) input := blobs.SetPropertiesInput{ - // TODO: other properties? + // TODO: other properties (Access Tier)? ContentType: utils.String(d.Get("content_type").(string)), } if _, err := blobsClient.SetProperties(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/resource_arm_storage_blob_legacy.go index 2f4b072109c4..1a255f9302b4 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/resource_arm_storage_blob_legacy.go @@ -27,13 +27,10 @@ type StorageBlobUpload struct { blobType string contentType string metaData map[string]string + parallelism int size int source string sourceUri string - - // TODO: deprecate/remove - attempts int - parallelism int } func (sbu StorageBlobUpload) Create(ctx context.Context) error { @@ -99,7 +96,6 @@ func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { ContentType: utils.String(sbu.contentType), MetaData: sbu.metaData, } - if err := sbu.client.PutBlockBlobFromFile(ctx, sbu.accountName, sbu.containerName, sbu.blobName, file, input); err != nil { return fmt.Errorf("Error PutBlockBlobFromFile: %s", err) } @@ -164,7 +160,7 @@ func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { return nil } -// TODO: remove below here +// TODO: move below here into Giovanni type storageBlobPage struct { offset int64 @@ -219,7 +215,7 @@ const ( ) func (sbu StorageBlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { - // whilst the file size can be any arbitary size, it must be uploaded in fixed-size pages + // whilst the file size can be any arbitrary size, it must be uploaded in fixed-size pages blobSize := fileSize if fileSize%minPageSize != 0 { blobSize = fileSize + (minPageSize - (fileSize % minPageSize)) From cab451fb2db9ab0d4ca84fefb218770cb169eaf0 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 11:01:21 +0100 Subject: [PATCH 29/39] r/storage_blob: documenting the deprecated fields --- azurerm/resource_arm_storage_blob.go | 11 ++++++----- .../docs/guides/2.0-upgrade-guide.html.markdown | 6 ++++++ website/docs/r/storage_blob.html.markdown | 15 +++++++-------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 612514d2386c..3797e37aec95 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -36,9 +36,6 @@ func resourceArmStorageBlob() *schema.Resource { // TODO: add validation }, - // TODO: this can be deprecated with the new sdk? - "resource_group_name": azure.SchemaResourceGroupName(), - "storage_account_name": { Type: schema.TypeString, Required: true, @@ -114,6 +111,8 @@ func resourceArmStorageBlob() *schema.Resource { Deprecated: "Retries are now handled by the Azure SDK as such this field is no longer necessary and will be removed in v2.0 of the Azure Provider", ValidateFunc: validation.IntAtLeast(1), }, + + "resource_group_name": azure.SchemaResourceGroupNameDeprecated(), }, } } @@ -175,7 +174,8 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro log.Printf("[DEBUG] Created Blob %q in Container %q within Storage Account %q.", name, containerName, accountName) d.SetId(id) - return resourceArmStorageBlobRead(d, meta) + + return resourceArmStorageBlobUpdate(d, meta) } func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) error { @@ -200,10 +200,11 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro return fmt.Errorf("Error building Blobs Client: %s", err) } + // TODO: changing the access tier + if d.HasChange("content_type") { log.Printf("[DEBUG] Updating Properties for Blob %q (Container %q / Account %q)...", id.BlobName, id.ContainerName, id.AccountName) input := blobs.SetPropertiesInput{ - // TODO: other properties (Access Tier)? ContentType: utils.String(d.Get("content_type").(string)), } if _, err := blobsClient.SetProperties(ctx, id.AccountName, id.ContainerName, id.BlobName, input); err != nil { diff --git a/website/docs/guides/2.0-upgrade-guide.html.markdown b/website/docs/guides/2.0-upgrade-guide.html.markdown index ce9bad625e29..a8e0ed69dccc 100644 --- a/website/docs/guides/2.0-upgrade-guide.html.markdown +++ b/website/docs/guides/2.0-upgrade-guide.html.markdown @@ -347,6 +347,12 @@ The deprecated `enable_filtering_messages_before_publishing` field will be remov The deprecated `account_type` field will be removed. This has been split into the fields `account_tier` and `account_replication_type`. +### Resource: `azurerm_storage_blob` + +The deprecated `attempts` field will be removed, since Terraform now handles retries automatically. + +The deprecated `resource_group_name` field will be removed, since this is no longer used. + ### Resource: `azurerm_storage_container` The deprecated `resource_group_name` field will be removed, since this is no longer used. diff --git a/website/docs/r/storage_blob.html.markdown b/website/docs/r/storage_blob.html.markdown index fd917d77f967..187d77d7fadd 100644 --- a/website/docs/r/storage_blob.html.markdown +++ b/website/docs/r/storage_blob.html.markdown @@ -3,12 +3,12 @@ layout: "azurerm" page_title: "Azure Resource Manager: azurerm_storage_blob" sidebar_current: "docs-azurerm-resource-storage-blob" description: |- - Manages a Azure Storage Blob. + Manages a Blob within a Storage Container. --- # azurerm_storage_blob -Manages an Azure Storage Blob. +Manages a Blob within a Storage Container. ## Example Usage @@ -21,7 +21,7 @@ resource "azurerm_resource_group" "test" { resource "azurerm_storage_account" "test" { name = "acctestaccs" resource_group_name = "${azurerm_resource_group.test.name}" - location = "westus" + location = "${azurerm_resource_group.test.location}" account_tier = "Standard" account_replication_type = "LRS" } @@ -51,9 +51,6 @@ The following arguments are supported: * `name` - (Required) The name of the storage blob. Must be unique within the storage container the blob is located. -* `resource_group_name` - (Required) The name of the resource group in which to - create the storage container. Changing this forces a new resource to be created. - * `storage_account_name` - (Required) Specifies the storage account in which to create the storage container. Changing this forces a new resource to be created. @@ -73,10 +70,12 @@ The following arguments are supported: * `parallelism` - (Optional) The number of workers per CPU core to run for concurrent uploads. Defaults to `8`. -* `attempts` - (Optional) The number of attempts to make per page or block when uploading. Defaults to `1`. - * `metadata` - (Optional) A map of custom blob metadata. +* `attempts` - (Optional / **Deprecated**) The number of attempts to make per page or block when uploading. Defaults to `1`. + +* `resource_group_name` - (Optional / **Deprecated**) The name of the resource group in which to create the storage container. + ## Attributes Reference The following attributes are exported in addition to the arguments listed above: From 39bf715d487e94d4a139bd205215cf9cf9227509 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:35:08 +0100 Subject: [PATCH 30/39] vendoring v0.4.0 of github.com/tombuildsstuff/giovanni --- go.mod | 4 +-- go.sum | 27 +++++++++++++++++++ .../go-autorest/autorest/azure/cli/go.mod | 5 ++-- .../go-autorest/autorest/azure/cli/go.sum | 12 +++++++++ .../autorest/azure/cli/go_mod_tidy_hack.go | 24 +++++++++++++++++ .../giovanni/version/version.go | 2 +- vendor/modules.txt | 4 +-- 7 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 vendor/github.com/Azure/go-autorest/autorest/azure/cli/go_mod_tidy_hack.go diff --git a/go.mod b/go.mod index c78c95e4c5a9..177678799531 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,12 @@ require ( github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-uuid v1.0.1 github.com/hashicorp/go-version v1.1.0 - github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/terraform v0.12.6 - github.com/pkg/errors v0.8.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/satori/uuid v0.0.0-20160927100844-b061729afc07 github.com/sirupsen/logrus v1.2.0 // indirect github.com/terraform-providers/terraform-provider-azuread v0.6.0 - github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 + github.com/tombuildsstuff/giovanni v0.4.0 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190502183928-7f726cade0ab gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 3fae28098d8c..44ab7c624a8c 100644 --- a/go.sum +++ b/go.sum @@ -38,10 +38,13 @@ github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIr github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo= github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/azure/auth v0.3.0/go.mod h1:CI4BQYBct8NS7BXNBBX+RchsFsUu5+oz+OSyR/ZIi7U= github.com/Azure/go-autorest/autorest/azure/cli v0.1.0 h1:YTtBrcb6mhA+PoSW8WxFDoIIyjp13XqJeX80ssQtri4= github.com/Azure/go-autorest/autorest/azure/cli v0.1.0/go.mod h1:Dk8CUAt/b/PzkfeRsWzVG9Yj3ps8mS8ECztu43rdU8U= github.com/Azure/go-autorest/autorest/azure/cli v0.2.0 h1:pSwNMF0qotgehbQNllUWwJ4V3vnrLKOzHrwDLEZK904= github.com/Azure/go-autorest/autorest/azure/cli v0.2.0/go.mod h1:WWTbGPvkAg3I4ms2j2s+Zr5xCGwGqTQh+6M2ZqOczkE= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.0 h1:5PAqnv+CSTwW9mlZWZAizmzrazFWEgZykEZXpr2hDtY= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.0/go.mod h1:rNYMNAefZMRowqCV0cVhr/YDW5dD7afFq9nXAXL4ykE= github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= @@ -71,6 +74,7 @@ github.com/Azure/go-ntlmssp v0.0.0-20180810175552-4a21cbd618b4/go.mod h1:chxPXzS github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022 h1:y8Gs8CzNfDF5AZvjr+5UyGQvQEBL7pwo+v+wX6q9JI8= github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Unknwon/com v0.0.0-20151008135407-28b053d5a292 h1:tuQ7w+my8a8mkwN7x2TSd7OzTjkZ7rAeSyH4xncuAMI= @@ -102,6 +106,7 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -139,10 +144,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -170,6 +177,7 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -306,6 +314,7 @@ github.com/hashicorp/vault v0.10.4/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bA github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -338,6 +347,7 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82 h1:wnfcqULT+N2seWf6y4yHzmi7GD2kNx4Ute0qArktD48= github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54tfGmO3NKssKveTEFFzH8C/akrSOy/iW9qEAUDV84= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9 h1:SmVbOZFWAlyQshuMfOkiAx1f5oUTsOGG5IXplAEYeeM= @@ -406,11 +416,13 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/packer-community/winrmcp v0.0.0-20180102160824-81144009af58 h1:m3CEgv3ah1Rhy82L+c0QG/U3VyY1UsvsIdkh0/rU97Y= github.com/packer-community/winrmcp v0.0.0-20180102160824-81144009af58/go.mod h1:f6Izs6JvFTdnRbziASagjZ2vmf55NSIkC/weStxCHqk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17 h1:chPfVn+gpAM5CTpTyVU9j8J+xgRGwmoDlNDLjKnJiYo= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= @@ -437,6 +449,7 @@ github.com/satori/uuid v0.0.0-20160927100844-b061729afc07/go.mod h1:B8HLsPLik/YN github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -467,9 +480,15 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:X github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -490,8 +509,11 @@ github.com/tombuildsstuff/giovanni v0.3.2 h1:uKt7irq74daF0lvaA0X9bEuFKSDhceVy4Dy github.com/tombuildsstuff/giovanni v0.3.2/go.mod h1:3UHcoRZCvWTjXMWCJ0epD1VROlPMwZeeof3vfP6NiWM= github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 h1:M5CRmx8+0gcgEG6qpPml39RS3Squ1zbFD0xVZ1owpzM= github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5/go.mod h1:3UHcoRZCvWTjXMWCJ0epD1VROlPMwZeeof3vfP6NiWM= +github.com/tombuildsstuff/giovanni v0.4.0 h1:MHJuSqcRkYqoNMIBpOjn3ljwAmABR5GPmtFH11bm/Vo= +github.com/tombuildsstuff/giovanni v0.4.0/go.mod h1:Xu/XU+DiRrKTDoCnJNGuh9ysD0eJyi/zU/naFh2aN9I= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5 h1:cMjKdf4PxEBN9K5HaD9UMW8gkTbM0kMzkTa9SJe0WNQ= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -502,6 +524,7 @@ github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0B github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557 h1:Jpn2j6wHkC9wJv5iMfJhKqrZJx3TahFx+7sbZ7zQdxs= github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zclconf/go-cty v0.0.0-20181129180422-88fbe721e0f8/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v0.0.0-20190516203816-4fecf87372ec h1:MSeYjmyjucsFbecMTxg63ASg23lcSARP/kr9sClTFfk= @@ -529,10 +552,12 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869 h1:kkXA53yGe04D0adEYJwEVQjeBppL01Exg+fnMjfUraU= golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -587,6 +612,7 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= @@ -594,6 +620,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.mod b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.mod index cef48f1ea1fa..03ad580d6148 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.mod +++ b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.mod @@ -3,8 +3,9 @@ module github.com/Azure/go-autorest/autorest/azure/cli go 1.12 require ( - github.com/Azure/go-autorest/autorest/adal v0.5.0 - github.com/Azure/go-autorest/autorest/date v0.1.0 + github.com/Azure/go-autorest/autorest v0.9.0 + github.com/Azure/go-autorest/autorest/adal v0.6.0 + github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/dimchansky/utfbom v1.1.0 github.com/mitchellh/go-homedir v1.1.0 ) diff --git a/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.sum b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.sum index 2d6636a33807..7a8b1f23f6f8 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.sum +++ b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go.sum @@ -1,9 +1,21 @@ +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo= +github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0 h1:Kx+AUU2Te+A3JIyYn6Dfs+cFgx5XorQKuIXrZGoq/SI= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= diff --git a/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go_mod_tidy_hack.go b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go_mod_tidy_hack.go new file mode 100644 index 000000000000..618bed392fc5 --- /dev/null +++ b/vendor/github.com/Azure/go-autorest/autorest/azure/cli/go_mod_tidy_hack.go @@ -0,0 +1,24 @@ +// +build modhack + +package cli + +// Copyright 2017 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file, and the github.com/Azure/go-autorest/autorest import, won't actually become part of +// the resultant binary. + +// Necessary for safely adding multi-module repo. +// See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository +import _ "github.com/Azure/go-autorest/autorest" diff --git a/vendor/github.com/tombuildsstuff/giovanni/version/version.go b/vendor/github.com/tombuildsstuff/giovanni/version/version.go index 79ab6f224636..a2582ab4fd47 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/version/version.go +++ b/vendor/github.com/tombuildsstuff/giovanni/version/version.go @@ -1,3 +1,3 @@ package version -const Number = "v0.3.2" +const Number = "v0.4.0" diff --git a/vendor/modules.txt b/vendor/modules.txt index 834729a112ac..0007eb48033b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -81,7 +81,7 @@ github.com/Azure/go-autorest/autorest github.com/Azure/go-autorest/autorest/azure # github.com/Azure/go-autorest/autorest/adal v0.6.0 github.com/Azure/go-autorest/autorest/adal -# github.com/Azure/go-autorest/autorest/azure/cli v0.2.0 +# github.com/Azure/go-autorest/autorest/azure/cli v0.3.0 github.com/Azure/go-autorest/autorest/azure/cli # github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/Azure/go-autorest/autorest/date @@ -328,7 +328,7 @@ github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/slices github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/tf github.com/terraform-providers/terraform-provider-azuread/azuread/helpers/validate github.com/terraform-providers/terraform-provider-azuread/version -# github.com/tombuildsstuff/giovanni v0.3.3-0.20190823094657-ff85ec2ac5d5 +# github.com/tombuildsstuff/giovanni v0.4.0 github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/blobs github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers github.com/tombuildsstuff/giovanni/storage/2018-11-09/file/directories From 37a23a3b67f7c4765b0a21d0b07a56346ba9f964 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:42:18 +0100 Subject: [PATCH 31/39] r/storage_blob: removing a completed todo --- azurerm/resource_arm_storage_blob.go | 1 - 1 file changed, 1 deletion(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 3797e37aec95..08cd4ccb6b8d 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -103,7 +103,6 @@ func resourceArmStorageBlob() *schema.Resource { // Deprecated fields "attempts": { - // TODO: @tombuildsstuff - add to the deprecation guide Type: schema.TypeInt, Optional: true, Default: 1, From c27642071afbeed13c1814f992090cbc20e835aa Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:45:41 +0100 Subject: [PATCH 32/39] r/storage_blob: moving the blobs helpers into the storage package --- .../services/storage/blobs.go} | 92 +++++++++---------- azurerm/resource_arm_storage_blob.go | 26 +++--- 2 files changed, 59 insertions(+), 59 deletions(-) rename azurerm/{resource_arm_storage_blob_legacy.go => internal/services/storage/blobs.go} (75%) diff --git a/azurerm/resource_arm_storage_blob_legacy.go b/azurerm/internal/services/storage/blobs.go similarity index 75% rename from azurerm/resource_arm_storage_blob_legacy.go rename to azurerm/internal/services/storage/blobs.go index 1a255f9302b4..6d893aee2b80 100644 --- a/azurerm/resource_arm_storage_blob_legacy.go +++ b/azurerm/internal/services/storage/blobs.go @@ -1,4 +1,4 @@ -package azurerm +package storage import ( "bytes" @@ -18,32 +18,32 @@ import ( const pollingInterval = time.Second * 15 type StorageBlobUpload struct { - client *blobs.Client - - accountName string - containerName string - blobName string - - blobType string - contentType string - metaData map[string]string - parallelism int - size int - source string - sourceUri string + Client *blobs.Client + + AccountName string + BlobName string + ContainerName string + + BlobType string + ContentType string + MetaData map[string]string + Parallelism int + Size int + Source string + SourceUri string } func (sbu StorageBlobUpload) Create(ctx context.Context) error { - if sbu.sourceUri != "" { + if sbu.SourceUri != "" { return sbu.copy(ctx) } - blobType := strings.ToLower(sbu.blobType) + blobType := strings.ToLower(sbu.BlobType) // TODO: new feature for 'append' blobs? if blobType == "block" { - if sbu.source != "" { + if sbu.Source != "" { return sbu.uploadBlockBlob(ctx) } @@ -51,7 +51,7 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { } if blobType == "page" { - if sbu.source != "" { + if sbu.Source != "" { return sbu.uploadPageBlob(ctx) } @@ -63,10 +63,10 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { func (sbu StorageBlobUpload) copy(ctx context.Context) error { input := blobs.CopyInput{ - CopySource: sbu.sourceUri, - MetaData: sbu.metaData, + CopySource: sbu.SourceUri, + MetaData: sbu.MetaData, } - if err := sbu.client.CopyAndWait(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input, pollingInterval); err != nil { + if err := sbu.Client.CopyAndWait(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input, pollingInterval); err != nil { return fmt.Errorf("Error copy/waiting: %s", err) } @@ -75,10 +75,10 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { func (sbu StorageBlobUpload) createEmptyBlockBlob(ctx context.Context) error { input := blobs.PutBlockBlobInput{ - ContentType: utils.String(sbu.contentType), - MetaData: sbu.metaData, + ContentType: utils.String(sbu.ContentType), + MetaData: sbu.MetaData, } - if _, err := sbu.client.PutBlockBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + if _, err := sbu.Client.PutBlockBlob(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("Error PutBlockBlob: %s", err) } @@ -86,17 +86,17 @@ func (sbu StorageBlobUpload) createEmptyBlockBlob(ctx context.Context) error { } func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { - file, err := os.Open(sbu.source) + file, err := os.Open(sbu.Source) if err != nil { return fmt.Errorf("Error opening: %s", err) } defer file.Close() input := blobs.PutBlockBlobInput{ - ContentType: utils.String(sbu.contentType), - MetaData: sbu.metaData, + ContentType: utils.String(sbu.ContentType), + MetaData: sbu.MetaData, } - if err := sbu.client.PutBlockBlobFromFile(ctx, sbu.accountName, sbu.containerName, sbu.blobName, file, input); err != nil { + if err := sbu.Client.PutBlockBlobFromFile(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, file, input); err != nil { return fmt.Errorf("Error PutBlockBlobFromFile: %s", err) } @@ -104,17 +104,17 @@ func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { } func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { - if sbu.size == 0 { + if sbu.Size == 0 { return fmt.Errorf("`size` cannot be zero for a page blob") } input := blobs.PutPageBlobInput{ - BlobContentLengthBytes: int64(sbu.size), - ContentType: utils.String(sbu.contentType), - MetaData: sbu.metaData, + BlobContentLengthBytes: int64(sbu.Size), + ContentType: utils.String(sbu.ContentType), + MetaData: sbu.MetaData, } // TODO: access tiers? - if _, err := sbu.client.PutPageBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + if _, err := sbu.Client.PutPageBlob(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("Error PutPageBlob: %s", err) } @@ -122,14 +122,14 @@ func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { } func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { - if sbu.size != 0 { + if sbu.Size != 0 { // the user shouldn't need to specify this since we infer it } // determine the details about the file - file, err := os.Open(sbu.source) + file, err := os.Open(sbu.Source) if err != nil { - return fmt.Errorf("Error opening source file for upload %q: %s", sbu.source, err) + return fmt.Errorf("Error opening source file for upload %q: %s", sbu.Source, err) } defer file.Close() @@ -145,11 +145,11 @@ func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { // first let's create a file of the specified file size input := blobs.PutPageBlobInput{ BlobContentLengthBytes: int64(fileSize), - ContentType: utils.String(sbu.contentType), - MetaData: sbu.metaData, + ContentType: utils.String(sbu.ContentType), + MetaData: sbu.MetaData, } // TODO: access tiers? - if _, err := sbu.client.PutPageBlob(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { + if _, err := sbu.Client.PutPageBlob(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("Error PutPageBlob: %s", err) } @@ -168,12 +168,12 @@ type storageBlobPage struct { } func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { - workerCount := sbu.parallelism * runtime.NumCPU() + workerCount := sbu.Parallelism * runtime.NumCPU() // first we chunk the file and assign them to 'pages' pageList, err := sbu.storageBlobPageSplit(file, fileSize) if err != nil { - return fmt.Errorf("Error splitting source file %q into pages: %s", sbu.source, err) + return fmt.Errorf("Error splitting source file %q into pages: %s", sbu.Source, err) } // finally we upload the contents of said file @@ -201,7 +201,7 @@ func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os. wg.Wait() if len(errors) > 0 { - return fmt.Errorf("Error while uploading source file %q: %s", sbu.source, <-errors) + return fmt.Errorf("Error while uploading source file %q: %s", sbu.Source, <-errors) } return nil @@ -215,7 +215,7 @@ const ( ) func (sbu StorageBlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { - // whilst the file size can be any arbitrary size, it must be uploaded in fixed-size pages + // whilst the file Size can be any arbitrary Size, it must be uploaded in fixed-Size pages blobSize := fileSize if fileSize%minPageSize != 0 { blobSize = fileSize + (minPageSize - (fileSize % minPageSize)) @@ -284,7 +284,7 @@ func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx chunk := make([]byte, size) _, err := page.section.Read(chunk) if err != nil && err != io.EOF { - uploadCtx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.source, page.offset, err) + uploadCtx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.Source, page.offset, err) uploadCtx.wg.Done() continue } @@ -295,8 +295,8 @@ func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx Content: chunk, } - if _, err = sbu.client.PutPageUpdate(ctx, sbu.accountName, sbu.containerName, sbu.blobName, input); err != nil { - uploadCtx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.source, err) + if _, err = sbu.Client.PutPageUpdate(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { + uploadCtx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.Source, err) uploadCtx.wg.Done() continue } diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 08cd4ccb6b8d..3a8fc9de342a 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -153,19 +153,19 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro log.Printf("[DEBUG] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) - blobInput := StorageBlobUpload{ - accountName: accountName, - containerName: containerName, - blobName: name, - client: blobsClient, - - blobType: d.Get("type").(string), - contentType: d.Get("content_type").(string), - metaData: storage.ExpandMetaData(metaDataRaw), - parallelism: d.Get("parallelism").(int), - size: d.Get("size").(int), - source: d.Get("source").(string), - sourceUri: d.Get("source_uri").(string), + blobInput := storage.StorageBlobUpload{ + AccountName: accountName, + ContainerName: containerName, + BlobName: name, + Client: blobsClient, + + BlobType: d.Get("type").(string), + ContentType: d.Get("content_type").(string), + MetaData: storage.ExpandMetaData(metaDataRaw), + Parallelism: d.Get("parallelism").(int), + Size: d.Get("size").(int), + Source: d.Get("source").(string), + SourceUri: d.Get("source_uri").(string), } if err := blobInput.Create(ctx); err != nil { return fmt.Errorf("Error creating Blob %q (Container %q / Account %q): %s", name, containerName, accountName, err) From 831b7aad87715227a68893234efd06146d13c6b5 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:46:12 +0100 Subject: [PATCH 33/39] r/storage_blob: ditching the 'storage' prefix --- azurerm/internal/services/storage/blobs.go | 20 ++++++++++---------- azurerm/resource_arm_storage_blob.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/azurerm/internal/services/storage/blobs.go b/azurerm/internal/services/storage/blobs.go index 6d893aee2b80..25a59db03d85 100644 --- a/azurerm/internal/services/storage/blobs.go +++ b/azurerm/internal/services/storage/blobs.go @@ -17,7 +17,7 @@ import ( const pollingInterval = time.Second * 15 -type StorageBlobUpload struct { +type BlobUpload struct { Client *blobs.Client AccountName string @@ -33,7 +33,7 @@ type StorageBlobUpload struct { SourceUri string } -func (sbu StorageBlobUpload) Create(ctx context.Context) error { +func (sbu BlobUpload) Create(ctx context.Context) error { if sbu.SourceUri != "" { return sbu.copy(ctx) } @@ -61,7 +61,7 @@ func (sbu StorageBlobUpload) Create(ctx context.Context) error { return fmt.Errorf("Unsupported Blob Type: %q", blobType) } -func (sbu StorageBlobUpload) copy(ctx context.Context) error { +func (sbu BlobUpload) copy(ctx context.Context) error { input := blobs.CopyInput{ CopySource: sbu.SourceUri, MetaData: sbu.MetaData, @@ -73,7 +73,7 @@ func (sbu StorageBlobUpload) copy(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) createEmptyBlockBlob(ctx context.Context) error { +func (sbu BlobUpload) createEmptyBlockBlob(ctx context.Context) error { input := blobs.PutBlockBlobInput{ ContentType: utils.String(sbu.ContentType), MetaData: sbu.MetaData, @@ -85,7 +85,7 @@ func (sbu StorageBlobUpload) createEmptyBlockBlob(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { +func (sbu BlobUpload) uploadBlockBlob(ctx context.Context) error { file, err := os.Open(sbu.Source) if err != nil { return fmt.Errorf("Error opening: %s", err) @@ -103,7 +103,7 @@ func (sbu StorageBlobUpload) uploadBlockBlob(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { +func (sbu BlobUpload) createEmptyPageBlob(ctx context.Context) error { if sbu.Size == 0 { return fmt.Errorf("`size` cannot be zero for a page blob") } @@ -121,7 +121,7 @@ func (sbu StorageBlobUpload) createEmptyPageBlob(ctx context.Context) error { return nil } -func (sbu StorageBlobUpload) uploadPageBlob(ctx context.Context) error { +func (sbu BlobUpload) uploadPageBlob(ctx context.Context) error { if sbu.Size != 0 { // the user shouldn't need to specify this since we infer it } @@ -167,7 +167,7 @@ type storageBlobPage struct { section *io.SectionReader } -func (sbu StorageBlobUpload) pageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { +func (sbu BlobUpload) pageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { workerCount := sbu.Parallelism * runtime.NumCPU() // first we chunk the file and assign them to 'pages' @@ -214,7 +214,7 @@ const ( maxPageSize int64 = 4 * 1024 * 1024 ) -func (sbu StorageBlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { +func (sbu BlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { // whilst the file Size can be any arbitrary Size, it must be uploaded in fixed-Size pages blobSize := fileSize if fileSize%minPageSize != 0 { @@ -272,7 +272,7 @@ type blobPageUploadContext struct { wg *sync.WaitGroup } -func (sbu StorageBlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPageUploadContext) { +func (sbu BlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPageUploadContext) { for page := range uploadCtx.pages { start := page.offset end := page.offset + page.section.Size() - 1 diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 3a8fc9de342a..3335f94762d8 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -153,7 +153,7 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro log.Printf("[DEBUG] Creating Blob %q in Container %q within Storage Account %q..", name, containerName, accountName) metaDataRaw := d.Get("metadata").(map[string]interface{}) - blobInput := storage.StorageBlobUpload{ + blobInput := storage.BlobUpload{ AccountName: accountName, ContainerName: containerName, BlobName: name, From d2e8cdb32766e230adbb59cf90f2bda98b9a82ef Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:47:35 +0100 Subject: [PATCH 34/39] r/storage_blob: adding a validation error --- azurerm/internal/services/storage/blobs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/internal/services/storage/blobs.go b/azurerm/internal/services/storage/blobs.go index 25a59db03d85..68a6a15791ac 100644 --- a/azurerm/internal/services/storage/blobs.go +++ b/azurerm/internal/services/storage/blobs.go @@ -123,7 +123,7 @@ func (sbu BlobUpload) createEmptyPageBlob(ctx context.Context) error { func (sbu BlobUpload) uploadPageBlob(ctx context.Context) error { if sbu.Size != 0 { - // the user shouldn't need to specify this since we infer it + return fmt.Errorf("`size` cannot be set for an uploaded page blob") } // determine the details about the file From bae71caeb75a364a4a0655e456c7e9fe7b185d36 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 28 Aug 2019 17:51:45 +0100 Subject: [PATCH 35/39] keeping the linter happy --- azurerm/internal/services/storage/blobs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azurerm/internal/services/storage/blobs.go b/azurerm/internal/services/storage/blobs.go index 68a6a15791ac..adb832f05a1e 100644 --- a/azurerm/internal/services/storage/blobs.go +++ b/azurerm/internal/services/storage/blobs.go @@ -144,7 +144,7 @@ func (sbu BlobUpload) uploadPageBlob(ctx context.Context) error { // first let's create a file of the specified file size input := blobs.PutPageBlobInput{ - BlobContentLengthBytes: int64(fileSize), + BlobContentLengthBytes: fileSize, ContentType: utils.String(sbu.ContentType), MetaData: sbu.MetaData, } @@ -167,7 +167,7 @@ type storageBlobPage struct { section *io.SectionReader } -func (sbu BlobUpload) pageUploadFromSource(ctx context.Context, file *os.File, fileSize int64) error { +func (sbu BlobUpload) pageUploadFromSource(ctx context.Context, file io.ReaderAt, fileSize int64) error { workerCount := sbu.Parallelism * runtime.NumCPU() // first we chunk the file and assign them to 'pages' @@ -214,7 +214,7 @@ const ( maxPageSize int64 = 4 * 1024 * 1024 ) -func (sbu BlobUpload) storageBlobPageSplit(file *os.File, fileSize int64) ([]storageBlobPage, error) { +func (sbu BlobUpload) storageBlobPageSplit(file io.ReaderAt, fileSize int64) ([]storageBlobPage, error) { // whilst the file Size can be any arbitrary Size, it must be uploaded in fixed-Size pages blobSize := fileSize if fileSize%minPageSize != 0 { From e897404ffed1b07be7da96e57b22aab19308a160 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Sun, 1 Sep 2019 08:14:58 +0200 Subject: [PATCH 36/39] r/storage_blob: conditionally setting `source_uri` if it's non-empty --- azurerm/resource_arm_storage_blob.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azurerm/resource_arm_storage_blob.go b/azurerm/resource_arm_storage_blob.go index 3335f94762d8..f122d83b236e 100644 --- a/azurerm/resource_arm_storage_blob.go +++ b/azurerm/resource_arm_storage_blob.go @@ -270,7 +270,12 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error d.Set("resource_group_name", resourceGroup) d.Set("content_type", props.ContentType) - d.Set("source_uri", props.CopySource) + + // The CopySource is only returned if the blob hasn't been modified (e.g. metadata configured etc) + // as such, we need to conditionally set this to ensure it's trackable if possible + if props.CopySource != "" { + d.Set("source_uri", props.CopySource) + } blobType := strings.ToLower(strings.Replace(string(props.BlobType), "Blob", "", 1)) d.Set("type", blobType) From 3344a4fe76602dc61476586fad6a81f9122ead4c Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Sun, 1 Sep 2019 08:18:27 +0200 Subject: [PATCH 37/39] r/storage_blob: fixing the requires import test --- azurerm/resource_arm_storage_blob_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/resource_arm_storage_blob_test.go b/azurerm/resource_arm_storage_blob_test.go index 377413223730..8e6d74bea7a8 100644 --- a/azurerm/resource_arm_storage_blob_test.go +++ b/azurerm/resource_arm_storage_blob_test.go @@ -435,7 +435,7 @@ func TestAccAzureRMStorageBlob_pageFromLocalFile(t *testing.T) { } func TestAccAzureRMStorageBlob_requiresImport(t *testing.T) { - if features.ShouldResourcesBeImported() { + if !features.ShouldResourcesBeImported() { t.Skip("Skipping since resources aren't required to be imported") return } From 28f97a62517e8cecda8209a59484dd42bca5cff1 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Sun, 1 Sep 2019 08:24:14 +0200 Subject: [PATCH 38/39] r/storage_blob: ignoring the source_uri field --- azurerm/resource_arm_storage_blob_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurerm/resource_arm_storage_blob_test.go b/azurerm/resource_arm_storage_blob_test.go index 8e6d74bea7a8..1e2ea6399252 100644 --- a/azurerm/resource_arm_storage_blob_test.go +++ b/azurerm/resource_arm_storage_blob_test.go @@ -180,7 +180,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicBlob(t *testing.T) { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source_uri", "type"}, }, }, }) @@ -207,7 +207,7 @@ func TestAccAzureRMStorageBlob_blockFromPublicFile(t *testing.T) { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source_uri", "type"}, }, }, }) @@ -234,7 +234,7 @@ func TestAccAzureRMStorageBlob_blockFromExistingBlob(t *testing.T) { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source_uri", "type"}, }, }, }) @@ -391,7 +391,7 @@ func TestAccAzureRMStorageBlob_pageFromExistingBlob(t *testing.T) { ResourceName: resourceName, ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "type"}, + ImportStateVerifyIgnore: []string{"attempts", "parallelism", "size", "source_uri", "type"}, }, }, }) From 91f460c44d82271ecc1ca4594181371cb03ad074 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 2 Sep 2019 07:34:40 +0200 Subject: [PATCH 39/39] r/storage_blob: refactoring from code review --- azurerm/internal/services/storage/blobs.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azurerm/internal/services/storage/blobs.go b/azurerm/internal/services/storage/blobs.go index adb832f05a1e..c292907aa6f8 100644 --- a/azurerm/internal/services/storage/blobs.go +++ b/azurerm/internal/services/storage/blobs.go @@ -113,7 +113,6 @@ func (sbu BlobUpload) createEmptyPageBlob(ctx context.Context) error { ContentType: utils.String(sbu.ContentType), MetaData: sbu.MetaData, } - // TODO: access tiers? if _, err := sbu.Client.PutPageBlob(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("Error PutPageBlob: %s", err) } @@ -282,8 +281,7 @@ func (sbu BlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPa size := end - start + 1 chunk := make([]byte, size) - _, err := page.section.Read(chunk) - if err != nil && err != io.EOF { + if _, err := page.section.Read(chunk); err != nil && err != io.EOF { uploadCtx.errors <- fmt.Errorf("Error reading source file %q at offset %d: %s", sbu.Source, page.offset, err) uploadCtx.wg.Done() continue @@ -295,7 +293,7 @@ func (sbu BlobUpload) blobPageUploadWorker(ctx context.Context, uploadCtx blobPa Content: chunk, } - if _, err = sbu.Client.PutPageUpdate(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { + if _, err := sbu.Client.PutPageUpdate(ctx, sbu.AccountName, sbu.ContainerName, sbu.BlobName, input); err != nil { uploadCtx.errors <- fmt.Errorf("Error writing page at offset %d for file %q: %s", page.offset, sbu.Source, err) uploadCtx.wg.Done() continue