storage: caching all of the account information

azurerm/internal/services/storage/client.go

@@ -47,85 +47,85 @@ func BuildClient(options *common.ClientOptions) *Client {
 	}
 }
 
-func (client Client) BlobsClient(ctx context.Context, resourceGroup, accountName string) (*blobs.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) BlobsClient(ctx context.Context, account accountDetails) (*blobs.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyAuthorizer(account.name, *accountKey)
 	blobsClient := blobs.NewWithEnvironment(client.environment)
 	blobsClient.Client.Authorizer = storageAuth
 	return &blobsClient, nil
 }
 
-func (client Client) ContainersClient(ctx context.Context, resourceGroup, accountName string) (*containers.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) ContainersClient(ctx context.Context, account accountDetails) (*containers.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyAuthorizer(account.name, *accountKey)
 	containersClient := containers.NewWithEnvironment(client.environment)
 	containersClient.Client.Authorizer = storageAuth
 	return &containersClient, nil
 }
 
-func (client Client) FileShareDirectoriesClient(ctx context.Context, resourceGroup, accountName string) (*directories.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) FileShareDirectoriesClient(ctx context.Context, account accountDetails) (*directories.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
 	directoriesClient := directories.NewWithEnvironment(client.environment)
 	directoriesClient.Client.Authorizer = storageAuth
 	return &directoriesClient, nil
 }
 
-func (client Client) FileSharesClient(ctx context.Context, resourceGroup, accountName string) (*shares.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) FileSharesClient(ctx context.Context, account accountDetails) (*shares.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
 	directoriesClient := shares.NewWithEnvironment(client.environment)
 	directoriesClient.Client.Authorizer = storageAuth
 	return &directoriesClient, nil
 }
 
-func (client Client) QueuesClient(ctx context.Context, resourceGroup, accountName string) (*queues.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) QueuesClient(ctx context.Context, account accountDetails) (*queues.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
 	queuesClient := queues.NewWithEnvironment(client.environment)
 	queuesClient.Client.Authorizer = storageAuth
 	return &queuesClient, nil
 }
 
-func (client Client) TableEntityClient(ctx context.Context, resourceGroup, accountName string) (*entities.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) TableEntityClient(ctx context.Context, account accountDetails) (*entities.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(account.name, *accountKey)
 	entitiesClient := entities.NewWithEnvironment(client.environment)
 	entitiesClient.Client.Authorizer = storageAuth
 	return &entitiesClient, nil
 }
 
-func (client Client) TablesClient(ctx context.Context, resourceGroup, accountName string) (*tables.Client, error) {
-	accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
+func (client Client) TablesClient(ctx context.Context, account accountDetails) (*tables.Client, error) {
+	accountKey, err := account.AccountKey(ctx, client)
 	if err != nil {
 		return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
 	}
 
-	storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(accountName, *accountKey)
+	storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(account.name, *accountKey)
 	tablesClient := tables.NewWithEnvironment(client.environment)
 	tablesClient.Client.Authorizer = storageAuth
 	return &tablesClient, nil

azurerm/internal/services/storage/helpers.go

@@ -4,97 +4,122 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"strings"
 	"sync"
 
-	"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
+	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
 )
 
 var (
-	accountKeysCache        = map[string]string{}
-	resourceGroupNamesCache = map[string]string{}
-	writeLock               = sync.RWMutex{}
-)
-
-func (client Client) ClearFromCache(resourceGroup, accountName string) {
-	writeLock.Lock()
+	storageAccountsCache = map[string]accountDetails{}
 
-	log.Printf("[DEBUG] Removing Account %q (Resource Group %q) from the cache", accountName, resourceGroup)
-	accountCacheKey := fmt.Sprintf("%s-%s", resourceGroup, accountName)
-	delete(accountKeysCache, accountCacheKey)
+	accountsLock    = sync.RWMutex{}
+	credentialsLock = sync.RWMutex{}
+)
 
-	resourceGroupsCacheKey := accountName
-	delete(resourceGroupNamesCache, resourceGroupsCacheKey)
+type accountDetails struct {
+	ID            string
+	ResourceGroup string
+	Properties    *storage.AccountProperties
 
-	log.Printf("[DEBUG] Removed Account %q (Resource Group %q) from the cache", accountName, resourceGroup)
-	writeLock.Unlock()
+	accountKey *string
+	name       string
 }
 
-func (client Client) FindResourceGroup(ctx context.Context, accountName string) (*string, error) {
-	cacheKey := accountName
-	if v, ok := resourceGroupNamesCache[cacheKey]; ok {
-		return &v, nil
+func (ad *accountDetails) AccountKey(ctx context.Context, client Client) (*string, error) {
+	credentialsLock.Lock()
+	defer credentialsLock.Unlock()
+
+	if ad.accountKey != nil {
+		return ad.accountKey, nil
 	}
 
-	log.Printf("[DEBUG] Cache Miss - looking up the resource group for storage account %q..", accountName)
-	writeLock.Lock()
-	accounts, err := client.AccountsClient.List(ctx)
+	log.Printf("[DEBUG] Cache Miss - looking up the account key for storage account %q..", ad.name)
+	props, err := client.AccountsClient.ListKeys(ctx, ad.ResourceGroup, ad.name)
 	if err != nil {
-		return nil, fmt.Errorf("Error listing Storage Accounts (to find Resource Group for %q): %s", accountName, err)
+		return nil, fmt.Errorf("Error Listing Keys for Storage Account %q (Resource Group %q): %+v", ad.name, ad.ResourceGroup, err)
 	}
 
-	if accounts.Values() == nil {
-		return nil, nil
+	if props.Keys == nil || len(*props.Keys) == 0 || (*props.Keys)[0].Value == nil {
+		return nil, fmt.Errorf("Keys were nil for Storage Account %q (Resource Group %q): %+v", ad.name, ad.ResourceGroup, err)
 	}
 
-	var resourceGroup *string
-	for _, account := range accounts.Values() {
-		if account.Name == nil || account.ID == nil {
-			continue
-		}
+	keys := *props.Keys
+	ad.accountKey = keys[0].Value
 
-		if strings.EqualFold(accountName, *account.Name) {
-			id, err := azure.ParseAzureResourceID(*account.ID)
-			if err != nil {
-				return nil, fmt.Errorf("Error parsing ID for Storage Account %q: %s", accountName, err)
-			}
+	// force-cache this
+	storageAccountsCache[ad.name] = *ad
 
-			resourceGroup = &id.ResourceGroup
-			break
-		}
-	}
+	return ad.accountKey, nil
+}
+
+func (client Client) AddToCache(accountName string, props storage.Account) error {
+	accountsLock.Lock()
+	defer accountsLock.Unlock()
 
-	if resourceGroup != nil {
-		resourceGroupNamesCache[cacheKey] = *resourceGroup
+	account, err := client.populateAccountDetails(accountName, props)
+	if err != nil {
+		return err
 	}
 
-	writeLock.Unlock()
+	storageAccountsCache[accountName] = *account
+
+	return nil
+}
 
-	return resourceGroup, nil
+func (client Client) RemoveAccountFromCache(accountName string) {
+	accountsLock.Lock()
+	delete(storageAccountsCache, accountName)
+	accountsLock.Unlock()
 }
 
-func (client Client) findAccountKey(ctx context.Context, resourceGroup, accountName string) (*string, error) {
-	cacheKey := fmt.Sprintf("%s-%s", resourceGroup, accountName)
-	if v, ok := accountKeysCache[cacheKey]; ok {
-		return &v, nil
+func (client Client) FindAccount(ctx context.Context, accountName string) (*accountDetails, error) {
+	accountsLock.Lock()
+	defer accountsLock.Unlock()
+
+	if existing, ok := storageAccountsCache[accountName]; ok {
+		return &existing, nil
 	}
 
-	writeLock.Lock()
-	log.Printf("[DEBUG] Cache Miss - looking up the account key for storage account %q..", accountName)
-	props, err := client.AccountsClient.ListKeys(ctx, resourceGroup, accountName)
+	accounts, err := client.AccountsClient.List(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("Error Listing Keys for Storage Account %q (Resource Group %q): %+v", accountName, resourceGroup, err)
+		return nil, fmt.Errorf("Error retrieving storage accounts: %+v", err)
 	}
 
-	if props.Keys == nil || len(*props.Keys) == 0 {
-		return nil, fmt.Errorf("Keys were nil for Storage Account %q (Resource Group %q): %+v", accountName, resourceGroup, err)
+	for _, v := range accounts.Values() {
+		if v.Name == nil {
+			continue
+		}
+
+		account, err := client.populateAccountDetails(*v.Name, v)
+		if err != nil {
+			return nil, err
+		}
+
+		storageAccountsCache[*v.Name] = *account
 	}
 
-	keys := *props.Keys
-	firstKey := keys[0].Value
+	if existing, ok := storageAccountsCache[accountName]; ok {
+		return &existing, nil
+	}
+
+	return nil, nil
+}
 
-	accountKeysCache[cacheKey] = *firstKey
-	writeLock.Unlock()
+func (client Client) populateAccountDetails(accountName string, props storage.Account) (*accountDetails, error) {
+	if props.ID == nil {
+		return nil, fmt.Errorf("`id` was nil for Account %q", accountName)
+	}
+
+	accountId := *props.ID
+	id, err := ParseAccountID(accountId)
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing %q as a Resource ID: %+v", accountId, err)
+	}
 
-	return firstKey, nil
+	return &accountDetails{
+		name:          accountName,
+		ID:            accountId,
+		ResourceGroup: id.ResourceGroup,
+		Properties:    props.AccountProperties,
+	}, nil
 }

azurerm/resource_arm_storage_account.go

@@ -735,6 +735,7 @@ func resourceArmStorageAccountCreate(d *schema.ResourceData, meta interface{}) e
 	log.Printf("[INFO] storage account %q ID: %q", storageAccountName, *account.ID)
 	d.SetId(*account.ID)
 
+	// TODO: deprecate & split this out into it's own resource in 2.0
 	// as this is not available in all regions, and presumably off by default
 	// lets only try to set this value when true
 	// TODO in 2.0 switch to guarding this with d.GetOkExists() ?
@@ -751,7 +752,16 @@ func resourceArmStorageAccountCreate(d *schema.ResourceData, meta interface{}) e
 	}
 
 	if val, ok := d.GetOk("queue_properties"); ok {
-		queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resourceGroupName, storageAccountName)
+		storageClient := meta.(*ArmClient).Storage
+		account, err := storageClient.FindAccount(ctx, storageAccountName)
+		if err != nil {
+			return fmt.Errorf("Error retrieving Account %q: %s", storageAccountName, err)
+		}
+		if account == nil {
+			return fmt.Errorf("Unable to locate Storage Account %q!", storageAccountName)
+		}
+
+		queueClient, err := storageClient.QueuesClient(ctx, *account)
 		if err != nil {
 			return fmt.Errorf("Error building Queues Client: %s", err)
 		}
@@ -945,7 +955,16 @@ func resourceArmStorageAccountUpdate(d *schema.ResourceData, meta interface{}) e
 	}
 
 	if d.HasChange("queue_properties") {
-		queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resourceGroupName, storageAccountName)
+		storageClient := meta.(*ArmClient).Storage
+		account, err := storageClient.FindAccount(ctx, storageAccountName)
+		if err != nil {
+			return fmt.Errorf("Error retrieving Account %q: %s", storageAccountName, err)
+		}
+		if account == nil {
+			return fmt.Errorf("Unable to locate Storage Account %q!", storageAccountName)
+		}
+
+		queueClient, err := storageClient.QueuesClient(ctx, *account)
 		if err != nil {
 			return fmt.Errorf("Error building Queues Client: %s", err)
 		}
@@ -1128,7 +1147,16 @@ func resourceArmStorageAccountRead(d *schema.ResourceData, meta interface{}) err
 		}
 	}
 
-	queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resGroup, name)
+	storageClient := meta.(*ArmClient).Storage
+	account, err := storageClient.FindAccount(ctx, name)
+	if err != nil {
+		return fmt.Errorf("Error retrieving Account %q: %s", name, err)
+	}
+	if account == nil {
+		return fmt.Errorf("Unable to locate Storage Account %q!", name)
+	}
+
+	queueClient, err := storageClient.QueuesClient(ctx, *account)
 	if err != nil {
 		return fmt.Errorf("Error building Queues Client: %s", err)
 	}
@@ -1202,7 +1230,7 @@ func resourceArmStorageAccountDelete(d *schema.ResourceData, meta interface{}) e
 	}
 
 	// remove this from the cache
-	storageClient.ClearFromCache(resourceGroup, name)
+	storageClient.RemoveAccountFromCache(name)
 
 	return nil
 }