Skip to content

Commit

Permalink
Merge pull request #4709 from terraform-providers/f/storage-caching
Browse files Browse the repository at this point in the history 
storage: caching all of the account information
  • Loading branch information
@tombuildsstuff
tombuildsstuff authored Oct 28, 2019
2 parents d8a5c97 + 1067ee0 commit 33174e7
Show file tree
Hide file tree
Showing 20 changed files with 433 additions and 395 deletions.
42 changes: 21 additions & 21 deletions azurerm/internal/services/storage/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,85 +47,85 @@ func BuildClient(options *common.ClientOptions) *Client {
}
}

func (client Client) BlobsClient(ctx context.Context, resourceGroup, accountName string) (*blobs.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) BlobsClient(ctx context.Context, account accountDetails) (*blobs.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyAuthorizer(account.name, *accountKey)
blobsClient := blobs.NewWithEnvironment(client.environment)
blobsClient.Client.Authorizer = storageAuth
return &blobsClient, nil
}

func (client Client) ContainersClient(ctx context.Context, resourceGroup, accountName string) (*containers.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) ContainersClient(ctx context.Context, account accountDetails) (*containers.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyAuthorizer(account.name, *accountKey)
containersClient := containers.NewWithEnvironment(client.environment)
containersClient.Client.Authorizer = storageAuth
return &containersClient, nil
}

func (client Client) FileShareDirectoriesClient(ctx context.Context, resourceGroup, accountName string) (*directories.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) FileShareDirectoriesClient(ctx context.Context, account accountDetails) (*directories.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
directoriesClient := directories.NewWithEnvironment(client.environment)
directoriesClient.Client.Authorizer = storageAuth
return &directoriesClient, nil
}

func (client Client) FileSharesClient(ctx context.Context, resourceGroup, accountName string) (*shares.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) FileSharesClient(ctx context.Context, account accountDetails) (*shares.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
directoriesClient := shares.NewWithEnvironment(client.environment)
directoriesClient.Client.Authorizer = storageAuth
return &directoriesClient, nil
}

func (client Client) QueuesClient(ctx context.Context, resourceGroup, accountName string) (*queues.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) QueuesClient(ctx context.Context, account accountDetails) (*queues.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyLiteAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyLiteAuthorizer(account.name, *accountKey)
queuesClient := queues.NewWithEnvironment(client.environment)
queuesClient.Client.Authorizer = storageAuth
return &queuesClient, nil
}

func (client Client) TableEntityClient(ctx context.Context, resourceGroup, accountName string) (*entities.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) TableEntityClient(ctx context.Context, account accountDetails) (*entities.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(account.name, *accountKey)
entitiesClient := entities.NewWithEnvironment(client.environment)
entitiesClient.Client.Authorizer = storageAuth
return &entitiesClient, nil
}

func (client Client) TablesClient(ctx context.Context, resourceGroup, accountName string) (*tables.Client, error) {
accountKey, err := client.findAccountKey(ctx, resourceGroup, accountName)
func (client Client) TablesClient(ctx context.Context, account accountDetails) (*tables.Client, error) {
accountKey, err := account.AccountKey(ctx, client)
if err != nil {
return nil, fmt.Errorf("Error retrieving Account Key: %s", err)
}

storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(accountName, *accountKey)
storageAuth := authorizers.NewSharedKeyLiteTableAuthorizer(account.name, *accountKey)
tablesClient := tables.NewWithEnvironment(client.environment)
tablesClient.Client.Authorizer = storageAuth
return &tablesClient, nil
Expand Down
143 changes: 84 additions & 59 deletions azurerm/internal/services/storage/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,97 +4,122 @@ import (
"context"
"fmt"
"log"
"strings"
"sync"

"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
)

var (
accountKeysCache = map[string]string{}
resourceGroupNamesCache = map[string]string{}
writeLock = sync.RWMutex{}
)

func (client Client) ClearFromCache(resourceGroup, accountName string) {
writeLock.Lock()
storageAccountsCache = map[string]accountDetails{}

log.Printf("[DEBUG] Removing Account %q (Resource Group %q) from the cache", accountName, resourceGroup)
accountCacheKey := fmt.Sprintf("%s-%s", resourceGroup, accountName)
delete(accountKeysCache, accountCacheKey)
accountsLock = sync.RWMutex{}
credentialsLock = sync.RWMutex{}
)

resourceGroupsCacheKey := accountName
delete(resourceGroupNamesCache, resourceGroupsCacheKey)
type accountDetails struct {
ID string
ResourceGroup string
Properties *storage.AccountProperties

log.Printf("[DEBUG] Removed Account %q (Resource Group %q) from the cache", accountName, resourceGroup)
writeLock.Unlock()
accountKey *string
name string
}

func (client Client) FindResourceGroup(ctx context.Context, accountName string) (*string, error) {
cacheKey := accountName
if v, ok := resourceGroupNamesCache[cacheKey]; ok {
return &v, nil
func (ad *accountDetails) AccountKey(ctx context.Context, client Client) (*string, error) {
credentialsLock.Lock()
defer credentialsLock.Unlock()

if ad.accountKey != nil {
return ad.accountKey, nil
}

log.Printf("[DEBUG] Cache Miss - looking up the resource group for storage account %q..", accountName)
writeLock.Lock()
accounts, err := client.AccountsClient.List(ctx)
log.Printf("[DEBUG] Cache Miss - looking up the account key for storage account %q..", ad.name)
props, err := client.AccountsClient.ListKeys(ctx, ad.ResourceGroup, ad.name)
if err != nil {
return nil, fmt.Errorf("Error listing Storage Accounts (to find Resource Group for %q): %s", accountName, err)
return nil, fmt.Errorf("Error Listing Keys for Storage Account %q (Resource Group %q): %+v", ad.name, ad.ResourceGroup, err)
}

if accounts.Values() == nil {
return nil, nil
if props.Keys == nil || len(*props.Keys) == 0 || (*props.Keys)[0].Value == nil {
return nil, fmt.Errorf("Keys were nil for Storage Account %q (Resource Group %q): %+v", ad.name, ad.ResourceGroup, err)
}

var resourceGroup *string
for _, account := range accounts.Values() {
if account.Name == nil || account.ID == nil {
continue
}
keys := *props.Keys
ad.accountKey = keys[0].Value

if strings.EqualFold(accountName, *account.Name) {
id, err := azure.ParseAzureResourceID(*account.ID)
if err != nil {
return nil, fmt.Errorf("Error parsing ID for Storage Account %q: %s", accountName, err)
}
// force-cache this
storageAccountsCache[ad.name] = *ad

resourceGroup = &id.ResourceGroup
break
}
}
return ad.accountKey, nil
}

func (client Client) AddToCache(accountName string, props storage.Account) error {
accountsLock.Lock()
defer accountsLock.Unlock()

if resourceGroup != nil {
resourceGroupNamesCache[cacheKey] = *resourceGroup
account, err := client.populateAccountDetails(accountName, props)
if err != nil {
return err
}

writeLock.Unlock()
storageAccountsCache[accountName] = *account

return nil
}

return resourceGroup, nil
func (client Client) RemoveAccountFromCache(accountName string) {
accountsLock.Lock()
delete(storageAccountsCache, accountName)
accountsLock.Unlock()
}

func (client Client) findAccountKey(ctx context.Context, resourceGroup, accountName string) (*string, error) {
cacheKey := fmt.Sprintf("%s-%s", resourceGroup, accountName)
if v, ok := accountKeysCache[cacheKey]; ok {
return &v, nil
func (client Client) FindAccount(ctx context.Context, accountName string) (*accountDetails, error) {
accountsLock.Lock()
defer accountsLock.Unlock()

if existing, ok := storageAccountsCache[accountName]; ok {
return &existing, nil
}

writeLock.Lock()
log.Printf("[DEBUG] Cache Miss - looking up the account key for storage account %q..", accountName)
props, err := client.AccountsClient.ListKeys(ctx, resourceGroup, accountName)
accounts, err := client.AccountsClient.List(ctx)
if err != nil {
return nil, fmt.Errorf("Error Listing Keys for Storage Account %q (Resource Group %q): %+v", accountName, resourceGroup, err)
return nil, fmt.Errorf("Error retrieving storage accounts: %+v", err)
}

if props.Keys == nil || len(*props.Keys) == 0 {
return nil, fmt.Errorf("Keys were nil for Storage Account %q (Resource Group %q): %+v", accountName, resourceGroup, err)
for _, v := range accounts.Values() {
if v.Name == nil {
continue
}

account, err := client.populateAccountDetails(*v.Name, v)
if err != nil {
return nil, err
}

storageAccountsCache[*v.Name] = *account
}

keys := *props.Keys
firstKey := keys[0].Value
if existing, ok := storageAccountsCache[accountName]; ok {
return &existing, nil
}

return nil, nil
}

accountKeysCache[cacheKey] = *firstKey
writeLock.Unlock()
func (client Client) populateAccountDetails(accountName string, props storage.Account) (*accountDetails, error) {
if props.ID == nil {
return nil, fmt.Errorf("`id` was nil for Account %q", accountName)
}

accountId := *props.ID
id, err := ParseAccountID(accountId)
if err != nil {
return nil, fmt.Errorf("Error parsing %q as a Resource ID: %+v", accountId, err)
}

return firstKey, nil
return &accountDetails{
name: accountName,
ID: accountId,
ResourceGroup: id.ResourceGroup,
Properties: props.AccountProperties,
}, nil
}
36 changes: 32 additions & 4 deletions azurerm/resource_arm_storage_account.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,7 @@ func resourceArmStorageAccountCreate(d *schema.ResourceData, meta interface{}) e
log.Printf("[INFO] storage account %q ID: %q", storageAccountName, *account.ID)
d.SetId(*account.ID)

// TODO: deprecate & split this out into it's own resource in 2.0
// as this is not available in all regions, and presumably off by default
// lets only try to set this value when true
// TODO in 2.0 switch to guarding this with d.GetOkExists() ?
Expand All @@ -751,7 +752,16 @@ func resourceArmStorageAccountCreate(d *schema.ResourceData, meta interface{}) e
}

if val, ok := d.GetOk("queue_properties"); ok {
queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resourceGroupName, storageAccountName)
storageClient := meta.(*ArmClient).Storage
account, err := storageClient.FindAccount(ctx, storageAccountName)
if err != nil {
return fmt.Errorf("Error retrieving Account %q: %s", storageAccountName, err)
}
if account == nil {
return fmt.Errorf("Unable to locate Storage Account %q!", storageAccountName)
}

queueClient, err := storageClient.QueuesClient(ctx, *account)
if err != nil {
return fmt.Errorf("Error building Queues Client: %s", err)
}
Expand Down Expand Up @@ -945,7 +955,16 @@ func resourceArmStorageAccountUpdate(d *schema.ResourceData, meta interface{}) e
}

if d.HasChange("queue_properties") {
queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resourceGroupName, storageAccountName)
storageClient := meta.(*ArmClient).Storage
account, err := storageClient.FindAccount(ctx, storageAccountName)
if err != nil {
return fmt.Errorf("Error retrieving Account %q: %s", storageAccountName, err)
}
if account == nil {
return fmt.Errorf("Unable to locate Storage Account %q!", storageAccountName)
}

queueClient, err := storageClient.QueuesClient(ctx, *account)
if err != nil {
return fmt.Errorf("Error building Queues Client: %s", err)
}
Expand Down Expand Up @@ -1128,7 +1147,16 @@ func resourceArmStorageAccountRead(d *schema.ResourceData, meta interface{}) err
}
}

queueClient, err := meta.(*ArmClient).Storage.QueuesClient(ctx, resGroup, name)
storageClient := meta.(*ArmClient).Storage
account, err := storageClient.FindAccount(ctx, name)
if err != nil {
return fmt.Errorf("Error retrieving Account %q: %s", name, err)
}
if account == nil {
return fmt.Errorf("Unable to locate Storage Account %q!", name)
}

queueClient, err := storageClient.QueuesClient(ctx, *account)
if err != nil {
return fmt.Errorf("Error building Queues Client: %s", err)
}
Expand Down Expand Up @@ -1202,7 +1230,7 @@ func resourceArmStorageAccountDelete(d *schema.ResourceData, meta interface{}) e
}

// remove this from the cache
storageClient.ClearFromCache(resourceGroup, name)
storageClient.RemoveAccountFromCache(name)

return nil
}
Expand Down
Loading

0 comments on commit 33174e7

Please sign in to comment.