From f4ad685ea4976d4e36e4a3b67550493f96247e4d Mon Sep 17 00:00:00 2001 From: Federico Feroldi Date: Fri, 12 Jan 2018 13:01:19 +0100 Subject: [PATCH] [#153676720] Refactors cosmosdb provisioning to be triggered by Terraform (#39) --- README.md | 5 +- infrastructure/azure.tf | 55 +++++++-- infrastructure/env/common/config.json | 6 - infrastructure/env/common/tfvars.json | 8 +- .../azurerm_cosmosdb_collection.ts} | 107 +++++++++++------- lib/login.ts | 13 +++ package.json | 26 ++--- yarn.lock | 4 + 8 files changed, 150 insertions(+), 74 deletions(-) rename infrastructure/{tasks/00-cosmosdb_setup.ts => local-provisioners/azurerm_cosmosdb_collection.ts} (57%) diff --git a/README.md b/README.md index abf9dbdd..75673985 100644 --- a/README.md +++ b/README.md @@ -246,8 +246,8 @@ deploying a new Azure resource or to make changes to the existing ones: The Terraform state is shared through an Azure [storage container](https://www.terraform.io/docs/state/remote.html). -The file `infrastructure/$ENVIRONMENT/backend.tf` contains -the name of the remote file, in the Azure Blob storage, +The file `infrastructure/$ENVIRONMENT/backend.tf` contains +the name of the remote file, in the Azure Blob storage, that stores the Terraform state for each environment. Before running any command involving Terraform you must request access to the @@ -314,7 +314,6 @@ script: | Command | Task | | -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `yarn resources:cosmosdb` | [Setup CosmosDB database and collections](./infrastructure/tasks/00-cosmosdb_setup.ts) | | `yarn resources:functions:setup` | [Create Functions resource and setup application settings](./infrastructure/tasks/10-functions_setup.ts) | | `yarn deploy:functions:sync` | [Deploy Functions code from the GitHub repository](./infrastructure/tasks/15-functions_sync.ts) | | `yarn resources:apim:setup` | [Create API management resource and setup configuration from template files](./infrastructure/tasks/20-apim_setup.ts) | diff --git a/infrastructure/azure.tf b/infrastructure/azure.tf index 96627e85..f611c2ff 100644 --- a/infrastructure/azure.tf +++ b/infrastructure/azure.tf @@ -9,6 +9,10 @@ provider "random" { version = "~> 1.1" } +provider "null" { + version = "~> 1.0" +} + # Set up an Azure backend to store Terraform state. # You *must* create the storage account and the container before running this script terraform { @@ -43,7 +47,7 @@ variable "azurerm_resource_group" { # Name of the storage account variable "azurerm_storage_account" { type = "string" -} +} # Name of the storage container resource variable "azurerm_storage_container" { @@ -70,6 +74,16 @@ variable "azurerm_cosmosdb" { type = "string" } +variable "azurerm_cosmosdb_documentdb" { + type = "string" + description = "Name of CosmosDB Database" +} + +variable "azurerm_cosmosdb_collections" { + type = "map" + description = "Name and partition keys of collections that must exist in the CosmosDB database" +} + # Name of the App Service Plan resource variable "azurerm_app_service_plan" { type = "string" @@ -112,28 +126,32 @@ variable "azurerm_application_insights" { # Name of Log Analytics resource variable "azurerm_log_analytics" { - type = "string" + type = "string" } # EventHub namespace variable "azurerm_eventhub_ns" { - type = "string" + type = "string" } # EventHub logger for API management variable "azurerm_apim_eventhub" { - type = "string" + type = "string" } # EventHub rule for API management variable "azurerm_apim_eventhub_rule" { - type = "string" + type = "string" } # module "variables" { # source = "./modules/variables" # } +variable "cosmosdb_collection_provisioner" { + default = "infrastructure/local-provisioners/azurerm_cosmosdb_collection.ts" +} + ## RESOURCE GROUP # Create a resource group if it doesn’t exist @@ -212,13 +230,13 @@ resource "azurerm_cosmosdb_account" "azurerm_cosmosdb" { name = "${var.azurerm_cosmosdb}" location = "${azurerm_resource_group.azurerm_resource_group.location}" resource_group_name = "${azurerm_resource_group.azurerm_resource_group.name}" - + # Possible values are GlobalDocumentDB and MongoDB kind = "GlobalDocumentDB" # Required - can be only set to Standard offer_type = "Standard" - + # Can be either BoundedStaleness, Eventual, Session or Strong # see https://docs.microsoft.com/en-us/azure/cosmos-db/consistency-levels # Note: with the default BoundedStaleness settings CosmosDB cannot perform failover / replication: @@ -244,6 +262,29 @@ resource "azurerm_cosmosdb_account" "azurerm_cosmosdb" { # } } +resource "null_resource" "azurerm_cosmosdb_collections" { + triggers = { + cosmosdb_id = "${azurerm_cosmosdb_account.azurerm_cosmosdb.id}" + + # serialize the collection data to json so that the provisioner will be + # triggered when collections get added or changed + # NOTE: when a collection gets removed from the config it will NOT be + # removed by the provisioner (the provisioner only creates collections) + collections_json = "${jsonencode(var.azurerm_cosmosdb_collections)}" + + # increment the following value when changing the provisioner script to + # trigger the re-execution of the script + # TODO: consider using the hash of the script content instead + provisioner_version = "5" + } + + count = "${length(keys(var.azurerm_cosmosdb_collections))}" + + provisioner "local-exec" { + command = "ts-node ${var.cosmosdb_collection_provisioner} --resource-group-name ${azurerm_resource_group.azurerm_resource_group.name} --cosmosdb-account-name ${azurerm_cosmosdb_account.azurerm_cosmosdb.name} --cosmosdb-documentdb-name ${var.azurerm_cosmosdb_documentdb} --cosmosdb-collection-name ${element(keys(var.azurerm_cosmosdb_collections), count.index)} -cosmosdb-collection-partition-key ${lookup(var.azurerm_cosmosdb_collections, element(keys(var.azurerm_cosmosdb_collections), count.index))}" + } +} + ## APPLICATION INSIGHTS resource "azurerm_application_insights" "azurerm_application_insights" { diff --git a/infrastructure/env/common/config.json b/infrastructure/env/common/config.json index 52b77fa5..88e34cda 100644 --- a/infrastructure/env/common/config.json +++ b/infrastructure/env/common/config.json @@ -1,10 +1,4 @@ { - "azurerm_cosmosdb_collections": [ - { "name": "messages", "partitionKey": "fiscalCode" }, - { "name": "profiles", "partitionKey": "fiscalCode" }, - { "name": "notifications", "partitionKey": "messageId" }, - { "name": "services", "partitionKey": "serviceId" } - ], "app_service_portal_git_repo": "https://github.com/teamdigitale/digital-citizenship-onboarding", "app_service_portal_git_branch": "master", diff --git a/infrastructure/env/common/tfvars.json b/infrastructure/env/common/tfvars.json index 0519d2fc..7d2fef57 100644 --- a/infrastructure/env/common/tfvars.json +++ b/infrastructure/env/common/tfvars.json @@ -1,4 +1,10 @@ { "azurerm_storage_queue_emailnotifications": "emailnotifications", - "azurerm_storage_queue_createdmessages": "createdmessages" + "azurerm_storage_queue_createdmessages": "createdmessages", + "azurerm_cosmosdb_collections": { + "messages": "fiscalCode", + "profiles": "fiscalCode", + "notifications": "messageId", + "services": "serviceId" + } } diff --git a/infrastructure/tasks/00-cosmosdb_setup.ts b/infrastructure/local-provisioners/azurerm_cosmosdb_collection.ts similarity index 57% rename from infrastructure/tasks/00-cosmosdb_setup.ts rename to infrastructure/local-provisioners/azurerm_cosmosdb_collection.ts index f524a182..e716d369 100644 --- a/infrastructure/tasks/00-cosmosdb_setup.ts +++ b/infrastructure/local-provisioners/azurerm_cosmosdb_collection.ts @@ -1,24 +1,13 @@ -/** - * Run this task to deploy CosmoDB database and collections: - * - * yarn resources:cosmosdb:setup - * - * This task assumes that the following resources are already created: - * - Resource group - * - CosmoDB database account - */ // tslint:disable:no-console -// tslint:disable:no-any import * as winston from "winston"; -import { login } from "../../lib/login"; - -import { IResourcesConfiguration, readConfig } from "../../lib/config"; -import { checkEnvironment } from "../../lib/environment"; +import { login, missingLoginEnvironment } from "../../lib/login"; import CosmosDBManagementClient = require("azure-arm-cosmosdb"); import * as documentdb from "documentdb"; +import yargs = require("yargs"); + const DocumentClient = documentdb.DocumentClient; const collectionNotExists = ( @@ -112,17 +101,25 @@ const createCollectionIfNotExists = ( }); }; -export const run = async (config: IResourcesConfiguration) => { - const loginCreds = await login(); +interface IRunParams { + readonly resourceGroup: string; + readonly cosmosdbAccountName: string; + readonly cosmosdbDatabaseName: string; + readonly cosmosdbCollectionName: string; + readonly cosmosdbCollectionPartitionKey: string; +} + +export const run = async (config: IRunParams) => { + const loginResult = await login(); const client = new CosmosDBManagementClient( - (loginCreds as any).creds, - loginCreds.subscriptionId + loginResult.creds, + loginResult.subscriptionId ); const databaseAccount = await client.databaseAccounts.get( - config.azurerm_resource_group, - config.azurerm_cosmosdb + config.resourceGroup, + config.cosmosdbAccountName ); if (databaseAccount.documentEndpoint === undefined) { @@ -130,35 +127,63 @@ export const run = async (config: IResourcesConfiguration) => { } const keys = await client.databaseAccounts.listKeys( - config.azurerm_resource_group, - config.azurerm_cosmosdb + config.resourceGroup, + config.cosmosdbAccountName ); const dbClient = new DocumentClient(databaseAccount.documentEndpoint, { masterKey: keys.primaryMasterKey }); - winston.info("Setup CosmosDB database"); + winston.info( + `Making sure database exists: name=${config.cosmosdbDatabaseName}` + ); - await createDatabaseIfNotExists(dbClient, config.azurerm_cosmosdb_documentdb); + await createDatabaseIfNotExists(dbClient, config.cosmosdbDatabaseName); - return Promise.all( - config.azurerm_cosmosdb_collections.map( - async collection => - await createCollectionIfNotExists( - dbClient, - config.azurerm_cosmosdb_documentdb, - collection.name, - collection.partitionKey - ) - ) + winston.info( + `Making sure collection exists: name=${ + config.cosmosdbCollectionName + } partitionKey=${config.cosmosdbCollectionPartitionKey}` + ); + return createCollectionIfNotExists( + dbClient, + config.cosmosdbDatabaseName, + config.cosmosdbCollectionName, + config.cosmosdbCollectionPartitionKey ); }; -checkEnvironment() - .then(() => readConfig(process.env.ENVIRONMENT)) - .then(run) - .then(() => - winston.info("Successfully deployed CosmosDB database and collections") - ) - .catch((e: Error) => console.error(process.env.VERBOSE ? e : e.message)); +// check whether all required environment variables are set +const missingEnvs = missingLoginEnvironment(); +if (missingEnvs.length > 0) { + console.error(`Missing required env vars: ${missingEnvs.join(", ")}`); + process.exit(-1); +} + +const argv = yargs + .alias("g", "resource-group-name") + .demandOption("g") + .string("g") + .alias("n", "cosmosdb-account-name") + .demandOption("n") + .string("n") + .alias("d", "cosmosdb-documentdb-name") + .demandOption("d") + .string("d") + .alias("c", "cosmosdb-collection-name") + .demandOption("c") + .string("c") + .alias("k", "cosmosdb-collection-partition-key") + .demandOption("k") + .string("k").argv; + +run({ + cosmosdbAccountName: argv.n as string, + cosmosdbCollectionName: argv.c as string, + cosmosdbCollectionPartitionKey: argv.k as string, + cosmosdbDatabaseName: argv.d as string, + resourceGroup: argv.g as string +}) + .then(() => winston.info("Completed")) + .catch((e: Error) => winston.error(e.message)); diff --git a/lib/login.ts b/lib/login.ts index d5925358..012b8d9a 100644 --- a/lib/login.ts +++ b/lib/login.ts @@ -8,6 +8,19 @@ export interface ICreds { readonly subscriptionId: string; } +/** + * Returns required env vars for logging in to Azure that are either undefined + * or empty. + */ +export const missingLoginEnvironment = (): ReadonlyArray => + [ + "ARM_SUBSCRIPTION_ID", + "ARM_CLIENT_ID", + "ARM_CLIENT_SECRET", + "ARM_TENANT_ID" + ] + .filter(e => process.env[e] == undefined || process.env[e] == "") + export const login = ( opts: msRestAzure.AzureTokenCredentialsOptions = {}, clientId = process.env.ARM_CLIENT_ID, diff --git a/package.json b/package.json index bafbfd6b..ae362283 100644 --- a/package.json +++ b/package.json @@ -16,24 +16,16 @@ "docs:api": "cp -r docs/api site", "docs:deploy": "gh-pages -t -d site", "docs:publish": "npm-run-all docs:build docs:nojekyll docs:api docs:deploy", - "resources:tf-init": - "env-cmd .env cross-var terraform init -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json -backend-config=\"infrastructure/env/$ENVIRONMENT/backend.tf\" infrastructure", - "resources:tf-plan": - "env-cmd .env cross-var terraform plan -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", - "resources:tf-apply": - "env-cmd .env cross-var terraform apply -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", - "resources:cosmosdb:setup": - "ts-node infrastructure/tasks/00-cosmosdb_setup.ts", - "resources:functions:setup": - "ts-node infrastructure/tasks/10-functions_setup.ts", - "deploy:functions:sync": - "ts-node infrastructure/tasks/15-functions_sync.ts", + "resources:tf-init": "env-cmd .env cross-var terraform init -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json -backend-config=\"infrastructure/env/$ENVIRONMENT/backend.tf\" infrastructure", + "resources:tf-plan": "env-cmd .env cross-var terraform plan -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", + "resources:tf-apply": "env-cmd .env cross-var terraform apply -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", + "resources:functions:setup": "ts-node infrastructure/tasks/10-functions_setup.ts", + "deploy:functions:sync": "ts-node infrastructure/tasks/15-functions_sync.ts", "resources:apim:setup": "ts-node infrastructure/tasks/20-apim_setup.ts", "resources:apim:logger": "ts-node infrastructure/tasks/21-apim_logger.ts", "resources:apim:adb2c": "ts-node infrastructure/tasks/22-apim_adb2c.ts", "resources:apim:api": "ts-node infrastructure/tasks/25-apim_api.ts", - "resources:devapp:apikey": - "ts-node --no-ignore infrastructure/tasks/30-devapp_apikey.ts", + "resources:devapp:apikey": "ts-node --no-ignore infrastructure/tasks/30-devapp_apikey.ts", "resources:devapp:setup": "ts-node infrastructure/tasks/31-devapp_setup.ts", "resources:devapp:git": "ts-node infrastructure/tasks/34-devapp_git.ts", "deploy:devapp:sync": "ts-node infrastructure/tasks/35-devapp_sync.ts", @@ -57,8 +49,7 @@ "azure-storage": "^2.5.0", "cross-env": "^5.1.1", "cross-var": "^1.1.0", - "digital-citizenship-functions": - "https://github.com/teamdigitale/digital-citizenship-functions#94f119d19", + "digital-citizenship-functions": "https://github.com/teamdigitale/digital-citizenship-functions#94f119d19", "documentdb": "^1.12.2", "dotenv": "^4.0.0", "env-cmd": "^7.0.0", @@ -79,5 +70,8 @@ "tslint-plugin-prettier": "^1.2.0", "typescript": "^2.5.2", "winston": "^2.4.0" + }, + "dependencies": { + "@types/yargs": "^10.0.1" } } diff --git a/yarn.lock b/yarn.lock index d7e1c1bf..c613ecd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -170,6 +170,10 @@ dependencies: "@types/node" "*" +"@types/yargs@^10.0.1": + version "10.0.1" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-10.0.1.tgz#f986e2b5d37f1fb8c13c0ed15f45d01bcc3fb3d6" + accepts@~1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f"