[#153676720] Refactors cosmosdb provisioning to be triggered by Terra…

…form (#39)
pagopa-archive · Jan 12, 2018 · f4ad685 · f4ad685
1 parent a429562
commit f4ad685
Show file tree

Hide file tree

Showing 8 changed files with 150 additions and 74 deletions.
diff --git a/README.md b/README.md
@@ -246,8 +246,8 @@ deploying a new Azure resource or to make changes to the existing ones:
 The Terraform state is shared through an Azure
 [storage container](https://www.terraform.io/docs/state/remote.html).
 
-The file `infrastructure/$ENVIRONMENT/backend.tf` contains 
-the name of the remote file, in the Azure Blob storage, 
+The file `infrastructure/$ENVIRONMENT/backend.tf` contains
+the name of the remote file, in the Azure Blob storage,
 that stores the Terraform state for each environment.
 
 Before running any command involving Terraform you must request access to the
@@ -314,7 +314,6 @@ script:
 
 | Command                          | Task                                                                                                                                                       |
 | -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `yarn resources:cosmosdb`        | [Setup CosmosDB database and collections](./infrastructure/tasks/00-cosmosdb_setup.ts)                                                                     |
 | `yarn resources:functions:setup` | [Create Functions resource and setup application settings](./infrastructure/tasks/10-functions_setup.ts)                                                   |
 | `yarn deploy:functions:sync`     | [Deploy Functions code from the GitHub repository](./infrastructure/tasks/15-functions_sync.ts)                                                            |
 | `yarn resources:apim:setup`      | [Create API management resource and setup configuration from template files](./infrastructure/tasks/20-apim_setup.ts)                                      |

diff --git a/infrastructure/azure.tf b/infrastructure/azure.tf
@@ -9,6 +9,10 @@ provider "random" {
   version = "~> 1.1"
 }
 
+provider "null" {
+  version = "~> 1.0"
+}
+
 # Set up an Azure backend to store Terraform state.
 # You *must* create the storage account and the container before running this script
 terraform {
@@ -43,7 +47,7 @@ variable "azurerm_resource_group" {
 # Name of the storage account
 variable "azurerm_storage_account" {
     type = "string"
-} 
+}
 
 # Name of the storage container resource
 variable "azurerm_storage_container" {
@@ -70,6 +74,16 @@ variable "azurerm_cosmosdb" {
     type = "string"
 }
 
+variable "azurerm_cosmosdb_documentdb" {
+  type = "string"
+  description = "Name of CosmosDB Database"
+}
+
+variable "azurerm_cosmosdb_collections" {
+  type = "map"
+  description = "Name and partition keys of collections that must exist in the CosmosDB database"
+}
+
 # Name of the App Service Plan resource
 variable "azurerm_app_service_plan" {
     type = "string"
@@ -112,28 +126,32 @@ variable "azurerm_application_insights" {
 
 # Name of Log Analytics resource
 variable "azurerm_log_analytics" {
-    type = "string"    
+    type = "string"
 }
 
 # EventHub namespace
 variable "azurerm_eventhub_ns" {
-    type = "string"    
+    type = "string"
 }
 
 # EventHub logger for API management
 variable "azurerm_apim_eventhub" {
-    type = "string"    
+    type = "string"
 }
 
 # EventHub rule for API management
 variable "azurerm_apim_eventhub_rule" {
-    type = "string"    
+    type = "string"
 }
 
 # module "variables" {
 #     source = "./modules/variables"
 # }
 
+variable "cosmosdb_collection_provisioner" {
+  default = "infrastructure/local-provisioners/azurerm_cosmosdb_collection.ts"
+}
+
 ## RESOURCE GROUP
 
 # Create a resource group if it doesn’t exist
@@ -212,13 +230,13 @@ resource "azurerm_cosmosdb_account" "azurerm_cosmosdb" {
     name                = "${var.azurerm_cosmosdb}"
     location            = "${azurerm_resource_group.azurerm_resource_group.location}"
     resource_group_name = "${azurerm_resource_group.azurerm_resource_group.name}"
-  
+
     # Possible values are GlobalDocumentDB and MongoDB
     kind = "GlobalDocumentDB"
 
     # Required - can be only set to Standard
     offer_type          = "Standard"
-  
+
     # Can be either BoundedStaleness, Eventual, Session or Strong
     # see https://docs.microsoft.com/en-us/azure/cosmos-db/consistency-levels
     # Note: with the default BoundedStaleness settings CosmosDB cannot perform failover / replication:
@@ -244,6 +262,29 @@ resource "azurerm_cosmosdb_account" "azurerm_cosmosdb" {
     # }
 }
 
+resource "null_resource" "azurerm_cosmosdb_collections" {
+  triggers = {
+    cosmosdb_id = "${azurerm_cosmosdb_account.azurerm_cosmosdb.id}"
+
+    # serialize the collection data to json so that the provisioner will be
+    # triggered when collections get added or changed
+    # NOTE: when a collection gets removed from the config it will NOT be
+    # removed by the provisioner (the provisioner only creates collections)
+    collections_json = "${jsonencode(var.azurerm_cosmosdb_collections)}"
+
+    # increment the following value when changing the provisioner script to
+    # trigger the re-execution of the script
+    # TODO: consider using the hash of the script content instead
+    provisioner_version = "5"
+  }
+
+  count = "${length(keys(var.azurerm_cosmosdb_collections))}"
+
+  provisioner "local-exec" {
+    command = "ts-node ${var.cosmosdb_collection_provisioner} --resource-group-name ${azurerm_resource_group.azurerm_resource_group.name} --cosmosdb-account-name ${azurerm_cosmosdb_account.azurerm_cosmosdb.name} --cosmosdb-documentdb-name ${var.azurerm_cosmosdb_documentdb} --cosmosdb-collection-name ${element(keys(var.azurerm_cosmosdb_collections), count.index)} -cosmosdb-collection-partition-key ${lookup(var.azurerm_cosmosdb_collections, element(keys(var.azurerm_cosmosdb_collections), count.index))}"
+  }
+}
+
 ## APPLICATION INSIGHTS
 
 resource "azurerm_application_insights" "azurerm_application_insights" {

diff --git a/infrastructure/env/common/config.json b/infrastructure/env/common/config.json
@@ -1,10 +1,4 @@
 {
-  "azurerm_cosmosdb_collections": [
-    { "name": "messages", "partitionKey": "fiscalCode" },
-    { "name": "profiles", "partitionKey": "fiscalCode" },
-    { "name": "notifications", "partitionKey": "messageId" },
-    { "name": "services", "partitionKey": "serviceId" }
-  ],
   "app_service_portal_git_repo":
     "https://github.com/teamdigitale/digital-citizenship-onboarding",
   "app_service_portal_git_branch": "master",

diff --git a/infrastructure/env/common/tfvars.json b/infrastructure/env/common/tfvars.json
@@ -1,4 +1,10 @@
 {
   "azurerm_storage_queue_emailnotifications": "emailnotifications",
-  "azurerm_storage_queue_createdmessages": "createdmessages"
+  "azurerm_storage_queue_createdmessages": "createdmessages",
+  "azurerm_cosmosdb_collections": {
+    "messages": "fiscalCode",
+    "profiles": "fiscalCode",
+    "notifications": "messageId",
+    "services": "serviceId"
+  }
 }
diff --git a/infrastructure/tasks/00-cosmosdb_setup.ts → ...ovisioners/azurerm_cosmosdb_collection.ts b/infrastructure/tasks/00-cosmosdb_setup.ts → ...ovisioners/azurerm_cosmosdb_collection.ts
@@ -1,24 +1,13 @@
-/**
- * Run this task to deploy CosmoDB database and collections:
- *
- * yarn resources:cosmosdb:setup
- *
- * This task assumes that the following resources are already created:
- *  - Resource group
- *  - CosmoDB database account
- */
 // tslint:disable:no-console
-// tslint:disable:no-any
 
 import * as winston from "winston";
-import { login } from "../../lib/login";
-
-import { IResourcesConfiguration, readConfig } from "../../lib/config";
-import { checkEnvironment } from "../../lib/environment";
+import { login, missingLoginEnvironment } from "../../lib/login";
 
 import CosmosDBManagementClient = require("azure-arm-cosmosdb");
 import * as documentdb from "documentdb";
 
+import yargs = require("yargs");
+
 const DocumentClient = documentdb.DocumentClient;
 
 const collectionNotExists = (
@@ -112,53 +101,89 @@ const createCollectionIfNotExists = (
   });
 };
 
-export const run = async (config: IResourcesConfiguration) => {
-  const loginCreds = await login();
+interface IRunParams {
+  readonly resourceGroup: string;
+  readonly cosmosdbAccountName: string;
+  readonly cosmosdbDatabaseName: string;
+  readonly cosmosdbCollectionName: string;
+  readonly cosmosdbCollectionPartitionKey: string;
+}
+
+export const run = async (config: IRunParams) => {
+  const loginResult = await login();
 
   const client = new CosmosDBManagementClient(
-    (loginCreds as any).creds,
-    loginCreds.subscriptionId
+    loginResult.creds,
+    loginResult.subscriptionId
   );
 
   const databaseAccount = await client.databaseAccounts.get(
-    config.azurerm_resource_group,
-    config.azurerm_cosmosdb
+    config.resourceGroup,
+    config.cosmosdbAccountName
   );
 
   if (databaseAccount.documentEndpoint === undefined) {
     throw new Error("Cannot get databaseAccount.documentEndpoint");
   }
 
   const keys = await client.databaseAccounts.listKeys(
-    config.azurerm_resource_group,
-    config.azurerm_cosmosdb
+    config.resourceGroup,
+    config.cosmosdbAccountName
   );
 
   const dbClient = new DocumentClient(databaseAccount.documentEndpoint, {
     masterKey: keys.primaryMasterKey
   });
 
-  winston.info("Setup CosmosDB database");
+  winston.info(
+    `Making sure database exists: name=${config.cosmosdbDatabaseName}`
+  );
 
-  await createDatabaseIfNotExists(dbClient, config.azurerm_cosmosdb_documentdb);
+  await createDatabaseIfNotExists(dbClient, config.cosmosdbDatabaseName);
 
-  return Promise.all(
-    config.azurerm_cosmosdb_collections.map(
-      async collection =>
-        await createCollectionIfNotExists(
-          dbClient,
-          config.azurerm_cosmosdb_documentdb,
-          collection.name,
-          collection.partitionKey
-        )
-    )
+  winston.info(
+    `Making sure collection exists: name=${
+      config.cosmosdbCollectionName
+    } partitionKey=${config.cosmosdbCollectionPartitionKey}`
+  );
+  return createCollectionIfNotExists(
+    dbClient,
+    config.cosmosdbDatabaseName,
+    config.cosmosdbCollectionName,
+    config.cosmosdbCollectionPartitionKey
   );
 };
 
-checkEnvironment()
-  .then(() => readConfig(process.env.ENVIRONMENT))
-  .then(run)
-  .then(() =>
-    winston.info("Successfully deployed CosmosDB database and collections")
-  )
-  .catch((e: Error) => console.error(process.env.VERBOSE ? e : e.message));
+// check whether all required environment variables are set
+const missingEnvs = missingLoginEnvironment();
+if (missingEnvs.length > 0) {
+  console.error(`Missing required env vars: ${missingEnvs.join(", ")}`);
+  process.exit(-1);
+}
+
+const argv = yargs
+  .alias("g", "resource-group-name")
+  .demandOption("g")
+  .string("g")
+  .alias("n", "cosmosdb-account-name")
+  .demandOption("n")
+  .string("n")
+  .alias("d", "cosmosdb-documentdb-name")
+  .demandOption("d")
+  .string("d")
+  .alias("c", "cosmosdb-collection-name")
+  .demandOption("c")
+  .string("c")
+  .alias("k", "cosmosdb-collection-partition-key")
+  .demandOption("k")
+  .string("k").argv;
+
+run({
+  cosmosdbAccountName: argv.n as string,
+  cosmosdbCollectionName: argv.c as string,
+  cosmosdbCollectionPartitionKey: argv.k as string,
+  cosmosdbDatabaseName: argv.d as string,
+  resourceGroup: argv.g as string
+})
+  .then(() => winston.info("Completed"))
+  .catch((e: Error) => winston.error(e.message));
diff --git a/lib/login.ts b/lib/login.ts
@@ -8,6 +8,19 @@ export interface ICreds {
   readonly subscriptionId: string;
 }
 
+/**
+ * Returns required env vars for logging in to Azure that are either undefined
+ * or empty.
+ */
+export const missingLoginEnvironment = (): ReadonlyArray<string> =>
+  [
+    "ARM_SUBSCRIPTION_ID",
+    "ARM_CLIENT_ID",
+    "ARM_CLIENT_SECRET",
+    "ARM_TENANT_ID"
+  ]
+  .filter(e => process.env[e] == undefined || process.env[e] == "")
+
 export const login = (
   opts: msRestAzure.AzureTokenCredentialsOptions = {},
   clientId = process.env.ARM_CLIENT_ID,

diff --git a/package.json b/package.json
@@ -16,24 +16,16 @@
     "docs:api": "cp -r docs/api site",
     "docs:deploy": "gh-pages -t -d site",
     "docs:publish": "npm-run-all docs:build docs:nojekyll docs:api docs:deploy",
-    "resources:tf-init":
-      "env-cmd .env cross-var terraform init -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json -backend-config=\"infrastructure/env/$ENVIRONMENT/backend.tf\" infrastructure",
-    "resources:tf-plan":
-      "env-cmd .env cross-var terraform plan -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure",
-    "resources:tf-apply":
-      "env-cmd .env cross-var terraform apply -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure",
-    "resources:cosmosdb:setup":
-      "ts-node infrastructure/tasks/00-cosmosdb_setup.ts",
-    "resources:functions:setup":
-      "ts-node infrastructure/tasks/10-functions_setup.ts",
-    "deploy:functions:sync":
-      "ts-node infrastructure/tasks/15-functions_sync.ts",
+    "resources:tf-init": "env-cmd .env cross-var terraform init -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json -backend-config=\"infrastructure/env/$ENVIRONMENT/backend.tf\" infrastructure",
+    "resources:tf-plan": "env-cmd .env cross-var terraform plan -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure",
+    "resources:tf-apply": "env-cmd .env cross-var terraform apply -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure",
+    "resources:functions:setup": "ts-node infrastructure/tasks/10-functions_setup.ts",
+    "deploy:functions:sync": "ts-node infrastructure/tasks/15-functions_sync.ts",
     "resources:apim:setup": "ts-node infrastructure/tasks/20-apim_setup.ts",
     "resources:apim:logger": "ts-node infrastructure/tasks/21-apim_logger.ts",
     "resources:apim:adb2c": "ts-node infrastructure/tasks/22-apim_adb2c.ts",
     "resources:apim:api": "ts-node infrastructure/tasks/25-apim_api.ts",
-    "resources:devapp:apikey":
-      "ts-node --no-ignore infrastructure/tasks/30-devapp_apikey.ts",
+    "resources:devapp:apikey": "ts-node --no-ignore infrastructure/tasks/30-devapp_apikey.ts",
     "resources:devapp:setup": "ts-node infrastructure/tasks/31-devapp_setup.ts",
     "resources:devapp:git": "ts-node infrastructure/tasks/34-devapp_git.ts",
     "deploy:devapp:sync": "ts-node infrastructure/tasks/35-devapp_sync.ts",
@@ -57,8 +49,7 @@
     "azure-storage": "^2.5.0",
     "cross-env": "^5.1.1",
     "cross-var": "^1.1.0",
-    "digital-citizenship-functions":
-      "https://github.com/teamdigitale/digital-citizenship-functions#94f119d19",
+    "digital-citizenship-functions": "https://github.com/teamdigitale/digital-citizenship-functions#94f119d19",
     "documentdb": "^1.12.2",
     "dotenv": "^4.0.0",
     "env-cmd": "^7.0.0",
@@ -79,5 +70,8 @@
     "tslint-plugin-prettier": "^1.2.0",
     "typescript": "^2.5.2",
     "winston": "^2.4.0"
+  },
+  "dependencies": {
+    "@types/yargs": "^10.0.1"
   }
 }
diff --git a/yarn.lock b/yarn.lock
@@ -170,6 +170,10 @@
   dependencies:
     "@types/node" "*"
 
+"@types/yargs@^10.0.1":
+  version "10.0.1"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-10.0.1.tgz#f986e2b5d37f1fb8c13c0ed15f45d01bcc3fb3d6"
+
 accepts@~1.3.4:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f"