diff --git a/README.md b/README.md index 5f17b120..032dd3a5 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,6 @@ script: | Command | Task | | -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `yarn resources:functions:setup` | [Create Functions resource and setup application settings](./infrastructure/tasks/10-functions_setup.ts) | | `yarn deploy:functions:sync` | [Deploy Functions code from the GitHub repository](./infrastructure/tasks/15-functions_sync.ts) | | `yarn resources:apim:setup` | [Create API management resource and setup configuration from template files](./infrastructure/tasks/20-apim_setup.ts) | | `yarn resources:apim:logger` | [Setup API management logging through EventHub](./infrastructure/tasks/21-apim_logger.ts) | @@ -395,7 +394,7 @@ INCLUDE_API_PRODUCTS=1 INCLUDE_API_POLICIES=1 # Mail service API key -SENDGRID_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +TF_VAR_SENDGRID_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX ``` ## Example output diff --git a/infrastructure/azure.tf b/infrastructure/azure.tf index f3c23209..fd008d2d 100644 --- a/infrastructure/azure.tf +++ b/infrastructure/azure.tf @@ -2,7 +2,7 @@ # Set up environment variables before running this script (see README.md) provider "azurerm" { - version = "~> 0.3" + version = "~> 1.0" } provider "random" { @@ -54,9 +54,29 @@ variable "azurerm_storage_container" { type = "string" } -# Name of the storage account for functions +variable "message_blob_container" { + default = "message-content" + description = "Name of the message container blob" +} + +variable "azurerm_functionapp" { + type = "string" + description = "Name of the main Functions application" +} + variable "azurerm_functionapp_storage_account" { - type = "string" + type = "string" + description = "Name of the storage account for functions" +} + +variable "azurerm_functionapp_git_repo" { + default = "https://github.com/teamdigitale/digital-citizenship-functions" + description = "The GitHub repository that must be associated to the function app" +} + +variable "azurerm_functionapp_git_branch" { + default = "funcpack-release-latest" + description = "The branch of the GitHub repository that must be associated to the function app" } # Name of the storage queue for email notifications @@ -154,6 +174,12 @@ variable "azurerm_apim_eventhub_rule" { type = "string" } +# This should be passed bya ENV var TF_VAR_SENDGRID_KEY +variable "SENDGRID_KEY" { + type = "string" + description = "The API key for the SendGrid service" +} + # module "variables" { # source = "./modules/variables" # } @@ -192,6 +218,8 @@ resource "azurerm_storage_account" "azurerm_storage_account" { # see https://docs.microsoft.com/en-us/azure/storage/common/storage-service-encryption enable_blob_encryption = true + enable_https_traffic_only = true + tags { environment = "${var.environment}" } @@ -210,6 +238,8 @@ resource "azurerm_storage_account" "azurerm_functionapp_storage_account" { # see https://docs.microsoft.com/en-us/azure/storage/common/storage-service-encryption enable_blob_encryption = true + enable_https_traffic_only = true + tags { environment = "${var.environment}" } @@ -238,6 +268,18 @@ resource "azurerm_storage_queue" "azurerm_storage_queue_createdmessages" { storage_account_name = "${azurerm_storage_account.azurerm_storage_account.name}" } +## BLOBS + +resource "azurerm_storage_blob" "azurerm_message_blob" { + name = "${var.message_blob_container}" + + resource_group_name = "${azurerm_resource_group.azurerm_resource_group.name}" + storage_account_name = "${azurerm_storage_account.azurerm_storage_account.name}" + storage_container_name = "${azurerm_storage_container.azurerm_storage_container.name}" + + type = "block" +} + ## DATABASE resource "azurerm_cosmosdb_account" "azurerm_cosmosdb" { @@ -328,6 +370,91 @@ resource "azurerm_app_service_plan" "azurerm_app_service_plan" { # } } +## FUNCTIONS + +resource "azurerm_function_app" "azurerm_function_app" { + name = "${var.azurerm_functionapp}" + location = "${azurerm_resource_group.azurerm_resource_group.location}" + resource_group_name = "${azurerm_resource_group.azurerm_resource_group.name}" + app_service_plan_id = "${azurerm_app_service_plan.azurerm_app_service_plan.id}" + storage_connection_string = "${azurerm_storage_account.azurerm_functionapp_storage_account.primary_connection_string}" + version = "~1" + + site_config = { + # We don't want the express server to idle + # so do not set `alwaysOn: false` in production + always_on = true + } + + app_settings = { + # "AzureWebJobsStorage" = "${azurerm_storage_account.azurerm_functionapp_storage_account.primary_connection_string}" + # "AzureWebJobsDashboard" = "${azurerm_storage_account.azurerm_functionapp_storage_account.primary_connection_string}" + + "COSMOSDB_NAME" = "${var.azurerm_cosmosdb_documentdb}" + + "QueueStorageConnection" = "${azurerm_storage_account.azurerm_storage_account.primary_connection_string}" + + "APPINSIGHTS_INSTRUMENTATIONKEY" = "${azurerm_application_insights.azurerm_application_insights.instrumentation_key}" + + # Avoid edit functions code from the Azure portal + "FUNCTION_APP_EDIT_MODE" = "readonly" + + # AzureWebJobsSecretStorageType may be `disabled` or `Blob` + # When set to `Blob` the API manager task won't be able + # to retrieve the master key + "AzureWebJobsSecretStorageType" = "disabled" + + "WEBSITE_HTTPLOGGING_RETENTION_DAYS" = "3" + + "DIAGNOSTICS_AZUREBLOBRETENTIONINDAYS" = "1" + + "WEBSITE_NODE_DEFAULT_VERSION" = "6.11.2" + + "SCM_USE_FUNCPACK_BUILD" = "1" + + "MESSAGE_CONTAINER_NAME" = "${azurerm_storage_blob.azurerm_message_blob.name}" + } + + connection_string = [ + # [#152800384] - TODO: change the following value + # when we'll migrate to production service + { + name = "SENDGRID_KEY" + type = "Custom" + value = "${var.SENDGRID_KEY}" + }, + { + name = "COSMOSDB_URI" + type = "Custom" + value = "https://${azurerm_cosmosdb_account.azurerm_cosmosdb.name}.documents.azure.com:443/" + }, + { + name = "COSMOSDB_KEY" + type = "Custom" + value = "${azurerm_cosmosdb_account.azurerm_cosmosdb.primary_master_key}" + } + ] +} + +resource "null_resource" "azurerm_function_app_git" { + triggers = { + azurerm_functionapp_id = "${azurerm_function_app.azurerm_function_app.id}" + + # trigger recreation of this resource when the following variables change + azurerm_functionapp_git_repo = "${var.azurerm_functionapp_git_repo}" + azurerm_functionapp_git_branch = "${var.azurerm_functionapp_git_branch}" + + # increment the following value when changing the provisioner script to + # trigger the re-execution of the script + # TODO: consider using the hash of the script content instead + provisioner_version = "1" + } + + provisioner "local-exec" { + command = "ts-node ${var.website_git_provisioner} --resource-group-name ${azurerm_resource_group.azurerm_resource_group.name} --app-name ${azurerm_function_app.azurerm_function_app.name} --git-repo ${var.azurerm_functionapp_git_repo} --git-branch ${var.azurerm_functionapp_git_branch}" + } +} + ### DEVELOPER PORTAL TASKS resource "azurerm_app_service_plan" "azurerm_app_service_plan_portal" { @@ -402,7 +529,7 @@ resource "null_resource" "azurerm_app_service_portal_git" { } provisioner "local-exec" { - command = "ts-node ${var.website_git_provisioner} --resource-group-name ${azurerm_resource_group.azurerm_resource_group.name} --appservice-portal-name ${azurerm_app_service.azurerm_app_service_portal.name} --git-repo ${var.app_service_portal_git_repo} --git-branch ${var.app_service_portal_git_branch}" + command = "ts-node ${var.website_git_provisioner} --resource-group-name ${azurerm_resource_group.azurerm_resource_group.name} --app-name ${azurerm_app_service.azurerm_app_service_portal.name} --git-repo ${var.app_service_portal_git_repo} --git-branch ${var.app_service_portal_git_branch}" } } diff --git a/infrastructure/env/common/config.json b/infrastructure/env/common/config.json index ba7d6a27..eefa943a 100644 --- a/infrastructure/env/common/config.json +++ b/infrastructure/env/common/config.json @@ -43,7 +43,6 @@ "policyFile": "admin.xml" } ], - "message_blob_container": "message-content", "azure_portal_ips": [ "104.42.195.92", "40.76.54.131", diff --git a/infrastructure/local-provisioners/azurerm_website_git.ts b/infrastructure/local-provisioners/azurerm_website_git.ts index 85e5835d..b9a9f139 100644 --- a/infrastructure/local-provisioners/azurerm_website_git.ts +++ b/infrastructure/local-provisioners/azurerm_website_git.ts @@ -1,16 +1,6 @@ /** * Run this task from the command line to set up deployment - * from the GitHub repository to the Azure App Service - * running the developer portal onboarding facilities: - * - * yarn resources:devapp:git - * - * https://github.com/teamdigitale/digital-citizenship-onboarding - * - * This task assumes that the following resources are already created: - * - Resource group - * - App Service Plan - * - App Service + * from the GitHub repository to an Azure App */ // tslint:disable:no-console // tslint:disable:no-any @@ -24,13 +14,13 @@ import webSiteManagementClient = require("azure-arm-website"); interface IRunParams { readonly resourceGroupName: string; - readonly appServicePortalName: string; - readonly appServicePortalGitBranch: string; - readonly appServicePortalGitRepo: string; + readonly appName: string; + readonly appGitBranch: string; + readonly appGitRepo: string; } export const run = async (config: IRunParams) => { - if (!config.appServicePortalGitRepo) { + if (!config.appGitRepo) { return Promise.reject( "Deployment from source control repository not configured, skipping." ); @@ -43,26 +33,26 @@ export const run = async (config: IRunParams) => { ); const siteSourceControl = { - branch: config.appServicePortalGitBranch, + branch: config.appGitBranch, deploymentRollbackEnabled: true, // [#152115927] TODO: setting `isManualIntegration: false` will fail trying to send an email // to the service principal user. I guess this is a bug in the Azure APIs isManualIntegration: true, isMercurial: false, - repoUrl: config.appServicePortalGitRepo, + repoUrl: config.appGitRepo, type: "GitHub" }; winston.info( - `Configuring Git integration for the Developer Portal application: ${ - config.appServicePortalGitRepo - }#${config.appServicePortalGitBranch}` + `Configuring Git integration for the application: ${ + config.appGitRepo + }#${config.appGitBranch}` ); // Create git integration return webSiteClient.webApps.createOrUpdateSourceControl( config.resourceGroupName, - config.appServicePortalName, + config.appName, siteSourceControl ); }; @@ -71,7 +61,7 @@ const argv = yargs .alias("g", "resource-group-name") .demandOption("g") .string("g") - .alias("n", "appservice-portal-name") + .alias("n", "app-name") .demandOption("n") .string("n") .alias("r", "git-repo") @@ -82,16 +72,14 @@ const argv = yargs .string("b").argv; run({ - appServicePortalGitBranch: argv.b as string, - appServicePortalGitRepo: argv.r as string, - appServicePortalName: argv.n as string, + appGitBranch: argv.b as string, + appGitRepo: argv.r as string, + appName: argv.n as string, resourceGroupName: argv.g as string }) .then(r => { if (r) { - winston.info( - "Successfully synced developer portal webapp with source control" - ); + winston.info("Successfully synced app with source control"); } else { winston.warn("Nothing happened"); } diff --git a/infrastructure/tasks/10-functions_setup.ts b/infrastructure/tasks/10-functions_setup.ts deleted file mode 100644 index 1c6c1dd4..00000000 --- a/infrastructure/tasks/10-functions_setup.ts +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Run this task to deploy Azure Functions: - * - * yarn resources:functions:setup - * - * This task assumes that the following resources are already created: - * - Resource group - * - CosmoDB database - * - App service plan - * - Storage account - * - Storage Blob container - * - AppInsights instance - * - */ -// tslint:disable:no-console -// tslint:disable:no-any - -import * as winston from "winston"; -import { login } from "../../lib/login"; - -import { IResourcesConfiguration, readConfig } from "../../lib/config"; -import { checkEnvironment } from "../../lib/environment"; - -import storageManagementClient = require("azure-arm-storage"); -import webSiteManagementClient = require("azure-arm-website"); - -import AppInsights = require("azure-arm-appinsights"); -import CosmosDBManagementClient = require("azure-arm-cosmosdb"); - -const getAppServicePlan = async ( - client: any, - config: IResourcesConfiguration -) => { - return await client.appServicePlans.get( - config.azurerm_resource_group, - config.azurerm_app_service_plan - ); -}; - -const getAppInsightsKey = async ( - appInsightsClient: AppInsights, - config: IResourcesConfiguration -) => { - const appInsightsInstance = await appInsightsClient.components.get( - config.azurerm_resource_group, - config.azurerm_application_insights - ); - return appInsightsInstance.instrumentationKey; -}; - -export const run = async (config: IResourcesConfiguration) => { - const loginCreds = await login(); - - winston.info("Get AppInsights key to populate Functions settings"); - - // Get AppInsights instrumentation key - const appInsightsClient = new AppInsights( - loginCreds.creds as any, - loginCreds.subscriptionId - ); - const appInsightsKey = await getAppInsightsKey(appInsightsClient, config); - - winston.info( - "Get Storage account (Functions) connection string to populate Functions settings" - ); - - // Needed to get storage account connection string - const storageClient = new storageManagementClient( - loginCreds.creds as any, - loginCreds.subscriptionId - ); - - const storageAccountKeys = await storageClient.storageAccounts.listKeys( - config.azurerm_resource_group, - config.azurerm_functionapp_storage_account - ); - if (!storageAccountKeys || !storageAccountKeys.keys) { - throw new Error("Functions storage account keys not found"); - } - // We finally got the storage account keys so we can build the connection string - // @see StorageAccountListKeysResult - const storageAccountMasterKey = storageAccountKeys.keys[0].value; - const storageConnectionString = - `DefaultEndpointsProtocol=https;AccountName=` + - `${config.azurerm_functionapp_storage_account};AccountKey=${ - storageAccountMasterKey - };EndpointSuffix=core.windows.net`; - - winston.info( - "Get Storage account (Queues, Blob) connection string to populate Functions settings" - ); - - // Functions storage account and Queues storage account differs - const queueStorageAccountKeys = await storageClient.storageAccounts.listKeys( - config.azurerm_resource_group, - config.azurerm_storage_account - ); - if (!queueStorageAccountKeys || !queueStorageAccountKeys.keys) { - throw new Error("Queues storage account keys not found"); - } - // We finally got the storage account keys so we can build the connection string - // @see StorageAccountListKeysResult - const queueStorageAccountMasterKey = queueStorageAccountKeys.keys[0].value; - const queueStorageConnectionString = - `DefaultEndpointsProtocol=https;AccountName=` + - `${config.azurerm_storage_account};AccountKey=${ - queueStorageAccountMasterKey - };EndpointSuffix=core.windows.net`; - - winston.info("Get CosmosDB connection string to populate Functions settings"); - - // Get CosmosDB key and url - const cosmosClient = new CosmosDBManagementClient( - (loginCreds as any).creds, - loginCreds.subscriptionId - ); - const keys = await cosmosClient.databaseAccounts.listKeys( - config.azurerm_resource_group, - config.azurerm_cosmosdb - ); - const cosmosdbKey = keys.primaryMasterKey; - const cosmosdbLink = `https://${ - config.azurerm_cosmosdb - }.documents.azure.com:443/`; - - winston.info("Create Function APP Service Plan"); - - // Create web app (functions) - const webSiteClient = new webSiteManagementClient( - loginCreds.creds as any, - loginCreds.subscriptionId - ); - const servicePlan = await getAppServicePlan(webSiteClient, config); - - const appConfig = { - kind: "functionapp", - location: config.location, - serverFarmId: servicePlan.id, - siteConfig: { - // We don't want the express server to idle, - // so do not set `alwaysOn: false` in production - alwaysOn: true, - // You may want to set up an `apiDefinition.url` as well - // to share OpenAPI specs with API manager - appSettings: [ - // Mandatory parameters - { name: "AzureWebJobsStorage", value: storageConnectionString }, - { name: "AzureWebJobsDashboard", value: storageConnectionString }, - // The following two have fixed values - { - name: "WEBSITE_NODE_DEFAULT_VERSION", - value: config.functionapp_nodejs_version - }, - { name: "FUNCTIONS_EXTENSION_VERSION", value: "~1" }, - // optional parameters - { - name: "COSMOSDB_NAME", - value: config.azurerm_cosmosdb_documentdb - }, - { name: "QueueStorageConnection", value: queueStorageConnectionString }, - { name: "APPINSIGHTS_INSTRUMENTATIONKEY", value: appInsightsKey }, - // Avoid edit functions code from the Azure portal - { name: "FUNCTION_APP_EDIT_MODE", value: "readonly" }, - // AzureWebJobsSecretStorageType may be `disabled` or `Blob` - // When set to `Blob` the API manager task won't be able - // to retrieve the master key - { name: "AzureWebJobsSecretStorageType", value: "disabled" }, - { name: "WEBSITE_HTTPLOGGING_RETENTION_DAYS", value: "3" }, - { name: "SCM_USE_FUNCPACK_BUILD", value: "1" }, - { - name: "MESSAGE_CONTAINER_NAME", - value: config.message_blob_container - } - ], - connectionStrings: [ - { - // [#152800384] - TODO: change the following value - // when we'll migrate to production service - connectionString: process.env.SENDGRID_KEY, - name: "SENDGRID_KEY", - type: "Custom" - }, - { - connectionString: cosmosdbLink, - name: "COSMOSDB_URI", - type: "Custom" - }, - { - connectionString: cosmosdbKey, - name: "COSMOSDB_KEY", - type: "Custom" - } - ] - } - }; - - winston.info("Create Function production slot"); - - // Create production slot - const createdFunction = await webSiteClient.webApps.createOrUpdate( - config.azurerm_resource_group, - config.azurerm_functionapp, - appConfig - ); - - winston.info("Create Function staging slot"); - - // Create staging slot - if (config.azurerm_functionapp_slot && createdFunction.id) { - await webSiteClient.webApps.createOrUpdateSlot( - config.azurerm_resource_group, - config.azurerm_functionapp, - appConfig, - config.azurerm_functionapp_slot - ); - } - - const siteSourceControl = { - branch: config.functionapp_git_branch, - deploymentRollbackEnabled: true, - // [#152115927] TODO: setting `isManualIntegration: false` will fail trying to send an email - // to the service principal user. I guess this is a bug in the Azure APIs - isManualIntegration: true, - isMercurial: false, - repoUrl: config.functionapp_git_repo, - type: config.functionapp_scm_type - }; - - winston.info("Setup Git integration for Function staging slot"); - - // Create git integration for the staging slot - if (config.azurerm_functionapp_slot && config.functionapp_git_repo) { - await webSiteClient.webApps.createOrUpdateSourceControlSlot( - config.azurerm_resource_group, - config.azurerm_functionapp, - siteSourceControl, - config.azurerm_functionapp_slot - ); - } - - winston.info("Setup Git integration for Function production slot"); - - // Create git integration for the production slot - if (createdFunction.id && config.functionapp_git_repo) { - await webSiteClient.webApps.createOrUpdateSourceControl( - config.azurerm_resource_group, - config.azurerm_functionapp, - siteSourceControl - ); - } -}; - -checkEnvironment() - .then(() => readConfig(process.env.ENVIRONMENT)) - .then(run) - .then(() => winston.info("Successfully deployed Functions app")) - .catch((e: Error) => console.error(process.env.VERBOSE ? e : e.message)); diff --git a/package.json b/package.json index 0af4ac18..3529d95e 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "resources:tf-init": "env-cmd .env cross-var terraform init -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json -backend-config=\"infrastructure/env/$ENVIRONMENT/backend.tf\" infrastructure", "resources:tf-plan": "env-cmd .env cross-var terraform plan -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", "resources:tf-apply": "env-cmd .env cross-var terraform apply -var-file=infrastructure/env/common/tfvars.json -var-file=infrastructure/env/$ENVIRONMENT/tfvars.json infrastructure", - "resources:functions:setup": "ts-node infrastructure/tasks/10-functions_setup.ts", "deploy:functions:sync": "ts-node infrastructure/tasks/15-functions_sync.ts", "resources:apim:setup": "ts-node infrastructure/tasks/20-apim_setup.ts", "resources:apim:logger": "ts-node infrastructure/tasks/21-apim_logger.ts", diff --git a/terraform.d/plugins/darwin_amd64/terraform-provider-azurerm_v1.0.1_x5 b/terraform.d/plugins/darwin_amd64/terraform-provider-azurerm_v1.0.1_x5 new file mode 100755 index 00000000..d1978d64 Binary files /dev/null and b/terraform.d/plugins/darwin_amd64/terraform-provider-azurerm_v1.0.1_x5 differ