diff --git a/mmv1/third_party/terraform/.teamcity/components/build_config_package.kt b/mmv1/third_party/terraform/.teamcity/components/build_config_package.kt index e5e0993449e5..d6136cb22dc4 100644 --- a/mmv1/third_party/terraform/.teamcity/components/build_config_package.kt +++ b/mmv1/third_party/terraform/.teamcity/components/build_config_package.kt @@ -3,11 +3,14 @@ import jetbrains.buildServer.configs.kotlin.* import jetbrains.buildServer.configs.kotlin.AbsoluteId -class packageDetails(name: String, displayName: String, environment: String) { +class packageDetails(name: String, displayName: String, environment: String, branchRef: String) { val packageName = name val displayName = displayName val environment = environment + val branchRef = branchRef + // buildConfiguration returns a BuildType for a service package + // For BuildType docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/build-type/index.html fun buildConfiguration(providerName : String, path : String, manualVcsRoot: AbsoluteId, nightlyTestsEnabled: Boolean, startHour: Int, parallelism: Int, daysOfWeek: String, daysOfMonth: String) : BuildType { return BuildType { // TC needs a consistent ID for dynamically generated packages @@ -50,12 +53,18 @@ class packageDetails(name: String, displayName: String, environment: String) { } triggers { - RunNightly(nightlyTestsEnabled, startHour, daysOfWeek, daysOfMonth) + RunNightly(nightlyTestsEnabled, startHour, daysOfWeek, daysOfMonth, branchRef) } } } fun uniqueID(provider : String) : String { - return "%s_SERVICE_%s_%s".format(provider.replace("-", "").toUpperCase(), environment.toUpperCase(), packageName.toUpperCase()) + // Replacing chars can be necessary, due to limitations on IDs + // "ID should start with a latin letter and contain only latin letters, digits and underscores (at most 225 characters)." + var pv = provider.replace("-", "").toUpperCase() + var env = environment.toUpperCase().replace("-", "").replace(".", "").toUpperCase() + var pkg = packageName.toUpperCase() + + return "%s_SERVICE_%s_%s".format(pv, env, pkg) } } diff --git a/mmv1/third_party/terraform/.teamcity/components/build_google.kt b/mmv1/third_party/terraform/.teamcity/components/build_google.kt index 4baaddfc5225..d0209eb36cdd 100644 --- a/mmv1/third_party/terraform/.teamcity/components/build_google.kt +++ b/mmv1/third_party/terraform/.teamcity/components/build_google.kt @@ -19,6 +19,9 @@ class ClientConfiguration(var custId: String, val identityUser : String ) { } +// ParametrizedWithType.ConfigureGoogleSpecificTestParameters allows build configs to be created +// with the environment variables needed to configure the provider and/or configure test code. +// Extension of ParametrizedWithType. For docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/parametrized-with-type/index.html fun ParametrizedWithType.ConfigureGoogleSpecificTestParameters(config: ClientConfiguration) { hiddenPasswordVariable("env.GOOGLE_CUST_ID", config.custId, "The ID of the Google Identity Customer") hiddenPasswordVariable("env.GOOGLE_ORG", config.org, "The Google Organization Id") diff --git a/mmv1/third_party/terraform/.teamcity/components/generated/build_components.erb b/mmv1/third_party/terraform/.teamcity/components/generated/build_components.erb index 2539ba3970e3..5e8b2e035109 100644 --- a/mmv1/third_party/terraform/.teamcity/components/generated/build_components.erb +++ b/mmv1/third_party/terraform/.teamcity/components/generated/build_components.erb @@ -13,6 +13,15 @@ import jetbrains.buildServer.configs.kotlin.triggers.schedule // // Until that changes, we'll continue to use `teamcity-go-test` to run // each test individually + +// NOTE: this file includes Extensions of Kotlin DSL classes +// See +// - BuildFeatures https://teamcity.jetbrains.com/app/dsl-documentation/root/build-features/index.html +// - BuildSteps https://teamcity.jetbrains.com/app/dsl-documentation/root/build-steps/index.html +// - ParametrizedWithType https://teamcity.jetbrains.com/app/dsl-documentation/root/parametrized-with-type/index.html +// - Triggers https://teamcity.jetbrains.com/app/dsl-documentation/root/triggers/index.html + + const val useTeamCityGoTest = false fun BuildFeatures.Golang() { @@ -121,10 +130,12 @@ fun ParametrizedWithType.hiddenPasswordVariable(name: String, value: String, des password(name, value, "", description, ParameterDisplay.HIDDEN) } -fun Triggers.RunNightly(nightlyTestsEnabled: Boolean, startHour: Int, daysOfWeek: String, daysOfMonth: String) { +fun Triggers.RunNightly(nightlyTestsEnabled: Boolean, startHour: Int, daysOfWeek: String, daysOfMonth: String, branchRef: String) { + val filter = "+:" + branchRef // e.g. "+:refs/heads/main" + schedule{ enabled = nightlyTestsEnabled - branchFilter = "+:refs/heads/main" + branchFilter = filter schedulingPolicy = cron { hours = startHour.toString() diff --git a/mmv1/third_party/terraform/.teamcity/components/generated/project.erb b/mmv1/third_party/terraform/.teamcity/components/generated/project.erb index 67ce34f9bc60..94997bbffcd5 100644 --- a/mmv1/third_party/terraform/.teamcity/components/generated/project.erb +++ b/mmv1/third_party/terraform/.teamcity/components/generated/project.erb @@ -7,28 +7,33 @@ import jetbrains.buildServer.configs.kotlin.AbsoluteId const val providerName = "google<%= "-" + version unless version == 'ga' -%>" -fun Google<%= version.capitalize unless version == 'ga' -%>(environment: String, manualVcsRoot: AbsoluteId, configuration: ClientConfiguration) : Project { +// Google<%= version.capitalize unless version == 'ga' -%> returns an instance of Project, +// which has multiple build configurations defined within it. +// See https://teamcity.jetbrains.com/app/dsl-documentation/root/project/index.html +fun Google<%= version.capitalize unless version == 'ga' -%>(environment: String, manualVcsRoot: AbsoluteId, branchRef: String, configuration: ClientConfiguration) : Project { return Project{ - var buildConfigs = buildConfigurationsForPackages(packages, providerName, "google<%= "-" + version unless version == 'ga' -%>", environment, manualVcsRoot, configuration) + var buildConfigs = buildConfigurationsForPackages(packages, providerName, "google<%= "-" + version unless version == 'ga' -%>", environment, manualVcsRoot, branchRef, configuration) buildConfigs.forEach { buildConfiguration -> buildType(buildConfiguration) } } } -fun buildConfigurationsForPackages(packages: Map, providerName : String, path : String, environment: String, manualVcsRoot: AbsoluteId, config: ClientConfiguration): List { +fun buildConfigurationsForPackages(packages: Map, providerName : String, path : String, environment: String, manualVcsRoot: AbsoluteId, branchRef: String, config: ClientConfiguration): List { var list = ArrayList() packages.forEach { (packageName, displayName) -> if (packageName == "services") { - var serviceList = buildConfigurationsForPackages(services, providerName, path+"/"+packageName, environment, manualVcsRoot, config) + // `services` is a folder containing packages, not a package itself; call buildConfigurationsForPackages to iterate through directories found within `services` + var serviceList = buildConfigurationsForPackages(services, providerName, path+"/"+packageName, environment, manualVcsRoot, branchRef, config) list.addAll(serviceList) } else { - var defaultTestConfig = testConfiguration() + // other folders assumed to be packages + var testConfig = testConfiguration(environment) - var pkg = packageDetails(packageName, displayName, environment) - var buildConfig = pkg.buildConfiguration(providerName, path, manualVcsRoot, true, defaultTestConfig.startHour, defaultTestConfig.parallelism, defaultTestConfig.daysOfWeek, defaultTestConfig.daysOfMonth) + var pkg = packageDetails(packageName, displayName, environment, branchRef) + var buildConfig = pkg.buildConfiguration(providerName, path, manualVcsRoot, true, testConfig.startHour, testConfig.parallelism, testConfig.daysOfWeek, testConfig.daysOfMonth) buildConfig.params.ConfigureGoogleSpecificTestParameters(config) @@ -39,9 +44,27 @@ fun buildConfigurationsForPackages(packages: Map, providerName : return list } -class testConfiguration(parallelism: Int = defaultParallelism, startHour: Int = defaultStartHour, daysOfWeek: String = defaultDaysOfWeek, daysOfMonth: String = defaultDaysOfMonth) { +class testConfiguration(environment: String, parallelism: Int = defaultParallelism, startHour: Int = defaultStartHour, daysOfWeek: String = defaultDaysOfWeek, daysOfMonth: String = defaultDaysOfMonth) { + + // Default values are present if init doesn't change them var parallelism = parallelism var startHour = startHour var daysOfWeek = daysOfWeek var daysOfMonth = daysOfMonth + + init { + // If the environment parameter is set to the value of MAJOR_RELEASE_TESTING, + // change the days of week to the day for v5.0.0 feature branch testing + if (environment == MAJOR_RELEASE_TESTING) { + this.parallelism = parallelism + this.startHour = startHour +<% if version == 'ga' -%> + this.daysOfWeek = "4" // Thursday for GA +<% elsif version == 'beta' -%> + this.daysOfWeek = "5" // Friday for Beta +<% end -%> + this.daysOfMonth = daysOfMonth + } + } + } \ No newline at end of file diff --git a/mmv1/third_party/terraform/.teamcity/components/generated/settings.erb b/mmv1/third_party/terraform/.teamcity/components/generated/settings.erb new file mode 100644 index 000000000000..5ac811536355 --- /dev/null +++ b/mmv1/third_party/terraform/.teamcity/components/generated/settings.erb @@ -0,0 +1,25 @@ +<% autogen_exception -%> +// this file is auto-generated with mmv1, any changes made here will be overwritten + +// specifies the default hour (UTC) at which tests should be triggered, if enabled +var defaultStartHour = 4 + +// specifies the default level of parallelism per-service-package +var defaultParallelism = 12 + +// specifies the default version of Terraform Core which should be used for testing +var defaultTerraformCoreVersion = "1.2.5" + +// This represents a cron view of days of the week +<% if version == 'ga' -%> +const val defaultDaysOfWeek = "1-3,5-7" // All nights except Thursday for GA; feature branch testing happens on Thursdays +<% elsif version == 'beta' -%> +const val defaultDaysOfWeek = "1-4,6,7" // All nights except Friday for Beta; feature branch testing happens on Fridays +<% end -%> + +// Cron value for any day of month +const val defaultDaysOfMonth = "*" + +// Values that `environment` parameter is checked against, +// when deciding to change how TeamCity objects are configured +const val MAJOR_RELEASE_TESTING = "major-release-5.0.0-testing" \ No newline at end of file diff --git a/mmv1/third_party/terraform/.teamcity/components/settings.kt b/mmv1/third_party/terraform/.teamcity/components/settings.kt deleted file mode 100644 index 44f98ded7b4d..000000000000 --- a/mmv1/third_party/terraform/.teamcity/components/settings.kt +++ /dev/null @@ -1,16 +0,0 @@ -// this file is copied from mmv1, any changes made here will be overwritten - -// specifies the default hour (UTC) at which tests should be triggered, if enabled -var defaultStartHour = 4 - -// specifies the default level of parallelism per-service-package -var defaultParallelism = 12 - -// specifies the default version of Terraform Core which should be used for testing -var defaultTerraformCoreVersion = "1.2.5" - -// This represents a cron view of days of the week, Monday - Friday. -const val defaultDaysOfWeek = "*" - -// Cron value for any day of month -const val defaultDaysOfMonth = "*" diff --git a/mmv1/third_party/terraform/.teamcity/generated/pom.xml.erb b/mmv1/third_party/terraform/.teamcity/generated/pom.xml.erb index 640a4a7f7530..d5c86d528953 100644 --- a/mmv1/third_party/terraform/.teamcity/generated/pom.xml.erb +++ b/mmv1/third_party/terraform/.teamcity/generated/pom.xml.erb @@ -96,7 +96,7 @@ kotlin target/generated-configs - public + default diff --git a/mmv1/third_party/terraform/.teamcity/generated/settings.kts.erb b/mmv1/third_party/terraform/.teamcity/generated/settings.kts.erb index 09072a04e5ac..e3b810f86a41 100644 --- a/mmv1/third_party/terraform/.teamcity/generated/settings.kts.erb +++ b/mmv1/third_party/terraform/.teamcity/generated/settings.kts.erb @@ -8,6 +8,12 @@ import jetbrains.buildServer.configs.kotlin.* version = "2023.05" +// The code below pulls context parameters from the TeamCity project. +// Context parameters aren't stored in VCS, and are managed manually. +// Due to this, the code needs to explicitly pull in values via the DSL and pass the values into other code. +// For DslContext docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/dsl-context/index.html + +// Values of these parameters are used to set ENVs needed for acceptance tests within the build configurations. var custId = DslContext.getParameter("custId", "") var org = DslContext.getParameter("org", "") var org2 = DslContext.getParameter("org2", "") @@ -21,7 +27,6 @@ var region = DslContext.getParameter("region", "") var serviceAccount = DslContext.getParameter("serviceAccount", "") var zone = DslContext.getParameter("zone", "") var credentials = DslContext.getParameter("credentials", "") -var environment = DslContext.getParameter("environment", "public") var firestoreProject = DslContext.getParameter("firestoreProject", "") var identityUser = DslContext.getParameter("identityUser", "") @@ -29,6 +34,12 @@ var identityUser = DslContext.getParameter("identityUser", "") // connected to the Project in TeamCity var manualVcsRoot = DslContext.settingsRootId +// Values of these parameters change configuration code behaviour. +var environment = DslContext.getParameter("environment", "default") +var branchRef = DslContext.getParameter("branch", "refs/heads/main") + var clientConfig = ClientConfiguration(custId, org, org2, billingAccount, billingAccount2, masterBillingAccount, credentials, project, orgDomain, projectNumber, region, serviceAccount, zone, firestoreProject, identityUser) -project(Google<%= version.capitalize unless version == 'ga' -%>(environment, manualVcsRoot, clientConfig)) +// This is the entry point of the code in .teamcity/ +// See https://teamcity.jetbrains.com/app/dsl-documentation/root/project.html +project(Google<%= version.capitalize unless version == 'ga' -%>(environment, manualVcsRoot, branchRef, clientConfig)) diff --git a/mmv1/third_party/terraform/.teamcity/tests/generated/configuration.erb b/mmv1/third_party/terraform/.teamcity/tests/generated/configuration.erb index ed0f2d066726..13f45f0fb640 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/generated/configuration.erb +++ b/mmv1/third_party/terraform/.teamcity/tests/generated/configuration.erb @@ -12,7 +12,7 @@ import useTeamCityGoTest class ConfigurationTests { @Test fun buildShouldFailOnError() { - val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), TestConfiguration()) + val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), "refs/heads/main", TestConfiguration()) project.buildTypes.forEach { bt -> assertTrue("Build '${bt.id}' should fail on errors!", bt.failureConditions.errorMessage) } @@ -20,7 +20,7 @@ class ConfigurationTests { @Test fun buildShouldHaveGoTestFeature() { - val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), TestConfiguration()) + val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), "refs/heads/main",TestConfiguration()) project.buildTypes.forEach{ bt -> var exists = false bt.features.items.forEach { f -> @@ -37,7 +37,7 @@ class ConfigurationTests { @Test fun buildShouldHaveTrigger() { - val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), TestConfiguration()) + val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), "refs/heads/main", TestConfiguration()) var exists = false project.buildTypes.forEach{ bt -> bt.triggers.items.forEach { t -> diff --git a/mmv1/third_party/terraform/.teamcity/tests/generated/vcs_roots.erb b/mmv1/third_party/terraform/.teamcity/tests/generated/vcs_roots.erb index 7fe6d4990337..9c8c01f5506d 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/generated/vcs_roots.erb +++ b/mmv1/third_party/terraform/.teamcity/tests/generated/vcs_roots.erb @@ -11,7 +11,7 @@ import org.junit.Test class VcsTests { @Test fun buildsHaveCleanCheckOut() { - val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), TestConfiguration()) + val project = Google<%= version.capitalize unless version == 'ga' -%>("public", TestVcsRootId(), "refs/heads/main", TestConfiguration()) project.buildTypes.forEach { bt -> assertTrue("Build '${bt.id}' doesn't use clean checkout", bt.vcs.cleanCheckout) }