From a730a9004b7819a06f898b0b5b4b2a8ae3bc8a25 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Wed, 15 Apr 2020 06:47:11 -0700 Subject: [PATCH] Update whitelist includes (#9912) * Update whitelist includes * Lock whitelist entries to a specific version * Rebase to get checkstyle update and fix a grammar issue * rebase to get pom restructure changes --- common/perf-test-core/pom.xml | 5 +- eng/versioning/pom_file_version_scanner.ps1 | 84 ++++++++++++++++++- eng/versioning/update_versions.py | 30 +++++-- eng/versioning/utils.py | 23 +++++ sdk/core/azure-core-amqp/pom.xml | 4 +- sdk/core/azure-core-http-netty/pom.xml | 18 ++-- sdk/core/azure-core-http-okhttp/pom.xml | 2 +- sdk/core/azure-core-test/pom.xml | 6 +- .../azure-core-tracing-opentelemetry/pom.xml | 2 +- sdk/core/azure-core/pom.xml | 14 ++-- sdk/core/ci.yml | 2 + sdk/cosmos/azure-cosmos-benchmark/pom.xml | 24 +++--- sdk/cosmos/azure-cosmos-examples/pom.xml | 12 +-- sdk/cosmos/azure-cosmos/pom.xml | 36 ++++---- sdk/e2e/pom.xml | 2 +- sdk/identity/azure-identity/pom.xml | 8 +- sdk/parents/azure-client-sdk-parent/pom.xml | 2 +- sdk/storage/ci.yml | 3 + 18 files changed, 197 insertions(+), 80 deletions(-) diff --git a/common/perf-test-core/pom.xml b/common/perf-test-core/pom.xml index 5b7e49aab699e..8869785ece398 100644 --- a/common/perf-test-core/pom.xml +++ b/common/perf-test-core/pom.xml @@ -66,11 +66,10 @@ com.azure:* - org.slf4j - com.fasterxml.jackson.* + com.fasterxml.jackson.core:jackson-databind:[2.10.1] - com.beust:jcommander + com.beust:jcommander:[1.58] diff --git a/eng/versioning/pom_file_version_scanner.ps1 b/eng/versioning/pom_file_version_scanner.ps1 index 2f995f1359cf9..02e33c9020876 100644 --- a/eng/versioning/pom_file_version_scanner.ps1 +++ b/eng/versioning/pom_file_version_scanner.ps1 @@ -44,6 +44,11 @@ $DependencyTypeForError = "$($DependencyTypeCurrent)|$($DependencyTypeDependency $UpdateTagFormat = "{x-version-update;:;$($DependencyTypeForError)}" $StartTime = $(get-date) +# This is the for the bannedDependencies include exceptions. All entries need to be of the +# form groupId:artifactId:[version] which locks to a specific version. The exception +# to this is the blanket, wildcard include for com.azure libraries. +$ComAzureWhitelistInclude = "com.azure:*" + function Write-Error-With-Color([string]$msg) { Write-Host "$($msg)" -ForegroundColor Red @@ -492,7 +497,84 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { $script:FoundError = $true Write-Error-With-Color "Error: Missing plugin version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be " } - } + } + + # This is for the whitelist dependencies. Fetch the banned dependencies + foreach($bannedDependencies in $xmlPomFile.GetElementsByTagName("bannedDependencies")) + { + # Include nodes will look like the following: + # groupId:artifactId:[version] + foreach($includeNode in $bannedDependencies.GetElementsByTagName("include")) + { + $rawIncludeText = $includeNode.InnerText.Trim() + $split = $rawIncludeText.Split(":") + if ($split.Count -eq 3) + { + $groupId = $split[0] + $artifactId = $split[1] + $version = $split[2] + # The groupId match has to be able to deal with _ for external dependency exceptions + if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment") + { + $script:FoundError = $true + Write-Error-With-Color "Error: is missing the update tag which should be " + } + elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(\w+)?$($groupId):$($artifactId);external_dependency}") + { + $script:FoundError = $true + Write-Error-With-Color "Error: version update tag for $($includeNode.InnerText) should be " + } + else + { + # verify that the version is formatted correctly + if (!$version.StartsWith("[") -or !$version.EndsWith("]")) + { + $script:FoundError = $true + Write-Error-With-Color "Error: the version entry '$($version)' for '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[]', the braces lock the include to a specific version for these entries. -->" + } + # verify the version has the correct value + else + { + $versionWithoutBraces = $version.Substring(1, $version.Length -2) + # the key into the dependency has needs to be created from the tag's group/artifact + # entries in case it's an external dependency entry. Because this has already + # been validated for format, grab the group:artifact + $depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1] + if ($extDepHash.ContainsKey($depKey)) + { + if ($versionWithoutBraces -ne $extDepHash[$depKey].ver) + { + $script:FoundError = $true + Write-Error-With-Color "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)" + } + } + else + { + $script:FoundError = $true + Write-Error-With-Color "Error: the groupId:artifactId entry '$($depKey)' for '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->" + } + } + } + } + # The only time a split count of 2 is allowed is in the following case. + # com.azure:* + # These entries will not and should not have an update tag + elseif ($split.Count -eq 2) + { + if ($rawIncludeText -ne $ComAzureWhitelistInclude) + { + $script:FoundError = $true + Write-Error-With-Color "Error: $($rawIncludeText) is not a valid entry. With the exception of the $($ComAzureWhitelistInclude), every entry must be of the form groupId:artifactId:[version]" + } + } + else + { + # At this point the include entry is wildly incorrect. + $script:FoundError = $true + Write-Error-With-Color "Error: $($rawIncludeText) is not a valid entry. Every entry must be of the form groupId:artifactId:[version]" + } + } + } } $ElapsedTime = $(get-date) - $StartTime $TotalRunTime = "{0:HH:mm:ss}" -f ([datetime]$ElapsedTime.Ticks) diff --git a/eng/versioning/update_versions.py b/eng/versioning/update_versions.py index 639db3fd7f94c..1ad60482f4756 100644 --- a/eng/versioning/update_versions.py +++ b/eng/versioning/update_versions.py @@ -40,8 +40,10 @@ from utils import BuildType from utils import CodeModule from utils import external_dependency_version_regex +from utils import external_dependency_include_regex from utils import run_check_call from utils import UpdateType +from utils import include_update_marker from utils import version_regex_str_no_anchor from utils import version_update_start_marker from utils import version_update_end_marker @@ -51,16 +53,22 @@ def update_versions(update_type, version_map, ext_dep_map, target_file, skip_readme, auto_version_increment): newlines = [] - repl_open, repl_thisline, file_changed = False, False, False + repl_open, repl_thisline, file_changed, is_include = False, False, False, False print('processing: ' + target_file) try: with open(target_file, encoding='utf-8') as f: for line in f: + is_include = False repl_thisline = repl_open match = version_update_marker.search(line) if match and not target_file.endswith('.md'): module_name, version_type = match.group(1), match.group(2) repl_thisline = True + elif include_update_marker.search(line): + match = include_update_marker.search(line) + module_name, version_type = match.group(1), match.group(2) + repl_thisline = True + is_include = True else: match = version_update_start_marker.search(line) if match: @@ -100,12 +108,20 @@ def update_versions(update_type, version_map, ext_dep_map, target_file, skip_rea if update_type == UpdateType.library: newlines.append(line) continue - try: - module = ext_dep_map[module_name] - new_version = module.external_dependency - newline = re.sub(external_dependency_version_regex, new_version, line) - except AttributeError: - raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line)) + if is_include: + try: + module = ext_dep_map[module_name] + new_include_version = module.string_for_whitelist_include() + newline = re.sub(external_dependency_include_regex, new_include_version, line) + except AttributeError: + raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line)) + else: + try: + module = ext_dep_map[module_name] + new_version = module.external_dependency + newline = re.sub(external_dependency_version_regex, new_version, line) + except AttributeError: + raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line)) else: raise ValueError('Invalid version type: {} for module: {}.\nFile={}\nLine={}'.format(version_type, module_name, target_file, line)) diff --git a/eng/versioning/utils.py b/eng/versioning/utils.py index 519365bf48a44..c8d49d7293175 100644 --- a/eng/versioning/utils.py +++ b/eng/versioning/utils.py @@ -8,6 +8,7 @@ import re from subprocess import check_call, CalledProcessError +include_update_marker = re.compile(r'\{x-include-update;([^;]+);([^}]+)\}') version_update_start_marker = re.compile(r'\{x-version-update-start;([^;]+);([^}]+)\}') version_update_end_marker = re.compile(r'\{x-version-update-end\}') version_update_marker = re.compile(r'\{x-version-update;([^;]+);([^}]+)\}') @@ -19,6 +20,10 @@ # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string version_regex_str_no_anchor = r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' +# External dependency versions do not have to match semver format and the semver regular expressions +# will partially match and produce some hilarious results. +external_dependency_include_regex = r'(?<=).+?(?=)' + # External dependency versions do not have to match semver format and the semver regular expressions # will partially match and produce some hilarious results. external_dependency_version_regex = r'(?<=).+?(?=)' @@ -99,6 +104,24 @@ def string_for_version_file(self): except AttributeError: return self.name + ';' + self.dependency + '\n' + # return the CodeModule string formatted for a whitelist include entry + # note: for whitelist includes the version needs to be braces in order for + # the version to be an explicit version. Without the braces a version + # would be treated as that version and above. For example: + # ::1.2 would be treated as 1.2 and above or equivalent to [1.2,) + def string_for_whitelist_include(self): + if hasattr(self, 'external_dependency'): + temp = self.name + # This is necessary to deal with the fact that external_dependencies can have + # '_' in them if they're an external dependency exception. Since the whitelist + # name needs to be the actual dependency, take everything after the _ which is + # the actual name + if '_' in temp: + temp = temp.split('_')[1] + return temp + ':[' + self.external_dependency + ']' + else: + raise ValueError('string_for_whitelist_include called on non-external_dependency: ' + self.name) + def run_check_call( command_array, working_directory, diff --git a/sdk/core/azure-core-amqp/pom.xml b/sdk/core/azure-core-amqp/pom.xml index 44cfc4a4f3dd0..aaa558def9d4a 100644 --- a/sdk/core/azure-core-amqp/pom.xml +++ b/sdk/core/azure-core-amqp/pom.xml @@ -128,8 +128,8 @@ com.azure:* - org.apache.qpid:proton-j - com.microsoft.azure:qpid-proton-j-extensions + com.microsoft.azure:qpid-proton-j-extensions:[1.2.2] + org.apache.qpid:proton-j:[0.33.2] diff --git a/sdk/core/azure-core-http-netty/pom.xml b/sdk/core/azure-core-http-netty/pom.xml index 92df5f4040d91..2555da3288b17 100644 --- a/sdk/core/azure-core-http-netty/pom.xml +++ b/sdk/core/azure-core-http-netty/pom.xml @@ -193,16 +193,14 @@ com.azure:* - - io.projectreactor.netty - - io.netty:netty-buffer - io.netty:netty-codec-http - io.netty:netty-codec-http2 - io.netty:netty-handler - io.netty:netty-handler-proxy - io.netty:netty-transport-native-unix-common - io.netty:netty-transport-native-epoll + io.netty:netty-buffer:[4.1.45.Final] + io.netty:netty-codec-http:[4.1.45.Final] + io.netty:netty-codec-http2:[4.1.45.Final] + io.netty:netty-handler:[4.1.45.Final] + io.netty:netty-handler-proxy:[4.1.45.Final] + io.netty:netty-transport-native-unix-common:[4.1.45.Final] + io.netty:netty-transport-native-epoll:[4.1.45.Final] + io.projectreactor.netty:reactor-netty:[0.9.5.RELEASE] diff --git a/sdk/core/azure-core-http-okhttp/pom.xml b/sdk/core/azure-core-http-okhttp/pom.xml index 8c6a778c07ec1..409787b3a7d4d 100644 --- a/sdk/core/azure-core-http-okhttp/pom.xml +++ b/sdk/core/azure-core-http-okhttp/pom.xml @@ -155,7 +155,7 @@ com.azure:* - com.squareup.okhttp3:okhttp + com.squareup.okhttp3:okhttp:[4.2.2] diff --git a/sdk/core/azure-core-test/pom.xml b/sdk/core/azure-core-test/pom.xml index 7dd010df23e7f..8e759cd8d8639 100644 --- a/sdk/core/azure-core-test/pom.xml +++ b/sdk/core/azure-core-test/pom.xml @@ -124,11 +124,9 @@ com.azure:* - org.slf4j - io.projectreactor - + io.projectreactor:reactor-test:[3.3.3.RELEASE] - org.junit.jupiter + org.junit.jupiter:junit-jupiter-api:[5.4.2] diff --git a/sdk/core/azure-core-tracing-opentelemetry/pom.xml b/sdk/core/azure-core-tracing-opentelemetry/pom.xml index c18bed373501a..084c745d0e11b 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/pom.xml +++ b/sdk/core/azure-core-tracing-opentelemetry/pom.xml @@ -95,7 +95,7 @@ com.azure:* - io.opentelemetry + io.opentelemetry:opentelemetry-api:[0.2.4] diff --git a/sdk/core/azure-core/pom.xml b/sdk/core/azure-core/pom.xml index 3bf7554c901d1..5e470ce2d8c4b 100644 --- a/sdk/core/azure-core/pom.xml +++ b/sdk/core/azure-core/pom.xml @@ -186,14 +186,12 @@ *:*:*:*:provided - org.slf4j:slf4j-api - io.projectreactor:reactor-core - io.netty:netty-tcnative-boringssl-static - - com.fasterxml.jackson.dataformat:jackson-dataformat-xml - com.fasterxml.jackson.datatype:jackson-datatype-jsr310 - - com.google.code.findbugs:jsr305 + io.netty:netty-tcnative-boringssl-static:[2.0.27.Final] + io.projectreactor:reactor-core:[3.3.3.RELEASE] + com.fasterxml.jackson.dataformat:jackson-dataformat-xml:[2.10.1] + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1] + com.google.code.findbugs:jsr305:[3.0.2] + org.slf4j:slf4j-api:[1.7.28] diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 05c5cd8ad9c17..7232c75107971 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -67,3 +67,5 @@ stages: AdditionalModules: - name: azure-data-appconfiguration groupId: com.azure + - name: perf-test-core + groupId: com.azure diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 8c8475169adfb..1735051c73d56 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -236,20 +236,20 @@ Licensed under the MIT License. com.azure:* - org.slf4j - io.dropwizard.metrics:metrics-core - com.beust:jcommander + com.beust:jcommander:[1.58] + io.dropwizard.metrics:metrics-core:[4.1.0] + org.slf4j:slf4j-api:[1.7.28] - com.google.guava:guava - io.micrometer:micrometer-registry-graphite - org.apache.commons:commons-lang3 - org.apache.logging.log4j:log4j-core - io.dropwizard.metrics:metrics-jvm - io.dropwizard.metrics:metrics-graphite - io.micrometer:micrometer-registry-azure-monitor - org.apache.logging.log4j:log4j-slf4j-impl - org.apache.logging.log4j:log4j-api:jar + com.google.guava:guava:[25.0-jre] + io.dropwizard.metrics:metrics-graphite:[4.1.0] + io.dropwizard.metrics:metrics-jvm:[4.1.0] + io.micrometer:micrometer-registry-azure-monitor:[1.2.0] + io.micrometer:micrometer-registry-graphite:[1.2.0] + org.apache.commons:commons-lang3:[3.8.1] + org.apache.logging.log4j:log4j-api:[2.11.1] + org.apache.logging.log4j:log4j-core:[2.11.1] + org.apache.logging.log4j:log4j-slf4j-impl:[2.13.0] diff --git a/sdk/cosmos/azure-cosmos-examples/pom.xml b/sdk/cosmos/azure-cosmos-examples/pom.xml index d7647a9d4d520..0ec10e90f719e 100644 --- a/sdk/cosmos/azure-cosmos-examples/pom.xml +++ b/sdk/cosmos/azure-cosmos-examples/pom.xml @@ -132,13 +132,15 @@ Licensed under the MIT License. com.azure:* - org.slf4j + org.slf4j:slf4j-api:[1.7.28] - com.google.guava:guava - commons-io:commons-io - org.apache.commons:commons-lang3 - org.apache.logging.log4j + com.google.guava:guava:[25.0-jre] + commons-io:commons-io:[2.5] + org.apache.commons:commons-lang3:[3.8.1] + org.apache.logging.log4j:log4j-api:[2.11.1] + org.apache.logging.log4j:log4j-core:[2.11.1] + org.apache.logging.log4j:log4j-slf4j-impl:[2.13.0] diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index 369afcb3e024a..f51f34df1f09b 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -313,26 +313,22 @@ Licensed under the MIT License. com.azure:* - org.slf4j:slf4j-api - io.projectreactor:reactor-core - - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-annotations - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.datatype:jackson-datatype-jsr310 - com.fasterxml.jackson.module:jackson-module-afterburner - - io.micrometer:micrometer-core - io.dropwizard.metrics:metrics-core - com.fasterxml.uuid:java-uuid-generator - io.projectreactor.netty - io.netty:netty-codec-http - io.netty:netty-codec-http2 - io.netty:netty-handler - io.netty:netty-handler-proxy - io.netty:netty-transport-native-epoll - - com.google.code.findbugs:jsr305 + com.fasterxml.jackson.core:jackson-core:[2.10.1] + com.fasterxml.jackson.core:jackson-annotations:[2.10.1] + com.fasterxml.jackson.core:jackson-databind:[2.10.1] + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1] + com.fasterxml.jackson.module:jackson-module-afterburner:[2.10.1] + com.google.code.findbugs:jsr305:[3.0.2] + io.dropwizard.metrics:metrics-core:[4.1.0] + io.micrometer:micrometer-core:[1.2.0] + io.netty:netty-codec-http:[4.1.45.Final] + io.netty:netty-codec-http2:[4.1.45.Final] + io.netty:netty-handler:[4.1.45.Final] + io.netty:netty-handler-proxy:[4.1.45.Final] + io.netty:netty-transport-native-epoll:[4.1.45.Final] + io.projectreactor:reactor-core:[3.3.3.RELEASE] + io.projectreactor.netty:reactor-netty:[0.9.5.RELEASE] + org.slf4j:slf4j-api:[1.7.28] diff --git a/sdk/e2e/pom.xml b/sdk/e2e/pom.xml index c9cd0ffe4c308..3b9e0e20f47ae 100644 --- a/sdk/e2e/pom.xml +++ b/sdk/e2e/pom.xml @@ -101,7 +101,7 @@ com.azure:* - org.slf4j:slf4j-api + org.slf4j:slf4j-api:[1.7.28] diff --git a/sdk/identity/azure-identity/pom.xml b/sdk/identity/azure-identity/pom.xml index a4526de7decc6..4023f54d1a427 100644 --- a/sdk/identity/azure-identity/pom.xml +++ b/sdk/identity/azure-identity/pom.xml @@ -120,10 +120,10 @@ com.azure:* - com.nimbusds:oauth2-oidc-sdk - com.microsoft.azure:msal4j - org.nanohttpd:nanohttpd - net.java.dev.jna + com.microsoft.azure:msal4j:[1.3.0] + com.nimbusds:oauth2-oidc-sdk:[6.14] + net.java.dev.jna:jna-platform:[5.4.0] + org.nanohttpd:nanohttpd:[2.3.1] diff --git a/sdk/parents/azure-client-sdk-parent/pom.xml b/sdk/parents/azure-client-sdk-parent/pom.xml index 2a571b8a21138..c6e36506fd7e3 100644 --- a/sdk/parents/azure-client-sdk-parent/pom.xml +++ b/sdk/parents/azure-client-sdk-parent/pom.xml @@ -485,7 +485,7 @@ com.azure:* - com.google.code.findbugs:jsr305 + com.google.code.findbugs:jsr305:[3.0.2] diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 94585e0dcd624..87d979c0738ad 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -79,3 +79,6 @@ stages: - name: azure-storage-queue groupId: com.azure safeName: azurestoragequeue + AdditionalModules: + - name: azure-storage-perf + groupId: com.azure