Update whitelist includes (#9912)

* Update whitelist includes

* Lock whitelist entries to a specific version

* Rebase to get checkstyle update and fix a grammar issue

* rebase to get pom restructure changes

common/perf-test-core/pom.xml

@@ -66,11 +66,10 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-                    <include>org.slf4j</include>
-                    <include>com.fasterxml.jackson.*</include>
+                    <include>com.fasterxml.jackson.core:jackson-databind:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.core:jackson-databind;external_dependency} -->
 
                     <!-- special allowance for perf-test-core as it is not a shipping library: -->
-                    <include>com.beust:jcommander</include>
+                    <include>com.beust:jcommander:[1.58]</include> <!-- {x-include-update;com.beust:jcommander;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

eng/versioning/pom_file_version_scanner.ps1

@@ -44,6 +44,11 @@ $DependencyTypeForError = "$($DependencyTypeCurrent)|$($DependencyTypeDependency
 $UpdateTagFormat = "{x-version-update;<groupId>:<artifactId>;$($DependencyTypeForError)}"
 $StartTime = $(get-date)
 
+# This is the for the bannedDependencies include exceptions. All <include> entries need to be of the
+# form <include>groupId:artifactId:[version]</include> which locks to a specific version. The exception
+# to this is the blanket, wildcard include for com.azure libraries.
+$ComAzureWhitelistInclude = "com.azure:*"
+
 function Write-Error-With-Color([string]$msg)
 {
     Write-Host "$($msg)" -ForegroundColor Red
@@ -492,7 +497,84 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
             $script:FoundError = $true
             Write-Error-With-Color "Error: Missing plugin version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
         }
-    }    
+    }
+
+    # This is for the whitelist dependencies. Fetch the banned dependencies
+    foreach($bannedDependencies in $xmlPomFile.GetElementsByTagName("bannedDependencies"))
+    {
+        # Include nodes will look like the following:
+        # <include>groupId:artifactId:[version]</include> <!-- {x-include-update;groupId:artifactId;external_dependency} -->
+        foreach($includeNode in $bannedDependencies.GetElementsByTagName("include"))
+        {
+            $rawIncludeText = $includeNode.InnerText.Trim()
+            $split = $rawIncludeText.Split(":")
+            if ($split.Count -eq 3)
+            {
+                $groupId = $split[0]
+                $artifactId = $split[1]
+                $version = $split[2]
+                # The groupId match has to be able to deal with <area>_ for external dependency exceptions
+                if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
+                {
+                    $script:FoundError = $true
+                    Write-Error-With-Color "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);external_dependency} -->"
+                }
+                elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(\w+)?$($groupId):$($artifactId);external_dependency}")
+                {
+                    $script:FoundError = $true
+                    Write-Error-With-Color "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);external_dependency} -->"
+                }
+                else
+                {
+                    # verify that the version is formatted correctly
+                    if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
+                    {
+                        $script:FoundError = $true
+                        Write-Error-With-Color "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
+                    }
+                    # verify the version has the correct value
+                    else
+                    {
+                        $versionWithoutBraces = $version.Substring(1, $version.Length -2)
+                        # the key into the dependency has needs to be created from the tag's group/artifact
+                        # entries in case it's an external dependency entry. Because this has already
+                        # been validated for format, grab the group:artifact
+                        $depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
+                        if ($extDepHash.ContainsKey($depKey))
+                        {
+                            if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
+                            {
+                                $script:FoundError = $true
+                                Write-Error-With-Color "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
+                            }
+                        }
+                        else
+                        {
+                            $script:FoundError = $true
+                            Write-Error-With-Color "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
+                        }
+                    }
+                }
+            }
+            # The only time a split count of 2 is allowed is in the following case.
+            # <include>com.azure:*</include>
+            # These entries will not and should not have an update tag
+            elseif ($split.Count -eq 2)
+            {
+                if ($rawIncludeText -ne $ComAzureWhitelistInclude)
+                {
+                    $script:FoundError = $true
+                    Write-Error-With-Color "Error:  $($rawIncludeText) is not a valid <include> entry. With the exception of the $($ComAzureWhitelistInclude), every <include> entry must be of the form <include>groupId:artifactId:[version]<include>"
+                }
+            }
+            else
+            {
+                # At this point the include entry is wildly incorrect.
+                $script:FoundError = $true
+                Write-Error-With-Color "Error:  $($rawIncludeText) is not a valid <include> entry. Every <include> entry must be of the form <include>groupId:artifactId:[version]<include>"
+            }
+        }
+    }
 }
 $ElapsedTime = $(get-date) - $StartTime
 $TotalRunTime = "{0:HH:mm:ss}" -f ([datetime]$ElapsedTime.Ticks)

eng/versioning/update_versions.py

@@ -40,8 +40,10 @@
 from utils import BuildType
 from utils import CodeModule
 from utils import external_dependency_version_regex
+from utils import external_dependency_include_regex
 from utils import run_check_call
 from utils import UpdateType
+from utils import include_update_marker
 from utils import version_regex_str_no_anchor
 from utils import version_update_start_marker
 from utils import version_update_end_marker
@@ -51,16 +53,22 @@
 def update_versions(update_type, version_map, ext_dep_map, target_file, skip_readme, auto_version_increment):
 
     newlines = []
-    repl_open, repl_thisline, file_changed = False, False, False
+    repl_open, repl_thisline, file_changed, is_include = False, False, False, False
     print('processing: ' + target_file)
     try:
         with open(target_file, encoding='utf-8') as f:
             for line in f:
+                is_include = False
                 repl_thisline = repl_open
                 match = version_update_marker.search(line)
                 if match and not target_file.endswith('.md'):
                     module_name, version_type = match.group(1), match.group(2)
                     repl_thisline = True
+                elif include_update_marker.search(line):
+                    match = include_update_marker.search(line)
+                    module_name, version_type = match.group(1), match.group(2)
+                    repl_thisline = True
+                    is_include = True
                 else:
                     match = version_update_start_marker.search(line)
                     if match:
@@ -100,12 +108,20 @@ def update_versions(update_type, version_map, ext_dep_map, target_file, skip_rea
                         if update_type == UpdateType.library:
                             newlines.append(line)
                             continue
-                        try:
-                            module = ext_dep_map[module_name]
-                            new_version = module.external_dependency
-                            newline = re.sub(external_dependency_version_regex, new_version, line)
-                        except AttributeError:
-                            raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
+                        if is_include:
+                            try:
+                                module = ext_dep_map[module_name]
+                                new_include_version = module.string_for_whitelist_include()
+                                newline = re.sub(external_dependency_include_regex, new_include_version, line)
+                            except AttributeError:
+                                raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
+                        else:
+                            try:
+                                module = ext_dep_map[module_name]
+                                new_version = module.external_dependency
+                                newline = re.sub(external_dependency_version_regex, new_version, line)
+                            except AttributeError:
+                                raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
                     else:
                         raise ValueError('Invalid version type: {} for module: {}.\nFile={}\nLine={}'.format(version_type, module_name, target_file, line))
 

eng/versioning/utils.py

@@ -8,6 +8,7 @@
 import re
 from subprocess import check_call, CalledProcessError
 
+include_update_marker = re.compile(r'\{x-include-update;([^;]+);([^}]+)\}')
 version_update_start_marker = re.compile(r'\{x-version-update-start;([^;]+);([^}]+)\}')	
 version_update_end_marker = re.compile(r'\{x-version-update-end\}')
 version_update_marker = re.compile(r'\{x-version-update;([^;]+);([^}]+)\}')
@@ -19,6 +20,10 @@
 # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
 version_regex_str_no_anchor = r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?'
 
+# External dependency versions do not have to match semver format and the semver regular expressions
+# will partially match and produce some hilarious results.
+external_dependency_include_regex = r'(?<=<include>).+?(?=</include>)'
+
 # External dependency versions do not have to match semver format and the semver regular expressions
 # will partially match and produce some hilarious results.
 external_dependency_version_regex = r'(?<=<version>).+?(?=</version>)'
@@ -99,6 +104,24 @@ def string_for_version_file(self):
         except AttributeError:
             return self.name + ';' + self.dependency + '\n'
 
+    # return the CodeModule string formatted for a whitelist include entry
+    # note: for whitelist includes the version needs to be braces in order for
+    # the version to be an explicit version. Without the braces a version
+    # would be treated as that version and above. For example: 
+    # <groupId>:<artifactId>:1.2 would be treated as 1.2 and above or equivalent to [1.2,)
+    def string_for_whitelist_include(self):
+        if hasattr(self, 'external_dependency'):
+            temp = self.name
+            # This is necessary to deal with the fact that external_dependencies can have
+            # '_' in them if they're an external dependency exception. Since the whitelist
+            # name needs to be the actual dependency, take everything after the _ which is
+            # the actual name
+            if '_' in temp:
+                temp = temp.split('_')[1]
+            return temp + ':[' + self.external_dependency + ']'
+        else:
+            raise ValueError('string_for_whitelist_include called on non-external_dependency: ' + self.name)
+
 def run_check_call(
     command_array,
     working_directory,

sdk/core/azure-core-amqp/pom.xml

@@ -128,8 +128,8 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-                    <include>org.apache.qpid:proton-j</include>
-                    <include>com.microsoft.azure:qpid-proton-j-extensions</include>
+                    <include>com.microsoft.azure:qpid-proton-j-extensions:[1.2.2]</include> <!-- {x-include-update;com.microsoft.azure:qpid-proton-j-extensions;external_dependency} -->
+                    <include>org.apache.qpid:proton-j:[0.33.2]</include> <!-- {x-include-update;org.apache.qpid:proton-j;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/azure-core-http-netty/pom.xml

@@ -193,16 +193,14 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-
-                    <include>io.projectreactor.netty</include>
-
-                    <include>io.netty:netty-buffer</include>
-                    <include>io.netty:netty-codec-http</include>
-                    <include>io.netty:netty-codec-http2</include>
-                    <include>io.netty:netty-handler</include>
-                    <include>io.netty:netty-handler-proxy</include>
-                    <include>io.netty:netty-transport-native-unix-common</include>
-                    <include>io.netty:netty-transport-native-epoll</include>
+                    <include>io.netty:netty-buffer:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-buffer;external_dependency} -->
+                    <include>io.netty:netty-codec-http:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http;external_dependency} -->
+                    <include>io.netty:netty-codec-http2:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http2;external_dependency} -->
+                    <include>io.netty:netty-handler:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-handler;external_dependency} -->
+                    <include>io.netty:netty-handler-proxy:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-handler-proxy;external_dependency} -->
+                    <include>io.netty:netty-transport-native-unix-common:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-unix-common;external_dependency} -->
+                    <include>io.netty:netty-transport-native-epoll:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-epoll;external_dependency} -->
+                    <include>io.projectreactor.netty:reactor-netty:[0.9.5.RELEASE]</include> <!-- {x-include-update;io.projectreactor.netty:reactor-netty;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/azure-core-http-okhttp/pom.xml

@@ -155,7 +155,7 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-                    <include>com.squareup.okhttp3:okhttp</include>
+                    <include>com.squareup.okhttp3:okhttp:[4.2.2]</include> <!-- {x-include-update;com.squareup.okhttp3:okhttp;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/azure-core-test/pom.xml

@@ -124,11 +124,9 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-                    <include>org.slf4j</include>
-                    <include>io.projectreactor</include>
-
+                    <include>io.projectreactor:reactor-test:[3.3.3.RELEASE]</include> <!-- {x-include-update;io.projectreactor:reactor-test;external_dependency} -->
                     <!-- special allowance for azure-core-test as it is not a shipping library: -->
-                    <include>org.junit.jupiter</include>
+                    <include>org.junit.jupiter:junit-jupiter-api:[5.4.2]</include> <!-- {x-include-update;org.junit.jupiter:junit-jupiter-api;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/azure-core-tracing-opentelemetry/pom.xml

@@ -95,7 +95,7 @@
                   </excludes>
                   <includes>
                     <include>com.azure:*</include>
-                    <include>io.opentelemetry</include>
+                    <include>io.opentelemetry:opentelemetry-api:[0.2.4]</include> <!-- {x-include-update;io.opentelemetry:opentelemetry-api;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/azure-core/pom.xml

@@ -186,14 +186,12 @@
                     <exclude>*:*:*:*:provided</exclude>
                   </excludes>
                   <includes>
-                    <include>org.slf4j:slf4j-api</include>
-                    <include>io.projectreactor:reactor-core</include>
-                    <include>io.netty:netty-tcnative-boringssl-static</include>
-
-                    <include>com.fasterxml.jackson.dataformat:jackson-dataformat-xml</include>
-                    <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310</include>
-
-                    <include>com.google.code.findbugs:jsr305</include>
+                    <include>io.netty:netty-tcnative-boringssl-static:[2.0.27.Final]</include> <!-- {x-include-update;io.netty:netty-tcnative-boringssl-static;external_dependency} -->
+                    <include>io.projectreactor:reactor-core:[3.3.3.RELEASE]</include> <!-- {x-include-update;io.projectreactor:reactor-core;external_dependency} -->
+                    <include>com.fasterxml.jackson.dataformat:jackson-dataformat-xml:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.dataformat:jackson-dataformat-xml;external_dependency} -->
+                    <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.datatype:jackson-datatype-jsr310;external_dependency} -->
+                    <include>com.google.code.findbugs:jsr305:[3.0.2]</include> <!-- {x-include-update;com.google.code.findbugs:jsr305;external_dependency} -->
+                    <include>org.slf4j:slf4j-api:[1.7.28]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->
                   </includes>
                 </bannedDependencies>
               </rules>

sdk/core/ci.yml

@@ -67,3 +67,5 @@ stages:
       AdditionalModules:
         - name: azure-data-appconfiguration
           groupId: com.azure
+        - name: perf-test-core
+          groupId: com.azure