From 80ca35cb96a8c48c6fb0e0b886a182c54bb1adb3 Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Wed, 2 Sep 2020 15:45:23 -0700 Subject: [PATCH 1/5] Add script that validates all files in the repo contain ASCII-only bytes and no UTF-8 BOM. --- .../templates/jobs/archetype-sdk-client.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index d12bf990c9..b4aef41c21 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -301,6 +301,25 @@ jobs: coverageThreshold: 70 condition: eq(variables['AZ_SDK_CODE_COV'], 1) + # Validate all the files are saved as ASCII only without a UTF-8 BOM. + - bash: | + shopt -s globstar + + files_with_non_ascii=0 + for file in /sdk/* sdk/**/* ; do + if [ ! -d "$file" ]; then + echo "Validating that $file file only contains ASCII bytes..." + if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then + ((files_with_non_ascii++)) + fi + fi + done + + echo Files found with non-ASCII characters: $files_with_non_ascii + exit $files_with_non_ascii + + displayName: Validate Characters are ASCII + - job: GenerateReleaseArtifacts pool: vmImage: windows-2019 From cc6b8b5c76631f144805e1f86820c63e9e1cc38f Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Wed, 2 Sep 2020 15:52:19 -0700 Subject: [PATCH 2/5] Update script to include all files in the repo. --- .../templates/jobs/archetype-sdk-client.yml | 2 +- script.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 script.sh diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index b4aef41c21..79d6b0ee10 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -306,7 +306,7 @@ jobs: shopt -s globstar files_with_non_ascii=0 - for file in /sdk/* sdk/**/* ; do + for file in . ./* ./**/* ; do if [ ! -d "$file" ]; then echo "Validating that $file file only contains ASCII bytes..." if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then diff --git a/script.sh b/script.sh new file mode 100644 index 0000000000..9308a08243 --- /dev/null +++ b/script.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +shopt -s globstar + +files_with_non_ascii=0 +for file in . ./* ./**/* ; do + if [ ! -d "$file" ]; then + echo "Validating that $file file only contains ASCII bytes..." + if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then + ((files_with_non_ascii++)) + fi + fi +done + +echo Files found with non-ASCII characters: $files_with_non_ascii +exit $files_with_non_ascii \ No newline at end of file From a5ab87fa5d840d5032fe85cfa48bfcc47db536f9 Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Wed, 2 Sep 2020 16:08:03 -0700 Subject: [PATCH 3/5] Update script and make sure it only runs once. --- .../templates/jobs/archetype-sdk-client.yml | 7 ++++++- script.sh | 16 ---------------- 2 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 script.sh diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 79d6b0ee10..e10d585b1c 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -310,7 +310,11 @@ jobs: if [ ! -d "$file" ]; then echo "Validating that $file file only contains ASCII bytes..." if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then - ((files_with_non_ascii++)) + if [[ $file == *"CodeCoverage.cmake"* ]]; then + echo "Skipping errors from $file since it contains a name." + else + ((files_with_non_ascii++)) + fi fi fi done @@ -319,6 +323,7 @@ jobs: exit $files_with_non_ascii displayName: Validate Characters are ASCII + condition: eq(variables['AZ_SDK_CODE_COV'], 1) - job: GenerateReleaseArtifacts pool: diff --git a/script.sh b/script.sh deleted file mode 100644 index 9308a08243..0000000000 --- a/script.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -shopt -s globstar - -files_with_non_ascii=0 -for file in . ./* ./**/* ; do - if [ ! -d "$file" ]; then - echo "Validating that $file file only contains ASCII bytes..." - if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then - ((files_with_non_ascii++)) - fi - fi -done - -echo Files found with non-ASCII characters: $files_with_non_ascii -exit $files_with_non_ascii \ No newline at end of file From 452d3c628a05b533971d52b2d5422a281a0e1ccc Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Fri, 4 Sep 2020 13:36:20 -0700 Subject: [PATCH 4/5] Address PR feedback, update script. --- .../templates/jobs/archetype-sdk-client.yml | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index e10d585b1c..987709557e 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -303,23 +303,19 @@ jobs: # Validate all the files are saved as ASCII only without a UTF-8 BOM. - bash: | - shopt -s globstar - - files_with_non_ascii=0 - for file in . ./* ./**/* ; do - if [ ! -d "$file" ]; then - echo "Validating that $file file only contains ASCII bytes..." - if grep -I --color='auto' -P -n "[^\x00-\x7F]" $file ; then - if [[ $file == *"CodeCoverage.cmake"* ]]; then - echo "Skipping errors from $file since it contains a name." - else - ((files_with_non_ascii++)) - fi - fi - fi - done - + # Run grep recursive excluding git folder and known expected files and save a file with results. + grep -I -P -n "[^\x00-\x7F]" -r --exclude-dir ".git" --exclude-dir "docs" --exclude "*CodeCoverage.cmake*" --exclude "grepResults" . > grepResults + + # Display results to console. + cat grepResults + + # Each result will produce one line, count how many lines were found. + files_with_non_ascii=($(wc -l < grepResults)) + + # Show info about the total files that needs attention. echo Files found with non-ASCII characters: $files_with_non_ascii + + # Return the count. When greater than 0, the step will fail. exit $files_with_non_ascii displayName: Validate Characters are ASCII From 333b85ce16022409f77b6eab42cfa181e39e5655 Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Fri, 4 Sep 2020 14:25:32 -0700 Subject: [PATCH 5/5] Skip the tools directory for now. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 3057d47636..c34b98ffb5 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -318,7 +318,7 @@ jobs: # Validate all the files are saved as ASCII only without a UTF-8 BOM. - bash: | # Run grep recursive excluding git folder and known expected files and save a file with results. - grep -I -P -n "[^\x00-\x7F]" -r --exclude-dir ".git" --exclude-dir "docs" --exclude "*CodeCoverage.cmake*" --exclude "grepResults" . > grepResults + grep -I -P -n "[^\x00-\x7F]" -r --exclude-dir ".git" --exclude-dir "docs" --exclude-dir "tools" --exclude "*CodeCoverage.cmake*" --exclude "grepResults" . > grepResults # Display results to console. cat grepResults