CDPD-32844: Speeding up HDFS scan by skipping folders not containing …

…jar or tar.gz files (#25)
cloudera · Dec 21, 2021 · 5094942 · 5094942
1 parent f277913
commit 5094942
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/hdp_support_scripts/patch_hdfs_tgz.sh b/hdp_support_scripts/patch_hdfs_tgz.sh
@@ -55,7 +55,7 @@ tmpdir=${TMPDIR:-/tmp}
 mkdir -p $tmpdir
 echo "Using tmp directory '$tmpdir'"
 
-for hdfs_file_path in $($user_option hdfs dfs -ls -R $hdfs_path | awk 'BEGIN {LAST=""} {if (match($8,LAST"/")>0) { print LAST; } LAST=$8}')
+for hdfs_file_path in $($user_option hdfs dfs -ls -R $hdfs_path | awk 'BEGIN {LAST=""} /^d/ {LAST=$8} /^-.*(jar|tar.gz)/ {if (LAST) { print LAST; } LAST=""}')
 do
   echo $hdfs_file_path