[Xamarin.Android.Build.Tasks] use CRC64 over SHA1 and MD5

Context: dotnet#1580

Bump to xamarin/java.interop/master@397013ed

Changes: dotnet/java-interop@be6048e...397013e

Use of MD5 results in an inability to use Xamarin.Android on
FIPS-enabled machines, because when the **System cryptography:
[Use FIPS compliant algorithms][0] for encryption,
hashing, and signing** setting is enabled on Windows:

!['Use Fips compliant algorithms' setting][1]

then the `System.Security.Cryptography.MD5.Create()` method will
throw `InvalidOperationException`.

In order to support FIPS-enabled machines, use a 64-bit
[Cyclic redundancy check][2] algorithm (CRC64) to instead generate
the Java package names, e.g. `crc64aae7d955a89b38db.Name` and
`crc64217c8705f00a5054.Name`.  A simple implementation using a
lookup table will suffice for our purposes, ported to C# from:

    https://github.com/gityf/crc/blob/8045f50ba6e4193d4ee5d2539025fef26e613c9f/crc/crc64.c

The use of CRC64 avoids the use of MD5, thus permitting execution on
FIPS-enabled Windows machines, and also results in *shorter*
directory names -- as the Java package name is a directory name --
which helps reduce Windows `MAX_PATH` issues.

~~ Changes so far ~~

All usage of `MD5.Create()` or `new SHA1Managed()` have been either:

* Switched to use `new Crc64()`
* Used `Files.HashString` if more appropriate

Calls to `jarsigner` now use:

    -sigalg SHA256withRSA -digestalg SHA-256

The previous default can be restored via
`$(AndroidApkSigningAlgorithm)` and `$(AndroidApkDigestAlgorithm)`
MSBuild properties.

[0]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/system-cryptography-use-fips-compliant-algorithms-for-encryption-hashing-and-signing
[1]: https://user-images.githubusercontent.com/24926263/38981221-7e0bf7c4-43dc-11e8-8f16-9b6d284a55fb.PNG
[2]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check

Documentation/guides/BuildProcess.md

@@ -222,7 +222,8 @@ when packaging Release applications.
 -   **AndroidApkDigestAlgorithm** – A string value which specifies
     the digest algorithm to use with `jarsigner -digestalg`.
 
-    The default value is `SHA1` for APKs and `SHA-256` for App Bundles.
+    The default value is `SHA-256`, which was `SHA-256` in previous
+    versions of Xamarin.Android.
 
     Added in Xamarin.Android 9.4.
 
@@ -234,7 +235,8 @@ when packaging Release applications.
 -   **AndroidApkSigningAlgorithm** – A string value which specifies
     the signing algorithm to use with `jarsigner -sigalg`.
 
-    The default value is `md5withRSA` for APKs and `SHA256withRSA` for App Bundles.
+    The default value is `SHA256withRSA`, which was `md5withRSA` in
+    previous versions of Xamarin.Android.
 
     Added in Xamarin.Android 8.2.
 

build-tools/xaprepare/xaprepare/ThirdPartyNotices/Java.Interop.cs

@@ -11,6 +11,7 @@ class JavaInterop_External_Dependencies_Group : ThirdPartyNoticeGroup
 
 		public override List<ThirdPartyNotice> Notices => new List <ThirdPartyNotice> {
 			new JavaInterop_xamarin_Java_Interop_TPN (),
+			new JavaInterop_gityf_crc_TPN (),
 			new JavaInterop_xamarin_mono_cecil_TPN (),
 			new JavaInterop_jonpryor_mono_linq_expressions_TPN (),
 			new JavaInterop_mono_csharp_TPN (),
@@ -31,6 +32,44 @@ class JavaInterop_xamarin_Java_Interop_TPN : ThirdPartyNotice
 		public override string LicenseText => null;
 	}
 
+	class JavaInterop_gityf_crc_TPN : ThirdPartyNotice
+	{
+		static readonly Uri    url         = new Uri ("https://github.com/gityf/crc");
+
+		public override string LicenseFile => null;
+		public override string Name        => "gityf/crc";
+		public override Uri    SourceUrl   => url;
+
+		public override string LicenseText => @"
+Copyright (c) 2012, Salvatore Sanfilippo <antirez at gmail dot com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+  * Neither the name of Redis nor the names of its contributors may be used
+    to endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS""
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+";
+	}
+
 	// git submodules of Java.Interop
 	class JavaInterop_xamarin_mono_cecil_TPN : ThirdPartyNotice
 	{

src/Mono.Android/Mono.Android.csproj

@@ -86,6 +86,9 @@
     <Compile Include="..\..\external\Java.Interop\src\Java.Interop.Tools.TypeNameMappings\Java.Interop.Tools.TypeNameMappings\JavaNativeTypeManager.cs">
       <Link>JavaNativeTypeManager.cs</Link>
     </Compile>
+    <Compile Include="..\..\external\Java.Interop\src\Java.Interop.Tools.JavaCallableWrappers\Java.Interop.Tools.JavaCallableWrappers\Crc64.cs">
+      <Link>Crc64.cs</Link>
+    </Compile>
     <Compile Include="Android.Runtime\DynamicMethodNameCounter.cs" />
     <Compile Include="Android.Runtime\IJavaObjectValueMarshaler.cs" />
   </ItemGroup>

src/Xamarin.Android.Build.Tasks/Tasks/AndroidSignPackage.cs

@@ -53,13 +53,13 @@ public class AndroidSignPackage : AndroidRunToolTask
 		public string TimestampAuthorityCertificateAlias { get; set; }
 
 		/// <summary>
-		/// -sigalg switch, should be md5withRSA for APKs or SHA256withRSA for App Bundles
+		/// -sigalg switch, which is SHA256withRSA by default. Previous versions of XA was md5withRSA.
 		/// </summary>
 		[Required]
 		public string SigningAlgorithm { get; set; }
 
 		/// <summary>
-		/// -digestalg switch, should be SHA1 for APKs or SHA-256 for App Bundles
+		/// -digestalg switch, which is SHA-256 by default. Previous versions of XA was SHA1.
 		/// </summary>
 		[Required]
 		public string DigestAlgorithm { get; set; }

src/Xamarin.Android.Build.Tasks/Tasks/CalculateAdditionalResourceCacheDirectories.cs

@@ -2,11 +2,9 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
-using System.Security.Cryptography;
-using System.Text.RegularExpressions;
-using System.Text;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using Xamarin.Android.Tools;
 
 namespace Xamarin.Android.Tasks
 {
@@ -28,15 +26,11 @@ public override bool RunTask ()
 			if (!AdditionalAndroidResourcePaths.Any ())
 				return true;
 
-			var md5 = MD5.Create ();
 			Directory.CreateDirectory (CacheDirectory);
 			var directories = new List<ITaskItem> ();
 
 			foreach (var path in AdditionalAndroidResourcePaths) {
-				var cacheSubDirectory = string.Concat (md5.ComputeHash (
-									Encoding.UTF8.GetBytes (path)).Select (b => b.ToString ("X02"))
-								);
-				var targetDir = Path.Combine (CacheDirectory, cacheSubDirectory);
+				var targetDir = Path.Combine (CacheDirectory, Files.HashString (path));
 				directories.Add (new TaskItem (Path.GetFullPath (targetDir).TrimEnd (Path.DirectorySeparatorChar)));
 			}
 			AdditionalResourceCachePaths = directories.ToArray ();

src/Xamarin.Android.Build.Tasks/Tasks/ComputeHash.cs

@@ -1,13 +1,14 @@
-using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Text;
 using System.Security.Cryptography;
 using Microsoft.Build.Utilities;
 using Microsoft.Build.Framework;
+using Java.Interop.Tools.JavaCallableWrappers;
 
-namespace Xamarin.Android.Tasks {
-	public class ComputeHash : AndroidTask {
+namespace Xamarin.Android.Tasks
+{
+	public class ComputeHash : AndroidTask
+	{
 		public override string TaskPrefix => "CPT";
 
 		List<ITaskItem> output = new List<ITaskItem> ();
@@ -22,11 +23,10 @@ public class ComputeHash : AndroidTask {
 
 		public override bool RunTask ()
 		{
-			using (var sha1 = SHA1.Create ()) {
-
+			using (var hashAlgorithm = new Crc64 ()) {
 				foreach (var item in Source) {
 					var newItem = new TaskItem(item.ItemSpec, new Dictionary<string, string>() {
-						{ "Hash", HashItemSpec (sha1, item.ItemSpec) }
+						{ "Hash", HashItemSpec (hashAlgorithm, item.ItemSpec) }
 					});
 					if (CopyMetaData)
 						item.CopyMetadataTo (newItem);
@@ -37,9 +37,9 @@ public override bool RunTask ()
 			}
 		}
 
-		string HashItemSpec (SHA1 sha1, string hashInput)
+		string HashItemSpec (HashAlgorithm hashAlgorithm, string hashInput)
 		{
-			var hash = sha1.ComputeHash (Encoding.UTF8.GetBytes (hashInput));
+			var hash = hashAlgorithm.ComputeHash (Encoding.UTF8.GetBytes (hashInput));
 			var hashResult = new StringBuilder (hash.Length * 2);
 
 			foreach (byte b in hash) {

src/Xamarin.Android.Build.Tasks/Tasks/Desugar.cs

@@ -1,11 +1,8 @@
 // Copyright (C) 2011 Xamarin, Inc. All rights reserved.
 
-using System;
-using System.Linq;
 using System.IO;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
-using System.Text;
 using System.Collections.Generic;
 using Xamarin.Android.Tools;
 
@@ -87,9 +84,8 @@ protected override string GenerateCommandLineCommands ()
 				cmd.AppendSwitch (DesugarExtraArguments); // it should contain "--dex".
 
 			var outputs = new List<string> ();
-			var md5 = System.Security.Cryptography.MD5.Create ();
 			foreach (var jar in InputJars) {
-				var output = Path.Combine (OutputDirectory, BitConverter.ToString (md5.ComputeHash (Encoding.UTF8.GetBytes (jar))) + Path.GetFileName (jar));
+				var output = Path.Combine (OutputDirectory, Files.HashString (jar) + Path.GetFileName (jar));
 				outputs.Add (output);
 				cmd.AppendSwitch ("--classpath_entry ");
 				cmd.AppendFileNameIfNotNull (jar);

src/Xamarin.Android.Build.Tasks/Tasks/GetAdditionalResourcesFromAssemblies.cs

@@ -6,12 +6,9 @@
 using System.Net;
 using System.Reflection.Metadata;
 using System.Reflection.PortableExecutable;
-using System.Security.Cryptography;
-using System.Text;
 using System.Text.RegularExpressions;
 using System.Xml.Linq;
 using Xamarin.Android.Tools;
-using Xamarin.Build;
 
 namespace Xamarin.Android.Tasks
 {
@@ -52,7 +49,6 @@ public class GetAdditionalResourcesFromAssemblies : AndroidAsyncTask
 		public string CacheFile { get; set; }
 
 		string CachePath;
-		MD5 md5 = MD5.Create ();
 
 		internal const string AndroidSkipResourceExtraction = "AndroidSkipResourceExtraction";
 
@@ -203,14 +199,14 @@ bool IsValidDownload(string file, string sha1)
 			if (string.IsNullOrEmpty (sha1))
 				return true;
 
-			var hashFile = file + ".sha1";
+			var hashFile = file + ".hash";
 			if (File.Exists (hashFile) && string.Compare (File.ReadAllText (hashFile), sha1, StringComparison.InvariantCultureIgnoreCase) == 0)
 				return true;
-			
-			var hash = Xamarin.Android.Tools.Files.HashFile (file).Replace ("-", String.Empty);
+
+			var hash = Files.HashFile (file);
 			LogDebugMessage ("File :{0}", file);
-			LogDebugMessage ("SHA1 : {0}", hash);
-			LogDebugMessage ("Expected SHA1 : {0}", sha1);
+			LogDebugMessage ("Hash : {0}", hash);
+			LogDebugMessage ("Expected Hash : {0}", sha1);
 
 			var isValid = string.Compare (hash, sha1, StringComparison.InvariantCultureIgnoreCase) == 0;
 			if (isValid)
@@ -291,7 +287,7 @@ string MakeSureLibraryIsInPlace (string destinationBase, string url, string vers
 			if (createDestinationDirectory)
 				Directory.CreateDirectory (destinationDir);
 
-			var hash = string.Concat (md5.ComputeHash (Encoding.UTF8.GetBytes (url)).Select (b => b.ToString ("X02")));
+			var hash = Files.HashString (url);
 			var uri = new Uri (url);
 
 			var extraPath = extraPaths.FirstOrDefault (x => File.Exists (Path.Combine (AndroidSdkDirectory, x, embeddedArchive ?? String.Empty)));

src/Xamarin.Android.Build.Tasks/Tasks/GetConvertedJavaLibraries.cs

@@ -1,11 +1,7 @@
-using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
 using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
+using Xamarin.Android.Tools;
 
 namespace Xamarin.Android.Tasks
 {
@@ -22,11 +18,10 @@ public class GetConvertedJavaLibraries : AndroidTask
 
 		public override bool RunTask ()
 		{
-			var md5 = MD5.Create ();
 			ConvertedFilesToBeGenerated =
 				(JarsToConvert ?? new string [0]).Select (
 					j => Path.Combine (OutputJackDirectory,
-					                   BitConverter.ToString (md5.ComputeHash (Encoding.UTF8.GetBytes (j))) + Path.ChangeExtension (Path.GetFileName (j), Extension)))
+					                   Files.HashString (j) + Path.ChangeExtension (Path.GetFileName (j), Extension)))
 				             .ToArray ();
 			Log.LogDebugTaskItems ("  ConvertedFilesToBeGenerated:", ConvertedFilesToBeGenerated);
 			return true;

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs

@@ -1,15 +1,13 @@
 using System;
-using Xamarin.ProjectTools;
-using NUnit.Framework;
-using System.Linq;
+using System.Collections.Generic;
 using System.IO;
-using System.Threading;
+using System.Linq;
 using System.Text;
-using System.Collections.Generic;
-using Microsoft.Build.Framework;
-using System.Xml.Linq;
-using System.Security.Cryptography;
 using System.Text.RegularExpressions;
+using System.Xml.Linq;
+using Microsoft.Build.Framework;
+using NUnit.Framework;
+using Xamarin.Android.Tools;
 using Xamarin.ProjectTools;
 
 namespace Xamarin.Android.Build.Tests
@@ -49,8 +47,7 @@ public void DesignTimeBuild ([Values(false, true)] bool isRelease, [Values (fals
 				{ "XAMARIN_CACHEPATH", cachePath },
 			};
 			var url = "http://dl-ssl.google.com/android/repository/build-tools_r24-macosx.zip";
-			var md5 = MD5.Create ();
-			var hash = string.Concat (md5.ComputeHash (Encoding.UTF8.GetBytes (url)).Select (b => b.ToString ("X02")));
+			var hash = Files.HashString (url);
 			var zipPath = Path.Combine (cachePath, "zips", $"{hash}.zip");
 			if (File.Exists (zipPath))
 				File.Delete (zipPath);

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/Aapt2Tests.cs

@@ -1,33 +1,17 @@
-using System;
 using System.Collections.Generic;
-using NUnit.Framework;
-using Xamarin.ProjectTools;
 using System.IO;
 using System.Linq;
 using Microsoft.Build.Framework;
-using System.Text;
-using Xamarin.Android.Tasks;
 using Microsoft.Build.Utilities;
-using System.Security.Cryptography;
-
-namespace Xamarin.Android.Build.Tests {
-	public class Aapt2Tests : BaseTest {
-
-		string GetHash (string hashInput)
-		{
-			using (var sha1 = SHA1.Create ())
-			{
-				var hash = sha1.ComputeHash (Encoding.UTF8.GetBytes (hashInput));
-				var hashResult = new StringBuilder (hash.Length * 2);
-
-				foreach (byte b in hash)
-				{
-					hashResult.Append (b.ToString ("x2"));
-				}
-				return hashResult.ToString ();
-			}
-		}
+using NUnit.Framework;
+using Xamarin.Android.Tasks;
+using Xamarin.Android.Tools;
+using Xamarin.ProjectTools;
 
+namespace Xamarin.Android.Build.Tests
+{
+	public class Aapt2Tests : BaseTest
+	{
 		void CallAapt2Compile (IBuildEngine engine, string dir, string outputPath, List<ITaskItem> output = null)
 		{
 			var errors = new List<BuildErrorEventArgs> ();
@@ -36,7 +20,7 @@ void CallAapt2Compile (IBuildEngine engine, string dir, string outputPath, List<
 				ToolPath = GetPathToAapt2 (),
 				ResourceDirectories = new ITaskItem [] {
 					new TaskItem(dir, new Dictionary<string, string> {
-						{ "Hash", output != null ? GetHash (dir) : "compiled" }
+						{ "Hash", output != null ? Files.HashString (dir) : "compiled" }
 					}),
 				},
 				FlatArchivesDirectory = outputPath,