fix: add workaround for android

knuddelsgmbh · Jul 3, 2023 · 9c55f2b · 9c55f2b
1 parent bd71a61
commit 9c55f2b
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/lib/src/main/java/com/knuddels/jtokkit/EncodingFactory.java b/lib/src/main/java/com/knuddels/jtokkit/EncodingFactory.java
@@ -124,7 +124,14 @@ private static Encoding fromPredefinedParameters(
 			final String fileName,
 			final Map<String, Integer> specialTokens
 	) {
-		final Pattern regex = Pattern.compile(patternString, Pattern.UNICODE_CHARACTER_CLASS);
+		Pattern regex;
+		try {
+			regex = Pattern.compile(patternString, Pattern.UNICODE_CHARACTER_CLASS);
+		} catch (final IllegalArgumentException exception) {
+			// Workaround for Android where an IllegalArgumentException is thrown when using UNICODE_CHARACTER_CLASS
+			regex = Pattern.compile(patternString);
+		}
+
 		final GptBytePairEncodingParams params = new GptBytePairEncodingParams(name, regex, loadMergeableRanks(fileName), specialTokens);
 		return fromParameters(params);
 	}