From aae377d6c792d94fde33248c1fb429f07e63243f Mon Sep 17 00:00:00 2001 From: Nik Clayton Date: Mon, 1 Jul 2024 23:31:31 +0200 Subject: [PATCH 1/5] feat: Warn the user if the posting language might be incorrect The user has to specify the language they're posting in, and sometimes they might get it wrong (e.g., replying to a post that also had the language set incorrectly, forgetfulness, etc). This has accessiblity issues (only following statuses in a given language fails, translation can fail, etc). Prevent this by trying to detect the language the status is written in when the user tries to post it. If the detected language and the set language do not match, and the detection is 60+% confident, warn the user the status language might be incorrect, and offer to correct it before posting. This is currently implemented using Google's ML Kit, and therefore only available in the `google` store flavour. To do this: - Add `LanguageIdentifier`, with methods to do the identification, and `LanguageIdentifier.Factory` to create the identifiers. - Inject the factory in `ComposeActivity` - Detect the language when the user posts, showing a dialog if there's a sufficiently large discrepency. The ML Kit dependencies (language models) will be installed by the Play libraries, so there's some machinery to check that they're installed, and kick off the installation if not. If they can't be installed then the language check is bypassed. Update the privacy policy, as the MLKit libraries may send some data to Google. --- PRIVACY.md | 14 +++ app/build.gradle.kts | 5 + .../di/LanguageIdentifierFactoryModule.kt | 34 ++++++ .../LanguageIdentifier.kt | 30 ++++++ .../di/LanguageIdentifierFactoryModule.kt | 34 ++++++ .../LanguageIdentifier.kt | 30 ++++++ .../di/LanguageIdentifierFactoryModule.kt | 41 +++++++ .../LanguageIdentifier.kt | 100 ++++++++++++++++++ app/src/main/AndroidManifest.xml | 4 + .../java/app/pachli/adapter/LocaleAdapter.kt | 9 ++ .../components/compose/ComposeActivity.kt | 89 +++++++++++++++- .../LanguageIdentification.kt | 60 +++++++++++ app/src/main/res/values/strings.xml | 4 + gradle/libs.versions.toml | 6 ++ 14 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt create mode 100644 app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt create mode 100644 app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt create mode 100644 app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt create mode 100644 app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt create mode 100644 app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt create mode 100644 app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt diff --git a/PRIVACY.md b/PRIVACY.md index c18d6de7d..672e45afd 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -28,8 +28,22 @@ You can not delete your Mastodon account using the application. Deleting your ac ## Data sharing +### If you have installed Pachli from F-Droid or GitHub releases + **None of the data used by the application is shared with the application developers or unrelated third parties.** Your data is only ever sent to your server, and handled in accordance with your server's privacy policy. +### If you have installed Pachli from Google Play + +Pachli uses Google's [ML Kit](https://developers.google.com/ml-kit) to provide features that: + +- Warn you if the language you have selected when editing a post appears to be different from the language you used + +When Pachli does this all of the data you have provided (e.g., the content you are posting) is processed on your device, and **ML Kit does not send that data to Google servers**. + +The ML Kit APIs may contact Google servers from time to time in order to receive things like bug fixes, updated models and hardware accelerator compatibility information. The ML Kit APIs also send metrics about the performance and utilization of the APIs in your app to Google. Google uses this metrics data to measure performance, debug, maintain and improve the APIs, and detect misuse or abuse, as further described in Google's [Privacy Policy](https://policies.google.com/privacy). + +The specific data collected by ML Kit is in Google's [data disclosure](https://developers.google.com/ml-kit/android-data-disclosure) description. + ## Data types The application processes the following types of data. diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 13b5de795..3bb537a6a 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -179,6 +179,11 @@ dependencies { googleImplementation(libs.app.update) googleImplementation(libs.app.update.ktx) + // Language detection + googleImplementation(libs.play.services.base) + googleImplementation(libs.mlkit.language.id) + googleImplementation(libs.kotlinx.coroutines.play.services) + implementation(libs.semver) debugImplementation(libs.leakcanary) diff --git a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt new file mode 100644 index 000000000..059aada69 --- /dev/null +++ b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.di + +import app.pachli.languageidentification.LanguageIdentifier +import app.pachli.languageidentification.LanguageIdentifierFactory +import dagger.Module +import dagger.Provides +import dagger.hilt.InstallIn +import dagger.hilt.components.SingletonComponent +import javax.inject.Singleton + +@InstallIn(SingletonComponent::class) +@Module +object LanguageIdentifierFactoryModule { + @Provides + @Singleton + fun providesLanguageIdentifierFactory(): LanguageIdentifierFactory = LanguageIdentifier.Factory +} diff --git a/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt new file mode 100644 index 000000000..837e60924 --- /dev/null +++ b/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.languageidentification + +/** + * LanguageIdentifer that always returns [UNDETERMINED_LANGUAGE_TAG]. + */ +class LanguageIdentifier : LanguageIdentifierBase { + override suspend fun identifyPossibleLanguages(text: String) = listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) + override fun close() { } + + companion object Factory : LanguageIdentifierFactory() { + override fun newInstance() = LanguageIdentifier() + } +} diff --git a/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt new file mode 100644 index 000000000..059aada69 --- /dev/null +++ b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.di + +import app.pachli.languageidentification.LanguageIdentifier +import app.pachli.languageidentification.LanguageIdentifierFactory +import dagger.Module +import dagger.Provides +import dagger.hilt.InstallIn +import dagger.hilt.components.SingletonComponent +import javax.inject.Singleton + +@InstallIn(SingletonComponent::class) +@Module +object LanguageIdentifierFactoryModule { + @Provides + @Singleton + fun providesLanguageIdentifierFactory(): LanguageIdentifierFactory = LanguageIdentifier.Factory +} diff --git a/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt new file mode 100644 index 000000000..837e60924 --- /dev/null +++ b/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.languageidentification + +/** + * LanguageIdentifer that always returns [UNDETERMINED_LANGUAGE_TAG]. + */ +class LanguageIdentifier : LanguageIdentifierBase { + override suspend fun identifyPossibleLanguages(text: String) = listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) + override fun close() { } + + companion object Factory : LanguageIdentifierFactory() { + override fun newInstance() = LanguageIdentifier() + } +} diff --git a/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt new file mode 100644 index 000000000..f28f4b160 --- /dev/null +++ b/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.di + +import android.content.Context +import app.pachli.core.common.di.ApplicationScope +import app.pachli.languageidentification.LanguageIdentifier +import app.pachli.languageidentification.LanguageIdentifierFactory +import dagger.Module +import dagger.Provides +import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext +import dagger.hilt.components.SingletonComponent +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope + +@InstallIn(SingletonComponent::class) +@Module +object LanguageIdentifierFactoryModule { + @Provides + @Singleton + fun providesLanguageIdentifierFactory( + @ApplicationScope externalScope: CoroutineScope, + @ApplicationContext context: Context, + ): LanguageIdentifierFactory = LanguageIdentifier.Factory(externalScope, context) +} diff --git a/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt new file mode 100644 index 000000000..379213c9d --- /dev/null +++ b/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.languageidentification + +import android.content.Context +import com.google.android.gms.common.moduleinstall.ModuleAvailabilityResponse.AvailabilityStatus.STATUS_READY_TO_DOWNLOAD +import com.google.android.gms.common.moduleinstall.ModuleInstall +import com.google.android.gms.common.moduleinstall.ModuleInstallClient +import com.google.android.gms.common.moduleinstall.ModuleInstallRequest +import com.google.mlkit.nl.languageid.LanguageIdentification +import com.google.mlkit.nl.languageid.LanguageIdentifier +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.tasks.await +import timber.log.Timber + +/** + * LanguageIdentifer that uses Google's ML Kit to perform the language identification. + * + * If the ML Kit module is not installed on the device it will be installed on + * first use. + */ +class LanguageIdentifier( + private val moduleInstallClient: ModuleInstallClient, + private val googleLanguageIdentifier: LanguageIdentifier, +) : LanguageIdentifierBase { + override suspend fun identifyPossibleLanguages(text: String): List { + val modulesAvailable = moduleInstallClient + .areModulesAvailable(googleLanguageIdentifier) + .await() + .areModulesAvailable() + + return if (modulesAvailable) { + googleLanguageIdentifier.identifyPossibleLanguages(text).await().map { + IdentifiedLanguage( + confidence = it.confidence, + languageTag = it.languageTag, + ) + } + } else { + Timber.d("langid module not installed yet") + listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) + } + } + + override fun close() { + googleLanguageIdentifier.close() + } + + /** + * Factory for LanguageIdentifer based on Google's ML Kit. + * + * When the factory is constructed a coroutine to check and install the + * language module is launched, increasing the chances the module will + * be installed before it is first used. + */ + class Factory( + externalScope: CoroutineScope, + context: Context, + ) : LanguageIdentifierFactory() { + private val moduleInstallClient = ModuleInstall.getClient(context) + + init { + externalScope.launch { + LanguageIdentification.getClient().use { langIdClient -> + val availablityStatus = moduleInstallClient + .areModulesAvailable(langIdClient) + .await() + .availabilityStatus + + if (availablityStatus != STATUS_READY_TO_DOWNLOAD) return@launch + Timber.d("langid module not installed, requesting download") + val moduleInstallRequest = ModuleInstallRequest.newBuilder() + .addApi(langIdClient) + .build() + moduleInstallClient.installModules(moduleInstallRequest) + .addOnSuccessListener { Timber.d("langid module installed") } + .addOnFailureListener { e -> Timber.d(e, "langid module install failed") } + } + } + } + + override fun newInstance() = LanguageIdentifier(moduleInstallClient, LanguageIdentification.getClient()) + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 17ecfe8d6..3050f5843 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -25,6 +25,10 @@ android:networkSecurityConfig="@xml/network_security_config" android:enableOnBackInvokedCallback="true"> + + ) : ArrayAdapter(context, resource, locales) { override fun getView(position: Int, convertView: View?, parent: ViewGroup): View { return (super.getView(position, convertView, parent) as TextView).apply { diff --git a/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt b/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt index 8ae61e7a8..682bec312 100644 --- a/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt +++ b/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt @@ -93,9 +93,12 @@ import app.pachli.core.network.model.Status import app.pachli.core.preferences.AppTheme import app.pachli.core.preferences.PrefKeys import app.pachli.core.preferences.SharedPreferencesRepository +import app.pachli.core.ui.extensions.await import app.pachli.core.ui.extensions.getErrorString import app.pachli.core.ui.makeIcon import app.pachli.databinding.ActivityComposeBinding +import app.pachli.languageidentification.LanguageIdentifierFactory +import app.pachli.languageidentification.UNDETERMINED_LANGUAGE_TAG import app.pachli.util.PickMediaFiles import app.pachli.util.getInitialLanguages import app.pachli.util.getLocaleList @@ -119,6 +122,7 @@ import java.io.File import java.io.IOException import java.text.DecimalFormat import java.util.Locale +import javax.inject.Inject import kotlin.math.max import kotlin.math.min import kotlinx.coroutines.flow.collect @@ -161,6 +165,12 @@ class ComposeActivity : private var maxUploadMediaNumber = DEFAULT_MAX_MEDIA_ATTACHMENTS + /** List of locales the user can choose from when posting. */ + private lateinit var locales: List + + @Inject + lateinit var languageIdentifierFactory: LanguageIdentifierFactory + private val takePicture = registerForActivityResult(ActivityResultContracts.TakePicture()) { success -> if (success) { pickMedia(photoUploadUri!!) @@ -602,7 +612,8 @@ class ComposeActivity : } } binding.composePostLanguageButton.apply { - adapter = LocaleAdapter(context, android.R.layout.simple_spinner_dropdown_item, getLocaleList(initialLanguages)) + locales = getLocaleList(initialLanguages) + adapter = LocaleAdapter(context, android.R.layout.simple_spinner_dropdown_item, locales) setSelection(0) } } @@ -950,7 +961,9 @@ class ComposeActivity : return binding.composeScheduleView.verifyScheduledTime(binding.composeScheduleView.getDateTime(viewModel.scheduledAt.value)) } - private fun onSendClicked() { + private fun onSendClicked() = lifecycleScope.launch { + confirmStatusLanguage() + if (verifyScheduledTime()) { sendStatus() } else { @@ -958,6 +971,78 @@ class ComposeActivity : } } + /** + * Check the status' language. + * + * Try and identify the language the status is written in, and compare that with + * the language the user selected. If the selected language is not in the top three + * detected languages, and the language was detected at 60+% confidence then prompt + * the user to change the language before posting. + */ + private suspend fun confirmStatusLanguage() { + // Note: There's some dancing around here because the language identifiers + // are BCP-47 codes (potentially with multiple components) and the Mastodon API wants ISO 639. + // See https://github.com/mastodon/mastodon/issues/23541 + + // Null check. Shouldn't be necessary + val currentLang = viewModel.language ?: return + + // Try and identify the language the status is written in. Limit to the + // first three possibilities. + val languageIdentifier = languageIdentifierFactory.newInstance() + val languages = languageIdentifier.use { + it.identifyPossibleLanguages(binding.composeEditField.text.toString()).take(3) + } + + // If there are no matches then bail + // Note: belt and braces, shouldn't happen per documented behaviour, as at + // least one item should always be returned. + if (languages.isEmpty()) return + + // Ignore results where the language could not be determined. + if (languages.first().languageTag == UNDETERMINED_LANGUAGE_TAG) return + + // If the current language is any of the ones detected then it's OK. + if (languages.any { it.languageTag.startsWith(currentLang) }) return + + // Warn the user about the language mismatch only if 60+% sure of the guess. + val detectedLang = languages.first() + if (detectedLang.confidence < 0.6) return + + // Viewmodel's language tag has just the first component (e.g., "zh"), the + // guessed language might more (e.g,. "-Hant"), so trim to just the first. + val detectedLangTruncatedTag = detectedLang.languageTag.split('-', limit = 2)[0] + + val localeList = getLocaleList(emptyList()).associateBy { it.modernLanguageCode } + + val detectedLocale = localeList[detectedLangTruncatedTag] ?: return + val detectedDisplayLang = detectedLocale.displayLanguage + val currentDisplayLang = localeList[viewModel.language]?.displayLanguage ?: return + + // Otherwise, show the dialog. + val dialog = AlertDialog.Builder(this@ComposeActivity) + .setTitle(R.string.compose_warn_language_dialog_title) + .setMessage( + getString( + R.string.compose_warn_language_dialog_fmt, + currentDisplayLang, + detectedDisplayLang, + ), + ) + .create() + .await( + getString(R.string.compose_warn_language_dialog_change_language_fmt, detectedDisplayLang), + getString(R.string.compose_warn_language_dialog_accept_language_fmt, currentDisplayLang), + ) + + if (dialog == AlertDialog.BUTTON_POSITIVE) { + viewModel.onLanguageChanged(detectedLangTruncatedTag) + locales.indexOf(detectedLocale).takeIf { it != -1 }?.let { + binding.composePostLanguageButton.setSelection(it) + } + } + } + /** This is for the fancy keyboards which can insert images and stuff, and drag&drop etc */ override fun onReceiveContent(view: View, contentInfo: ContentInfoCompat): ContentInfoCompat? { if (contentInfo.clip.description.hasMimeType("image/*")) { diff --git a/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt new file mode 100644 index 000000000..09d50d5d4 --- /dev/null +++ b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.languageidentification + +import java.io.Closeable + +/** The BCP 47 language tag for "undetermined language". */ +const val UNDETERMINED_LANGUAGE_TAG = "und" + +/** + * A language identified by [LanguageIdentifierBase.identifyPossibleLanguages]. + */ +data class IdentifiedLanguage( + /** Confidence score associated with the identification. */ + val confidence: Float, + /** BCP 47 language tag for the identified language. */ + val languageTag: String, +) + +interface LanguageIdentifierBase : + Closeable, + AutoCloseable { + /** + * Identifies the language in [text] and returns a list of possible + * languages. + * + * @return A non-empty list of identified languages in [text]. If no + * languages were identified a list with an item with the languageTag + * set to [UNDETERMINED_LANGUAGE_TAG] is used. + */ + suspend fun identifyPossibleLanguages(text: String): List +} + +// Language identifiers may consume a lot of resources while in use, so they +// cannot be treated as singletons that can be injected and can persist as long +// as an activity remains. They may also require resource cleanup, which is +// impossible to guarantee using activity lifecycle methods (onDestroy etc). +// +// So instead of injecting the language identifier, inject a factory for creating +// language identifiers. It is the responsibility of the calling code to use the +// factory to create the language identifier, and then close the language +// identifier when finished. +abstract class LanguageIdentifierFactory { + abstract fun newInstance(): LanguageIdentifier +} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index ed45a4b49..5d21c8499 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -693,4 +693,8 @@ At least one filter context is required Title is required + Check post\'s language + The post\'s language is set to %1$s but you might have written it in %2$s. + Change language to "%1$s" and post + Post as-is (%1$s) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 14f4dd906..3301d9221 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -34,6 +34,7 @@ auto-service-ksp = "1.2.0" bouncycastle = "1.70" conscrypt = "2.5.2" coroutines = "1.8.1" +coroutines-play-services = "1.4.1" desugar_jdk_libs = "2.0.4" diffx = "1.1.1" emoji2 = "1.3.0" @@ -54,6 +55,7 @@ material = "1.12.0" material-drawer = "9.0.2" material-iconics = "5.5.0-compose01" material-typeface = "4.0.0.3-kotlin" +mlkit-language-id = "17.0.0" mockito-inline = "5.2.0" mockito-kotlin = "5.3.1" moshi = "1.15.1" @@ -61,6 +63,7 @@ moshix = "0.27.1" networkresult-calladapter = "1.0.0" okhttp = "4.12.0" okio = "3.9.0" +play-services-base = "18.5.0" quadrant = "1.9.1" retrofit = "2.11.0" robolectric = "4.12.2" @@ -178,6 +181,7 @@ junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", vers kotlin-result = { module = "com.michael-bull.kotlin-result:kotlin-result", version.ref = "kotlin-result" } kotlin-result-coroutines = { module = "com.michael-bull.kotlin-result:kotlin-result-coroutines", version.ref = "kotlin-result" } kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref = "coroutines" } +kotlinx-coroutines-play-services = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-play-services", version.ref = "coroutines-play-services" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" } kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" } image-cropper = { module = "com.github.CanHub:Android-Image-Cropper", version.ref = "image-cropper" } @@ -190,6 +194,7 @@ material-drawer-core = { module = "com.mikepenz:materialdrawer", version.ref = " material-drawer-iconics = { module = "com.mikepenz:materialdrawer-iconics", version.ref = "material-drawer" } material-iconics = { module = "com.mikepenz:iconics-core", version.ref="material-iconics" } material-typeface = { module = "com.mikepenz:google-material-typeface", version.ref = "material-typeface" } +mlkit-language-id = { module = "com.google.android.gms:play-services-mlkit-language-id", version.ref = "mlkit-language-id" } mockito-kotlin = { module = "org.mockito.kotlin:mockito-kotlin", version.ref = "mockito-kotlin" } mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockito-inline" } mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" } @@ -203,6 +208,7 @@ okhttp-core = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } okhttp-logging-interceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" } okhttp-tls = { module = "com.squareup.okhttp3:okhttp-tls", version.ref = "okhttp" } okio = { module = "com.squareup.okio:okio", version.ref = "okio" } +play-services-base = { module = "com.google.android.gms:play-services-base", version.ref = "play-services-base" } retrofit-converter-moshi = { module = "com.squareup.retrofit2:converter-moshi", version.ref = "retrofit" } retrofit-core = { module = "com.squareup.retrofit2:retrofit", version.ref = "retrofit" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } From 03fbf146618732066fd3da237a374958aee48be5 Mon Sep 17 00:00:00 2001 From: Nik Clayton Date: Tue, 2 Jul 2024 00:14:02 +0200 Subject: [PATCH 2/5] wip: Use TextClassification service on API29+ / non-Google builds --- .../di/LanguageIdentifierFactoryModule.kt | 11 +++- .../LanguageIdentifier.kt | 57 +++++++++++++++++-- .../LanguageIdentification.kt | 2 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index 059aada69..c47ed6d6e 100644 --- a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -17,18 +17,25 @@ package app.pachli.di -import app.pachli.languageidentification.LanguageIdentifier +import android.content.Context +import app.pachli.core.common.di.ApplicationScope +import app.pachli.languageidentification.Factory import app.pachli.languageidentification.LanguageIdentifierFactory import dagger.Module import dagger.Provides import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope @InstallIn(SingletonComponent::class) @Module object LanguageIdentifierFactoryModule { @Provides @Singleton - fun providesLanguageIdentifierFactory(): LanguageIdentifierFactory = LanguageIdentifier.Factory + fun providesLanguageIdentifierFactory( + @ApplicationScope externalScope: CoroutineScope, + @ApplicationContext context: Context, + ): LanguageIdentifierFactory = Factory(externalScope, context) } diff --git a/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt index 837e60924..f9edaf1ae 100644 --- a/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt +++ b/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt @@ -17,14 +17,61 @@ package app.pachli.languageidentification +import android.content.Context +import android.os.Build +import android.view.textclassifier.TextClassificationManager +import android.view.textclassifier.TextLanguage +import androidx.annotation.RequiresApi +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.plus + +@RequiresApi(Build.VERSION_CODES.Q) +class Api29LanguageIdentifier( + private val externalScope: CoroutineScope, + val context: Context, +) : LanguageIdentifierBase { + override suspend fun identifyPossibleLanguages(text: String): List = (externalScope + Dispatchers.IO).async { + val textClassificationManager = context.getSystemService(TextClassificationManager::class.java) + val textClassifier = textClassificationManager.textClassifier + + val textRequest = TextLanguage.Request.Builder(text).build() + val detectedLanguage = textClassifier.detectLanguage(textRequest) + + buildList { + for (i in 0 until detectedLanguage.localeHypothesisCount) { + val localeDetected = detectedLanguage.getLocale(i) + val confidence = detectedLanguage.getConfidenceScore(localeDetected) + + add( + IdentifiedLanguage( + confidence = confidence, + languageTag = localeDetected.toLanguageTag(), + ), + ) + } + } + }.await() + + override fun close() { } +} + +class Factory( + private val externalScope: CoroutineScope, + val context: Context, +) : LanguageIdentifierFactory() { + override fun newInstance(): LanguageIdentifierBase = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + Api29LanguageIdentifier(externalScope, context) + } else { + NopLanguageIdentifier + } +} + /** * LanguageIdentifer that always returns [UNDETERMINED_LANGUAGE_TAG]. */ -class LanguageIdentifier : LanguageIdentifierBase { +object NopLanguageIdentifier : LanguageIdentifierBase { override suspend fun identifyPossibleLanguages(text: String) = listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) override fun close() { } - - companion object Factory : LanguageIdentifierFactory() { - override fun newInstance() = LanguageIdentifier() - } } diff --git a/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt index 09d50d5d4..839938bc4 100644 --- a/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt +++ b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt @@ -56,5 +56,5 @@ interface LanguageIdentifierBase : // factory to create the language identifier, and then close the language // identifier when finished. abstract class LanguageIdentifierFactory { - abstract fun newInstance(): LanguageIdentifier + abstract fun newInstance(): LanguageIdentifierBase } From 58466e8140c19bf5fc0c4c7dfed4eb4742a220dc Mon Sep 17 00:00:00 2001 From: Nik Clayton Date: Tue, 2 Jul 2024 17:25:35 +0200 Subject: [PATCH 3/5] Cleanup further - Move the language identifiers for API 29 and NOP in to the primary code, as they don't have dependency issues. - Provide a default language detector factory for all devices. - Return Result<> from language detection to cover possible exceptions. - Pay attention to trying to use the language detector after close. --- .../di/LanguageIdentifierFactoryModule.kt | 5 +- .../LanguageIdentifier.kt | 77 ---------- .../di/LanguageIdentifierFactoryModule.kt | 12 +- .../LanguageIdentifier.kt | 30 ---- .../di/LanguageIdentifierFactoryModule.kt | 4 +- .../LanguageIdentifier.kt | 110 ++++++++------- .../components/compose/ComposeActivity.kt | 17 ++- .../LanguageIdentification.kt | 131 +++++++++++++++--- 8 files changed, 197 insertions(+), 189 deletions(-) delete mode 100644 app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt delete mode 100644 app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt diff --git a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index c47ed6d6e..11d25e7d4 100644 --- a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -19,8 +19,7 @@ package app.pachli.di import android.content.Context import app.pachli.core.common.di.ApplicationScope -import app.pachli.languageidentification.Factory -import app.pachli.languageidentification.LanguageIdentifierFactory +import app.pachli.languageidentification.DefaultLanguageIdentifierFactory import dagger.Module import dagger.Provides import dagger.hilt.InstallIn @@ -37,5 +36,5 @@ object LanguageIdentifierFactoryModule { fun providesLanguageIdentifierFactory( @ApplicationScope externalScope: CoroutineScope, @ApplicationContext context: Context, - ): LanguageIdentifierFactory = Factory(externalScope, context) + ): LanguageIdentifier.Factory = DefaultLanguageIdentifierFactory(externalScope, context) } diff --git a/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt deleted file mode 100644 index f9edaf1ae..000000000 --- a/app/src/fdroid/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2024 Pachli Association - * - * This file is a part of Pachli. - * - * This program is free software; you can redistribute it and/or modify it under the terms of the - * GNU General Public License as published by the Free Software Foundation; either version 3 of the - * License, or (at your option) any later version. - * - * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even - * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General - * Public License for more details. - * - * You should have received a copy of the GNU General Public License along with Pachli; if not, - * see . - */ - -package app.pachli.languageidentification - -import android.content.Context -import android.os.Build -import android.view.textclassifier.TextClassificationManager -import android.view.textclassifier.TextLanguage -import androidx.annotation.RequiresApi -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.plus - -@RequiresApi(Build.VERSION_CODES.Q) -class Api29LanguageIdentifier( - private val externalScope: CoroutineScope, - val context: Context, -) : LanguageIdentifierBase { - override suspend fun identifyPossibleLanguages(text: String): List = (externalScope + Dispatchers.IO).async { - val textClassificationManager = context.getSystemService(TextClassificationManager::class.java) - val textClassifier = textClassificationManager.textClassifier - - val textRequest = TextLanguage.Request.Builder(text).build() - val detectedLanguage = textClassifier.detectLanguage(textRequest) - - buildList { - for (i in 0 until detectedLanguage.localeHypothesisCount) { - val localeDetected = detectedLanguage.getLocale(i) - val confidence = detectedLanguage.getConfidenceScore(localeDetected) - - add( - IdentifiedLanguage( - confidence = confidence, - languageTag = localeDetected.toLanguageTag(), - ), - ) - } - } - }.await() - - override fun close() { } -} - -class Factory( - private val externalScope: CoroutineScope, - val context: Context, -) : LanguageIdentifierFactory() { - override fun newInstance(): LanguageIdentifierBase = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - Api29LanguageIdentifier(externalScope, context) - } else { - NopLanguageIdentifier - } -} - -/** - * LanguageIdentifer that always returns [UNDETERMINED_LANGUAGE_TAG]. - */ -object NopLanguageIdentifier : LanguageIdentifierBase { - override suspend fun identifyPossibleLanguages(text: String) = listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) - override fun close() { } -} diff --git a/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index 059aada69..11d25e7d4 100644 --- a/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -17,18 +17,24 @@ package app.pachli.di -import app.pachli.languageidentification.LanguageIdentifier -import app.pachli.languageidentification.LanguageIdentifierFactory +import android.content.Context +import app.pachli.core.common.di.ApplicationScope +import app.pachli.languageidentification.DefaultLanguageIdentifierFactory import dagger.Module import dagger.Provides import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope @InstallIn(SingletonComponent::class) @Module object LanguageIdentifierFactoryModule { @Provides @Singleton - fun providesLanguageIdentifierFactory(): LanguageIdentifierFactory = LanguageIdentifier.Factory + fun providesLanguageIdentifierFactory( + @ApplicationScope externalScope: CoroutineScope, + @ApplicationContext context: Context, + ): LanguageIdentifier.Factory = DefaultLanguageIdentifierFactory(externalScope, context) } diff --git a/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt deleted file mode 100644 index 837e60924..000000000 --- a/app/src/github/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2024 Pachli Association - * - * This file is a part of Pachli. - * - * This program is free software; you can redistribute it and/or modify it under the terms of the - * GNU General Public License as published by the Free Software Foundation; either version 3 of the - * License, or (at your option) any later version. - * - * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even - * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General - * Public License for more details. - * - * You should have received a copy of the GNU General Public License along with Pachli; if not, - * see . - */ - -package app.pachli.languageidentification - -/** - * LanguageIdentifer that always returns [UNDETERMINED_LANGUAGE_TAG]. - */ -class LanguageIdentifier : LanguageIdentifierBase { - override suspend fun identifyPossibleLanguages(text: String) = listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) - override fun close() { } - - companion object Factory : LanguageIdentifierFactory() { - override fun newInstance() = LanguageIdentifier() - } -} diff --git a/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index f28f4b160..8497dcc3c 100644 --- a/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/google/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -20,7 +20,7 @@ package app.pachli.di import android.content.Context import app.pachli.core.common.di.ApplicationScope import app.pachli.languageidentification.LanguageIdentifier -import app.pachli.languageidentification.LanguageIdentifierFactory +import app.pachli.languageidentification.MlKitLanguageIdentifier import dagger.Module import dagger.Provides import dagger.hilt.InstallIn @@ -37,5 +37,5 @@ object LanguageIdentifierFactoryModule { fun providesLanguageIdentifierFactory( @ApplicationScope externalScope: CoroutineScope, @ApplicationContext context: Context, - ): LanguageIdentifierFactory = LanguageIdentifier.Factory(externalScope, context) + ): LanguageIdentifier.Factory = MlKitLanguageIdentifier.Factory(externalScope, context) } diff --git a/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt index 379213c9d..d44ce5662 100644 --- a/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt +++ b/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt @@ -18,83 +18,91 @@ package app.pachli.languageidentification import android.content.Context -import com.google.android.gms.common.moduleinstall.ModuleAvailabilityResponse.AvailabilityStatus.STATUS_READY_TO_DOWNLOAD +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Result +import com.github.michaelbull.result.coroutines.runSuspendCatching +import com.github.michaelbull.result.mapError import com.google.android.gms.common.moduleinstall.ModuleInstall -import com.google.android.gms.common.moduleinstall.ModuleInstallClient import com.google.android.gms.common.moduleinstall.ModuleInstallRequest import com.google.mlkit.nl.languageid.LanguageIdentification -import com.google.mlkit.nl.languageid.LanguageIdentifier +import com.google.mlkit.nl.languageid.LanguageIdentifier as GoogleLanguageIdentifier import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.launch import kotlinx.coroutines.tasks.await import timber.log.Timber /** - * LanguageIdentifer that uses Google's ML Kit to perform the language identification. - * - * If the ML Kit module is not installed on the device it will be installed on - * first use. + * [LanguageIdentifier] that uses Google's ML Kit to perform the language + * identification. */ -class LanguageIdentifier( - private val moduleInstallClient: ModuleInstallClient, - private val googleLanguageIdentifier: LanguageIdentifier, -) : LanguageIdentifierBase { - override suspend fun identifyPossibleLanguages(text: String): List { - val modulesAvailable = moduleInstallClient - .areModulesAvailable(googleLanguageIdentifier) - .await() - .areModulesAvailable() +class MlKitLanguageIdentifier private constructor() : LanguageIdentifier { + private var client: GoogleLanguageIdentifier? = null - return if (modulesAvailable) { - googleLanguageIdentifier.identifyPossibleLanguages(text).await().map { - IdentifiedLanguage( - confidence = it.confidence, - languageTag = it.languageTag, - ) - } - } else { - Timber.d("langid module not installed yet") - listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG)) - } + init { + client = LanguageIdentification.getClient() + } + + override suspend fun identifyPossibleLanguages(text: String): Result, LanguageIdentifierError> { + return client?.let { client -> + // May throw an MlKitException, so catch and map error + runSuspendCatching { + client.identifyPossibleLanguages(text).await().map { + IdentifiedLanguage( + confidence = it.confidence, + languageTag = it.languageTag, + ) + } + }.mapError { LanguageIdentifierError.Unknown(it) } + } ?: Err(LanguageIdentifierError.UseAfterClose) } override fun close() { - googleLanguageIdentifier.close() + client?.close() + client = null } /** * Factory for LanguageIdentifer based on Google's ML Kit. * - * When the factory is constructed a coroutine to check and install the - * language module is launched, increasing the chances the module will - * be installed before it is first used. + * When the factory is constructed a [com.google.android.gms.tasks.Task] + * to check and install the language module is launched, increasing the + * chances the module will be installed before it is first used. */ class Factory( - externalScope: CoroutineScope, - context: Context, - ) : LanguageIdentifierFactory() { + private val externalScope: CoroutineScope, + private val context: Context, + ) : LanguageIdentifier.Factory() { private val moduleInstallClient = ModuleInstall.getClient(context) init { - externalScope.launch { - LanguageIdentification.getClient().use { langIdClient -> - val availablityStatus = moduleInstallClient - .areModulesAvailable(langIdClient) - .await() - .availabilityStatus + LanguageIdentification.getClient().use { langIdClient -> + val moduleInstallRequest = ModuleInstallRequest.newBuilder() + .addApi(langIdClient) + .build() - if (availablityStatus != STATUS_READY_TO_DOWNLOAD) return@launch - Timber.d("langid module not installed, requesting download") - val moduleInstallRequest = ModuleInstallRequest.newBuilder() - .addApi(langIdClient) - .build() - moduleInstallClient.installModules(moduleInstallRequest) - .addOnSuccessListener { Timber.d("langid module installed") } - .addOnFailureListener { e -> Timber.d(e, "langid module install failed") } - } + moduleInstallClient.installModules(moduleInstallRequest) } } - override fun newInstance() = LanguageIdentifier(moduleInstallClient, LanguageIdentification.getClient()) + /** + * Returns a [MlKitLanguageIdentifier] if the relevant modules are + * installed, defers to [DefaultLanguageIdentifierFactory] if not. + */ + override suspend fun newInstance(): LanguageIdentifier { + LanguageIdentification.getClient().use { langIdClient -> + val modulesAreAvailable = moduleInstallClient + .areModulesAvailable(langIdClient) + .await() + .areModulesAvailable() + + return if (modulesAreAvailable) { + Timber.d("mlkit langid module available") + MlKitLanguageIdentifier() + } else { + Timber.d("mlkit langid module *not* available") + DefaultLanguageIdentifierFactory(externalScope, context) + .newInstance() + } + } + } } } diff --git a/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt b/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt index 682bec312..37af76d2f 100644 --- a/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt +++ b/app/src/main/java/app/pachli/components/compose/ComposeActivity.kt @@ -97,7 +97,7 @@ import app.pachli.core.ui.extensions.await import app.pachli.core.ui.extensions.getErrorString import app.pachli.core.ui.makeIcon import app.pachli.databinding.ActivityComposeBinding -import app.pachli.languageidentification.LanguageIdentifierFactory +import app.pachli.languageidentification.LanguageIdentifier import app.pachli.languageidentification.UNDETERMINED_LANGUAGE_TAG import app.pachli.util.PickMediaFiles import app.pachli.util.getInitialLanguages @@ -110,6 +110,7 @@ import app.pachli.util.setDrawableTint import com.canhub.cropper.CropImage import com.canhub.cropper.CropImageContract import com.canhub.cropper.options +import com.github.michaelbull.result.getOrElse import com.google.android.material.bottomsheet.BottomSheetBehavior import com.google.android.material.color.MaterialColors import com.google.android.material.snackbar.Snackbar @@ -169,7 +170,7 @@ class ComposeActivity : private lateinit var locales: List @Inject - lateinit var languageIdentifierFactory: LanguageIdentifierFactory + lateinit var languageIdentifierFactory: LanguageIdentifier.Factory private val takePicture = registerForActivityResult(ActivityResultContracts.TakePicture()) { success -> if (success) { @@ -988,10 +989,14 @@ class ComposeActivity : val currentLang = viewModel.language ?: return // Try and identify the language the status is written in. Limit to the - // first three possibilities. - val languageIdentifier = languageIdentifierFactory.newInstance() - val languages = languageIdentifier.use { - it.identifyPossibleLanguages(binding.composeEditField.text.toString()).take(3) + // first three possibilities. Don't show errors to the user, just bail, + // as there's nothing they can do to resolve any error. + val languages = languageIdentifierFactory.newInstance().use { + it.identifyPossibleLanguages(binding.composeEditField.text.toString()) + .getOrElse { + Timber.d("error when identifying languages: %s", it) + return + } } // If there are no matches then bail diff --git a/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt index 839938bc4..8926cfe06 100644 --- a/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt +++ b/app/src/main/java/app/pachli/languageidentification/LanguageIdentification.kt @@ -17,13 +17,28 @@ package app.pachli.languageidentification -import java.io.Closeable +import android.content.Context +import android.os.Build +import android.view.textclassifier.TextClassificationManager +import android.view.textclassifier.TextClassifier +import android.view.textclassifier.TextLanguage +import androidx.annotation.RequiresApi +import androidx.annotation.StringRes +import app.pachli.core.common.PachliError +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.Result +import com.github.michaelbull.result.toResultOr +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.plus /** The BCP 47 language tag for "undetermined language". */ const val UNDETERMINED_LANGUAGE_TAG = "und" /** - * A language identified by [LanguageIdentifierBase.identifyPossibleLanguages]. + * A language identified by [LanguageIdentifier.identifyPossibleLanguages]. */ data class IdentifiedLanguage( /** Confidence score associated with the identification. */ @@ -32,9 +47,21 @@ data class IdentifiedLanguage( val languageTag: String, ) -interface LanguageIdentifierBase : - Closeable, - AutoCloseable { +sealed class LanguageIdentifierError( + @StringRes override val resourceId: Int = -1, + override val formatArgs: Array? = null, + override val cause: PachliError? = null, +) : PachliError { + + /** + * Called [LanguageIdentifier.identifyPossibleLanguages] after calling + * [LanguageIdentifier.close]. + */ + data object UseAfterClose : LanguageIdentifierError() + data class Unknown(val throwable: Throwable) : LanguageIdentifierError() +} + +interface LanguageIdentifier : AutoCloseable { /** * Identifies the language in [text] and returns a list of possible * languages. @@ -43,18 +70,88 @@ interface LanguageIdentifierBase : * languages were identified a list with an item with the languageTag * set to [UNDETERMINED_LANGUAGE_TAG] is used. */ - suspend fun identifyPossibleLanguages(text: String): List + suspend fun identifyPossibleLanguages(text: String): Result, LanguageIdentifierError> + + // Language identifiers may consume a lot of resources while in use, so they + // cannot be treated as singletons that can be injected and can persist as long + // as an activity remains. They may also require resource cleanup, which is + // impossible to guarantee using activity lifecycle methods (onDestroy etc). + // + // So instead of injecting the language identifier, inject a factory for creating + // language identifiers. It is the responsibility of the calling code to use the + // factory to create the language identifier, and then close the language + // identifier when finished. + abstract class Factory { + abstract suspend fun newInstance(): LanguageIdentifier + } +} + +/** + * [LanguageIdentifier] that uses Android's [TextClassificationManager], available + * on API 29 and above, to identify the language. + */ +@RequiresApi(Build.VERSION_CODES.Q) +class Api29LanguageIdentifier( + private val externalScope: CoroutineScope, + val context: Context, +) : LanguageIdentifier { + private val textClassificationManager: TextClassificationManager = context.getSystemService(TextClassificationManager::class.java) + private var textClassifier: TextClassifier? = textClassificationManager.textClassifier + + override suspend fun identifyPossibleLanguages(text: String): Result, LanguageIdentifierError> = (externalScope + Dispatchers.IO).async { + val textRequest = TextLanguage.Request.Builder(text).build() + + textClassifier?.detectLanguage(textRequest)?.let { detectedLanguage -> + buildList { + for (i in 0 until detectedLanguage.localeHypothesisCount) { + val localeDetected = detectedLanguage.getLocale(i) + val confidence = detectedLanguage.getConfidenceScore(localeDetected) + + add( + IdentifiedLanguage( + confidence = confidence, + languageTag = localeDetected.toLanguageTag(), + ), + ) + } + } + }.toResultOr { LanguageIdentifierError.UseAfterClose } + }.await() + + override fun close() { + textClassifier = null + } } -// Language identifiers may consume a lot of resources while in use, so they -// cannot be treated as singletons that can be injected and can persist as long -// as an activity remains. They may also require resource cleanup, which is -// impossible to guarantee using activity lifecycle methods (onDestroy etc). -// -// So instead of injecting the language identifier, inject a factory for creating -// language identifiers. It is the responsibility of the calling code to use the -// factory to create the language identifier, and then close the language -// identifier when finished. -abstract class LanguageIdentifierFactory { - abstract fun newInstance(): LanguageIdentifierBase +/** + * [LanguageIdentifier] that always returns [UNDETERMINED_LANGUAGE_TAG]. + * + * Use when no other language identifier is available. + */ +object NopLanguageIdentifier : LanguageIdentifier { + private var closed = false + override suspend fun identifyPossibleLanguages(text: String): Result, LanguageIdentifierError> = if (closed) { + Err(LanguageIdentifierError.UseAfterClose) + } else { + Ok(listOf(IdentifiedLanguage(confidence = 1f, languageTag = UNDETERMINED_LANGUAGE_TAG))) + } + + override fun close() { + closed = true + } +} + +/** + * [LanguageIdentifier.Factory] that creates [Api29LanguageIdentifier] if available, + * [NopLanguageIdentifier] otherwise. + */ +class DefaultLanguageIdentifierFactory( + private val externalScope: CoroutineScope, + val context: Context, +) : LanguageIdentifier.Factory() { + override suspend fun newInstance(): LanguageIdentifier = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + Api29LanguageIdentifier(externalScope, context) + } else { + NopLanguageIdentifier + } } From 5d48f2a15f6edb2034ec643a808e2d43e074f7c5 Mon Sep 17 00:00:00 2001 From: Nik Clayton Date: Tue, 2 Jul 2024 17:39:44 +0200 Subject: [PATCH 4/5] Lint and build --- .../kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt | 1 + .../kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt | 1 + .../{LanguageIdentifier.kt => MlKitLanguageIdentifier.kt} | 0 3 files changed, 2 insertions(+) rename app/src/google/kotlin/app/pachli/languageidentification/{LanguageIdentifier.kt => MlKitLanguageIdentifier.kt} (100%) diff --git a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index 11d25e7d4..4cb7d620a 100644 --- a/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/fdroid/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -20,6 +20,7 @@ package app.pachli.di import android.content.Context import app.pachli.core.common.di.ApplicationScope import app.pachli.languageidentification.DefaultLanguageIdentifierFactory +import app.pachli.languageidentification.LanguageIdentifier import dagger.Module import dagger.Provides import dagger.hilt.InstallIn diff --git a/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt index 11d25e7d4..4cb7d620a 100644 --- a/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt +++ b/app/src/github/kotlin/app/pachli/di/LanguageIdentifierFactoryModule.kt @@ -20,6 +20,7 @@ package app.pachli.di import android.content.Context import app.pachli.core.common.di.ApplicationScope import app.pachli.languageidentification.DefaultLanguageIdentifierFactory +import app.pachli.languageidentification.LanguageIdentifier import dagger.Module import dagger.Provides import dagger.hilt.InstallIn diff --git a/app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt b/app/src/google/kotlin/app/pachli/languageidentification/MlKitLanguageIdentifier.kt similarity index 100% rename from app/src/google/kotlin/app/pachli/languageidentification/LanguageIdentifier.kt rename to app/src/google/kotlin/app/pachli/languageidentification/MlKitLanguageIdentifier.kt From 3db818bf573153e74eef3ac92626bf8288c5262b Mon Sep 17 00:00:00 2001 From: Nik Clayton Date: Tue, 2 Jul 2024 18:11:05 +0200 Subject: [PATCH 5/5] Get tests to pass with fake language detector --- .../di/FakeLanguageIdentifierFactoryModule.kt | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 app/src/test/java/app/pachli/di/FakeLanguageIdentifierFactoryModule.kt diff --git a/app/src/test/java/app/pachli/di/FakeLanguageIdentifierFactoryModule.kt b/app/src/test/java/app/pachli/di/FakeLanguageIdentifierFactoryModule.kt new file mode 100644 index 000000000..b464556cb --- /dev/null +++ b/app/src/test/java/app/pachli/di/FakeLanguageIdentifierFactoryModule.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.di + +import android.content.Context +import app.pachli.core.common.di.ApplicationScope +import app.pachli.languageidentification.DefaultLanguageIdentifierFactory +import app.pachli.languageidentification.LanguageIdentifier +import dagger.Module +import dagger.Provides +import dagger.hilt.android.qualifiers.ApplicationContext +import dagger.hilt.components.SingletonComponent +import dagger.hilt.testing.TestInstallIn +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope + +@TestInstallIn( + components = [SingletonComponent::class], + replaces = [LanguageIdentifierFactoryModule::class], +) +@Module +class FakeLanguageIdentifierFactoryModule { + @Provides + @Singleton + fun providesLanguageIdentifierFactory( + @ApplicationScope externalScope: CoroutineScope, + @ApplicationContext context: Context, + ): LanguageIdentifier.Factory = DefaultLanguageIdentifierFactory(externalScope, context) +}