fix: Calculate length of posts and polls with emojis correctly

Mastodon counts post lengths by considering emojis to be single
characters, no matter how many unicode code points they are composed
of. So "😜" has length 1.

Pachli was using `String.length`, which considers "😜" as length 2.

Correct the calculation by using a BreakIterator to count the characters
in the string, which treats multi-character emojis as a length 1.

Poll options had a similar problem, exacerbated by the Mastodon web UI
also having the same problem, see mastodon/mastodon#28336.

Fix that by creating `MastodonLengthFilter`, an `InputFilter` that does
the right thing for regular text that may contain emojis.

See also tuskyapp/Tusky#4152, which has the fix
for status length but not polls.

Co-authored-by: Konrad Pozniak <[email protected]>

app/src/main/java/app/pachli/components/compose/ComposeActivity.kt

@@ -28,6 +28,7 @@ import android.net.Uri
 import android.os.Build
 import android.os.Bundle
 import android.provider.MediaStore
+import android.text.InputFilter
 import android.text.Spanned
 import android.text.style.URLSpan
 import android.view.KeyEvent
@@ -116,6 +117,7 @@ import kotlinx.coroutines.launch
 import timber.log.Timber
 import java.io.File
 import java.io.IOException
+import java.text.BreakIterator
 import java.text.DecimalFormat
 import java.util.Locale
 import kotlin.math.max
@@ -1323,6 +1325,7 @@ class ComposeActivity :
          *   (https://docs.joinmastodon.org/user/posting/#mentions)
          * - Hashtags are always treated as their actual length, including the "#"
          *   (https://docs.joinmastodon.org/user/posting/#hashtags)
+         * - Emojis are treated as a single character
          *
          * Content warning text is always treated as its full length, URLs and other entities
          * are not treated differently.
@@ -1332,9 +1335,8 @@ class ComposeActivity :
          * @param urlLength the number of characters attributed to URLs
          * @return the effective status length
          */
-        @JvmStatic
         fun statusLength(body: Spanned, contentWarning: Spanned?, urlLength: Int): Int {
-            var length = body.length - body.getSpans(0, body.length, URLSpan::class.java)
+            var length = body.toString().mastodonLength() - body.getSpans(0, body.length, URLSpan::class.java)
                 .fold(0) { acc, span ->
                     // Accumulate a count of characters to be *ignored* in the final length
                     acc + when (span) {
@@ -1347,15 +1349,64 @@ class ComposeActivity :
                         }
                         else -> {
                             // Expected to be negative if the URL length < maxUrlLength
-                            span.url.length - urlLength
+                            span.url.mastodonLength() - urlLength
                         }
                     }
                 }
 
             // Content warning text is treated as is, URLs or mentions there are not special
-            contentWarning?.let { length += it.length }
+            contentWarning?.let { length += it.toString().mastodonLength() }
 
             return length
         }
+
+        /**
+         * @return the "Mastodon" length of a string. [String.length] counts emojis as
+         * multiple characters, but Mastodon treats them as a single character.
+         */
+        private fun String.mastodonLength(): Int {
+            val breakIterator = BreakIterator.getCharacterInstance()
+            breakIterator.setText(this)
+            var count = 0
+            while (breakIterator.next() != BreakIterator.DONE) {
+                count++
+            }
+            return count
+        }
+
+        /**
+         * [InputFilter] that uses the "Mastodon" length of a string, where emojis always
+         * count as a single character.
+         *
+         * Unlike [InputFilter.LengthFilter] the source input is not trimmed if it is
+         * too long, it's just rejected.
+         */
+        class MastodonLengthFilter(private val maxLength: Int) : InputFilter {
+            override fun filter(
+                source: CharSequence?,
+                start: Int,
+                end: Int,
+                dest: Spanned?,
+                dstart: Int,
+                dend: Int,
+            ): CharSequence? {
+                val destRemovedLength = dest?.subSequence(dstart, dend).toString().mastodonLength()
+                val available = maxLength - dest.toString().mastodonLength() + destRemovedLength
+                val sourceLength = source?.subSequence(start, end).toString().mastodonLength()
+
+                // Not enough space to insert the new text
+                if (sourceLength > available) return REJECT
+
+                return ALLOW
+            }
+
+            companion object {
+                /** Filter result allowing the changes */
+                val ALLOW = null
+
+                /** Filter result preventing the changes */
+                const val REJECT = ""
+            }
+        }
     }
 }

app/src/main/java/app/pachli/components/compose/dialog/AddPollOptionsAdapter.kt

@@ -16,13 +16,13 @@
 
 package app.pachli.components.compose.dialog
 
-import android.text.InputFilter
 import android.view.LayoutInflater
 import android.view.View
 import android.view.ViewGroup
 import androidx.core.widget.doOnTextChanged
 import androidx.recyclerview.widget.RecyclerView
 import app.pachli.R
+import app.pachli.components.compose.ComposeActivity.Companion.MastodonLengthFilter
 import app.pachli.databinding.ItemAddPollOptionBinding
 import app.pachli.util.BindingHolder
 import app.pachli.util.visible
@@ -45,7 +45,7 @@ class AddPollOptionsAdapter(
     override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): BindingHolder<ItemAddPollOptionBinding> {
         val binding = ItemAddPollOptionBinding.inflate(LayoutInflater.from(parent.context), parent, false)
         val holder = BindingHolder(binding)
-        binding.optionEditText.filters = arrayOf(InputFilter.LengthFilter(maxOptionLength))
+        binding.optionEditText.filters = arrayOf(MastodonLengthFilter(maxOptionLength))
 
         binding.optionEditText.doOnTextChanged { s, _, _, _ ->
             val pos = holder.bindingAdapterPosition

app/src/test/java/app/pachli/SpanUtilsTest.kt

@@ -1,6 +1,6 @@
 package app.pachli
 
-import android.text.Spannable
+import app.pachli.core.testing.fakes.FakeSpannable
 import app.pachli.util.highlightSpans
 import org.junit.Assert
 import org.junit.Test
@@ -127,55 +127,4 @@ class SpanUtilsTest {
             Assert.assertEquals(expectedEndIndex, span.end)
         }
     }
-
-    class FakeSpannable(private val text: String) : Spannable {
-        val spans = mutableListOf<BoundedSpan>()
-
-        override fun setSpan(what: Any?, start: Int, end: Int, flags: Int) {
-            spans.add(BoundedSpan(what, start, end))
-        }
-
-        override fun <T : Any> getSpans(start: Int, end: Int, type: Class<T>): Array<T> {
-            return spans.filter { it.start >= start && it.end <= end && type.isInstance(it.span) }
-                .map { it.span }
-                .toTypedArray() as Array<T>
-        }
-
-        override fun removeSpan(what: Any?) {
-            spans.removeIf { span -> span.span == what }
-        }
-
-        override fun toString(): String {
-            return text
-        }
-
-        override val length: Int
-            get() = text.length
-
-        class BoundedSpan(val span: Any?, val start: Int, val end: Int)
-
-        override fun nextSpanTransition(start: Int, limit: Int, type: Class<*>?): Int {
-            throw NotImplementedError()
-        }
-
-        override fun getSpanEnd(tag: Any?): Int {
-            throw NotImplementedError()
-        }
-
-        override fun getSpanFlags(tag: Any?): Int {
-            throw NotImplementedError()
-        }
-
-        override fun get(index: Int): Char {
-            throw NotImplementedError()
-        }
-
-        override fun subSequence(startIndex: Int, endIndex: Int): CharSequence {
-            throw NotImplementedError()
-        }
-
-        override fun getSpanStart(tag: Any?): Int {
-            throw NotImplementedError()
-        }
-    }
 }

app/src/test/java/app/pachli/components/compose/ComposeActivityTest.kt

@@ -238,6 +238,49 @@ class ComposeActivityTest {
         }
     }
 
+    @Test
+    fun whenTextContainsEmoji_emojisAreCountedAsOneCharacter() {
+        val content = "Test 😜"
+        rule.launch()
+        rule.getScenario().onActivity {
+            insertSomeTextInContent(it, content)
+            assertEquals(6, it.calculateTextLength())
+        }
+    }
+
+    @Test
+    fun whenTextContainsConesecutiveEmoji_emojisAreCountedAsSeparateCharacters() {
+        val content = "Test 😜😜"
+        rule.launch()
+        rule.getScenario().onActivity {
+            insertSomeTextInContent(it, content)
+            assertEquals(7, it.calculateTextLength())
+        }
+    }
+
+    @Test
+    fun whenTextContainsUrlWithEmoji_ellipsizedUrlIsCountedCorrectly() {
+        val content = "https://🤪.com"
+        rule.launch()
+        rule.getScenario().onActivity {
+            insertSomeTextInContent(it, content)
+            assertEquals(
+                InstanceInfoRepository.DEFAULT_CHARACTERS_RESERVED_PER_URL,
+                it.calculateTextLength(),
+            )
+        }
+    }
+
+    @Test
+    fun whenTextContainsNonEnglishCharacters_lengthIsCountedCorrectly() {
+        val content = "こんにちは. General Kenobi" // "Hello there. General Kenobi"
+        rule.launch()
+        rule.getScenario().onActivity {
+            insertSomeTextInContent(it, content)
+            assertEquals(21, it.calculateTextLength())
+        }
+    }
+
     @Test
     fun whenTextContainsUrl_onlyEllipsizedURLIsCounted() {
         val url = "https://www.google.dk/search?biw=1920&bih=990&tbm=isch&sa=1&ei=bmDrWuOoKMv6kwWOkIaoDQ&q=indiana+jones+i+hate+snakes+animated&oq=indiana+jones+i+hate+snakes+animated&gs_l=psy-ab.3...54174.55443.0.55553.9.7.0.0.0.0.255.333.1j0j1.2.0....0...1c.1.64.psy-ab..7.0.0....0.40G-kcDkC6A#imgdii=PSp15hQjN1JqvM:&imgrc=H0hyE2JW5wrpBM:"