From af298e5281c14336c9087f79e10ddc583881209a Mon Sep 17 00:00:00 2001
From: Levi Bard <taktaktaktaktaktaktaktaktaktak@gmail.com>
Date: Fri, 19 Oct 2018 17:44:46 +0200
Subject: [PATCH] Account for underscores when tokenizing mentions for
autocompletion (#888)
* Account for underscores when tokenizing mentions for autocompletion
Fixes #743
* Migrate MentionTokenizer to kotlin
* Add tests for mention tokenizer
---
.../tusky/util/MentionTokenizer.java | 67 -----------------
.../tusky/util/MentionTokenizer.kt | 72 +++++++++++++++++++
.../tusky/MentionTokenizerTest.kt | 61 ++++++++++++++++
3 files changed, 133 insertions(+), 67 deletions(-)
delete mode 100644 app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java
create mode 100644 app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt
create mode 100644 app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt
diff --git a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java
deleted file mode 100644
index fd8748e3..00000000
--- a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright 2017 Andrew Dawson
- *
- * This file is a part of Tusky.
- *
- * This program is free software; you can redistribute it and/or modify it under the terms of the
- * GNU General Public License as published by the Free Software Foundation; either version 3 of the
- * License, or (at your option) any later version.
- *
- * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
- * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
- * Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with Tusky; if not,
- * see <http://www.gnu.org/licenses>. */
-
-package com.keylesspalace.tusky.util;
-
-import android.text.SpannableString;
-import android.text.Spanned;
-import android.text.TextUtils;
-import android.widget.MultiAutoCompleteTextView;
-
-public class MentionTokenizer implements MultiAutoCompleteTextView.Tokenizer {
- @Override
- public int findTokenStart(CharSequence text, int cursor) {
- int i = cursor;
- while (i > 0 && text.charAt(i - 1) != '@') {
- if (!Character.isLetterOrDigit(text.charAt(i - 1))) return cursor;
- i--;
- }
- if (i < 1 || text.charAt(i - 1) != '@') {
- return cursor;
- }
- return i;
- }
-
- @Override
- public int findTokenEnd(CharSequence text, int cursor) {
- int i = cursor;
- int length = text.length();
- while (i < length) {
- if (text.charAt(i) == ' ') {
- return i;
- } else {
- i++;
- }
- }
- return length;
- }
-
- @Override
- public CharSequence terminateToken(CharSequence text) {
- int i = text.length();
- while (i > 0 && text.charAt(i - 1) == ' ') {
- i--;
- }
- if (i > 0 && text.charAt(i - 1) == ' ') {
- return text;
- } else if (text instanceof Spanned) {
- SpannableString s = new SpannableString(text + " ");
- TextUtils.copySpansFrom((Spanned) text, 0, text.length(), Object.class, s, 0);
- return s;
- } else {
- return text + " ";
- }
- }
-}
diff --git a/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt
new file mode 100644
index 00000000..cb996c0e
--- /dev/null
+++ b/app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt
@@ -0,0 +1,72 @@
+/* Copyright 2017 Andrew Dawson
+ *
+ * This file is a part of Tusky.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Tusky; if not,
+ * see <http://www.gnu.org/licenses>. */
+
+package com.keylesspalace.tusky.util
+
+import android.text.SpannableString
+import android.text.Spanned
+import android.text.TextUtils
+import android.widget.MultiAutoCompleteTextView
+
+class MentionTokenizer : MultiAutoCompleteTextView.Tokenizer {
+ override fun findTokenStart(text: CharSequence, cursor: Int): Int {
+ if (cursor == 0) {
+ return cursor
+ }
+ var i = cursor
+ var character = text[i - 1]
+ while (i > 0 && character != '@') {
+ // See SpanUtils.MENTION_REGEX
+ if (!Character.isLetterOrDigit(character) && character != '_') {
+ return cursor
+ }
+ i--
+ character = if (i == 0) ' ' else text[i - 1]
+ }
+ if (i < 1 || character != '@') {
+ return cursor
+ }
+ return i
+ }
+
+ override fun findTokenEnd(text: CharSequence, cursor: Int): Int {
+ var i = cursor
+ val length = text.length
+ while (i < length) {
+ if (text[i] == ' ') {
+ return i
+ } else {
+ i++
+ }
+ }
+ return length
+ }
+
+ override fun terminateToken(text: CharSequence): CharSequence {
+ var i = text.length
+ while (i > 0 && text[i - 1] == ' ') {
+ i--
+ }
+ return if (i > 0 && text[i - 1] == ' ') {
+ text
+ } else if (text is Spanned) {
+ val s = SpannableString(text.toString() + " ")
+ TextUtils.copySpansFrom(text, 0, text.length, Object::class.java, s, 0)
+ s
+ } else {
+ text.toString() + " "
+ }
+ }
+}
diff --git a/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt b/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt
new file mode 100644
index 00000000..dbcdc53f
--- /dev/null
+++ b/app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt
@@ -0,0 +1,61 @@
+/* Copyright 2018 Levi Bard
+ *
+ * This file is a part of Tusky.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Tusky; if not,
+ * see <http://www.gnu.org/licenses>. */
+
+package com.keylesspalace.tusky
+
+import com.keylesspalace.tusky.util.MentionTokenizer
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@RunWith(Parameterized::class)
+class MentionTokenizerTest(private val text: CharSequence,
+ private val expectedStartIndex: Int,
+ private val expectedEndIndex: Int) {
+
+ companion object {
+ @Parameterized.Parameters(name = "{0}")
+ @JvmStatic
+ fun data(): Iterable<Any> {
+ return listOf(
+ arrayOf("@mention", 1, 8),
+ arrayOf("@ment10n", 1, 8),
+ arrayOf("@ment10n_", 1, 9),
+ arrayOf("@ment10n_n", 1, 10),
+ arrayOf("@ment10n_9", 1, 10),
+ arrayOf(" @mention", 2, 9),
+ arrayOf(" @ment10n", 2, 9),
+ arrayOf(" @ment10n_", 2, 10),
+ arrayOf(" @ment10n_ @", 12, 12),
+ arrayOf(" @ment10n_ @ment20n", 12, 19),
+ arrayOf(" @ment10n_ @ment20n_", 12, 20),
+ arrayOf(" @ment10n_ @ment20n_n", 12, 21),
+ arrayOf(" @ment10n_ @ment20n_9", 12, 21),
+ arrayOf("mention", 7, 7),
+ arrayOf("ment10n", 7, 7),
+ arrayOf("mentio_", 7, 7)
+ )
+ }
+ }
+
+ private val tokenizer = MentionTokenizer()
+
+ @Test
+ fun tokenIndices_matchExpectations() {
+ Assert.assertEquals(expectedStartIndex, tokenizer.findTokenStart(text, text.length))
+ Assert.assertEquals(expectedEndIndex, tokenizer.findTokenEnd(text, text.length))
+ }
+}
\ No newline at end of file