Repo created

library/TokenAutoComplete/README.md

@@ -0,0 +1,9 @@
+# TokenAutoComplete
+
+Gmail style `MultiAutoCompleteTextView` for Android.
+
+---
+
+Forked from https://github.com/splitwise/TokenAutoComplete (licensed under the Apache License, Version 2.0).
+
+Based on https://github.com/splitwise/TokenAutoComplete/commit/bb51c96b39d90d43e74b2b8cf709ec58dd633c45

library/TokenAutoComplete/build.gradle.kts

@@ -0,0 +1,14 @@
+plugins {
+    id(ThunderbirdPlugins.Library.android)
+}
+
+android {
+    namespace = "app.k9mail.library.tokenautocomplete"
+}
+
+dependencies {
+    implementation(libs.androidx.annotation)
+    implementation(libs.androidx.appcompat)
+
+    testImplementation(libs.junit)
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/CharacterTokenizer.java

@@ -0,0 +1,121 @@
+package com.tokenautocomplete;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+import androidx.annotation.NonNull;
+import android.text.SpannableString;
+import android.text.Spanned;
+import android.text.TextUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tokenizer with configurable array of characters to tokenize on.
+ *
+ * Created on 2/3/15.
+ * @author mgod
+ */
+public class CharacterTokenizer implements Tokenizer {
+    private ArrayList<Character> splitChar;
+    private String tokenTerminator;
+
+    @SuppressWarnings("WeakerAccess")
+    public CharacterTokenizer(List<Character> splitChar, String tokenTerminator){
+        super();
+        this.splitChar = new ArrayList<>(splitChar);
+        this.tokenTerminator = tokenTerminator;
+    }
+
+    @Override
+    public boolean containsTokenTerminator(CharSequence charSequence) {
+        for (int i = 0; i < charSequence.length(); ++i) {
+            if (splitChar.contains(charSequence.charAt(i))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    @NonNull
+    public List<Range> findTokenRanges(CharSequence charSequence, int start, int end) {
+        ArrayList<Range>result = new ArrayList<>();
+
+        if (start == end) {
+            //Can't have a 0 length token
+            return result;
+        }
+
+        int tokenStart = start;
+
+        for (int cursor = start; cursor < end; ++cursor) {
+            char character = charSequence.charAt(cursor);
+
+            //Avoid including leading whitespace, tokenStart will match the cursor as long as we're at the start
+            if (tokenStart == cursor && Character.isWhitespace(character)) {
+                tokenStart = cursor + 1;
+            }
+
+            //Either this is a split character, or we contain some content and are at the end of input
+            if (splitChar.contains(character) || cursor == end - 1) {
+                boolean hasTokenContent =
+                        //There is token content befor the current character
+                        cursor > tokenStart ||
+                        //If the current single character is valid token content, not a split char or whitespace
+                        (cursor == tokenStart && !splitChar.contains(character));
+                if (hasTokenContent) {
+                    //There is some token content
+                    //Add one to range end as the end of the ranges is not inclusive
+                    result.add(new Range(tokenStart, cursor + 1));
+                }
+
+                tokenStart = cursor + 1;
+            }
+        }
+
+        return result;
+    }
+
+    @Override
+    @NonNull
+    public CharSequence wrapTokenValue(CharSequence text) {
+        CharSequence wrappedText = text + tokenTerminator;
+
+        if (text instanceof Spanned) {
+            SpannableString sp = new SpannableString(wrappedText);
+            TextUtils.copySpansFrom((Spanned) text, 0, text.length(),
+                    Object.class, sp, 0);
+            return sp;
+        } else {
+            return wrappedText;
+        }
+    }
+
+    public static final Parcelable.Creator<CharacterTokenizer> CREATOR = new Parcelable.Creator<CharacterTokenizer>() {
+        @SuppressWarnings("unchecked")
+        public CharacterTokenizer createFromParcel(Parcel in) {
+            return new CharacterTokenizer(in);
+        }
+
+        public CharacterTokenizer[] newArray(int size) {
+            return new CharacterTokenizer[size];
+        }
+    };
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @SuppressWarnings({"WeakerAccess", "unchecked"})
+    CharacterTokenizer(Parcel in) {
+        this(in.readArrayList(Character.class.getClassLoader()), in.readString());
+    }
+
+    @Override
+    public void writeToParcel(Parcel parcel, int i) {
+        parcel.writeList(splitChar);
+        parcel.writeString(tokenTerminator);
+    }
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/CountSpan.java

@@ -0,0 +1,44 @@
+package com.tokenautocomplete;
+
+import android.text.Layout;
+import android.text.TextPaint;
+import android.text.style.CharacterStyle;
+
+import java.util.Locale;
+
+/**
+ * Span that displays +[x]
+ *
+ * Created on 2/3/15.
+ * @author mgod
+ */
+
+class CountSpan extends CharacterStyle {
+    private String countText;
+
+    CountSpan() {
+        super();
+        countText = "";
+    }
+
+    @Override
+    public void updateDrawState(TextPaint textPaint) {
+        //Do nothing, we are using this span as a location marker
+    }
+
+    void setCount(int c) {
+        if (c > 0) {
+            countText = String.format(Locale.getDefault(), " +%d", c);
+        } else {
+            countText = "";
+        }
+    }
+
+    String getCountText() {
+        return countText;
+    }
+
+    float getCountTextWidthForPaint(TextPaint paint) {
+        return Layout.getDesiredWidth(countText, 0, countText.length(), paint);
+    }
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/DummySpan.java

@@ -0,0 +1,23 @@
+package com.tokenautocomplete;
+
+import android.text.TextPaint;
+import android.text.style.MetricAffectingSpan;
+
+import androidx.annotation.NonNull;
+
+/**
+ * Invisible MetricAffectingSpan that will trigger a redraw when it is being added to or removed from an Editable.
+ *
+ * @see TokenCompleteTextView#redrawTokens()
+ */
+class DummySpan extends MetricAffectingSpan {
+    static final DummySpan INSTANCE = new DummySpan();
+
+    private DummySpan() {}
+
+    @Override
+    public void updateMeasureState(@NonNull TextPaint textPaint) {}
+
+    @Override
+    public void updateDrawState(TextPaint tp) {}
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/FilteredArrayAdapter.java

@@ -0,0 +1,144 @@
+package com.tokenautocomplete;
+
+import android.content.Context;
+import androidx.annotation.NonNull;
+import android.widget.ArrayAdapter;
+import android.widget.Filter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Simplified custom filtered ArrayAdapter
+ * override keepObject with your test for filtering
+ * <p>
+ * Based on gist <a href="https://gist.github.com/tobiasschuerg/3554252/raw/30634bf9341311ac6ad6739ef094222fc5f07fa8/FilteredArrayAdapter.java">
+ * FilteredArrayAdapter</a> by Tobias Schürg
+ * <p>
+ * Created on 9/17/13.
+ * @author mgod
+ */
+
+abstract public class FilteredArrayAdapter<T> extends ArrayAdapter<T> {
+
+    private List<T> originalObjects;
+    private Filter filter;
+
+    /**
+     * Constructor
+     *
+     * @param context The current context.
+     * @param resource The resource ID for a layout file containing a TextView to use when
+     *                 instantiating views.
+     * @param objects The objects to represent in the ListView.
+     */
+    public FilteredArrayAdapter(Context context, int resource, T[] objects) {
+        this(context, resource, 0, objects);
+    }
+
+    /**
+     * Constructor
+     *
+     * @param context The current context.
+     * @param resource The resource ID for a layout file containing a layout to use when
+     *                 instantiating views.
+     * @param textViewResourceId The id of the TextView within the layout resource to be populated
+     * @param objects The objects to represent in the ListView.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public FilteredArrayAdapter(Context context, int resource, int textViewResourceId, T[] objects) {
+        this(context, resource, textViewResourceId, new ArrayList<>(Arrays.asList(objects)));
+    }
+
+    /**
+     * Constructor
+     *
+     * @param context The current context.
+     * @param resource The resource ID for a layout file containing a TextView to use when
+     *                 instantiating views.
+     * @param objects The objects to represent in the ListView.
+     */
+    @SuppressWarnings("unused")
+    public FilteredArrayAdapter(Context context, int resource, List<T> objects) {
+        this(context, resource, 0, objects);
+    }
+
+    /**
+     * Constructor
+     *
+     * @param context The current context.
+     * @param resource The resource ID for a layout file containing a layout to use when
+     *                 instantiating views.
+     * @param textViewResourceId The id of the TextView within the layout resource to be populated
+     * @param objects The objects to represent in the ListView.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public FilteredArrayAdapter(Context context, int resource, int textViewResourceId, List<T> objects) {
+        super(context, resource, textViewResourceId, new ArrayList<>(objects));
+        this.originalObjects = objects;
+    }
+
+    @NonNull 
+    @Override
+    public Filter getFilter() {
+        if (filter == null)
+            filter = new AppFilter();
+        return filter;
+    }
+
+    /**
+     * Filter method used by the adapter. Return true if the object should remain in the list
+     *
+     * @param obj object we are checking for inclusion in the adapter
+     * @param mask current text in the edit text we are completing against
+     * @return true if we should keep the item in the adapter
+     */
+    abstract protected boolean keepObject(T obj, String mask);
+
+    /**
+     * Class for filtering Adapter, relies on keepObject in FilteredArrayAdapter
+     *
+     * based on gist by Tobias Schürg
+     * in turn inspired by inspired by Alxandr
+     *         (http://stackoverflow.com/a/2726348/570168)
+     */
+    private class AppFilter extends Filter {
+
+        @Override
+        protected FilterResults performFiltering(CharSequence chars) {
+            ArrayList<T> sourceObjects = new ArrayList<>(originalObjects);
+
+            FilterResults result = new FilterResults();
+            if (chars != null && chars.length() > 0) {
+                String mask = chars.toString();
+                ArrayList<T> keptObjects = new ArrayList<>();
+
+                for (T object : sourceObjects) {
+                    if (keepObject(object, mask))
+                        keptObjects.add(object);
+                }
+                result.count = keptObjects.size();
+                result.values = keptObjects;
+            } else {
+                // add all objects
+                result.values = sourceObjects;
+                result.count = sourceObjects.size();
+            }
+            return result;
+        }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        protected void publishResults(CharSequence constraint, FilterResults results) {
+            clear();
+            if (results.count > 0) {
+                FilteredArrayAdapter.this.addAll((Collection)results.values);
+                notifyDataSetChanged();
+            } else {
+                notifyDataSetInvalidated();
+            }
+        }
+    }
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/Range.java

@@ -0,0 +1,36 @@
+package com.tokenautocomplete;
+
+import java.util.Locale;
+
+class Range {
+    public final int start;
+    public final int end;
+
+    Range(int start, int end) {
+        if (start > end) {
+            throw new IllegalArgumentException(String.format(Locale.ENGLISH,
+                    "Start (%d) cannot be greater than end (%d)", start, end));
+        }
+        this.start = start;
+        this.end = end;
+    }
+
+    public int length() {
+        return end - start;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj || !(obj instanceof Range)) {
+            return false;
+        }
+
+        Range other = (Range) obj;
+        return other.start == start && other.end == end;
+    }
+
+    @Override
+    public String toString() {
+        return String.format(Locale.US, "[%d..%d]", start, end);
+    }
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/SpanUtils.java

@@ -0,0 +1,57 @@
+package com.tokenautocomplete;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import android.text.SpannableStringBuilder;
+import android.text.Spanned;
+import android.text.TextPaint;
+import android.text.TextUtils;
+
+public class SpanUtils {
+
+    private static class EllipsizeCallback implements TextUtils.EllipsizeCallback {
+        int start = 0;
+        int end = 0;
+
+        @Override
+        public void ellipsized(int ellipsedStart, int ellipsedEnd) {
+            start = ellipsedStart;
+            end = ellipsedEnd;
+        }
+    }
+
+    @Nullable
+    public static Spanned ellipsizeWithSpans(@Nullable CountSpan countSpan,
+                                           int tokenCount, @NonNull TextPaint paint,
+                                           @NonNull CharSequence originalText, float maxWidth) {
+
+        float countWidth = 0;
+        if (countSpan != null) {
+            //Assume the largest possible number of items for measurement
+            countSpan.setCount(tokenCount);
+            countWidth = countSpan.getCountTextWidthForPaint(paint);
+        }
+
+        EllipsizeCallback ellipsizeCallback = new EllipsizeCallback();
+        CharSequence tempEllipsized = TextUtils.ellipsize(originalText, paint, maxWidth - countWidth,
+                TextUtils.TruncateAt.END, false, ellipsizeCallback);
+        SpannableStringBuilder ellipsized = new SpannableStringBuilder(tempEllipsized);
+        if (tempEllipsized instanceof Spanned) {
+            TextUtils.copySpansFrom((Spanned)tempEllipsized, 0, tempEllipsized.length(), Object.class, ellipsized, 0);
+        }
+
+        if (ellipsizeCallback.start != ellipsizeCallback.end) {
+
+            if (countSpan != null) {
+                int visibleCount = ellipsized.getSpans(0, ellipsized.length(), TokenCompleteTextView.TokenImageSpan.class).length;
+                countSpan.setCount(tokenCount - visibleCount);
+                ellipsized.replace(ellipsizeCallback.start, ellipsized.length(), countSpan.getCountText());
+                ellipsized.setSpan(countSpan, ellipsizeCallback.start, ellipsized.length(),
+                        Spanned.SPAN_EXCLUSIVE_EXCLUSIVE);
+            }
+            return ellipsized;
+        }
+        //No ellipses necessary
+        return null;
+    }
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/Tokenizer.java

@@ -0,0 +1,39 @@
+package com.tokenautocomplete;
+
+import android.os.Parcelable;
+import androidx.annotation.NonNull;
+
+import java.util.List;
+
+public interface Tokenizer extends Parcelable {
+    /**
+     * Find all ranges that can be tokenized. This system should detect possible tokens
+     * both with and without having had wrapTokenValue called on the token string representation
+     *
+     * @param charSequence the string to search in
+     * @param start where the tokenizer should start looking for tokens
+     * @param end where the tokenizer should stop looking for tokens
+     * @return all ranges of characters that are valid tokens
+     */
+    @NonNull
+    List<Range> findTokenRanges(CharSequence charSequence, int start, int end);
+
+    /**
+     * Return a complete string representation of the token. Often used to add commas after email
+     * addresses when creating tokens
+     *
+     * This value must NOT include any leading or trailing whitespace
+     *
+     * @param unwrappedTokenValue the value to wrap
+     * @return the token value with any expected delimiter characters
+     */
+    @NonNull
+    CharSequence wrapTokenValue(CharSequence unwrappedTokenValue);
+
+    /**
+     * Return true if there is a character in the charSequence that should trigger token detection
+     * @param charSequence source text to look at
+     * @return true if charSequence contains a value that should end a token
+     */
+    boolean containsTokenTerminator(CharSequence charSequence);
+}

library/TokenAutoComplete/src/main/java/com/tokenautocomplete/ViewSpan.java

@@ -0,0 +1,84 @@
+package com.tokenautocomplete;
+
+import android.graphics.Canvas;
+import android.graphics.Paint;
+import androidx.annotation.IntRange;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import android.text.style.ReplacementSpan;
+import android.view.View;
+import android.view.ViewGroup;
+
+/**
+ * Span that holds a view it draws when rendering
+ *
+ * Created on 2/3/15.
+ * @author mgod
+ */
+public class ViewSpan extends ReplacementSpan {
+    protected View view;
+    private ViewSpan.Layout layout;
+    private int cachedMaxWidth = -1;
+
+    @SuppressWarnings("WeakerAccess")
+    public ViewSpan(View view, ViewSpan.Layout layout) {
+        super();
+        this.layout = layout;
+        this.view = view;
+        this.view.setLayoutParams(new ViewGroup.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT,
+                                                             ViewGroup.LayoutParams.WRAP_CONTENT));
+    }
+
+    private void prepView() {
+        if (layout.getMaxViewSpanWidth() != cachedMaxWidth || view.isLayoutRequested()) {
+            cachedMaxWidth = layout.getMaxViewSpanWidth();
+
+            int spec = View.MeasureSpec.AT_MOST;
+            if (cachedMaxWidth == 0) {
+                //If the width is 0, allow the view to choose it's own content size
+                spec = View.MeasureSpec.UNSPECIFIED;
+            }
+            int widthSpec = View.MeasureSpec.makeMeasureSpec(cachedMaxWidth, spec);
+            int heightSpec = View.MeasureSpec.makeMeasureSpec(0, View.MeasureSpec.UNSPECIFIED);
+
+            view.measure(widthSpec, heightSpec);
+            view.layout(0, 0, view.getMeasuredWidth(), view.getMeasuredHeight());
+        }
+    }
+
+    @Override
+    public void draw(@NonNull Canvas canvas, CharSequence text, @IntRange(from = 0) int start,
+                     @IntRange(from = 0) int end, float x, int top, int y, int bottom, @NonNull Paint paint) {
+        prepView();
+
+        canvas.save();
+        canvas.translate(x, top);
+        view.draw(canvas);
+        canvas.restore();
+    }
+
+    @Override
+    public int getSize(@NonNull Paint paint, CharSequence charSequence, @IntRange(from = 0) int start,
+                       @IntRange(from = 0) int end, @Nullable Paint.FontMetricsInt fontMetricsInt) {
+        prepView();
+
+        if (fontMetricsInt != null) {
+            //We need to make sure the layout allots enough space for the view
+            int height = view.getMeasuredHeight();
+
+            int adjustedBaseline = view.getBaseline();
+            //-1 means the view doesn't support baseline alignment, so align bottom to font baseline
+            if (adjustedBaseline == -1) {
+                adjustedBaseline = height;
+            }
+            fontMetricsInt.ascent = fontMetricsInt.top = -adjustedBaseline;
+            fontMetricsInt.descent = fontMetricsInt.bottom = height - adjustedBaseline;
+        }
+
+        return view.getRight();
+    }
+
+    public interface Layout {
+        int getMaxViewSpanWidth();
+    }
+}

library/TokenAutoComplete/src/test/java/com/tokenautocomplete/CharacterTokenizerTest.java

@@ -0,0 +1,128 @@
+package com.tokenautocomplete;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * Make sure the tokenizer finds the right boundaries
+ *
+ * Created by mgod on 8/24/17.
+ */
+
+public class CharacterTokenizerTest {
+
+    @Test
+    public void handleWhiteSpaceWithCommaTokens() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), ",");
+        String text = "bears, ponies";
+
+        assertTrue(tokenizer.containsTokenTerminator(text));
+
+        assertEquals(2, tokenizer.findTokenRanges(text, 0, text.length()).size());
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 6), new Range(7, 13)), ranges);
+        assertEquals("bears,", text.subSequence(ranges.get(0).start, ranges.get(0).end));
+        assertEquals("ponies", text.subSequence(ranges.get(1).start, ranges.get(1).end));
+
+        ranges = tokenizer.findTokenRanges(text, 5, text.length());
+        assertEquals(", ponies", text.substring(5));
+        assertEquals(Collections.singletonList(new Range(7, 13)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 1, text.length());
+        assertEquals(Arrays.asList(new Range(1, 6), new Range(7, 13)), ranges);
+
+        assertEquals(Collections.singletonList(new Range(7, 13)),
+                tokenizer.findTokenRanges(text, 7, text.length()));
+        assertEquals(Collections.singletonList(new Range(8, 13)),
+                tokenizer.findTokenRanges(text, 8, text.length()));
+        assertEquals(Collections.singletonList(new Range(11, 13)),
+                tokenizer.findTokenRanges(text, 11, text.length()));
+    }
+
+    @Test
+    public void handleWhiteSpaceWithWhitespaceTokens() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(' '), "");
+        String text = "bears ponies";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 6), new Range(6, 12)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 1, text.length());
+        assertEquals(Arrays.asList(new Range(1, 6), new Range(6, 12)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 4, text.length());
+        assertEquals(Arrays.asList(new Range(4, 6), new Range(6, 12)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 6, text.length());
+        assertEquals(Collections.singletonList(new Range(6, 12)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 7, text.length());
+        assertEquals(Collections.singletonList(new Range(7, 12)), ranges);
+
+        ranges = tokenizer.findTokenRanges(text, 0, text.length() - 3);
+        assertEquals(Arrays.asList(new Range(0, 6), new Range(6, 9)), ranges);
+    }
+
+    @Test
+    public void handleLotsOfWhitespace() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "bears,      ponies     ,another";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 6), new Range(12, 24), new Range(24, 31)), ranges);
+    }
+
+    @Test
+    public void handleLotsOfWhitespaceWithWhitespaceTokenizer() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(' '), "");
+        String text = "bears,  \t   ponies  \n  ,another";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 7), new Range(12, 19), new Range(23, 31)), ranges);
+    }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatches() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "a";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(0,1)), ranges);
+    }
+
+    @Test
+    public void allowsOneCharacterCandidateRangeMatchesWithWhitespace() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = " a";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(1,2)), ranges);
+    }
+
+    @Test
+    public void doesntMatchWhitespaceAsCandidateTokenRange() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "test, ";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Collections.singletonList(new Range(0, 5)), ranges);
+    }
+
+    @Test
+    public void matchesSingleLetterTokens() {
+        CharacterTokenizer tokenizer = new CharacterTokenizer(Collections.singletonList(','), "");
+        String text = "t,r, a,,b";
+
+        List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length());
+        assertEquals(Arrays.asList(new Range(0, 2), new Range(2,4), new Range(5,7), new Range(8,9)), ranges);
+    }
+}