Fixed #53

2018-04-10 14:43:53 +03:00 · 2018-04-10 14:43:53 +03:00 · 9523813541
parent 251ccfdeca
commit 9523813541
1 changed files with 1 additions and 1 deletions
--- a/src/tokenizer.py
+++ b/src/tokenizer.py
@ -55,7 +55,7 @@ class Tokenizer:

        result += utf16bytes[cur_pos:]

-        return utf16bytes.decode(encoding)
+        return result.decode(encoding)

    def prettify(self, word):
        lowercase_word = word.lower().strip()