Lower word once

This commit is contained in:
Laurent Monin
2019-03-01 13:55:12 +01:00
parent b86a687e15
commit dc035f951f

View File

@@ -394,8 +394,9 @@ class ClusterDict(object):
return self.id
def tokenize(self, word):
token = self.regexp.sub('', word.lower())
return token if token else self.spaces.sub('', word.lower())
word = word.lower()
token = self.regexp.sub('', word)
return token if token else self.spaces.sub('', word)
def add(self, word):
"""