Python容错的前缀树实现中文纠错

介绍

本文使用 python 实现了前缀树，并且支持编辑距离容错的查询。文中的前缀树只存储了三个分词，格式为 (分词字符串,频率) ，如：('中海晋西园', 2)、('中海西园', 24)、('*', 4)，可以换成自己的文件进行数据的替换。在查询的时候要指定一个字符串和最大的容错编辑距离。

实现

class word:
    def __init__(self, word, freq):
        self.word = word
        self.freq = freq

class trie:
    def __init__(self):
        self.root = letternode('')
        self.start = 3

    def insert(self, word, freq):
        self.root.insert(word, freq, 0)

    def findall(self, query, maxdistance):
        suggestions = self.root.recommend(query, maxdistance, self.start)
        return sorted(set(suggestions), key=lambda x: x.freq)


class letternode:
    def __init__(self, char):
        self.remove = -1
        self.add = 1
        self.same = 0
        self.change = 2
        self.start = 3
        self.pointers = []
        self.char = char
        self.word = none

    def charis(self, c):
        return self.char == c

    def insert(self, word, freq, depth):
        if ' ' in word:
            word = [i for i in word.split(' ')]
        if depth < len(word):
            c = word[depth].lower()
            for next in self.pointers:
                if next.charis(c):
                    return next.insert(word, freq, depth + 1)
            nextnode = letternode(c)
            self.pointers.append(nextnode)
            return nextnode.insert(word, freq, depth + 1)
        else:
            self.word = word(word, freq)

    def recommend(self, query, movesleft, lastaction):
        suggestions = []
        length = len(query)

        if length >= 0 and movesleft - length >= 0 and self.word:
            suggestions.append(self.word)

        if movesleft == 0 and length > 0:
            for next in self.pointers:
                if next.charis(query[0]):
                    suggestions += next.recommend(query[1:], movesleft, self.same)
                    break

        elif movesleft > 0:
            for next in self.pointers:
                if length > 0:
                    if next.charis(query[0]):
                        suggestions += next.recommend(query[1:], movesleft, self.same)
                    else:
                        suggestions += next.recommend(query[1:], movesleft - 1, self.change)
                        if lastaction != self.change and lastaction != self.remove:
                            suggestions += next.recommend(query, movesleft - 1, self.add)
                        if lastaction != self.add and lastaction != self.change:
                            if length > 1 and next.charis(query[1]):
                                suggestions += next.recommend(query[2:], movesleft - 1, self.remove)
                            elif length > 2 and next.charis(query[2]) and movesleft == 2:
                                suggestions += next.recommend(query[3:], movesleft - 2, self.remove)
                else:
                    if lastaction != self.change and lastaction != self.remove:
                        suggestions += next.recommend(query, movesleft - 1, self.add)
        return suggestions



def buildtriefromfile():
    trie = trie()
    rows = [('中海晋西园', 2),('中海西园', 24),('*', 4)]
    for row in rows:
        trie.insert(row[0], int(row[1]))
    return trie


def suggestor(trie, s, maxdistance):
    if ' ' in s:
        s = [x for x in s.split(' ')]
    suggestions = trie.findall(s, maxdistance)
    return [str(x.word) for x in suggestions]


if __name__ == "__main__":
    trie = buildtriefromfile()
    r = suggestor(trie, '中海晋西园', 1)
    print(r)

分析

结果打印：
['中海晋西园', '中海西园']

可以看出“中海晋西园”是和输入完全相同的字符串，编辑距离为 0 ，所以符合最大编辑距离为 1 的要求，直接返回。

“中海西园”是“中海晋西园”去掉“晋”字之后的结果，编辑距离为 1， 所以符合最大编辑距离为 1 的要求，直接返回。

另外，“*”和“中海晋西园”的编辑距离为 4 ，不符合最大编辑距离为 1 的要求，所以结果中没有出现。

参考

https://github.com/leoross/autocorrecttrie

到此这篇关于python容错的前缀树实现中文纠错的文章就介绍到这了,更多相关python 中文纠错内容请搜索以前的文章或继续浏览下面的相关文章希望大家以后多多支持！