如何为nltk.word_tokenize定义特殊的“不可标记化”单词

import nltk

class CustomTokenizer(nltk.tokenize.api.Tokenizer):
    def __init__(self, untokenizable_words):
        self.untokenizable_words = untokenizable_words

    def tokenize(self, text):
        words = nltk.word_tokenize(text)
        tokens = []
        i = 0
        while i < len(words):
            if words[i] in self.untokenizable_words:
                tokens.append(words[i])
                i += 1
            else:
                j = i + 1
                while j < len(words) and words[j] in self.untokenizable_words:
                    j += 1
                tokens.append("".join(words[i:j]))
                i = j
        return tokens

# 定义不可标记化的单词列表
untokenizable_words = ["don't", "can't", "won't"]

# 创建自定义分词器实例
tokenizer = CustomTokenizer(untokenizable_words)

# 使用自定义分词器进行分词
text = "I don't like this book. It can't be true."
tokens = tokenizer.tokenize(text)
print(tokens)