在Spacy中使用自定义标记器对不带空格的字符串进行标记化

import re
from spacy.tokenizer import Tokenizer
from spacy.util import compile_prefix_regex, compile_infix_regex, compile_suffix_regex

class CustomTokenizer(object):
    def __init__(self, nlp):
        self.tokenizer = Tokenizer(nlp.vocab, 
                                   prefix_search=compile_prefix_regex([]),
                                   suffix_search=compile_suffix_regex([]),
                                   infix_finditer=compile_infix_regex([]).finditer)

    def __call__(self, text):
        words = re.findall(r'\w+', text)  # 使用正则表达式提取单词
        return [self.tokenizer.tokenizer(word) for word in words]

import spacy

nlp = spacy.load('en_core_web_sm')
custom_tokenizer = CustomTokenizer(nlp)
nlp.tokenizer = custom_tokenizer

text = "Thisisateststringwithoutspaces."
doc = nlp(text)
tokens = [token.text for token in doc]
print(tokens)