Spacy - NLTK:语言检测

pip install langdetect

from langdetect import detect, DetectorFactory

# 为了确保结果的一致性
DetectorFactory.seed = 0

# 示例文本
texts = [
    "Hello, how are you?",
    "Bonjour, comment ça va?",
    "Hola, ¿cómo estás?",
    "你好，你好吗？"
]

# 检测语言
for text in texts:
    lang = detect(text)
    print(f'Text: "{text}" is detected as language: {lang}')

import spacy
from langdetect import detect, DetectorFactory

# 为了确保结果的一致性
DetectorFactory.seed = 0

# 加载 spaCy 模型
nlp_en = spacy.load("en_core_web_sm")
nlp_fr = spacy.load("fr_core_news_sm")
nlp_es = spacy.load("es_core_news_sm")
nlp_zh = spacy.load("zh_core_web_sm")

# 示例文本
texts = [
    "Hello, how are you?",
    "Bonjour, comment ça va?",
    "Hola, ¿cómo estás?",
    "你好，你好吗？"
]

# 检测语言并使用 spaCy 处理文本
for text in texts:
    lang = detect(text)
    print(f'Text: "{text}" is detected as language: {lang}')
    
    # 根据检测到的语言选择相应的 spaCy 模型
    if lang == 'en':
        doc = nlp_en(text)
    elif lang == 'fr':
        doc = nlp_fr(text)
    elif lang == 'es':
        doc = nlp_es(text)
    elif lang == 'zh':
        doc = nlp_zh(text)
    else:
        print("Language not supported for further processing.")
        continue

    # 进行进一步的处理，例如提取名词
    nouns = [token.text for token in doc if token.pos_ == "NOUN"]
    print(f'Nouns in the text: {nouns}')

import nltk
from langdetect import detect, DetectorFactory

# 为了确保结果的一致性
DetectorFactory.seed = 0

# 示例文本
texts = [
    "Hello, how are you?",
    "Bonjour, comment ça va?",
    "Hola, ¿cómo estás?",
    "你好，你好吗？"
]

# 检测语言并使用 NLTK 处理文本
for text in texts:
    lang = detect(text)
    print(f'Text: "{text}" is detected as language: {lang}')
    
    # 进行进一步的处理，例如分词
    tokens = nltk.word_tokenize(text)
    print(f'Tokens: {tokens}')