问自然语言处理

EN

import json
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

with open('data_full.json','r') as f:
    data0 = f.read()
rawdata = json.loads(data0)

for dataset,instances in rawdata.items():
    for instance in instances:
        sentence = instance[0]
        label = instance[1]
        tokens = word_tokenize(sentence)
        print('in ',dataset,': ', '|'.join(tokens),'; label:', instance[1])
        lemmas = [ lemmatizer.lemmatize(token) for token in tokens ]
        print('          lemmas = ','|'.join(lemmas))