在Python中计算n-gram的逐点互信息(PMI)得分

import nltk
from nltk.util import ngrams
from collections import Counter
import numpy as np

def pmi_scorer(text, n=2):
    # 计算n-gram
    ngram_list = list(ngrams(nltk.word_tokenize(text), n))
    
    # 计算n-gram出现次数
    ngram_counter = Counter(ngram_list)
    
    # 计算n-gram总数
    total_ngrams = sum(ngram_counter.values())
    
    # 计算每个n-gram的PMI得分
    pmi_scores = {}
    for ngram, count in ngram_counter.items():
        p_x = count / total_ngrams
        p_y = 0
        p_xy = 0
        for i, word in enumerate(ngram):
            if i == 0:
                p_y = sum([1 for ngram_list in ngram_counter.keys() if word in ngram_list]) / total_ngrams
            else:
                p_xy += ngram_counter[ngram] / total_ngrams
        pmi = np.log2((p_xy * total_ngrams) / (p_x * p_y))
        pmi_scores[ngram] = pmi
    
    return pmi_scores

text = "Python是一种解释型、高级、通用的编程语言。"
pmi_scores = pmi_scorer(text)
print(pmi_scores)