ValueError: np.nan是无效文档

import numpy as np
import pandas as pd

# 假设 df 是一个包含文本数据的 DataFrame
df = pd.DataFrame({'text': ['Hello world', np.nan, 'Another sentence']})

# 移除包含 NaN 的行
df_cleaned = df.dropna(subset=['text'])

# 将 NaN 值替换为空字符串或其他默认值
df['text'].fillna('', inplace=True)

try:
    # 尝试处理文档
    processed_docs = [process_document(doc) for doc in documents]
except ValueError as e:
    print(f"Error processing documents: {e}")
    # 处理异常情况，例如记录日志或跳过有问题的文档

def preprocess_documents(documents):
    cleaned_docs = []
    for doc in documents:
        if pd.isna(doc):
            continue  # 跳过 NaN 值
        cleaned_docs.append(process_document(doc))
    return cleaned_docs

from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

def process_document(doc):
    return tokenizer.encode(doc, add_special_tokens=True)

documents = ['Hello world', np.nan, 'Another sentence']

# 预处理文档
cleaned_documents = preprocess_documents(documents)
print(cleaned_documents)