25_T5的统一框架：文本到文本转换的创新范式

Attention(Q, K, V) = softmax(QK^T / √d_k)V

MultiHead(Q, K, V) = Concat(head_1, ..., head_h)W^O
where head_i = Attention(QW_i^Q, KW_i^K, VW_i^V)

from transformers import T5Tokenizer, T5ForConditionalGeneration

# 加载T5模型和tokenizer
tokenizer = T5Tokenizer.from_pretrained('t5-base')
model = T5ForConditionalGeneration.from_pretrained('t5-base')

# 设置模型为评估模式
model.eval()

# 示例文本
input_text = "translate English to French: The weather is nice today."

# 编码输入
input_ids = tokenizer.encode(input_text, return_tensors='pt')

# 生成输出
outputs = model.generate(input_ids)

# 解码输出
output_text = tokenizer.decode(outputs[0], skip_special_tokens=True)

print(f"输入: {input_text}")
print(f"输出: {output_text}")

from transformers import T5Tokenizer, T5ForConditionalGeneration
import torch

# 加载模型和tokenizer
model_name = 't5-base'
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForConditionalGeneration.from_pretrained(model_name)

# 移动模型到GPU（如果可用）
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

# 定义摘要生成函数
def generate_summary(text, max_length=150, min_length=30):
    # 准备输入文本，添加摘要任务前缀
    input_text = f"summarize: {text}"
    
    # 编码输入
    inputs = tokenizer(input_text, return_tensors='pt', truncation=True, max_length=512)
    input_ids = inputs.input_ids.to(device)
    attention_mask = inputs.attention_mask.to(device)
    
    # 生成摘要
    outputs = model.generate(
        input_ids,
        attention_mask=attention_mask,
        max_length=max_length,
        min_length=min_length,
        num_beams=4,
        early_stopping=True
    )
    
    # 解码输出
    summary = tokenizer.decode(outputs[0], skip_special_tokens=True)
    
    return summary

# 示例文本
long_text = ""
"Transformer-based models have revolutionized natural language processing in recent years. These models, including BERT, GPT, and T5, have achieved state-of-the-art results on various NLP tasks. T5, or Text-to-Text Transfer Transformer, is particularly notable for its unified approach to NLP tasks, treating all tasks as text-to-text conversion problems.

The key innovation of T5 is its text-to-text framework, which allows the same model architecture and training objective to be used for all NLP tasks. This simplifies model design and training, and facilitates knowledge transfer between tasks. T5 uses a standard encoder-decoder Transformer architecture, with relative positional encodings and a denoising autoencoder pre-training objective.

T5 has been pre-trained on a massive text corpus called C4 (Colossal Clean Crawled Corpus), which contains trillions of tokens of web text. This extensive pre-training enables T5 to learn rich language representations that can be fine-tuned for specific tasks with minimal task-specific adaptations.

In practice, using T5 for a specific task involves formatting the input as text with a task-specific prefix, and training the model to generate the desired output text. For example, for translation, the input might be "translate English to French: Hello world", and the expected output would be "Bonjour monde".

T5 has been shown to achieve excellent performance on a wide range of NLP tasks, including translation, summarization, question answering, and text classification. Its unified approach and strong performance have made it a popular choice for NLP practitioners and researchers alike."
""

# 生成摘要
summary = generate_summary(long_text)

print("原始文本:")
print(long_text)
print("\n摘要:")
print(summary)

from transformers import T5Tokenizer, T5ForConditionalGeneration
import torch

# 加载模型和tokenizer
model_name = 't5-base'
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForConditionalGeneration.from_pretrained(model_name)

# 移动模型到GPU（如果可用）
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

# 定义问答函数
def answer_question(question, context=None):
    # 根据是否有上下文决定输入格式
    if context:
        input_text = f"question: {question} context: {context}"
    else:
        input_text = f"question: {question}"
    
    # 编码输入
    inputs = tokenizer(input_text, return_tensors='pt', truncation=True, max_length=512)
    input_ids = inputs.input_ids.to(device)
    attention_mask = inputs.attention_mask.to(device)
    
    # 生成答案
    outputs = model.generate(
        input_ids,
        attention_mask=attention_mask,
        max_length=100,
        num_beams=4,
        early_stopping=True
    )
    
    # 解码输出
    answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
    
    return answer

# 示例1：有上下文的问答
context = ""
"T5 is a transformer-based language model developed by Google Research. It was introduced in 2019 and uses a unified text-to-text approach to natural language processing. T5 has been pre-trained on a large text corpus and can be fine-tuned for various NLP tasks including translation, summarization, question answering, and text classification."
""
question1 = "Who developed T5?"
answer1 = answer_question(question1, context)

print(f"问题: {question1}")
print(f"上下文: {context}")
print(f"答案: {answer1}")
print()

# 示例2：无上下文的问答
question2 = "What is machine learning?"
answer2 = answer_question(question2)

print(f"问题: {question2}")
print(f"答案: {answer2}")

from transformers import T5Tokenizer, T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
from datasets import load_dataset
import torch
import numpy as np

# 加载模型和tokenizer
model_name = 't5-base'
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForConditionalGeneration.from_pretrained(model_name)

# 加载数据集（这里使用example数据集，实际应用中替换为自己的数据）
dataset = load_dataset('cnn_dailymail', '3.0.0', split='train[:1%]')  # 使用小部分数据作为示例

def preprocess_function(examples):
    # 将摘要任务转换为T5的文本到文本格式
    inputs = [f"summarize: {text}" for text in examples['article']]
    model_inputs = tokenizer(inputs, max_length=512, truncation=True)
    
    # 标记目标文本
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(examples['highlights'], max_length=150, truncation=True)
    
    model_inputs["labels"] = labels["input_ids"]
    return model_inputs

# 预处理数据集
tokenized_dataset = dataset.map(preprocess_function, batched=True)

# 分割数据集
train_test_split = tokenized_dataset.train_test_split(test_size=0.2)
train_dataset = train_test_split['train']
eval_dataset = train_test_split['test']

# 定义评估指标
def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
    
    # 替换标签中的-100（填充标记）
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
    
    # 计算ROUGE分数（这里简化为计算长度比率）
   rouge_scores = {}
    avg_ratio = sum(len(pred) / max(len(label), 1) for pred, label in zip(decoded_preds, decoded_labels)) / len(decoded_preds)
    rouge_scores['length_ratio'] = avg_ratio
    
    return rouge_scores

# 定义训练参数
training_args = Seq2SeqTrainingArguments(
    output_dir="./t5-finetuned-summarization",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=3,
    predict_with_generate=True,
)

# 初始化Trainer
trainer = Seq2SeqTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)

# 开始微调
trainer.train()

# 保存微调后的模型
trainer.save_model("./t5-finetuned-summarization-final")
tokenizer.save_pretrained("./t5-finetuned-summarization-final")

# 使用微调后的模型生成摘要
def generate_summary_with_finetuned_model(text):
    # 加载微调后的模型
    finetuned_model = T5ForConditionalGeneration.from_pretrained("./t5-finetuned-summarization-final")
    finetuned_model.to(device)
    
    input_text = f"summarize: {text}"
    inputs = tokenizer(input_text, return_tensors='pt', truncation=True, max_length=512).to(device)
    
    outputs = finetuned_model.generate(
        inputs.input_ids,
        max_length=150,
        num_beams=4,
        early_stopping=True
    )
    
    return tokenizer.decode(outputs[0], skip_special_tokens=True)