24_BERT模型详解：从预训练到微调的全方位指南

BERT模型发展时间线：
2018年10月 → BERT原始论文发表
2019年 → BERT在GLUE基准测试中刷新多项纪录
2019年 → RoBERTa、ALBERT等BERT变体相继提出
2020年 → BERT在工业界广泛应用
2022年 → BERT成为大型语言模型（LLM）的重要基础
2025年 → BERT技术持续演进，应用场景不断扩展

BERT模型架构：
输入序列 → 词嵌入+位置编码+段嵌入 → 多层Transformer编码器 → 任务特定输出

输入表示 = 词嵌入 + 位置嵌入 + 段嵌入

pip install transformers datasets torch evaluate

import evaluate
import numpy as np
from datasets import load_dataset
from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments

def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

# 加载数据集
dataset = load_dataset("yelp_review_full")

# 加载预训练模型和分词器
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)

# 处理数据集
tokenized_datasets = dataset.map(tokenize_function, batched=True)
small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))

# 加载评估指标
metric = evaluate.load("accuracy")

# 设置训练参数
training_args = TrainingArguments(
    output_dir="test_trainer",
    num_train_epochs=3,
    logging_steps=50,
    report_to="none",  # 不使用任何报告工具
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
)

# 创建Trainer实例
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=small_train_dataset,
    eval_dataset=small_eval_dataset,
    compute_metrics=compute_metrics,
)

# 开始训练
trainer.train()

# 保存模型
model.save_pretrained("bert-yelp-classification")
tokenizer.save_pretrained("bert-yelp-classification")

from datasets import load_dataset
from transformers import AutoModelForTokenClassification, AutoTokenizer, DataCollatorForTokenClassification, Trainer, TrainingArguments
import numpy as np
import evaluate

# 加载数据集
dataset = load_dataset("conll2003")

# 加载预训练模型和分词器
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
label_list = dataset["train"].features["ner_tags"].feature.names
model = AutoModelForTokenClassification.from_pretrained("bert-base-cased", num_labels=len(label_list))

# 处理数据集中的token和标签
def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
    labels = []
    for i, label in enumerate(examples["ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:
            # 将子词的标签设置为-100，这样它们会被忽略
            if word_idx is None:
                label_ids.append(-100)
            # 为每个词的第一个token分配标签
            elif word_idx != previous_word_idx:
                label_ids.append(label[word_idx])
            # 为其他子词也分配相同的标签
            else:
                label_ids.append(label[word_idx])
            previous_word_idx = word_idx
        labels.append(label_ids)
    tokenized_inputs["labels"] = labels
    return tokenized_inputs

# 应用处理函数
tokenized_datasets = dataset.map(tokenize_and_align_labels, batched=True)

# 数据收集器
data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)

# 评估指标
metric = evaluate.load("seqeval")

def compute_metrics(p):
    predictions, labels = p
    predictions = np.argmax(predictions, axis=2)
    
    # 移除被忽略的标签（-100）
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    true_labels = [
        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    
    results = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": results["overall_precision"],
        "recall": results["overall_recall"],
        "f1": results["overall_f1"],
        "accuracy": results["overall_accuracy"],
    }

# 设置训练参数
training_args = TrainingArguments(
    output_dir="test_ner",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    load_best_model_at_end=True,
)

# 创建Trainer实例
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

# 开始训练
trainer.train()

# 保存模型
model.save_pretrained("bert-ner")
tokenizer.save_pretrained("bert-ner")

from datasets import load_dataset
from transformers import AutoModelForQuestionAnswering, AutoTokenizer, TrainingArguments, Trainer
import torch

def preprocess_function(examples):
    questions = [q.strip() for q in examples["question"]]
    inputs = tokenizer(
        questions,
        examples["context"],
        max_length=384,
        truncation="only_second",
        return_offsets_mapping=True,
        padding="max_length",
    )
    
    offset_mapping = inputs.pop("offset_mapping")
    answers = examples["answers"]
    start_positions = []
    end_positions = []
    
    for i, offset in enumerate(offset_mapping):
        answer = answers[i]
        start_char = answer["answer_start"][0]
        end_char = start_char + len(answer["text"][0])
        sequence_ids = inputs.sequence_ids(i)
        
        # 找到context的开始和结束位置
        idx = 0
        while sequence_ids[idx] != 1:
            idx += 1
        context_start = idx
        while sequence_ids[idx] == 1:
            idx += 1
        context_end = idx - 1
        
        # 如果答案不在context范围内，则标记为context的开始位置
        if offset[context_end][1] < start_char or offset[context_start][0] > end_char:
            start_positions.append(0)
            end_positions.append(0)
        else:
            # 找到答案在token中的开始位置
            idx = context_start
            while idx <= context_end and offset[idx][0] <= start_char:
                start_positions.append(idx)
                idx += 1
            
            # 找到答案在token中的结束位置
            idx = context_end
            while idx >= context_start and offset[idx][1] >= end_char:
                end_positions.append(idx)
                idx -= 1
    
    inputs["start_positions"] = start_positions
    inputs["end_positions"] = end_positions
    return inputs

# 加载数据集
dataset = load_dataset("squad")

# 加载预训练模型和分词器
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
model = AutoModelForQuestionAnswering.from_pretrained("bert-base-cased")

# 处理数据集
tokenized_datasets = dataset.map(preprocess_function, batched=True)

# 设置训练参数
training_args = TrainingArguments(
    output_dir="./results",
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    learning_rate=2e-5,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
)

# 创建Trainer实例
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    tokenizer=tokenizer,
)

# 开始训练
trainer.train()

# 保存模型
model.save_pretrained("bert-squad")
tokenizer.save_pretrained("bert-squad")

from transformers import BertTokenizer, BertForSequenceClassification
import torch

def analyze_sentiment(text):
    # 加载模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased-finetuned-sst-2-english')
    model = BertForSequenceClassification.from_pretrained('bert-base-uncased-finetuned-sst-2-english')
    
    # 处理输入
    inputs = tokenizer(text, return_tensors="pt")
    
    # 进行预测
    with torch.no_grad():
        outputs = model(**inputs)
    
    # 获取结果
    predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
    sentiment = torch.argmax(predictions).item()
    
    # 返回结果
    if sentiment == 0:
        return "消极", predictions[0][0].item()
    else:
        return "积极", predictions[0][1].item()

# 使用示例
text = "这部电影非常精彩，演员的表演令人印象深刻。"
sentiment, score = analyze_sentiment(text)
print(f"情感: {sentiment}, 置信度: {score:.4f}")

from transformers import BertTokenizer, BertForTokenClassification
import torch

def recognize_entities(text):
    # 加载模型和分词器
    tokenizer = BertTokenizer.from_pretrained('dslim/bert-base-NER')
    model = BertForTokenClassification.from_pretrained('dslim/bert-base-NER')
    
    # 标签映射
    label_names = ['O', 'B-MISC', 'I-MISC', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']
    
    # 处理输入
    inputs = tokenizer(text, return_tensors="pt")
    tokens = tokenizer.tokenize(text)
    
    # 进行预测
    with torch.no_grad():
        outputs = model(**inputs)
    
    # 获取结果
    predictions = torch.argmax(outputs.logits, dim=2)[0].tolist()
    
    # 提取实体
    entities = []
    current_entity = None
    current_entity_type = None
    
    for i, (token, pred) in enumerate(zip(tokens, predictions[1:-1])):
        label = label_names[pred]
        
        if label.startswith('B-'):
            # 开始一个新实体
            if current_entity:
                entities.append((current_entity, current_entity_type))
            current_entity = token
            current_entity_type = label[2:]
        elif label.startswith('I-') and current_entity and label[2:] == current_entity_type:
            # 继续当前实体
            current_entity += token.lstrip('#')
        else:
            # 实体结束
            if current_entity:
                entities.append((current_entity, current_entity_type))
                current_entity = None
                current_entity_type = None
    
    # 添加最后一个实体
    if current_entity:
        entities.append((current_entity, current_entity_type))
    
    return entities

# 使用示例
text = "苹果公司总部位于加利福尼亚州库比蒂诺，由史蒂夫·乔布斯创立。"
entities = recognize_entities(text)
for entity, entity_type in entities:
    print(f"实体: {entity}, 类型: {entity_type}")

from transformers import BertTokenizer, BertForQuestionAnswering
import torch

def answer_question(question, context):
    # 加载模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
    model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
    
    # 处理输入
    inputs = tokenizer(question, context, return_tensors="pt")
    
    # 进行预测
    with torch.no_grad():
        outputs = model(**inputs)
    
    # 获取答案的开始和结束位置
    start_idx = torch.argmax(outputs.start_logits)
    end_idx = torch.argmax(outputs.end_logits) + 1
    
    # 转换为文本
    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs.input_ids[0][start_idx:end_idx]))
    
    # 计算置信度
    start_confidence = torch.softmax(outputs.start_logits, dim=1)[0][start_idx].item()
    end_confidence = torch.softmax(outputs.end_logits, dim=1)[0][end_idx-1].item()
    confidence = (start_confidence + end_confidence) / 2
    
    return answer, confidence

# 使用示例
question = "BERT模型是由哪家公司开发的？"
context = "BERT（Bidirectional Encoder Representations from Transformers）是由Google AI在2018年推出的预训练语言模型。它通过双向Transformer编码器，能够捕捉词语在上下文环境中的完整语义信息。"

answer, confidence = answer_question(question, context)
print(f"问题: {question}")
print(f"答案: {answer}")
print(f"置信度: {confidence:.4f}")

from transformers import BertTokenizer, BertForSequenceClassification
import torch

def natural_language_inference(premise, hypothesis):
    # 加载模型和分词器
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased-finetuned-mnli')
    model = BertForSequenceClassification.from_pretrained('bert-base-uncased-finetuned-mnli')
    
    # 处理输入
    inputs = tokenizer(premise, hypothesis, return_tensors="pt")
    
    # 进行预测
    with torch.no_grad():
        outputs = model(**inputs)
    
    # 获取结果
    predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
    
    # 标签映射
    labels = ['蕴含', '矛盾', '中性']
    predicted_label = labels[torch.argmax(predictions).item()]
    
    return predicted_label, predictions[0].tolist()

# 使用示例
premise = "小明今天去了公园。"
hypothesis1 = "小明今天外出了。"
hypothesis2 = "小明今天一直待在家里。"
hypothesis3 = "小明今天读了一本书。"

label1, scores1 = natural_language_inference(premise, hypothesis1)
label2, scores2 = natural_language_inference(premise, hypothesis2)
label3, scores3 = natural_language_inference(premise, hypothesis3)

print(f"前提: {premise}")
print(f"假设1: {hypothesis1}, 关系: {label1}, 置信度: {scores1}")
print(f"假设2: {hypothesis2}, 关系: {label2}, 置信度: {scores2}")
print(f"假设3: {hypothesis3}, 关系: {label3}, 置信度: {scores3}")