2025 Transformer技术全景：位置编码到三线性注意力的革命性突破​​

from transformers import AutoTokenizer, AutoModelForCausalLM
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3-8B")
model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3-8B")
text = "大模型如何理解语言"
inputs = tokenizer(text, return_tensors="pt")
# 计算下一个token概率分布
outputs = model(**inputs, labels=inputs["input_ids"])
logits = outputs.logits[0, -1]  # 最后一个位置的logits
probs = torch.softmax(logits, dim=-1)
# 输出最可能的前5个token
top_tokens = torch.topk(probs, 5)
for token_id, prob in zip(top_tokens.indices, top_tokens.values):
    print(f"Token: {tokenizer.decode(token_id)} \t Probability: {prob:.4f}")

Token: ？       Probability: 0.4021
Token: 的       Probability: 0.1987
Token: 呢       Probability: 0.1012
Token: 本质     Probability: 0.0873
Token: 过程     Probability: 0.0562

from sentence_transformers import SentenceTransformer
model = SentenceTransformer('BAAI/bge-base-zh')
embeddings = model.encode([
    "深度学习模型", 
    "神经网络架构"
])
# 计算余弦相似度
from sklearn.metrics.pairwise import cosine_similarity
similarity = cosine_similarity([embeddings[0]], [embeddings[1]])
print(f"语义相似度: {similarity[0][0]:.2f}")  # 输出：0.87

import torch
import torch.nn as nn
class MultiHeadAttention(nn.Module):
    def __init__(self, d_model=512, heads=8):
        super().__init__()
        self.d_k = d_model // heads
        self.heads = heads
        self.WQ = nn.Linear(d_model, d_model)  # 查询矩阵
        self.WK = nn.Linear(d_model, d_model)  # 键矩阵
        self.WV = nn.Linear(d_model, d_model)  # 值矩阵
        self.out = nn.Linear(d_model, d_model)
    def forward(self, X, mask=None):
        Q = self.WQ(X)  # [batch, seq, d_model]
        K = self.WK(X)
        V = self.WV(X)
        
        # 分头处理
        Q = Q.view(X.size(0), -1, self.heads, self.d_k).transpose(1,2)
        K, V = ... # 类似处理
        
        # 注意力分数
        scores = torch.matmul(Q, K.transpose(-2,-1)) / torch.sqrt(self.d_k)
        if mask is not None:
            scores = scores.masked_fill(mask==0, -1e9)
        attn_weights = torch.softmax(scores, dim=-1)
        
        # 加权聚合
        context = torch.matmul(attn_weights, V)
        context = context.transpose(1,2).contiguous().view(X.size(0), -1, self.d_model)
        return self.out(context)

# 使用BertViz可视化注意力
from bertviz.transformers_neuron_view import BertModel, BertTokenizer
from bertviz.neuron_view import show
model = BertModel.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
sentence = "The cat sat on the mat"
show(model, 'bert', tokenizer, sentence, layer=4, head=3)  # 展示第4层第3头注意力

from sentence_transformers import SentenceTransformer, losses
from torch.utils.data import DataLoader
# 1. 加载预训练模型
model = SentenceTransformer('BAAI/bge-base-zh')
# 2. 准备领域数据（问答对）
train_data = [
    ('量子计算原理', '利用量子比特叠加态并行计算'),
    ('Transformer架构', '基于自注意力的编码-解码结构')
]
# 3. 定义对比损失
train_dataloader = DataLoader(train_data, batch_size=16)
loss = losses.MultipleNegativesRankingLoss(model)
# 4. 微调训练
model.fit(
    train_objectives=[(train_dataloader, loss)],
    epochs=3,
    output_path='my_domain_embedding_model'
)

# 基于Meta 2-simplicial attention的简化实现
class SimplicialAttention(nn.Module):
    def __init__(self, d_model, window1=512, window2=32):
        super().__init__()
        self.WK_prime = nn.Linear(d_model, d_model)  # 第二键矩阵
        self.register_buffer("mask", self._create_mask(window1, window2))
    def forward(self, Q, K, V):
        K_prime = self.WK_prime(K)  # 第二键投影 [B, L, D]
        
        # 三线性注意力得分
        sim_tensor = torch.einsum('bqd,bkd,bkl->bqkl', Q, K, K_prime)
        sim_tensor = sim_tensor / (Q.size(-1) ** (1/3))
        
        # 应用局部窗口掩码
        sim_tensor += self.mask[:Q.size(1), :K.size(1), :K_prime.size(1)]
        attn_weights = torch.softmax(sim_tensor, dim=-1)
        
        # Hadamard乘积加权
        output = torch.einsum('bqkl,bld,bld->bqd', attn_weights, V, V_prime)
        return output
    def _create_mask(self, w1, w2):
        # 创建滑动窗口掩码（局部注意力）
        mask = torch.full((L, L, L), float('-inf'))
        for q_pos in range(L):
            start_k = max(0, q_pos - w1//2)
            end_k = min(L, q_pos + w1//2)
            start_kp = max(0, q_pos - w2//2)
            end_kp = min(L, q_pos + w2//2)
            mask[q_pos, start_k:end_k, start_kp:end_kp] = 0
        return mask