多模态大模型深度实践：从统一架构到端侧部署的全链路解析

class DynamicRoutingAttention(nn.Module):
    def __init__(self, dim, num_heads):
        super().__init__()
        self.attn = nn.MultiheadAttention(dim, num_heads)
        self.gate = nn.Sequential(nn.Linear(dim*2, dim), nn.Sigmoid())
    
    def forward(self, x_text, x_image):
        # 计算模态间相似度
        sim = torch.bmm(x_text, x_image.transpose(1,2))
        gate_score = self.gate(torch.cat([x_text, x_image], dim=-1))
        # 动态权重分配
        weighted_text = x_text * gate_score
        weighted_image = x_image * (1 - gate_score)
        combined = torch.cat([weighted_text, weighted_image], dim=1)
        return self.attn(combined, combined, combined)[0]

from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from PIL import Image
import torch

model = Qwen2VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2.5-VL-7B-Instruct",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct")

# 加载图像
image = Image.open("example.jpg")

# 构建多模态对话
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": image},
            {"type": "text", "text": "这张图片里有什么？"}
        ]
    }
]

# 处理输入并生成
text = processor.apply_chat_template(messages, tokenize=False)
inputs = processor(text, images=[image], return_tensors="pt").to(model.device)

output_ids = model.generate(**inputs, max_new_tokens=128)
response = processor.decode(output_ids[0], skip_special_tokens=True)
print(response)

from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM

lora_config = LoraConfig(
    r=8,
    lora_alpha=16,
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.05,
    task_type="CAUSAL_LM"
)
model = AutoModelForCausalLM.from_pretrained("OpenBMB/MiniCPM-o-2.6")
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # 仅约0.1%参数可训练

from vllm import LLM, SamplingParams

llm = LLM(
    model="Qwen/Qwen2.5-VL-7B-Instruct",
    max_model_len=8192,
    limit_mm_per_prompt={"image": 2}
)

sampling_params = SamplingParams(temperature=0.2, max_tokens=256)
outputs = llm.generate(
    [{"prompt": "<image>\n描述这张图片", "multi_modal_data": {"image": image_path}}],
    sampling_params
)