DINOv3上手指南：改变视觉模型使用方式，一个模型搞定分割、检测、深度估计

 # pip install torch torchvision transformers pillow  
from transformers import AutoModel, AutoImageProcessor  
from transformers.image_utils import load_image  
import torch  

model_id = "facebook/dinov3-vitb16-pretrain-lvd1689m"  # 如果你有访问权限，可以换成vitl/7b
processor = AutoImageProcessor.from_pretrained(model_id)  
model = AutoModel.from_pretrained(model_id, device_map="auto")  
img = load_image("http://images.cocodataset.org/val2017/000000039769.jpg")  
inputs = processor(images=img, return_tensors="pt").to(model.device)  
with torch.inference_mode():  
    outputs = model(**inputs)  
cls = outputs.last_hidden_state[:, 0]  # 全局（[CLS]）
num_regs = model.config.num_register_tokens  
patch_flat = outputs.last_hidden_state[:, 1 + num_regs:, :]  
# 重塑为[B, C, H, W]，步长=16
B, N, C = patch_flat.shape  
H = W = int((N) ** 0.5)  
 feat_map = patch_flat.reshape(B, H, W, C).permute(0, 3, 1, 2)

 import torch.nn as nn  

class LinearSegHead(nn.Module):  
    def __init__(self, in_ch, n_classes):  
        super().__init__()  
        self.proj = nn.Conv2d(in_ch, n_classes, 1)  
        self.up = nn.Upsample(scale_factor=16, mode="bilinear", align_corners=False)  
    def forward(self, fmap):  # fmap: [B, C, H, W]（步长16）
        return self.up(self.proj(fmap))  # 输入分辨率的logits
head = LinearSegHead(in_ch=feat_map.shape[1], n_classes=150).to(model.device)  
 for p in model.parameters(): p.requires_grad_(False)  # 保持骨干网络冻结

 """  
train_seg_head.py — 在冻结DINOv3特征上的最小训练循环。
用ADE20K/Cityscapes等替换`YourDataset`（图像、掩码）。
"""  
import torch, torch.nn as nn, torch.optim as optim  
from torch.utils.data import DataLoader  
from transformers import AutoModel, AutoImageProcessor  

MODEL_ID = "facebook/dinov3-vitb16-pretrain-lvd1689m"  
N_CLASSES = 150  
class LinearSegHead(nn.Module):  
    def __init__(self, in_ch, n_classes):  
        super().__init__()  
        self.proj = nn.Conv2d(in_ch, n_classes, 1)  
        self.up = nn.Upsample(scale_factor=16, mode="bilinear", align_corners=False)  
    def forward(self, fmap): return self.up(self.proj(fmap))  
def extract_fmap(model, proc, image):  
    inputs = proc(images=image, return_tensors="pt").to(model.device)  
    with torch.inference_mode():  
        out = model(**inputs)  
    num_regs = model.config.num_register_tokens  
    grid = out.last_hidden_state[:, 1 + num_regs:, :]  # 丢弃CLS+寄存器
    B, N, C = grid.shape  
    H = W = int(N ** 0.5)  
    return grid.reshape(B, H, W, C).permute(0,3,1,2)  
def main(train_set, val_set, epochs=10, lr=1e-3):  
    proc = AutoImageProcessor.from_pretrained(MODEL_ID)  
    model = AutoModel.from_pretrained(MODEL_ID, device_map="auto")  
    for p in model.parameters(): p.requires_grad_(False)  
    # 推断通道数一次
    x0, _ = train_set[0]  
    fmap0 = extract_fmap(model, proc, x0)  
    head = LinearSegHead(fmap0.shape[1], N_CLASSES).to(model.device)  
    opt = optim.AdamW(head.parameters(), lr=lr)  
    loss = nn.CrossEntropyLoss()  
    train = DataLoader(train_set, batch_size=4, shuffle=True)  
    val = DataLoader(val_set, batch_size=4)  
    for ep in range(epochs):  
        head.train()  
        for img, mask in train:  
            fmap = extract_fmap(model, proc, img)  
            logits = head(fmap)  
            L = loss(logits, mask.to(logits.device).long())  
            opt.zero_grad(); L.backward(); opt.step()  
        # TODO: 在`val`上添加简单的mIoU
     torch.save(head.state_dict(), "dinov3_seg_head.pth")