GroundAnything：一个 4B 的"指哪打哪"模型，凭什么敢说比 GPT 级大模型还能打？

<|object_ref_start|>car<|object_ref_end|><|box_start|><100><200><500><650><|box_end|>

git clone https://github.com/groundingpi/GroundAnything.git
cd GroundAnything
python3 -m pip install -r requirements.txt huggingface_hub

# 拉扩散版权重（已经是完整 bundle，不用再 prepare-model）
hf download GroundingPI/GroundAnything --local-dir weights/dlm_bundle

# 装服务环境，然后起服务
python3 run.py setup serve
python3 run.py serve --decoder denoise      # 熵引导
# python3 run.py serve --decoder speculative  # 想换自推测，先停掉上面这个

hf download GroundingPI/GroundAnything-VLM --local-dir weights/vlm
python3 run.py serve --config configs/release/vlm_sglang.yaml

from grounding_anything import GroundingAnything, visualize

client = GroundingAnything(
    base_url="http://127.0.0.1:8101/v1",
    model="groundinganything",
)

# 指代框
result = client.predict("street.jpg", "the red car", task="bbox")
print(result.to_dict())

if result.valid:
    visualize("street.jpg", result).save("prediction.png")

# 指代点
point = client.predict("street.jpg", "the center of the red car", task="point")
print(point.to_dict())

import base64, json, re, requests

def encode_image(path: str) -> str:
    with open(path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

def call_groundanything(image_path: str, prompt: str,
                        base_url="http://127.0.0.1:8101/v1",
                        model="groundinganything"):
    payload = {
        "model": model,
        "messages": [{
            "role": "user",
            "content": [
                {"type": "image_url",
                 "image_url": {"url": f"data:image/jpeg;base64,{encode_image(image_path)}"}},
                {"type": "text", "text": prompt},
            ],
        }],
        "max_tokens": 4096,
        # 坑 1：千万别开 skip_special_tokens，坐标全靠这些特殊 token 撑着
        "skip_special_tokens": False,
        "temperature": 0.0,
    }
    r = requests.post(f"{base_url}/chat/completions", json=payload, timeout=120)
    r.raise_for_status()
    return r.json()["choices"][0]["message"]["content"]

BOX_RE = re.compile(
    r"<\|object_ref_start\|>(.*?)<\|object_ref_end\|>"
    r"<\|box_start\|>(.*?)<\|box_end\|>",
    re.S,
)

def parse_gam(text: str):
    """把 GAM 串解析成 [(label, [x1,y1,x2,y2] 或 [x,y] 或 None)]"""
    out = []
    for label, coords in BOX_RE.findall(text):
        nums = re.findall(r"<(\d+)>", coords)
        if not nums or "None" in coords:
            out.append((label.strip(), None))     # 目标不存在
            continue
        vals = [int(n) for n in nums]
        out.append((label.strip(), vals))
    return out

def to_pixels(vals, W, H):
    """0-999 网格 → 原图像素坐标"""
    if vals is None:
        return None
    if len(vals) == 2:
        return [vals[0] / 999 * W, vals[1] / 999 * H]
    x1, y1, x2, y2 = vals
    return [x1 / 999 * W, y1 / 999 * H, x2 / 999 * W, y2 / 999 * H]

import json, os
from pathlib import Path

CATEGORIES = ["car", "person", "bicycle", "traffic sign"]
IMAGE_DIR = Path("data/street")

def build_manifest(image_dir: Path, out_path: str, categories=CATEGORIES):
    cats = "</c>".join(categories)
    prompt = f"Locate all the instances that match the following categories: {cats}."
    with open(out_path, "w", encoding="utf-8") as f:
        for p in sorted(image_dir.glob("*.jpg")):
            f.write(json.dumps({
                "image": str(p),
                "prompt": prompt,
                "task": "bbox",
            }, ensure_ascii=False) + "\n")

build_manifest(IMAGE_DIR, "requests.jsonl")
print("总条数:", sum(1 for _ in open("requests.jsonl", encoding="utf-8")))

python3 examples/batch_predict.py \
  --input requests.jsonl --output predictions.jsonl \
  --base-url http://127.0.0.1:8101/v1 --model groundinganything

# 假设 4 张卡，切 4 份
split -n l/4 -d requests.jsonl part_
for i in 0 1 2 3; do
  python3 examples/batch_predict.py \
    --input part_0$i --output pred_$i.jsonl \
    --base-url http://127.0.0.1:810${i}/v1 --model groundinganything &
done
wait
cat pred_*.jsonl > predictions.jsonl

import json
from PIL import Image

def load_jsonl(p):
    with open(p, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                yield json.loads(line)

def quality_check(boxes, W, H, min_area_ratio=1e-4, max_area_ratio=0.9):
    """滤掉空结果、越界框、面积离谱的框"""
    keep = []
    for label, coords in boxes:
        if coords is None or len(coords) != 4:
            continue
        x1, y1, x2, y2 = to_pixels(coords, W, H)
        x1, x2 = sorted([x1, x2]); y1, y2 = sorted([y1, y2])
        x1, y1 = max(0, x1), max(0, y1)
        x2, y2 = min(W, x2), min(H, y2)
        if x2 <= x1 or y2 <= y1:
            continue
        area_ratio = ((x2 - x1) * (y2 - y1)) / (W * H)
        if not (min_area_ratio <= area_ratio <= max_area_ratio):
            continue
        keep.append((label, [x1, y1, x2, y2]))
    return keep

def export_coco(pred_path, out_path, categories=CATEGORIES):
    cat2id = {c: i + 1 for i, c in enumerate(categories)}
    images, annotations = [], []
    ann_id, img_id = 1, 1
    stats = {"total": 0, "valid": 0, "skipped": 0, "boxes": 0}

    for rec in load_jsonl(pred_path):
        stats["total"] += 1
        if not rec.get("valid", False):
            stats["skipped"] += 1
            continue
        p = rec["image"]
        try:
            W, H = Image.open(p).size
        except Exception:
            stats["skipped"] += 1
            continue

        parsed = parse_gam(rec.get("raw_output", ""))
        kept = quality_check(parsed, W, H)
        if not kept:
            stats["skipped"] += 1
            continue

        images.append({"id": img_id, "file_name": p, "width": W, "height": H})
        for label, (x1, y1, x2, y2) in kept:
            if label not in cat2id:
                continue
            annotations.append({
                "id": ann_id, "image_id": img_id, "category_id": cat2id[label],
                "bbox": [round(x1, 1), round(y1, 1),
                         round(x2 - x1, 1), round(y2 - y1, 1)],
                "area": round((x2 - x1) * (y2 - y1), 1),
                "iscrowd": 0,
            })
            ann_id += 1
        stats["valid"] += 1
        stats["boxes"] += len(kept)
        img_id += 1

    coco = {
        "images": images,
        "annotations": annotations,
        "categories": [{"id": i + 1, "name": c} for i, c in enumerate(categories)],
    }
    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(coco, f, ensure_ascii=False)
    return stats

print(export_coco("predictions.jsonl", "street_coco.json"))
# {'total': 10000, 'valid': 9432, 'skipped': 568, 'boxes': 71238}