25：【HuggingFace】model下载超级慢 / ConnectionError → 国内镜像 & hf.co缓存

# 下载慢错误示例
Downloading model.safetensors: 1.2%|▏| 12.3M/10.5G [02:34<3:45:12, 789kB/s]

# ConnectionError错误示例
OSError: Failed to download (trying to open file https://huggingface.co/.../pytorch_model.bin): HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: ... (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f8c...>: Failed to establish a new connection: [Errno 101] Network is unreachable'))

# 设置国内镜像源（方法1：环境变量）
import os
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'

# 方法2：代码中设置
from huggingface_hub import set_proxy_and_mirror
set_proxy_and_mirror(mirror='https://hf-mirror.com')

# 方法3：使用huggingface-cli
# 命令行执行
huggingface-cli login --token your_token --endpoint https://hf-mirror.com

# 设置缓存目录
import os
os.environ['HF_HOME'] = '/path/to/your/cache/directory'

# 或在代码中设置
from huggingface_hub import login
login(token="your_token", cache_dir="/path/to/your/cache/directory")

# 检查缓存状态
from huggingface_hub import HfFolder
print("Cache directory:", HfFolder.path_cache)
print("Cache size:", sum(os.path.getsize(os.path.join(dirpath, filename)) for dirpath, dirnames, filenames in os.walk(HfFolder.path_cache) for filename in filenames) // (1024*1024*1024), "GB")

#!/usr/bin/env python3
"""
HuggingFace镜像源自动切换工具
"""
import os
import time
import requests
from huggingface_hub import set_proxy_and_mirror

# 镜像源列表
MIRRORS = [
    'https://hf-mirror.com',
    'https://mirror.sjtu.edu.cn/huggingface',
    'https://modelscope.cn',
    'https://huggingface.co'  # 官方源作为备选
]

def test_mirror_speed(mirror):
    """测试镜像源速度"""
    try:
        start_time = time.time()
        response = requests.get(f"{mirror}/api/models", timeout=10)
        if response.status_code == 200:
            return time.time() - start_time
        else:
            return float('inf')
    except:
        return float('inf')

def get_best_mirror():
    """获取最佳镜像源"""
    print("正在测试镜像源速度...")
    speeds = []
    for mirror in MIRRORS:
        speed = test_mirror_speed(mirror)
        if speed < float('inf'):
            speeds.append((speed, mirror))
            print(f"{mirror}: {speed:.2f}s")
        else:
            print(f"{mirror}: 不可用")
    
    if not speeds:
        print("所有镜像源都不可用，请检查网络连接")
        return None
    
    # 选择速度最快的镜像源
    best_speed, best_mirror = min(speeds, key=lambda x: x[0])
    print(f"最佳镜像源: {best_mirror} (响应时间: {best_speed:.2f}s)")
    return best_mirror

def set_hf_mirror():
    """设置HuggingFace镜像源"""
    best_mirror = get_best_mirror()
    if best_mirror:
        # 设置环境变量
        os.environ['HF_ENDPOINT'] = best_mirror
        # 设置huggingface_hub
        set_proxy_and_mirror(mirror=best_mirror)
        print(f"已设置HuggingFace镜像源为: {best_mirror}")
        return best_mirror
    return None

if __name__ == "__main__":
    set_hf_mirror()

#!/usr/bin/env python3
"""
HuggingFace缓存管理工具
"""
import os
import shutil
from huggingface_hub import HfFolder

def get_cache_size(cache_dir):
    """获取缓存目录大小"""
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(cache_dir):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            if os.path.exists(filepath):
                total_size += os.path.getsize(filepath)
    return total_size

def clean_cache(cache_dir, max_size_gb=50):
    """清理缓存，保持在指定大小以下"""
    max_size = max_size_gb * 1024 * 1024 * 1024
    current_size = get_cache_size(cache_dir)
    
    print(f"当前缓存大小: {current_size / (1024*1024*1024):.2f} GB")
    print(f"最大缓存大小: {max_size_gb} GB")
    
    if current_size <= max_size:
        print("缓存大小在限制范围内，无需清理")
        return
    
    # 获取所有缓存文件，按修改时间排序
    files = []
    for dirpath, dirnames, filenames in os.walk(cache_dir):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            if os.path.exists(filepath):
                mtime = os.path.getmtime(filepath)
                files.append((mtime, filepath))
    
    # 按修改时间排序，删除最旧的文件
    files.sort(key=lambda x: x[0])
    
    while current_size > max_size and files:
        mtime, filepath = files.pop(0)
        try:
            file_size = os.path.getsize(filepath)
            os.remove(filepath)
            current_size -= file_size
            print(f"已删除: {filepath} ({file_size / (1024*1024):.2f} MB)")
        except Exception as e:
            print(f"删除文件失败: {filepath}, 错误: {e}")
    
    print(f"清理后缓存大小: {current_size / (1024*1024*1024):.2f} GB")

def main():
    """主函数"""
    cache_dir = HfFolder.path_cache
    print(f"HuggingFace缓存目录: {cache_dir}")
    
    # 清理缓存
    clean_cache(cache_dir)

if __name__ == "__main__":
    main()

#!/bin/bash

# 设置HuggingFace国内镜像源
export HF_ENDPOINT="https://hf-mirror.com"

# 设置缓存目录（可选）
# export HF_HOME="/path/to/your/cache/directory"

# 测试下载速度
echo "测试HuggingFace模型下载速度..."
python -c "
from huggingface_hub import snapshot_download
import time

start_time = time.time()
# 下载一个小模型测试速度
model_id = 'gpt2'
snapshot_download(model_id, local_dir='./test_model', max_workers=8)
end_time = time.time()
print(f'下载完成，耗时: {end_time - start_time:.2f}秒')
"

# 清理测试文件
rm -rf ./test_model

echo "配置完成！"