用Go的resty库批量下载公开网站视频

package main
​
import (
    "fmt"
    "io"
    "os"
    "path/filepath"
    "sync"
    "time"
​
    "github.com/go-resty/resty/v2"
)
​
const (
    concurrency    = 3                // 并发下载数量
    downloadDir    = "./videos"       // 下载目录
    userAgent      = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    requestTimeout = 30 * time.Second // 请求超时时间
)
​
// 视频下载任务结构
type DownloadTask struct {
    URL      string
    FileName string
}
​
func main() {
    // 创建下载目录
    if err := os.MkdirAll(downloadDir, 0755); err != nil {
        panic(err)
    }
​
    // 示例视频URL列表（替换为实际需要下载的URL）
    videoUrls := []string{
        "https://example.com/video1.mp4",
        "https://example.com/video2.mp4",
        "https://example.com/video3.mp4",
    }
​
    // 创建任务队列
    tasks := make(chan DownloadTask, len(videoUrls))
    
    // 添加下载任务
    go func() {
        defer close(tasks)
        for _, url := range videoUrls {
            tasks <- DownloadTask{
                URL:      url,
                FileName: filepath.Base(url),
            }
        }
    }()
​
    var wg sync.WaitGroup
    wg.Add(concurrency)
​
    // 启动工作池
    for i := 0; i < concurrency; i++ {
        go func(workerID int) {
            defer wg.Done()
            client := resty.New().
                SetTimeout(requestTimeout).
                SetHeader("User-Agent", userAgent).
                SetRetryCount(2).
                SetRetryWaitTime(5 * time.Second)
​
            for task := range tasks {
                downloadVideo(client, task, workerID)
            }
        }(i + 1)
    }
​
    wg.Wait()
    fmt.Println("\n所有下载任务完成！")
}
​
// 下载视频文件
func downloadVideo(client *resty.Client, task DownloadTask, workerID int) {
    filePath := filepath.Join(downloadDir, task.FileName)
    
    // 创建文件
    file, err := os.Create(filePath)
    if err != nil {
        fmt.Printf("[Worker %d] 创建文件失败: %s\n", workerID, err)
        return
    }
    defer file.Close()
​
    // 发起请求
    resp, err := client.R().
        SetDoNotParseResponse(true).
        Get(task.URL)
​
    if err != nil {
        fmt.Printf("[Worker %d] 请求失败: %s\n", workerID, err)
        return
    }
    defer resp.RawBody().Close()
​
    // 获取文件大小（可能不可用）
    contentLength := resp.Size()
    
    // 创建进度写入器
    counter := &WriteCounter{
        Total:     contentLength,
        WorkerID:  workerID,
        FileName:  task.FileName,
        StartTime: time.Now(),
    }
​
    // 写入文件并显示进度
    _, err = io.Copy(file, io.TeeReader(resp.RawBody(), counter))
    if err != nil {
        fmt.Printf("\n[Worker %d] 下载失败: %s\n", workerID, err)
        os.Remove(filePath) // 删除不完整文件
        return
    }
​
    fmt.Printf("\n[Worker %d] 下载完成: %s\n", workerID, task.FileName)
}
​
// 进度统计结构
type WriteCounter struct {
    Total     int64
    Downloaded int64
    WorkerID  int
    FileName  string
    StartTime time.Time
}
​
// 实现Write接口
func (wc *WriteCounter) Write(p []byte) (int, error) {
    n := len(p)
    wc.Downloaded += int64(n)
    wc.printProgress()
    return n, nil
}
​
// 显示下载进度
func (wc *WriteCounter) printProgress() {
    duration := time.Since(wc.StartTime).Seconds()
    speed := float64(wc.Downloaded) / 1024 / 1024 / duration
​
    progress := ""
    if wc.Total > 0 {
        percent := float64(wc.Downloaded) / float64(wc.Total) * 100
        progress = fmt.Sprintf("%.2f%% [%.2f MB/s]", percent, speed)
    } else {
        progress = fmt.Sprintf("%.2f MB [%.2f MB/s]", 
            float64(wc.Downloaded)/1024/1024, speed)
    }
​
    fmt.Printf("\r[Worker %d] %s => %s", 
        wc.WorkerID, wc.FileName, progress)
}

# 安装依赖
go get github.com/go-resty/resty/v2
​
# 运行程序
go run main.go

const (
    concurrency    = 5      // 根据网络情况调整并发数
    requestTimeout = 60 * time.Second  // 大文件需要更长时间
)

client.SetProxy("http://proxy-server:port")

client.SetHeaders(map[string]string{
    "Referer":    "https://example.com",
    "Accept":     "video/webm,video/ogg,video/*;q=0.9",
})

// 检查已下载大小
if info, err := os.Stat(filePath); err == nil {
    client.SetHeader("Range", fmt.Sprintf("bytes=%d-", info.Size()))
}

// 下载完成后验证
if resp.Size() > 0 && fileInfo.Size() != resp.Size() {
    fmt.Println("文件大小不匹配")
}