网络爬虫的最佳实践：结合 set_time_limit() 与 setTrafficLimit() 抓取云盘数据

<?php

// 设置脚本最大执行时间为10000秒
set_time_limit(10000);

// 引入多线程库
use Thread;

// 爬虫代理加强版 配置信息 
$proxy_ip = 'proxy.16yun.cn';
$proxy_port = '代理服务器端口';
$proxy_username = '代理用户名';
$proxy_password = '代理密码';

// 设置流量限制（示例）
function setTrafficLimit($limit) {
    echo "设定每秒流量限制为 {$limit} KB\n";
}

// 百度云盘爬虫类
class BaiduYunCrawler extends Thread {
    private $url;

    public function __construct($url) {
        $this->url = $url;
    }

    public function run() {
        // 初始化CURL
        $ch = curl_init();
        
        // 设置CURL选项
        curl_setopt($ch, CURLOPT_URL, $this->url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_PROXY, $GLOBALS['proxy_ip']);
        curl_setopt($ch, CURLOPT_PROXYPORT, $GLOBALS['proxy_port']);
        curl_setopt($ch, CURLOPT_PROXYUSERPWD, "{$GLOBALS['proxy_username']}:{$GLOBALS['proxy_password']}");
        
        // 执行爬虫
        $output = curl_exec($ch);
        
        // 错误处理
        if (curl_errno($ch)) {
            echo 'CURL 错误: ' . curl_error($ch);
        } else {
            // 处理爬取的内容（这里只示例输出部分内容）
            echo "爬取的内容：\n" . substr($output, 0, 200); // 示例：仅显示前200个字符
        }

        curl_close($ch);
    }
}

// 定义需要爬取的百度云盘公开分享URL列表
$urls = [
    'https://pan.baidu.com/s/1example_link1',  // 示例百度云盘公开分享链接
    'https://pan.baidu.com/s/1example_link2',
    // 添加更多链接
];

// 创建并运行多线程爬虫
$threads = [];
foreach ($urls as $url) {
    $crawler = new BaiduYunCrawler($url);
    $crawler->start();  // 启动线程
    $threads[] = $crawler;
}

// 等待所有线程结束
foreach ($threads as $thread) {
    $thread->join();  // 等待线程完成
}

// 设定流量限制
setTrafficLimit(50);  // 示例：限制流量为50KB/s

?>

<?php

// 示例数据分类统计
$statistics = [
    'total_pages' => count($urls),
    'successful' => 0,
    'failed' => 0,
    'total_data_size' => 0
];

foreach ($threads as $thread) {
    if ($thread->isRunning()) {
        // 假设每个线程中会返回成功与否、数据大小等信息
        $statistics['successful']++;
        $statistics['total_data_size'] += strlen($thread->output);
    } else {
        $statistics['failed']++;
    }
}

echo "总页数: " . $statistics['total_pages'] . "\n";
echo "成功抓取: " . $statistics['successful'] . "\n";
echo "失败抓取: " . $statistics['failed'] . "\n";
echo "总数据大小: " . $statistics['total_data_size'] . " 字节\n";

?>