C++舆情监控爬虫程序实现

#include <iostream>
#include <string>
#include <vector>
#include <thread>
#include <mutex>
#include <curl/curl.h>
#include <gumbo.h>
​
using namespace std;
​
// 全局互斥锁
mutex g_mtx;
​
// 存储结果的数据结构
vector<string> g_results;
​
// libcurl写回调函数
static size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* userp) {
    ((string*)userp)->append((char*)contents, size * nmemb);
    return size * nmemb;
}
​
// 使用Gumbo解析HTML
void parse_html(const string& html) {
    GumboOutput* output = gumbo_parse(html.c_str());
    
    // 示例：提取所有<p>标签内容
    GumboNode* root = output->root;
    GumboVector* children = &root->v.element.children;
    
    for (unsigned int i = 0; i < children->length; ++i) {
        GumboNode* node = static_cast<GumboNode*>(children->data[i]);
        if (node->type == GUMBO_NODE_ELEMENT &&
            node->v.element.tag == GUMBO_TAG_P) {
            GumboNode* text = static_cast<GumboNode*>(
                node->v.element.children.data[0]);
            if (text->type == GUMBO_NODE_TEXT) {
                lock_guard<mutex> lock(g_mtx);
                g_results.push_back(text->v.text.text);
            }
        }
    }
    
    gumbo_destroy_output(&kGumboDefaultOptions, output);
}
​
// 爬取单个URL
void crawl_url(const string& url) {
    CURL* curl = curl_easy_init();
    if (curl) {
        CURLcode res;
        string readBuffer;
        
        // 设置curl参数
        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
        curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
​
        res = curl_easy_perform(curl);
        if (res == CURLE_OK) {
            parse_html(readBuffer);
        }
        
        curl_easy_cleanup(curl);
    }
}
​
// 舆情关键词过滤
bool contains_keywords(const string& text, const vector<string>& keywords) {
    for (const auto& kw : keywords) {
        if (text.find(kw) != string::npos) {
            return true;
        }
    }
    return false;
}
​
int main() {
    curl_global_init(CURL_GLOBAL_DEFAULT);
    
    // 配置参数
    vector<string> urls = {
        "https://news.example.com",
        "https://forum.example.org",
        "https://blog.test.net"
    };
    
    vector<string> keywords = {"疫情", "经济", "政策"};
    vector<thread> threads;
​
    // 创建爬取线程
    for (const auto& url : urls) {
        threads.emplace_back([url]() {
            crawl_url(url);
        });
    }
​
    // 等待所有线程完成
    for (auto& t : threads) {
        t.join();
    }
​
    // 过滤和输出结果
    cout << "舆情分析结果：" << endl;
    for (const auto& text : g_results) {
        if (contains_keywords(text, keywords)) {
            cout << "- " << text << endl;
        }
    }
​
    curl_global_cleanup();
    return 0;
}

// 随机延迟
void random_delay() {
    this_thread::sleep_for(chrono::milliseconds(rand() % 3000 + 1000));
}
​
// 在crawl_url函数中添加
random_delay();

// 设置代理服务器
curl_easy_setopt(curl, CURLOPT_PROXY, "http://proxy.example.com:8080");

// SQLite3存储示例
#include <sqlite3.h>
​
void save_to_db(const string& text) {
    sqlite3* db;
    sqlite3_open("results.db", &db);
    
    string sql = "INSERT INTO results(content) VALUES('" + text + "');";
    sqlite3_exec(db, sql.c_str(), 0, 0, 0);
    
    sqlite3_close(db);
}

g++ -std=c++11舆情爬虫.cpp -lcurl -lgumbo -lsqlite3 -lpthread -o monitor