C++在数据分析和网络爬虫的应用

#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <boost/beast/core.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/version.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/asio/thread_pool.hpp>

namespace beast = boost::beast;
namespace http = beast::http;
namespace asio = boost::asio;
using tcp = asio::ip::tcp;

void scrapeMovieData(asio::io_context& ioContext, const std::string& host, const std::string& port, const std::string& proxyHost, const std::string& proxyPort, const std::string& proxyUsername, const std::string& proxyPassword)
{
    asio::ssl::context sslContext(asio::ssl::context::tlsv12_client);

    tcp::resolver resolver(ioContext);
    tcp::resolver::query query(proxyHost, proxyPort);
    tcp::resolver::iterator endpointIterator = resolver.resolve(query);

    asio::ssl::stream<tcp::socket> socket(ioContext, sslContext);
    asio::connect(socket.next_layer(), endpointIterator);

    if (!proxyUsername.empty() && !proxyPassword.empty())
    {
        std::string authString = proxyUsername + ":" + proxyPassword;
        std::string encodedAuthString = beast::detail::base64_encode(authString);
        std::string proxyAuthHeader = "Proxy-Authorization: Basic " + encodedAuthString;
        socket.next_layer().write(asio::buffer(proxyAuthHeader + "\r\n"));
    }

    socket.handshake(asio::ssl::stream_base::client);

    http::request<http::string_body> request(http::verb::get, "/", 11);
    request.set(http::field::host, host);
    request.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING);

    http::write(socket, request);

    beast::flat_buffer buffer;
    http::response<http::dynamic_body> response;

    http::read(socket, buffer, response);

    std::vector<std::string> movieNames;
    std::vector<std::string> movieRatings;

    std::size_t startPos = 0;
    std::size_t endPos = 0;

    std::string responseBody = boost::beast::buffers_to_string(response.body().data());
    while ((startPos = responseBody.find("<a class=\"\">", endPos)) != std::string::npos)
    {
        startPos += 13;
        endPos = responseBody.find("</a>", startPos);
        std::string movieName = responseBody.substr(startPos, endPos - startPos);

        startPos = responseBody.find("<span class=\"rating_nums\">", endPos) + 25;
        endPos = responseBody.find("</span>", startPos);
        std::string movieRating = responseBody.substr(startPos, endPos - startPos);

        movieNames.push_back(movieName);
        movieRatings.push_back(movieRating);
    }

    std::ofstream outputFile("movie_data.xlsx");
    if (outputFile)
    {
        outputFile << "电影名称\t评分\n";
        for (std::size_t i = 0; i < movieNames.size(); ++i)
        {
            outputFile << movieNames[i] << "\t" << movieRatings[i] << "\n";
        }
        outputFile.close();
        std::cout << "数据采集成功，并保存到'movie_data.xlsx'文件中。" << std::endl;
    }
    else
    {
        std::cout << "无法打开输出文件。" << std::endl;
    }
}

int main()
{
    const std::string host = "movie.douban.com";
    const std::string port = "443";
    //亿牛云 爬虫代理加强版
    //设置代理IP信息，主机、端口、用户名和密码
    const std::string proxyHost = "www.16yun.cn";
    const std::string proxyPort = "8080";
    const std::string proxyUsername = "16YUN";
    const std::string proxyPassword = "16IP";

    asio::io_context ioContext;
    asio::thread_pool threadPool;

    for (int i = 0; i < 10; ++i)
    {
        asio::post(threadPool, [=, &ioContext]() {
            scrapeMovieData(ioContext, host, port, proxyHost, proxyPort, proxyUsername, proxyPassword);
        });
    }

    threadPool.join();

    return 0;
}