高效爬取Reddit：C#与RestSharp的完美结合

using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using RestSharp;

namespace RedditCrawler
{
    class Program
    {
        // 爬虫代理加强版***代理服务器信息
        private static string proxyHost = "www.Host.cn";
        private static int proxyPort = 31111; // 替换为实际端口
        private static string proxyUser = "用户名";
        private static string proxyPass = "密码";

        static void Main(string[] args)
        {
            // 要爬取的URL列表
            string[] urls = { "https://www.reddit.com/r/programming/", "https://www.reddit.com/r/technology/" };

            // 并发爬取每个URL的内容
            Parallel.ForEach(urls, url =>
            {
                FetchRedditContent(url);
            });

            Console.WriteLine("所有爬取任务已完成。");
        }

        // 爬取Reddit内容的方法
        private static void FetchRedditContent(string url)
        {
            // 创建RestClient实例，并设置爬虫代理服务器
            var client = new RestClient(url)
            {
                Proxy = new WebProxy(proxyHost, proxyPort)
                {
                    Credentials = new NetworkCredential(proxyUser, proxyPass)
                }
            };

            // 创建Request对象，设置GET请求
            var request = new RestRequest("api/frontpage.json", Method.GET);
            
            // 设置User-Agent和Cookie
            request.AddHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
            request.AddHeader("Cookie", "your_cookie_here");

            // 执行请求，获取响应
            var response = client.Execute(request);

            if (response.IsSuccessful)
            {
                AnalyzeResponse(response.Content);
            }
            else
            {
                Console.WriteLine($"请求失败：{response.ErrorMessage}");
            }
        }

        // 分析响应内容的方法
        private static void AnalyzeResponse(string content)
        {
            var json = JObject.Parse(content);
            var posts = json["data"]["children"];

            var postTitles = new List<string>();
            var postScores = new List<int>();
            var postComments = new List<int>();

            foreach (var post in posts)
            {
                postTitles.Add(post["data"]["title"].ToString());
                postScores.Add((int)post["data"]["score"]);
                postComments.Add((int)post["data"]["num_comments"]);
            }

            // 统计数据
            Console.WriteLine("统计结果：");
            Console.WriteLine($"帖子数量: {postTitles.Count}");
            Console.WriteLine($"平均得分: {GetAverage(postScores)}");
            Console.WriteLine($"平均评论数: {GetAverage(postComments)}");

            // 输出部分帖子标题
            Console.WriteLine("部分帖子标题:");
            foreach (var title in postTitles.Take(5))
            {
                Console.WriteLine(title);
            }
        }

        // 计算平均值的方法
        private static double GetAverage(List<int> values)
        {
            if (values.Count == 0) return 0;
            double sum = 0;
            foreach (var value in values)
            {
                sum += value;
            }
            return sum / values.Count;
        }
    }
}