C#编写HttpClient爬虫程序示例

using System;
using System.Net.Http;
using System.Threading.Tasks;
using HtmlAgilityPack; // 需要安装NuGet包
​
class WebCrawler
{
    private static readonly HttpClient _httpClient = new HttpClient();
​
    static async Task Main(string[] args)
    {
        // 配置HTTP客户端
        _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
        _httpClient.Timeout = TimeSpan.FromSeconds(30);
​
        try
        {
            string url = "https://example.com";
            
            // 发送HTTP GET请求
            HttpResponseMessage response = await _httpClient.GetAsync(url);
            response.EnsureSuccessStatusCode(); // 检查状态码
​
            // 读取响应内容
            string htmlContent = await response.Content.ReadAsStringAsync();
            
            // 解析HTML内容
            var htmlDoc = new HtmlDocument();
            htmlDoc.LoadHtml(htmlContent);
​
            // 示例：提取所有链接
            foreach (HtmlNode link in htmlDoc.DocumentNode.SelectNodes("//a[@href]"))
            {
                string hrefValue = link.GetAttributeValue("href", string.Empty);
                Console.WriteLine($"发现链接: {hrefValue}");
            }
​
            // 示例：提取页面标题
            string pageTitle = htmlDoc.DocumentNode.SelectSingleNode("//title")?.InnerText;
            Console.WriteLine($"页面标题: {pageTitle}");
        }
        catch (HttpRequestException ex)
        {
            Console.WriteLine($"HTTP请求失败: {ex.Message}");
        }
        catch (TaskCanceledException ex)
        {
            Console.WriteLine($"请求超时: {ex.Message}");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"发生错误: {ex.Message}");
        }
    }
}

dotnet add package HtmlAgilityPack