Java 异步爬虫高效获取小红书短视频内容

<dependencies>
    <!-- Jackson JSON解析 -->
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>2.15.2</version>
    </dependency>
    <!-- Lombok -->
    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
        <version>1.18.30</version>
        <scope>provided</scope>
    </dependency>
    <!-- 字符串工具 -->
    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-lang3</artifactId>
        <version>3.14.0</version>
    </dependency>
</dependencies>

import lombok.Data;

/**
 * 小红书短视频基础信息实体类
 */
@Data
public class XiaohongshuVideo {
    // 视频ID
    private String videoId;
    // 视频标题
    private String title;
    // 发布者昵称
    private String authorName;
    // 播放量
    private long playCount;
    // 视频封面URL
    private String coverUrl;
    // 视频播放URL
    private String playUrl;
    // 发布时间
    private String publishTime;
}

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/**
 * 小红书异步爬虫核心类
 */
public class XiaohongshuAsyncCrawler {
    // 异步HttpClient实例（线程安全，全局复用）
    private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder()
            .connectTimeout(Duration.ofSeconds(10)) // 连接超时
            .followRedirects(HttpClient.Redirect.NORMAL) // 自动重定向
            .build();
    
    // JSON解析器
    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
    
    // 请求头（模拟移动端请求，需替换为自己的Cookie）
    private static final String USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1";
    private static final String COOKIE = "your_cookie_here"; // 替换为真实Cookie
    
    // 并发控制：单次最大异步请求数（避免请求过多被风控）
    private static final int MAX_CONCURRENT_REQUEST = 20;

    /**
     * 异步获取单页小红书短视频数据
     * @param keyword 搜索关键词
     * @param page 页码
     * @return 异步结果：该页视频列表
     */
    public CompletableFuture<List<XiaohongshuVideo>> crawlVideoPageAsync(String keyword, int page) {
        // 1. 构建请求URL（示例接口，需抓包获取最新接口）
        String url = String.format("https://edith.xiaohongshu.com/api/sns/web/v1/feed?keyword=%s&page=%d&page_size=20", 
                keyword, page);
        
        // 2. 构建HTTP请求
        HttpRequest request = HttpRequest.newBuilder()
                .uri(URI.create(url))
                .header("User-Agent", USER_AGENT)
                .header("Cookie", COOKIE)
                .header("Referer", "https://www.xiaohongshu.com/")
                .header("Accept", "application/json")
                .timeout(Duration.ofSeconds(15))
                .GET()
                .build();
        
        // 3. 异步发送请求并处理响应
        return HTTP_CLIENT.sendAsync(request, HttpResponse.BodyHandlers.ofString())
                .thenApply(this::parseVideoResponse) // 解析响应为视频列表
                .exceptionally(e -> { // 异常处理
                    System.err.println("抓取第" + page + "页失败：" + e.getMessage());
                    return new ArrayList<>();
                });
    }

    /**
     * 解析JSON响应为视频列表
     * @param response HTTP响应
     * @return 视频列表
     */
    private List<XiaohongshuVideo> parseVideoResponse(HttpResponse<String> response) {
        List<XiaohongshuVideo> videoList = new ArrayList<>();
        if (response.statusCode() != 200 || StringUtils.isEmpty(response.body())) {
            return videoList;
        }
        
        try {
            // 解析JSON根节点
            JsonNode rootNode = OBJECT_MAPPER.readTree(response.body());
            // 定位到视频列表节点（需根据实际接口调整路径）
            JsonNode dataNode = rootNode.get("data");
            if (dataNode == null || !dataNode.has("items")) {
                return videoList;
            }
            
            JsonNode itemsNode = dataNode.get("items");
            // 遍历每个视频节点
            for (JsonNode itemNode : itemsNode) {
                XiaohongshuVideo video = new XiaohongshuVideo();
                // 提取核心字段（需根据实际接口调整字段名）
                video.setVideoId(itemNode.get("id").asText());
                video.setTitle(itemNode.get("title").asText());
                video.setAuthorName(itemNode.get("user").get("nickname").asText());
                video.setPlayCount(itemNode.get("play_count").asLong());
                video.setCoverUrl(itemNode.get("cover").get("url").asText());
                video.setPlayUrl(itemNode.get("video").get("url").asText());
                video.setPublishTime(itemNode.get("create_time").asText());
                
                videoList.add(video);
            }
        } catch (JsonProcessingException e) {
            System.err.println("JSON解析失败：" + e.getMessage());
        }
        return videoList;
    }

    /**
     * 批量异步抓取多页视频数据
     * @param keyword 搜索关键词
     * @param totalPages 总抓取页数
     * @return 所有视频列表
     */
    public List<XiaohongshuVideo> crawlBatchVideos(String keyword, int totalPages) {
        List<CompletableFuture<List<XiaohongshuVideo>>> futureList = new ArrayList<>();
        List<XiaohongshuVideo> allVideos = new ArrayList<>();
        
        // 1. 提交所有异步请求
        for (int page = 1; page <= totalPages; page++) {
            // 控制并发数：每提交MAX_CONCURRENT_REQUEST个请求，等待一次
            if (futureList.size() >= MAX_CONCURRENT_REQUEST) {
                waitAndCollectResults(futureList, allVideos);
                futureList.clear();
            }
            futureList.add(crawlVideoPageAsync(keyword, page));
        }
        
        // 2. 处理剩余的异步请求
        waitAndCollectResults(futureList, allVideos);
        
        return allVideos;
    }

    /**
     * 等待异步请求完成并收集结果
     * @param futureList 异步请求列表
     * @param allVideos 最终结果容器
     */
    private void waitAndCollectResults(List<CompletableFuture<List<XiaohongshuVideo>>> futureList, 
                                       List<XiaohongshuVideo> allVideos) {
        // 等待所有异步请求完成
        CompletableFuture<Void> allFutures = CompletableFuture.allOf(
                futureList.toArray(new CompletableFuture[0])
        );
        
        try {
            // 等待完成（超时时间30秒）
            allFutures.get(30, TimeUnit.SECONDS);
            // 收集每个请求的结果
            for (CompletableFuture<List<XiaohongshuVideo>> future : futureList) {
                allVideos.addAll(future.get());
            }
        } catch (InterruptedException | ExecutionException | TimeoutException e) {
            System.err.println("批量抓取超时/异常：" + e.getMessage());
        }
    }

    // 测试主方法
    public static void main(String[] args) {
        XiaohongshuAsyncCrawler crawler = new XiaohongshuAsyncCrawler();
        // 抓取关键词“旅行vlog”的前5页视频
        List<XiaohongshuVideo> videos = crawler.crawlBatchVideos("旅行vlog", 5);
        
        // 输出结果
        System.out.println("共抓取到" + videos.size() + "条短视频数据：");
        for (XiaohongshuVideo video : videos) {
            System.out.println("标题：" + video.getTitle() + " | 播放量：" + video.getPlayCount());
        }
    }
}