R语言中的rvest库写个视频爬虫通用代码

# 加载必要库
library(rvest)  # 网页抓取
library(httr)   # 处理HTTP请求
library(tools)  # 处理文件扩展名
​
# 设置通用视频爬虫函数
video_crawler <- function(
  url,                  # 目标页面URL
  video_selector,       # 视频链接的CSS选择器 (e.g., "video source", ".video-link")
  referer = NULL,       # 需要设置的Referer头
  file_prefix = "video",# 保存文件前缀
  delay = 1             # 请求延迟(秒，防止被封)
{
  # 设置浏览器标识
  user_agent <- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
  
  # 发送HTTP请求
  response <- GET(url, add_headers(
    'User-Agent' = user_agent,
    'Referer' = ifelse(is.null(referer), url, referer)
  ))
  
  # 检查响应状态
  if (status_code(response) != 200) {
    stop(paste("请求失败，状态码：", status_code(response)))
  }
  
  # 解析网页内容
  page <- content(response, as = "parsed")
  
  # 提取视频链接
  video_elements <- html_nodes(page, video_selector)
  video_links <- html_attr(video_elements, "src")
  
  # 处理相对链接
  video_links <- url_absolute(video_links, url)
  
  # 去重处理
  video_links <- unique(video_links)
  
  # 检查是否找到视频
  if (length(video_links) == 0) {
    stop("未找到视频链接，请检查选择器设置")
  }
  
  # 创建保存目录
  dir.create("downloads", showWarnings = FALSE)
  
  # 下载视频
  for (i in seq_along(video_links)) {
    tryCatch({
      # 获取视频文件信息
      video_url <- video_links[i]
      file_ext <- file_ext(video_url)
      
      # 生成文件名
      filename <- sprintf("downloads/%s_%03d.%s", 
                         file_prefix, i, 
                         ifelse(nchar(file_ext) > 0, file_ext, "mp4"))
      
      # 添加延迟
      Sys.sleep(delay)
      
      # 下载文件
      message(sprintf("正在下载第 %d 个视频：%s", i, video_url))
      GET(video_url, 
          add_headers(Referer = url),
          write_disk(filename, overwrite = TRUE),
          user_agent(user_agent))
      
    }, error = function(e) {
      message(sprintf("下载失败：%s", e$message))
    })
  }
  
  message(sprintf("\n成功下载 %d/%d 个视频", 
                 sum(file.exists(sprintf("downloads/%s_%03d.*", file_prefix, seq_along(video_links)))),
                 length(video_links)))
}
​
# 使用示例 (需要替换实际参数)
# video_crawler(
#   url = "https://example.com/videos",
#   video_selector = "video source",  # 根据目标网站结构调整
#   referer = "https://example.com/", # 有些网站需要验证来源
#   delay = 2  # 更长的延迟更安全
# )

# 如需处理JavaScript动态加载内容，建议配合RSelenium：
library(RSelenium)
remDr <- remoteDriver(browserName = "chrome")
remDr$open()
remDr$navigate(url)
page <- remDr$getPageSource()[[1]] %>% read_html()

response <- GET(url, use_proxy("http://proxy.com:8080"))