挖掘网络宝藏：R和XML库助你轻松抓取 www.sohu.com 图片

install.packages("XML")

# 加载所需库
library(XML)
library(httr)
library(foreach)
library(doParallel)

# 爬虫代理 设置IP信息
proxy <- list(
  http = "http://www.18XXXX.cn:36986:user:pass",
  https = "http://www.18XXXX.cn:36986:user:pass"
)

# 设置目标网址
url <- "https://www.sohu.com"

# 获取页面内容的函数
get_page <- function(url) {
  tryCatch({
    response <- httr::GET(url, use_proxy(proxy))
    content <- httr::content(response, as = "text")
    return(content)
  }, error = function(e) {
    cat("Error fetching page:", conditionMessage(e), "\n")
    return(NULL)
  })
}

# 解析页面内容，提取图片链接
parse_page <- function(content) {
  doc <- XML::htmlParse(content, useInternalNodes = TRUE)
  img_nodes <- XML::getNodeSet(doc, "//img")
  img_links <- sapply(img_nodes, function(node) XML::xmlGetAttr(node, "src"))
  return(img_links)
}

# 多线程下载图片
download_images <- function(img_links) {
  registerDoParallel(cores = 4)  # 设置线程数
  foreach(link = img_links, .combine = c) %dopar% {
    tryCatch({
      img <- httr::GET(link, use_proxy(proxy))
      filename <- basename(link)
      writeBin(content(img, "raw"), file.path("images", filename))
      cat("Downloaded", filename, "\n")
    }, error = function(e) {
      cat("Error downloading image:", conditionMessage(e), "\n")
    })
  }
  stopImplicitCluster()
}

# 主函数
main <- function() {
  content <- get_page(url)
  if (!is.null(content)) {
    img_links <- parse_page(content)
    if (length(img_links) > 0) {
      dir.create("images", showWarnings = FALSE)
      download_images(img_links)
    } else {
      cat("No images found on the page.\n")
    }
  } else {
    cat("Failed to fetch page content.\n")
  }
}

# 执行主函数
main()