无缝调用Java生态：用Jsoup轻松解析网页

#!/usr/bin/env groovy
​
// 导入必要的库
@Grab('org.jsoup:jsoup:1.15.3')
@Grab('org.apache.httpcomponents:httpclient:4.5.13')
@Grab('com.fasterxml.jackson.core:jackson-databind:2.14.2')
​
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.config.RequestConfig
import org.apache.http.util.EntityUtils
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
​
/**
 * 图书信息类
 */
class Book {
    String title
    String author
    BigDecimal price
    String description
    Integer rating
    String url
    
    String toString() {
        "《${title}》- ${author} - 价格: ${price}元 - 评分: ${rating}星"
    }
}
​
/**
 * 网页爬虫类
 */
class BookScraper {
    // HTTP客户端配置
    private def httpClient
    private def config
    private def objectMapper
    
    BookScraper() {
        // 配置HTTP客户端（设置超时和代理等）
        config = RequestConfig.custom()
            .setConnectTimeout(5000)
            .setSocketTimeout(5000)
            .build()
        
        httpClient = HttpClients.custom()
            .setDefaultRequestConfig(config)
            .build()
            
        objectMapper = new ObjectMapper()
        objectMapper.enable(SerializationFeature.INDENT_OUTPUT)
    }
    
    /**
     * 获取网页内容
     */
    String fetchPage(String url) {
        try {
            println "正在抓取: $url"
            def httpGet = new HttpGet(url)
            // 设置请求头，模拟浏览器
            httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
            httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
            
            def response = httpClient.execute(httpGet)
            def entity = response.getEntity()
            def content = EntityUtils.toString(entity)
            EntityUtils.consume(entity)
            
            return content
        } catch (Exception e) {
            println "抓取网页时出错: ${e.message}"
            return null
        }
    }
    
    /**
     * 解析图书列表页
     */
    List<Book> parseBookList(String html, String baseUrl) {
        def books = []
        try {
            Document doc = Jsoup.parse(html)
            Elements bookElements = doc.select(".product_pod") // 假设每本书都在这个CSS类中
            
            bookElements.each { Element element ->
                try {
                    Book book = new Book()
                    
                    // 提取标题
                    book.title = element.select("h3 a").attr("title")
                    if (!book.title) {
                        book.title = element.select("h3 a").text()
                    }
                    
                    // 提取作者
                    book.author = element.select(".author").text() ?: "未知作者"
                    
                    // 提取价格
                    def priceText = element.select(".price_color").text()
                    book.price = priceText ? new BigDecimal(priceText.replaceAll("[^\\d.]", "")) : 0.0
                    
                    // 提取评分（假设评分用CSS类表示，如star-rating-Five）
                    def ratingClass = element.select(".star-rating").attr("class")
                    def ratingMatch = ratingClass =~ /star-rating-(\w+)/
                    if (ratingMatch) {
                        def ratingMap = [One:1, Two:2, Three:3, Four:4, Five:5]
                        book.rating = ratingMap[ratingMatch[0][1]] ?: 0
                    }
                    
                    // 提取详情页链接
                    def relativeUrl = element.select("h3 a").attr("href")
                    book.url = baseUrl + relativeUrl
                    
                    // 获取图书详情
                    fetchBookDetails(book)
                    
                    books << book
                    println "已解析: ${book.title}"
                } catch (Exception e) {
                    println "解析单本书时出错: ${e.message}"
                }
            }
        } catch (Exception e) {
            println "解析图书列表时出错: ${e.message}"
        }
        
        return books
    }
    
    /**
     * 获取图书详细信息
     */
    void fetchBookDetails(Book book) {
        try {
            def html = fetchPage(book.url)
            if (html) {
                Document doc = Jsoup.parse(html)
                
                // 提取描述
                def descriptionElement = doc.select("#product_description + p")
                if (descriptionElement) {
                    book.description = descriptionElement.text()
                }
            }
        } catch (Exception e) {
            println "获取图书详情时出错: ${e.message}"
        }
    }
    
    /**
     * 保存结果到JSON文件
     */
    void saveToJson(List<Book> books, String filename) {
        try {
            objectMapper.writeValue(new File(filename), books)
            println "结果已保存到: ${filename}"
        } catch (Exception e) {
            println "保存结果时出错: ${e.message}"
        }
    }
    
    /**
     * 关闭HTTP客户端
     */
    void close() {
        httpClient.close()
    }
}
​
/**
 * 主程序
 */
def main() {
    // 初始化爬虫
    def scraper = new BookScraper()
    
    try {
        // 要爬取的网站URL（这里以虚构的图书网站为例）
        def baseUrl = "https://books.example.com"
        def startUrl = "${baseUrl}/catalogue/page-1.html"
        
        // 获取网页内容
        def html = scraper.fetchPage(startUrl)
        
        if (html) {
            // 解析图书列表
            def books = scraper.parseBookList(html, baseUrl)
            
            // 打印结果
            println "\n爬取结果:"
            books.eachWithIndex { book, index ->
                println "${index + 1}. ${book}"
                if (book.description) {
                    println "   描述: ${book.description.length() > 100 ? book.description.substring(0, 100) + '...' : book.description}"
                }
                println()
            }
            
            // 保存结果到JSON文件
            scraper.saveToJson(books, "books.json")
            
            println "共爬取 ${books.size()} 本书籍"
        } else {
            println "未能获取网页内容"
        }
    } finally {
        // 确保关闭HTTP客户端
        scraper.close()
    }
}
​
// 运行主程序
main()