利用Selenium和PhantomJS提升网页内容抓取与分析的效率

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.proxy import Proxy, ProxyType

# 配置PhantomJS驱动路径
phantomjs_path = '/path/to/phantomjs'  # 替换为你的PhantomJS路径

# 代理信息
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 设置代理
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = f"{proxyHost}:{proxyPort}"
proxy.ssl_proxy = f"{proxyHost}:{proxyPort}"

# 设置代理认证信息（如果需要）
proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)

# 初始化WebDriver
driver = webdriver.PhantomJS(executable_path=phantomjs_path, desired_capabilities=webdriver.DesiredCapabilities.PHANTOMJS)

# 打开目标网页
url = 'https://example.com'  # 替换为目标网页地址
driver.get(url)

# 等待页面加载完成
try:
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.TAG_NAME, 'body'))
    )
    print("页面加载完成！")
except Exception as e:
    print("页面加载失败：", str(e))

# 模拟用户交互：点击按钮
try:
    button = driver.find_element(By.ID, 'example-button')  # 替换为目标按钮的ID
    button.click()
    print("按钮点击成功！")
except Exception as e:
    print("按钮点击失败：", str(e))

# 捕获页面内容
time.sleep(2)  # 等待动态内容加载
html_content = driver.page_source

# 关闭浏览器
driver.quit()

# 分析页面内容
# 示例：提取页面标题
from bs4 import BeautifulSoup

soup = BeautifulSoup(html_content, 'html.parser')
title = soup.find('title').text
print("页面标题：", title)

# 示例：提取所有链接
links = soup.find_all('a')
for link in links:
    print("链接：", link.get('href'))

# 示例：提取特定内容
try:
    target_content = soup.find('div', {'class': 'example-class'}).text
    print("目标内容：", target_content)
except Exception as e:
    print("目标内容提取失败：", str(e))