2024,Python爬虫系统入门与多领域实战指南fx

pip install requests beautifulsoup4 lxml selenium

import requests

def get_page(url):
    response = requests.get(url)
    return response.text

page = get_page('http://example.com')
print(page)

from bs4 import BeautifulSoup

soup = BeautifulSoup(page, 'html.parser')
print(soup.title.string)  # 打印网页标题

session = requests.Session()
response = session.get('http://example.com/login', data={'username': 'user', 'password': 'pass'})

from selenium import webdriver

driver = webdriver.Chrome()
driver.get('http://example.com')
element = driver.find_element_by_id('dynamic-content')
print(element.text)
driver.quit()

try:
    response = requests.get('http://example.com')
    response.raise_for_status()  # 检查请求是否成功
    soup = BeautifulSoup(response.text, 'html.parser')
except requests.exceptions.RequestException as e:
    print(e)

def scrape_books(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    books = soup.find_all('div', class_='book')
    for book in books:
        title = book.find('h3').text
        author = book.find('span', class_='author').text
        print(f'Title: {title}, Author: {author}')

scrape_books('http://books.example.com')

def scrape_dynamic_data(url):
    driver = webdriver.Chrome()
    driver.get(url)
    # 假设需要点击一个按钮来加载数据
    button = driver.find_element_by_id('load-data-button')
    button.click()
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    data = soup.find('div', id='data-container').text
    driver.quit()
    return data

data = scrape_dynamic_data('http://dynamic.example.com')
print(data)

def save_data(data, filename):
    with open(filename, 'w', encoding='utf-8') as file:
        file.write(data)

save_data(page, 'scraped_data.html')

import requests
from bs4 import BeautifulSoup

def fetch_html(url):
    response = requests.get(url)
    return response.text

url = 'http://example.com'
html_content = fetch_html(url)
print(html_content)

import tweepy
import json

# 配置Twitter API的认证信息
consumer_key = 'YOUR_CONSUMER_KEY'
consumer_secret = 'YOUR_CONSUMER_SECRET'
access_token = 'YOUR_ACCESS_TOKEN'
access_token_secret = 'YOUR_ACCESS_TOKEN_SECRET'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth)

# 获取用户的时间线
public_tweets = api.home_timeline()
for tweet in public_tweets:
    print(json.dumps(tweet._json, indent=4))

from selenium import webdriver

# 设置Selenium使用的WebDriver
driver = webdriver.Chrome('/path/to/chromedriver')

# 访问网页
driver.get('http://example.com')

# 等待页面加载完成
driver.implicitly_wait(10)

# 获取页面源代码
html_content = driver.page_source

# 关闭浏览器
driver.quit()

print(html_content)

import scrapy
from scrapy.crawler import CrawlerProcess

class ProductSpider(scrapy.Spider):
    name = 'product_spider'
    start_urls = ['http://example.com/products']

    def parse(self, response):
        for product in response.css('div.product'):
            yield {
                'name': product.css('h3::text').get(),
                'price': product.css('p.price::text').get(),
                'url': product.css('a::attr(href)').get(),
            }

# 运行爬虫
process = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0 (compatible; Scrapy/1.2; +http://example.com)'
})
process.crawl(ProductSpider)
process.start()

import requests
from fake_useragent import UserAgent

ua = UserAgent()
headers = {'User-Agent': ua.random}

proxies = {
    'http': 'http://10.10.1.10:3128',
    'https': 'https://10.10.1.10:1080',
}

def fetch_html_with_proxies(url):
    response = requests.get(url, headers=headers, proxies=proxies)
    return response.text

html_content = fetch_html_with_proxies('http://example.com')
print(html_content)