Python爬虫程序网络请求及内容解析

import requests
from bs4 import BeautifulSoup

# 爬虫爬虫IP信息
proxy_host = 'duoip'
proxy_port = '8000'

# 请求URL
url = '目标网站'

# 创建一个requests的Session对象，并设置爬虫IP
session = requests.Session()
session.proxies = {
    'http': f'http://{proxy_host}:{proxy_port}',
    'https': f'https://{proxy_host}:{proxy_port}'
}

# 发送GET请求，获取网页内容
response = session.get(url)
response.encoding = 'utf-8'

# 使用BeautifulSoup解析网页内容
soup = BeautifulSoup(response.text, 'html.parser')

# 找到所有的商品内容
items = soup.find_all('div', class_='item')

# 打印商品内容
for item in items:
    print(item.text)