linux如何获取网页数据

curl http://example.com

curl http://example.com -o output.html

curl -H "User-Agent: MyCustomUserAgent" http://example.com

wget http://example.com

wget --mirror -p --convert-links -R index.html -N -nd http://example.com

import requests

url = 'http://example.com'
response = requests.get(url)

if response.status_code == 200:
    html_content = response.text
    # 进一步处理html_content
    print(html_content)
else:
    print(f'Failed to retrieve the webpage. Status code: {response.status_code}')

pip install scrapy

import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ['http://example.com']

    def parse(self, response):
        self.log('Visited %s' % response.url)
        # 提取数据
        title = response.xpath('//title/text()').get()
        print(title)