用Dispatch库的爬虫程序爬取图片网站

import os
import aiohttp
import asyncio
from bs4 import BeautifulSoup
​
async def fetch(session, url):
    async with session.get(url) as response:
        return await response.text()
​
async def download_image(session, url, folder):
    try:
        async with session.get(url) as response:
            if response.status == 200:
                content = await response.read()
                filename = os.path.join(folder, url.split("/")[-1])
                with open(filename, 'wb') as f:
                    f.write(content)
                print(f"Downloaded: {filename}")
    except Exception as e:
        print(f"Error downloading {url}: {e}")
​
async def main():
    target_url = "https://example.com/images"
    save_folder = "downloaded_images"
    os.makedirs(save_folder, exist_ok=True)
​
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    }
​
    async with aiohttp.ClientSession(headers=headers) as session:
        html = await fetch(session, target_url)
        soup = BeautifulSoup(html, 'lxml')
        img_tags = soup.find_all('img')
​
        tasks = []
        for img in img_tags:
            img_url = img.get('src')
            if img_url and img_url.startswith('http'):
                tasks.append(download_image(session, img_url, save_folder))
        
        await asyncio.gather(*tasks)
​
if __name__ == "__main__":
    asyncio.run(main())