基于 selenium 实现网站图片采集

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@File    :   dow_img_file.py
@Time    :   2023/11/15 20:53:40
@Author  :   Li Ruilong
@Version :   1.0
@Contact :   liruilonger@gmail.com
@Desc    :   批量图片采集
"""

# here put the import lib
import requests
import base64
import pandas as pd
import time
import io
import uuid
from selenium import webdriver
from selenium.webdriver.common.by import By
from PIL import Image



"""


"""


def get_img_url_base64(url):
    """
    @Time    :   2023/05/29 21:50:42
    @Author  :   liruilonger@gmail.com
    @Version :   1.0
    @Desc    :   图片 url 解析为 base64 编码
                 Args:
                   url
                 Returns:
                   base64_bytes
    """
    response = requests.get(url)
    image_bytes = response.content
    base64_bytes = base64.b64encode(image_bytes)
    return base64_bytes.decode('utf-8')


def save_base64_image(base64_data, output_file):
    """
    @Time    :   2023/11/15 22:17:15
    @Author  :   liruilonger@gmail.com
    @Version :   1.0
    @Desc    :   保存 b64 编码为 图片
    """
    
    # 解析 Base64 编码字符串
    format, data = base64_data.split(";base64,")
    image_format = format.split("/")[-1]

    # 解码 Base64 数据
    image_data = base64.b64decode(data)

    # 将字节数据读取为图像
    image = Image.open(io.BytesIO(image_data))
    image = image.convert("RGB")
    # 保存图像为文件
    image.save(output_file, image_format)


def get_img_url_byte(url):
    """
    @Time    :   2023/10/15 23:49:10
    @Author  :   liruilonger@gmail.com
    @Version :   1.0
    @Desc    :   图片 url 解析为 字节
    """
    response = requests.get(url)
    image_bytes = response.content
    return image_bytes


driver = webdriver.Chrome()

driver.get('https://image.baidu.com/')


driver.find_element(By.XPATH, "//input[@id='kw']").send_keys("K8s")
time.sleep(3)
driver.find_element(By.XPATH, "//input[@class='s_newBtn']").click()
time.sleep(5)

# 懒加载数据处理，点击 10 次加载更多
for page in range(0,2):
    # 跳转的页底部，触发懒加载
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    time.sleep(2)
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    time.sleep(2)
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    time.sleep(3)

img_elements = driver.find_elements(By.TAG_NAME,'img')
time.sleep(1) 


# 对采集处理数据进行加工
imgs = []
data = {
        "URI":[],
    }
for img_element in img_elements:
    img_id = img_element.get_attribute('id')
    img_src = img_element.get_attribute('src')
    if img_src is not None and len(img_src) > 10:
        imgs.append((img_id,img_src))
        data['URI'].append(img_src)



# 这里可以根据实际清理输出表格
df = pd.DataFrame(data)
file_name = "img_url"
df.to_csv(f'{file_name}.csv', index=False) 

# 批量下载图片
for img in  imgs:
    if 'base64' in img[1]:
        save_base64_image(img[1],f"{str(uuid.uuid4()).replace('-', '')}.jpg")
    else:    
        image_bytes = get_img_url_byte(img[1])
        image = Image.open(io.BytesIO(image_bytes))
        image = image.convert("RGB")
        image.save(f"{str(uuid.uuid4()).replace('-', '')}.jpg")