scrapy 数据存入mysql

pip install pymysql

CREATE DATABASE scrapy_db;
USE scrapy_db;

CREATE TABLE items (
    id INT AUTO_INCREMENT PRIMARY KEY,
    title VARCHAR(255),
    url VARCHAR(255)
);

import pymysql

class MySQLPipeline(object):

    def __init__(self, db_host, db_user, db_password, db_name):
        self.db_host = db_host
        self.db_user = db_user
        self.db_password = db_password
        self.db_name = db_name

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            db_host=crawler.settings.get('MYSQL_HOST'),
            db_user=crawler.settings.get('MYSQL_USER'),
            db_password=crawler.settings.get('MYSQL_PASSWORD'),
            db_name=crawler.settings.get('MYSQL_DBNAME'),
        )

    def open_spider(self, spider):
        self.connection = pymysql.connect(host=self.db_host,
                                         user=self.db_user,
                                         password=self.db_password,
                                         db=self.db_name,
                                         charset='utf8mb4',
                                         cursorclass=pymysql.cursors.DictCursor)
        self.cursor = self.connection.cursor()

    def close_spider(self, spider):
        self.connection.close()

    def process_item(self, item, spider):
        query = "INSERT INTO items (title, url) VALUES (%s, %s)"
        self.cursor.execute(query, (item['title'], item['url']))
        self.connection.commit()
        return item

# settings.py

# MySQL settings
MYSQL_HOST = 'localhost'
MYSQL_USER = 'your_username'
MYSQL_PASSWORD = 'your_password'
MYSQL_DBNAME = 'scrapy_db'

ITEM_PIPELINES = {
    'your_project_name.pipelines.MySQLPipeline': 300,
}