数据库太大怎么导入

import pandas as pd
from sqlalchemy import create_engine

def batch_import(file_path, table_name, batch_size=10000):
    engine = create_engine('your_database_connection_string')
    for chunk in pd.read_csv(file_path, chunksize=batch_size):
        chunk.to_sql(table_name, engine, if_exists='append', index=False)

# 使用示例
batch_import('large_dataset.csv', 'your_table')

-- MySQL 示例
LOAD DATA INFILE 'large_dataset.csv'
INTO TABLE your_table
FIELDS TERMINATED BY ','
ENCLOSED BY '"'
LINES TERMINATED BY '\n';

import concurrent.futures
import pandas as pd
from sqlalchemy import create_engine

def import_chunk(chunk, table_name, engine):
    chunk.to_sql(table_name, engine, if_exists='append', index=False)

def parallel_import(file_path, table_name, batch_size=10000, max_workers=4):
    engine = create_engine('your_database_connection_string')
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = []
        for chunk in pd.read_csv(file_path, chunksize=batch_size):
            futures.append(executor.submit(import_chunk, chunk, table_name, engine))
        concurrent.futures.wait(futures)

# 使用示例
parallel_import('large_dataset.csv', 'your_table')

# 假设有一个时间戳字段 'last_updated' 来标识新数据
new_data = pd.read_csv('new_data.csv')
new_data.to_sql('your_table', engine, if_exists='append', index=False)