import pandas as pd
import os

# 示例数据帧
data = {
    'Category': ['A', 'B', 'A', 'C', 'B', 'C'],
    'Value': [10, 20, 30, 40, 50, 60],
    'Date': ['2023-01-01', '2023-01-02', '2023-01-01', '2023-01-03', '2023-01-02', '2023-01-03']
}
df = pd.DataFrame(data)

# 按Category列分组保存到不同文件夹
for category, group in df.groupby('Category'):
    # 创建文件夹（如果不存在）
    folder_path = f'output/{category}'
    os.makedirs(folder_path, exist_ok=True)
    
    # 保存数据到CSV文件
    file_path = f'{folder_path}/data.csv'
    group.to_csv(file_path, index=False)
    print(f'Saved {len(group)} rows to {file_path}')

# 按Category和Date两列分组保存
for (category, date), group in df.groupby(['Category', 'Date']):
    folder_path = f'output/{category}/{date}'
    os.makedirs(folder_path, exist_ok=True)
    
    file_path = f'{folder_path}/data.csv'
    group.to_csv(file_path, index=False)

# 每2行保存到一个文件夹
chunk_size = 2
for i, chunk in enumerate(range(0, len(df), chunk_size)):
    folder_path = f'output/chunk_{i+1}'
    os.makedirs(folder_path, exist_ok=True)
    
    df_chunk = df.iloc[chunk:chunk+chunk_size]
    file_path = f'{folder_path}/data.csv'
    df_chunk.to_csv(file_path, index=False)

import os
import stat

try:
    os.makedirs(folder_path, exist_ok=True)
    # 设置文件夹权限（如果需要）
    os.chmod(folder_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
except PermissionError:
    print(f"Permission denied: cannot create directory {folder_path}")

from datetime import datetime

timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
file_path = f'{folder_path}/data_{timestamp}.csv'

# 分块处理大数据集
chunk_size = 10000
for i in range(0, len(df), chunk_size):
    df_chunk = df.iloc[i:i+chunk_size]
    # 处理并保存分块数据

# 保存为Parquet格式（更高效）
df.to_parquet('output/data.parquet', partition_cols=['Category', 'Date'])