import dask.dataframe as dd

ddf = dd.read_csv('large_dataset.csv')
result = ddf.groupby(['category', 'subcategory']).sum().compute()

# 读取时指定数据类型
dtypes = {'id': 'int32', 'value': 'float32'}
df = pd.read_csv('large.csv', dtype=dtypes)

# 使用分类类型减少内存
df['category'] = df['category'].astype('category')

chunksize = 10**6
chunks = pd.read_csv('very_large.csv', chunksize=chunksize)

result = pd.DataFrame()
for chunk in chunks:
    temp = chunk.pivot_table(index='date', columns='product', values='sales', aggfunc='sum')
    result = result.add(temp, fill_value=0)

import pandas as pd
import numpy as np

# 创建大型示例数据
size = 10**6
dates = pd.date_range('20200101', periods=365)
df = pd.DataFrame({
    'date': np.random.choice(dates, size),
    'product': np.random.choice(['A', 'B', 'C', 'D'], size),
    'region': np.random.choice(['North', 'South', 'East', 'West'], size),
    'sales': np.random.randint(100, 10000, size),
    'cost': np.random.randint(50, 5000, size)
})

# 基本透视表
pivot1 = pd.pivot_table(df, 
                       index=['date', 'region'],
                       columns='product',
                       values='sales',
                       aggfunc=['sum', 'mean'])

# 多值聚合
pivot2 = pd.pivot_table(df,
                       index='region',
                       columns='product',
                       values=['sales', 'cost'],
                       aggfunc={'sales': ['sum', 'mean'],
                               'cost': ['sum', np.std]})

# 添加总计
pivot3 = pd.pivot_table(df,
                       index='region',
                       columns='product',
                       values='sales',
                       aggfunc='sum',
                       margins=True,
                       margins_name='Total')