借助 DeepSeek 优化 Python 脚本：从数据处理到性能提升

import h5py
import numpy as np
import os

def process_hdf5_files(input_dir, output_dir):
    """处理目录下的所有HDF5文件"""
    files = [f for f in os.listdir(input_dir) if f.endswith('.h5')]
    
    for file in files:
        with h5py.File(os.path.join(input_dir, file), 'r') as hdf_file:
            # 读取所有波段
            bands = {}
            for band_name in ['B01', 'B02', 'B03', 'B04', 'B05']:
                bands[band_name] = hdf_file[band_name][:]
            
            # 逐个像素处理
            height, width = bands['B01'].shape
            result = np.zeros((height, width))
            
            for i in range(height):
                for j in range(width):
                    # 复杂的像素级计算
                    pixel_values = [bands[band][i, j] for band in bands]
                    result[i, j] = calculate_ndvi(pixel_values)
            
            # 保存结果
            output_path = os.path.join(output_dir, f'processed_{file}')
            with h5py.File(output_path, 'w') as out_file:
                out_file.create_dataset('result', data=result)

def calculate_ndvi(values):
    """计算NDVI指数"""
    # 简化的NDVI计算公式
    return (values[3] - values[2]) / (values[3] + values[2] + 1e-10)

def calculate_ndvi_vectorized(red_band, nir_band):
    """向量化的NDVI计算"""
    return (nir_band - red_band) / (nir_band + red_band + 1e-10)

def process_hdf5_files_optimized(input_dir, output_dir):
    """优化后的处理函数"""
    files = [f for f in os.listdir(input_dir) if f.endswith('.h5')]
    
    for file in files:
        with h5py.File(os.path.join(input_dir, file), 'r') as hdf_file:
            # 只加载需要的波段，减少内存占用
            nir_band = hdf_file['B04'][:]
            red_band = hdf_file['B03'][:]
            
            # 向量化计算
            result = calculate_ndvi_vectorized(red_band, nir_band)
            
            output_path = os.path.join(output_dir, f'processed_{file}')
            with h5py.File(output_path, 'w') as out_file:
                out_file.create_dataset('result', data=result)

def process_large_hdf5(file_path, output_path, chunk_size=1024):
    """分块处理大文件"""
    with h5py.File(file_path, 'r') as hdf_file, \
         h5py.File(output_path, 'w') as out_file:
        
        nir_dataset = hdf_file['B04']
        red_dataset = hdf_file['B03']
        height, width = nir_dataset.shape
        
        # 创建可扩展的输出数据集
        result_dataset = out_file.create_dataset(
            'result', (height, width), 
            dtype=np.float32, 
            chunks=(chunk_size, chunk_size),
            compression='gzip'
        )
        
        # 分块处理
        for i in range(0, height, chunk_size):
            for j in range(0, width, chunk_size):
                # 计算当前块的边界
                i_end = min(i + chunk_size, height)
                j_end = min(j + chunk_size, width)
                
                # 读取当前块的数据
                nir_chunk = nir_dataset[i:i_end, j:j_end]
                red_chunk = red_dataset[i:i_end, j:j_end]
                
                # 向量化计算
                ndvi_chunk = calculate_ndvi_vectorized(red_chunk, nir_chunk)
                
                # 写入结果
                result_dataset[i:i_end, j:j_end] = ndvi_chunk

from multiprocessing import Pool
import functools

def process_single_file(file, input_dir, output_dir):
    """处理单个文件"""
    input_path = os.path.join(input_dir, file)
    output_path = os.path.join(output_dir, f'processed_{file}')
    
    try:
        with h5py.File(input_path, 'r') as hdf_file:
            nir_band = hdf_file['B04'][:]
            red_band = hdf_file['B03'][:]
            result = calculate_ndvi_vectorized(red_band, nir_band)
            
        with h5py.File(output_path, 'w') as out_file:
            out_file.create_dataset('result', data=result, compression='gzip')
        
        return True
    except Exception as e:
        print(f"处理文件 {file} 时出错: {e}")
        return False

def process_parallel(input_dir, output_dir, num_processes=None):
    """并行处理所有文件"""
    files = [f for f in os.listdir(input_dir) if f.endswith('.h5')]
    
    # 使用部分函数固定参数
    process_func = functools.partial(
        process_single_file,
        input_dir=input_dir,
        output_dir=output_dir
    )
    
    with Pool(processes=num_processes) as pool:
        results = pool.map(process_func, files)
    
    return sum(results)  # 返回成功处理的文件数量