大附件迁移优化断点续传与增量同步实现

class ResumableUploader {
    constructor(options) {
        this.fileId = options.fileId;
        this.file = options.file;
        this.chunkSize = options.chunkSize || 1024 * 1024;
        this.concurrency = options.concurrency || 3;
        this.retryCount = options.retryCount || 3;
        
        this.chunks = [];
        this.uploadedChunks = new Set();
    }
    
    async init() {
        const status = await this.checkStatus();
        if (status.exists) {
            this.uploadedChunks = new Set(status.chunks);
        }
        this.splitFile();
    }
    
    splitFile() {
        const totalChunks = Math.ceil(this.file.size / this.chunkSize);
        for (let i = 0; i < totalChunks; i++) {
            const start = i * this.chunkSize;
            const end = Math.min(start + this.chunkSize, this.file.size);
            this.chunks.push({
                index: i,
                start,
                end,
                data: this.file.slice(start, end)
            });
        }
    }
    
    async checkStatus() {
        const response = await fetch(`/api/file/status?fileId=${this.fileId}`);
        return response.json();
    }
    
    async upload() {
        const pendingChunks = this.chunks.filter(c => !this.uploadedChunks.has(c.index));
        const batches = this.chunkArray(pendingChunks, this.concurrency);
        
        for (const batch of batches) {
            await Promise.all(batch.map(chunk => this.uploadChunk(chunk)));
        }
        
        await this.mergeFile();
    }
    
    async uploadChunk(chunk) {
        for (let attempt = 0; attempt < this.retryCount; attempt++) {
            try {
                const formData = new FormData();
                formData.append('fileId', this.fileId);
                formData.append('chunkIndex', chunk.index);
                formData.append('chunkTotal', this.chunks.length);
                formData.append('chunkSize', chunk.data.size);
                formData.append('data', chunk.data);
                
                const response = await fetch('/api/file/upload/chunk', {
                    method: 'POST',
                    body: formData
                });
                
                const result = await response.json();
                if (result.success) {
                    this.uploadedChunks.add(chunk.index);
                    return;
                }
            } catch (error) {
                console.error(`Chunk ${chunk.index} upload failed:`, error);
            }
        }
        
        throw new Error(`Chunk ${chunk.index} upload failed after ${this.retryCount} attempts`);
    }
    
    async mergeFile() {
        const response = await fetch('/api/file/upload/merge', {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({
                fileId: this.fileId,
                fileName: this.file.name,
                chunkTotal: this.chunks.length
            })
        });
        
        return response.json();
    }
    
    chunkArray(array, size) {
        const chunks = [];
        for (let i = 0; i < array.length; i += size) {
            chunks.push(array.slice(i, i + size));
        }
        return chunks;
    }
}

import os
import hashlib
import requests

class IncrementalSync:
    def __init__(self, source_dir, target_api):
        self.source_dir = source_dir
        self.target_api = target_api
    
    def calculate_md5(self, file_path):
        md5_hash = hashlib.md5()
        with open(file_path, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), b''):
                md5_hash.update(chunk)
        return md5_hash.hexdigest()
    
    def scan_files(self, directory, recursive=True):
        files = []
        for root, dirs, filenames in os.walk(directory):
            for filename in filenames:
                file_path = os.path.join(root, filename)
                relative_path = os.path.relpath(file_path, directory)
                
                files.append({
                    'filePath': relative_path,
                    'fileName': filename,
                    'size': os.path.getsize(file_path),
                    'modifiedTime': os.path.getmtime(file_path),
                    'md5': self.calculate_md5(file_path)
                })
            
            if not recursive:
                break
        
        return files
    
    def get_target_files(self):
        response = requests.post(
            f'{self.target_api}/api/files/scan',
            json={'directory': '/data', 'recursive': True}
        )
        return response.json().get('files', [])
    
    def compare_files(self, source_files, target_files):
        source_map = {f['filePath']: f for f in source_files}
        target_map = {f['filePath']: f for f in target_files}
        
        added = [f for path, f in source_map.items() if path not in target_map]
        deleted = [f for path, f in target_map.items() if path not in source_map]
        
        modified = []
        unchanged = []
        
        for path, source in source_map.items():
            if path in target_map:
                target = target_map[path]
                if source['md5'] != target['md5']:
                    modified.append(source)
                else:
                    unchanged.append(source)
        
        return {
            'added': added,
            'modified': modified,
            'deleted': deleted,
            'unchanged': unchanged
        }
    
    def sync_files(self, comparison):
        all_files = comparison['added'] + comparison['modified']
        
        if not all_files:
            return {'success': True, 'message': 'No files to sync'}
        
        response = requests.post(
            f'{self.target_api}/api/files/sync',
            json={
                'syncType': 'all',
                'files': all_files,
                'deleteMissing': True
            }
        )
        
        return response.json()
    
    def run(self):
        print(f"Scanning source directory: {self.source_dir}")
        source_files = self.scan_files(self.source_dir)
        print(f"Found {len(source_files)} files")
        
        print("Getting target files...")
        target_files = self.get_target_files()
        print(f"Target has {len(target_files)} files")
        
        print("Comparing files...")
        comparison = self.compare_files(source_files, target_files)
        
        print(f"Added: {len(comparison['added'])}")
        print(f"Modified: {len(comparison['modified'])}")
        print(f"Deleted: {len(comparison['deleted'])}")
        print(f"Unchanged: {len(comparison['unchanged'])}")
        
        print("Starting sync...")
        result = self.sync_files(comparison)
        print(f"Sync result: {result}")
        
        return result