2025年多模态图像处理与生成技术完全指南：从入门到精通

# 2025年基于视觉-语言预训练的图像到文本生成示例实现
import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import requests

class AdvancedImageToTextGenerator:
    def __init__(self, model_name="microsoft/git-base-coco"):
        # 加载预训练的图像到文本生成模型
        self.processor = AutoProcessor.from_pretrained(model_name)
        self.model = AutoModelForCausalLM.from_pretrained(model_name)
        
        # 移至GPU（如果可用）
        if torch.cuda.is_available():
            self.model.to("cuda")
        
        # 设置默认生成参数
        self.default_params = {
            "max_length": 50,
            "num_beams": 5,
            "early_stopping": True,
            "no_repeat_ngram_size": 2
        }
    
    def generate_caption(self, image, **kwargs):
        # 合并默认参数和用户提供的参数
        params = {**self.default_params, **kwargs}
        
        # 处理图像
        inputs = self.processor(images=image, return_tensors="pt")
        
        # 移至GPU（如果可用）
        if torch.cuda.is_available():
            inputs = {k: v.to("cuda") for k, v in inputs.items()}
        
        # 生成文本描述
        with torch.no_grad():
            output = self.model.generate(
                pixel_values=inputs["pixel_values"],
                max_length=params["max_length"],
                num_beams=params["num_beams"],
                early_stopping=params["early_stopping"],
                no_repeat_ngram_size=params["no_repeat_ngram_size"]
            )
        
        # 解码生成的文本
        caption = self.processor.decode(output[0], skip_special_tokens=True)
        
        return caption
    
    def generate_multiple_captions(self, image, num_captions=3):
        # 生成多个文本描述变体
        captions = []
        for i in range(num_captions):
            # 使用不同的生成参数生成不同的描述
            params = {
                "num_beams": 3 + i,
                "temperature": 0.7 + i * 0.1,
                "do_sample": True
            }
            caption = self.generate_caption(image, **params)
            captions.append(caption)
        return captions
    
    def generate_detailed_description(self, image):
        # 生成更详细的图像描述
        detailed_params = {
            "max_length": 100,
            "num_beams": 8,
            "temperature": 0.9
        }
        detailed_description = self.generate_caption(image, **detailed_params)
        return detailed_description
    
    def load_image_from_url(self, url):
        # 从URL加载图像
        image = Image.open(requests.get(url, stream=True).raw)
        return image

# 示例使用代码
def demo_image_to_text():
    image_to_text = AdvancedImageToTextGenerator()
    
    # 从URL加载图像（示例URL）
    image_url = "https://images.unsplash.com/photo-1507525428034-b723cf961d3e"
    image = image_to_text.load_image_from_url(image_url)
    
    # 生成基本描述
    basic_caption = image_to_text.generate_caption(image)
    print(f"基本描述: {basic_caption}")
    
    # 生成多个描述变体
    multiple_captions = image_to_text.generate_multiple_captions(image, num_captions=3)
    for i, caption in enumerate(multiple_captions):
        print(f"变体 {i+1}: {caption}")
    
    # 生成详细描述
    detailed_description = image_to_text.generate_detailed_description(image)
    print(f"详细描述: {detailed_description}")

# 如需运行示例，请取消注释下面的行
# demo_image_to_text()

# 2025年基于扩散模型的高级文本到图像生成示例
from diffusers import DiffusionPipeline, StableDiffusionXLPipeline
import torch
import os

# 加载高级文本到图像模型
def load_advanced_text_to_image_model(model_name="stabilityai/stable-diffusion-4", use_4bit=True):
    print(f"正在加载模型: {model_name}")
    
    # 根据模型类型选择合适的管道
    if "xl" in model_name.lower() or "sd4" in model_name.lower():
        pipeline = StableDiffusionXLPipeline.from_pretrained(
            model_name,
            torch_dtype=torch.bfloat16,
            use_safetensors=True,
            variant="fp16"
        )
    else:
        pipeline = DiffusionPipeline.from_pretrained(
            model_name,
            torch_dtype=torch.bfloat16,
            use_safetensors=True,
            variant="fp16"
        )
    
    # 应用4bit量化（如果启用）
    if use_4bit:
        from transformers import BitsAndBytesConfig
        quantization_config = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.bfloat16
        )
        # 对于SDXL，我们需要分别量化文本编码器
        if hasattr(pipeline, "text_encoder"):
            if isinstance(pipeline.text_encoder, list):
                pipeline.text_encoder = torch.nn.ModuleList([
                    torch.quantization.quantize_dynamic(
                        encoder, {torch.nn.Linear}, dtype=torch.qint8
                    ) for encoder in pipeline.text_encoder
                ])
            else:
                pipeline.text_encoder = torch.quantization.quantize_dynamic(
                    pipeline.text_encoder, {torch.nn.Linear}, dtype=torch.qint8
                )
    
    # 将模型移至GPU（如果可用）
    if torch.cuda.is_available():
        pipeline = pipeline.to("cuda")
        # 启用内存优化
        pipeline.enable_model_cpu_offload()
    
    return pipeline

# 高级文本到图像生成函数
def generate_advanced_image(pipeline, prompt, negative_prompt=None, width=1024, height=1024, 
                           num_inference_steps=30, guidance_scale=7.5, num_images_per_prompt=1, 
                           seed=None):
    # 设置随机种子以获得可重复的结果
    if seed is not None:
        generator = torch.Generator(device="cuda" if torch.cuda.is_available() else "cpu").manual_seed(seed)
    else:
        generator = None
    
    # 生成图像
    images = pipeline(
        prompt=prompt,
        negative_prompt=negative_prompt,
        width=width,
        height=height,
        num_inference_steps=num_inference_steps,
        guidance_scale=guidance_scale,
        num_images_per_prompt=num_images_per_prompt,
        generator=generator
    ).images
    
    return images

# 保存生成的图像
def save_generated_images(images, output_dir="output_images", prefix="generated"):
    os.makedirs(output_dir, exist_ok=True)
    saved_paths = []
    
    for i, image in enumerate(images):
        path = os.path.join(output_dir, f"{prefix}_{i}.png")
        image.save(path)
        saved_paths.append(path)
    
    return saved_paths

# 示例使用代码
def demo_text_to_image():
    # 加载模型
    pipeline = load_advanced_text_to_image_model(model_name="stabilityai/stable-diffusion-4", use_4bit=True)
    
    # 定义提示词和负面提示词
    prompt = "a beautiful landscape with mountains, lake, trees, and a small cabin, detailed, 4k, hyper realistic"
    negative_prompt = "blurry, low quality, bad anatomy, extra limbs, missing limbs"
    
    # 生成图像
    images = generate_advanced_image(
        pipeline,
        prompt=prompt,
        negative_prompt=negative_prompt,
        width=1024,
        height=1024,
        num_inference_steps=30,
        guidance_scale=7.5,
        num_images_per_prompt=1,
        seed=42
    )
    
    # 保存图像
    saved_paths = save_generated_images(images, prefix="landscape")
    print(f"图像已保存至: {saved_paths}")

# 如需运行示例，请取消注释下面的行
# demo_text_to_image()

# 2025年基于扩散模型的图像到图像转换示例实现
import torch
from transformers import StableDiffusionImg2ImgPipeline
from PIL import Image
import numpy as np

class AdvancedImageToImageGenerator:
    def __init__(self, model_name="stabilityai/stable-diffusion-xl-base-1.0"):
        # 加载预训练的图像到图像转换模型
        self.pipeline = StableDiffusionImg2ImgPipeline.from_pretrained(
            model_name,
            torch_dtype=torch.float16,
            use_safetensors=True
        )
        
        # 移至GPU（如果可用）
        if torch.cuda.is_available():
            self.pipeline.to("cuda")
        
        # 设置默认生成参数
        self.default_params = {
            "strength": 0.75,
            "guidance_scale": 7.5,
            "num_inference_steps": 30,
            "seed": 42
        }
    
    def transform_image(self, prompt, image, negative_prompt=None, **kwargs):
        # 合并默认参数和用户提供的参数
        params = {**self.default_params, **kwargs}
        
        # 调整图像大小和格式
        image = self._prepare_image(image)
        
        # 生成转换后的图像
        with torch.autocast("cuda", enabled=torch.cuda.is_available()):
            transformed_image = self.pipeline(
                prompt=prompt,
                image=image,
                negative_prompt=negative_prompt,
                strength=params["strength"],
                guidance_scale=params["guidance_scale"],
                num_inference_steps=params["num_inference_steps"],
                generator=torch.manual_seed(params["seed"])
            ).images[0]
        
        return transformed_image
    
    def _prepare_image(self, image):
        # 准备输入图像
        if isinstance(image, str):
            # 如果是文件路径，加载图像
            image = Image.open(image)
        
        # 确保图像是RGB格式
        if image.mode != "RGB":
            image = image.convert("RGB")
        
        # 调整图像大小以提高处理效率
        max_size = 1024
        width, height = image.size
        if width > max_size or height > max_size:
            ratio = min(max_size / width, max_size / height)
            new_width = int(width * ratio)
            new_height = int(height * ratio)
            image = image.resize((new_width, new_height), Image.LANCZOS)
        
        return image
    
    def generate_variations(self, prompt, image, num_variations=4, variation_strength=0.1):
        # 生成同一图像的多个变体
        variations = []
        
        # 先进行基础转换
        base_image = self.transform_image(prompt, image)
        variations.append(base_image)
        
        # 生成变体
        for i in range(1, num_variations):
            # 使用不同的种子和略微调整的强度
            variation_params = {
                "seed": self.default_params["seed"] + i,
                "strength": self.default_params["strength"] * (1 + variation_strength * (-1 if i % 2 == 0 else 1))
            }
            variation = self.transform_image(prompt, image, **variation_params)
            variations.append(variation)
        
        return variations
    
    def style_transfer(self, style_prompt, content_image):
        # 实现风格迁移
        style_params = {
            "strength": 0.85,  # 更高的强度以更好地捕捉风格
            "num_inference_steps": 50,  # 更多的推理步骤以提高质量
            "guidance_scale": 8.5
        }
        styled_image = self.transform_image(style_prompt, content_image, **style_params)
        return styled_image

# 示例使用代码
def demo_image_to_image():
    image_to_image = AdvancedImageToImageGenerator()
    
    # 加载内容图像（示例URL）
    from io import BytesIO
    import requests
    
    image_url = "https://images.unsplash.com/photo-1507525428034-b723cf961d3e"
    response = requests.get(image_url)
    content_image = Image.open(BytesIO(response.content))
    
    # 基本图像转换
    prompt = "a beautiful landscape painted by Claude Monet"
    transformed_image = image_to_image.transform_image(prompt, content_image)
    transformed_image.save("monet_style.jpg")
    
    # 生成多个变体
    variations = image_to_image.generate_variations(prompt, content_image, num_variations=4)
    for i, variation in enumerate(variations):
        variation.save(f"variation_{i}.jpg")
    
    # 风格迁移
    style_prompt = "a digital painting in cyberpunk style, neon lights, futuristic city, detailed, high quality"
    cyberpunk_image = image_to_image.style_transfer(style_prompt, content_image)
    cyberpunk_image.save("cyberpunk_style.jpg")

# 如需运行示例，请取消注释下面的行
# demo_image_to_image()

# Image-Text-to-Text模型示例实现
import torch
import torch.nn as nn
from transformers import VisionEncoderDecoderModel, ViTModel, GPT2Model, AutoProcessor

class ImageTextToTextModel(nn.Module):
    def __init__(self, image_model_name, text_model_name):
        super().__init__()
        # 初始化图像编码器
        self.image_encoder = ViTModel.from_pretrained(image_model_name)
        # 初始化文本编码器
        self.text_encoder = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(
                d_model=768,  # 与ViT输出维度匹配
                nhead=12,
                dim_feedforward=3072
            ),
            num_layers=6
        )
        # 初始化解码器
        self.decoder = GPT2Model.from_pretrained(text_model_name)
        # 投影层，将图像特征映射到解码器的输入维度
        self.image_proj = nn.Linear(self.image_encoder.config.hidden_size, self.decoder.config.hidden_size)
        # 输出层，生成文本概率分布
        self.output_layer = nn.Linear(self.decoder.config.hidden_size, self.decoder.config.vocab_size)
        
    def forward(self, image_inputs, text_inputs, decoder_input_ids=None):
        # 提取图像特征
        image_outputs = self.image_encoder(**image_inputs)
        image_features = image_outputs.last_hidden_state
        
        # 提取文本特征
        text_features = self.text_encoder(text_inputs)
        
        # 将图像特征投影到解码器的输入维度
        image_features_proj = self.image_proj(image_features)
        
        # 融合图像和文本特征
        # 在序列维度上拼接特征
        combined_features = torch.cat([image_features_proj, text_features], dim=1)
        
        # 准备解码器输入
        if decoder_input_ids is None:
            # 生成模式下，使用开始标记作为第一个输入
            batch_size = image_inputs['pixel_values'].shape[0]
            decoder_input_ids = torch.tensor([[50256]] * batch_size, device=image_inputs['pixel_values'].device)  # GPT2的开始标记ID
        
        # 解码器前向传播
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            encoder_hidden_states=combined_features
        )
        
        # 生成文本概率分布
        logits = self.output_layer(decoder_outputs.last_hidden_state)
        
        return logits

    def generate(self, image_inputs, text_inputs, max_length=100, temperature=1.0):
        # 生成模式
        batch_size = image_inputs['pixel_values'].shape[0]
        device = image_inputs['pixel_values'].device
        
        # 初始化解码器输入
        decoder_input_ids = torch.tensor([[50256]] * batch_size, device=device)
        
        # 生成文本
        for _ in range(max_length - 1):
            # 获取当前的logits
            logits = self.forward(image_inputs, text_inputs, decoder_input_ids)
            
            # 应用温度缩放
            logits = logits[:, -1, :] / temperature
            
            # 采样下一个token
            next_token = torch.multinomial(torch.softmax(logits, dim=-1), num_samples=1)
            
            # 将新token添加到输入中
            decoder_input_ids = torch.cat([decoder_input_ids, next_token], dim=-1)
            
            # 检查是否生成了结束标记
            if (next_token == 50256).all():
                break
        
        return decoder_input_ids

# 示例使用代码
def demo_image_text_to_text():
    from transformers import AutoTokenizer
    import requests
    from PIL import Image
    
    # 初始化处理器和分词器
    processor = AutoProcessor.from_pretrained("google/vit-base-patch16-224")
    tokenizer = AutoTokenizer.from_pretrained("gpt2")
    
    # 准备图像输入（示例URL）
    image_url = "https://images.unsplash.com/photo-1507525428034-b723cf961d3e"
    image = Image.open(requests.get(image_url, stream=True).raw)
    image_inputs = processor(images=image, return_tensors="pt")
    
    # 准备文本输入
    text = "describe this image in detail"
    text_inputs = torch.tensor(tokenizer.encode(text)).unsqueeze(0)
    
    # 初始化模型
    model = ImageTextToTextModel(
        image_model_name="google/vit-base-patch16-224",
        text_model_name="gpt2"
    )
    
    # 生成文本
    generated_text_ids = model.generate(image_inputs, text_inputs, max_length=100)
    
    # 解码生成的文本
    decoded_text = tokenizer.decode(generated_text_ids[0], skip_special_tokens=True)
    print("生成的文本:", decoded_text)

# 如需运行示例，请取消注释下面的行
# demo_image_text_to_text()

# 多模态图像处理模型量化技术示例
import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from transformers import BitsAndBytesConfig

class QuantizedImageModel:
    def __init__(self, model_name, quantization_type="4bit"):
        # 设置量化配置
        if quantization_type == "4bit":
            quantization_config = BitsAndBytesConfig(
                load_in_4bit=True,
                bnb_4bit_quant_type="nf4",
                bnb_4bit_compute_dtype=torch.float16,
                bnb_4bit_use_double_quant=True
            )
        elif quantization_type == "8bit":
            quantization_config = BitsAndBytesConfig(
                load_in_8bit=True,
                llm_int8_threshold=6.0
            )
        else:
            quantization_config = None
        
        # 加载处理器
        self.processor = AutoProcessor.from_pretrained(model_name)
        
        # 加载预训练模型（带量化）
        if quantization_config:
            # 使用量化模型
            self.model = AutoModelForCausalLM.from_pretrained(
                model_name,
                quantization_config=quantization_config,
                device_map="auto",
                torch_dtype=torch.float16
            )
        else:
            # 使用普通模型
            self.model = AutoModelForCausalLM.from_pretrained(model_name)
            # 移至GPU（如果可用）
            if torch.cuda.is_available():
                self.model.to("cuda")
        
    def get_model_size(self):
        # 获取模型大小（近似值）
        total_params = sum(p.numel() for p in self.model.parameters())
        # 假设每个参数是4字节（float32）
        model_size_mb = total_params * 4 / (1024 * 1024)
        return model_size_mb

# 示例使用代码
def demo_model_quantization():
    # 创建量化模型
    quantized_model = QuantizedImageModel(
        model_name="microsoft/git-base-coco",
        quantization_type="4bit"
    )
    
    # 创建非量化模型用于比较
    regular_model = QuantizedImageModel(
        model_name="microsoft/git-base-coco",
        quantization_type=None
    )
    
    # 获取模型大小
    quantized_size = quantized_model.get_model_size()
    regular_size = regular_model.get_model_size()
    print(f"量化模型大小: {quantized_size:.2f} MB")
    print(f"非量化模型大小: {regular_size:.2f} MB")
    print(f"压缩比例: {regular_size/quantized_size:.2f}x")

# 如需运行示例，请取消注释下面的行
# demo_model_quantization()