构建AI智能体：中文新闻智能分类：K-Means聚类K与Qwen主题生成的融合应用

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
 
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
 
# 生成示例数据
np.random.seed(42)
X, y_true = make_blobs(n_samples=300, centers=4, cluster_std=0.8, random_state=42)
 
# K-Means 算法实现
def k_means(X, k, max_iter=100, tol=1e-4):
    # 1. 初始化中心点
    centers = X[np.random.choice(X.shape[0], k, replace=False)]
    
    # 存储历史信息用于可视化
    history = {'centers': [centers.copy()], 'labels': []}
    
    for i in range(max_iter):
        # 2. 分配步骤
        distances = np.sqrt(((X - centers[:, np.newaxis])**2).sum(axis=2))
        labels = np.argmin(distances, axis=0)
        history['labels'].append(labels.copy())
        
        # 3. 更新步骤
        new_centers = np.array([X[labels == j].mean(axis=0) for j in range(k)])
        
        # 4. 收敛判断
        if np.all(np.linalg.norm(new_centers - centers, axis=1) < tol):
            break
            
        centers = new_centers
        history['centers'].append(centers.copy())
    
    return labels, centers, history
 
# 应用 K-Means 算法
k = 4
labels, centers, history = k_means(X, k)
 
# 可视化结果
plt.figure(figsize=(15, 10))
 
# 绘制初始状态
plt.subplot(2, 3, 1)
plt.scatter(X[:, 0], X[:, 1], c='gray', s=30, alpha=0.5)
plt.scatter(history['centers'][0][:, 0], history['centers'][0][:, 1], 
            c='red', s=100, marker='X')
plt.title('初始化: 随机选择中心点')
 
# 绘制迭代过程
for i in range(min(4, len(history['labels']))):
    plt.subplot(2, 3, i+2)
    for j in range(k):
        cluster_points = X[history['labels'][i] == j]
        plt.scatter(cluster_points[:, 0], cluster_points[:, 1], s=30, alpha=0.5)
    plt.scatter(history['centers'][i][:, 0], history['centers'][i][:, 1], 
                c='red', s=100, marker='X')
    plt.scatter(history['centers'][i+1][:, 0], history['centers'][i+1][:, 1], 
                c='green', s=100, marker='X')
    plt.title(f'迭代 {i+1}: 分配和更新中心点')
 
# 绘制最终结果
plt.subplot(2, 3, 6)
for j in range(k):
    cluster_points = X[labels == j]
    plt.scatter(cluster_points[:, 0], cluster_points[:, 1], s=30, alpha=0.5)
plt.scatter(centers[:, 0], centers[:, 1], c='red', s=100, marker='X')
plt.title('最终结果: 算法收敛')
 
plt.tight_layout()
plt.show()
 
# 打印最终中心点位置
print("最终中心点位置:")
for i, center in enumerate(centers):
    print(f"中心点 {i+1}: ({center[0]:.2f}, {center[1]:.2f})")

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
 
# 设置中文字体支持
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False    # 用来正常显示负号
 
# 生成示例数据
np.random.seed(42)
X = np.random.randn(300, 2)
 
# 自己实现 K-Means 算法
def kmeans(X, k, init_method='random', max_iter=100, tol=1e-4):
    n_samples, n_features = X.shape
    
    # 初始化中心点
    if init_method == 'random':
        # 随机初始化
        centers = X[np.random.choice(n_samples, k, replace=False)]
    elif init_method == 'kmeans++':
        # K-Means++ 初始化
        centers = np.zeros((k, n_features))
        # 第一个中心点随机选择
        centers[0] = X[np.random.randint(n_samples)]
        
        for i in range(1, k):
            # 计算每个点到最近中心点的距离
            distances = np.min(np.sqrt(((X - centers[:i, np.newaxis])**2).sum(axis=2)), axis=0)
            # 按距离平方的概率选择下一个中心点
            probabilities = distances**2 / np.sum(distances**2)
            next_center_idx = np.random.choice(n_samples, p=probabilities)
            centers[i] = X[next_center_idx]
    
    # 存储历史信息
    history = {'centers': [centers.copy()], 'labels': [], 'wcss': []}
    
    for iteration in range(max_iter):
        # 计算每个点到每个中心的距离
        distances = np.sqrt(((X - centers[:, np.newaxis])**2).sum(axis=2))
        
        # 分配步骤：将每个点分配到最近的中心
        labels = np.argmin(distances, axis=0)
        history['labels'].append(labels.copy())
        
        # 计算 WCSS (Within-Cluster Sum of Squares)
        wcss = 0
        for j in range(k):
            cluster_points = X[labels == j]
            if len(cluster_points) > 0:
                wcss += np.sum((cluster_points - centers[j])**2)
        history['wcss'].append(wcss)
        
        # 更新步骤：重新计算中心点
        new_centers = np.zeros((k, n_features))
        for j in range(k):
            cluster_points = X[labels == j]
            if len(cluster_points) > 0:
                new_centers[j] = np.mean(cluster_points, axis=0)
            else:
                # 如果簇为空，重新初始化该中心点
                new_centers[j] = X[np.random.randint(n_samples)]
        
        # 检查收敛
        if np.all(np.linalg.norm(new_centers - centers, axis=1) < tol):
            break
            
        centers = new_centers
        history['centers'].append(centers.copy())
    
    return labels, centers, history
 
# 运行随机初始化的 K-Means
labels_random, centers_random, history_random = kmeans(X, k=4, init_method='random')
wcss_random = history_random['wcss'][-1]
 
# 运行 K-Means++ 初始化的 K-Means
labels_plus, centers_plus, history_plus = kmeans(X, k=4, init_method='kmeans++')
wcss_plus = history_plus['wcss'][-1]
 
print(f"随机初始化的误差: {wcss_random:.2f}")
print(f"K-Means++ 初始化的误差: {wcss_plus:.2f}")
print(f"改进: {wcss_random - wcss_plus:.2f} (越小越好)")
 
# 可视化结果
plt.figure(figsize=(15, 5))
 
# 随机初始化的结果
plt.subplot(1, 3, 1)
colors = list(mcolors.TABLEAU_COLORS.values())
for j in range(4):
    cluster_points = X[labels_random == j]
    plt.scatter(cluster_points[:, 0], cluster_points[:, 1], c=colors[j], s=30, alpha=0.8)
plt.scatter(centers_random[:, 0], centers_random[:, 1], c='red', marker='X', s=200, edgecolors='black')
plt.title('随机初始化\n误差: {:.2f}'.format(wcss_random))
 
# K-Means++ 初始化的结果
plt.subplot(1, 3, 2)
for j in range(4):
    cluster_points = X[labels_plus == j]
    plt.scatter(cluster_points[:, 0], cluster_points[:, 1], c=colors[j], s=30, alpha=0.8)
plt.scatter(centers_plus[:, 0], centers_plus[:, 1], c='red', marker='X', s=200, edgecolors='black')
plt.title('K-Means++ 初始化\n误差: {:.2f}'.format(wcss_plus))
 
# 误差对比
plt.subplot(1, 3, 3)
methods = ['随机初始化', 'K-Means++']
wcss_values = [wcss_random, wcss_plus]
bars = plt.bar(methods, wcss_values, color=['lightblue', 'lightgreen'])
plt.ylabel('WCSS 误差')
plt.title('初始化方法对比')
 
# 在柱状图上添加数值标签
for bar, value in zip(bars, wcss_values):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 5, 
             f'{value:.2f}', ha='center', va='bottom')
 
plt.tight_layout()
plt.show()
 
# 打印迭代次数和最终中心点
print(f"\n随机初始化迭代次数: {len(history_random['wcss'])}")
print(f"K-Means++ 迭代次数: {len(history_plus['wcss'])}")
print("\n随机初始化最终中心点:")
for i, center in enumerate(centers_random):
    print(f"中心点 {i+1}: ({center[0]:.2f}, {center[1]:.2f})")
print("\nK-Means++ 最终中心点:")
for i, center in enumerate(centers_plus):
    print(f"中心点 {i+1}: ({center[0]:.2f}, {center[1]:.2f})")

import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import requests
import json
import os
import time
 
# 设置中文字体支持
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False    # 用来正常显示负号
 
# 1. 准备中文新闻标题数据
news_titles = [
    "股市在经济复苏中创历史新高",
    "新研究显示地中海饮食对心脏健康的益处",
    "地方选举结果公布，现任政党保持席位",
    "科技巨头发布具有先进AI功能的最新智能手机",
    "科学家在亚马逊雨林发现新物种",
    "美联储暗示下季度可能降息",
    "COVID-19疫苗加强针现已向所有成年人开放",
    "足球队在戏剧性决赛后赢得冠军",
    "气候峰会达成碳排放新协议",
    "苹果发布改进相机系统的新iPhone",
    "央行宣布新货币政策以遏制通胀",
    "研究将空气污染与痴呆症风险增加联系起来",
    "市议会批准新公共交通系统预算",
    "三星推出新款可折叠手机参与市场竞争",
    "当地公园发现稀有鸟类，观鸟者蜂拥而至",
    "财政部长评论数字货币未来",
    "癌症治疗突破在临床试验中显示希望",
    "市长候选人承诺解决无家可归危机",
    "谷歌发布搜索算法更新",
    "飓风逼近海岸，居民被敦促撤离"
]
 
# 2. 加载嵌入模型
embedder = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')  # 多语言模型，支持中文
 
# 3. 将文本转换为嵌入向量
corpus_embeddings = embedder.encode(news_titles)
 
# 4. 自己实现 K-Means 算法，避免使用 sklearn
def kmeans_custom(X, n_clusters, max_iter=100, tol=1e-4):
    # 随机初始化中心点
    centers = X[np.random.choice(X.shape[0], n_clusters, replace=False)]
    
    for _ in range(max_iter):
        # 计算每个点到每个中心的距离
        distances = np.sqrt(((X - centers[:, np.newaxis])**2).sum(axis=2))
        
        # 分配步骤：将每个点分配到最近的中心
        labels = np.argmin(distances, axis=0)
        
        # 更新步骤：重新计算中心点
        new_centers = np.array([X[labels == j].mean(axis=0) for j in range(n_clusters)])
        
        # 检查收敛
        if np.all(np.linalg.norm(new_centers - centers, axis=1) < tol):
            break
            
        centers = new_centers
    
    return labels, centers
 
# 使用自定义 K-Means 进行聚类
num_clusters = 5  # 假设有5个主要类别
clusters, centers = kmeans_custom(corpus_embeddings, num_clusters)
 
# 5. 可视化（使用PCA将高维向量降维到2D）
pca = PCA(n_components=2)
reduced_embeddings = pca.fit_transform(corpus_embeddings)
 
# 计算每个簇在2D空间中的中心点
cluster_centers_2d = []
for i in range(num_clusters):
    cluster_points = reduced_embeddings[clusters == i]
    if len(cluster_points) > 0:
        center_2d = np.mean(cluster_points, axis=0)
        cluster_centers_2d.append(center_2d)
    else:
        cluster_centers_2d.append([0, 0])
 
# 6. 打印聚类结果
clustered_titles = {}
for idx, cluster_id in enumerate(clusters):
    if cluster_id not in clustered_titles:
        clustered_titles[cluster_id] = []
    clustered_titles[cluster_id].append(news_titles[idx])
 
print("=== 聚类结果 ===")
for cluster_id, titles in clustered_titles.items():
    print(f"\n--- 簇 #{cluster_id} (包含 {len(titles)} 个标题) ---")
    for title in titles:
        print(f" - {title}")
 
# 7. 使用 Qwen API 总结每个簇的主题
# 设置你的 Qwen API 密钥和端点
QWEN_API_KEY = os.environ.get("DASHSCOPE_API_KEY")
QWEN_API_URL = "https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation"
 
def summarize_cluster_with_qwen(titles):
    """使用 Qwen API 总结一个簇的主题"""
    # 构建提示词 - 使用字符串连接而不是 f-string 中的反斜杠
    prompt_lines = [
        "以下是一组被算法归为一类的新闻标题。",
        "请分析它们的共同主题或话题，并用非常简短的中文标签（3-5个词）进行概括。",
        "",
        "新闻标题："
    ]
    prompt_lines.extend(titles[:3])  # 只使用前3个标题
    prompt_lines.extend(["", "共同主题：", ""])
    prompt = "\n".join(prompt_lines)
    
    try:
        # 构建请求
        headers = {
            "Authorization": f"Bearer {QWEN_API_KEY}",
            "Content-Type": "application/json"
        }
        
        payload = {
            "model": "qwen-max",  # 使用 Qwen 模型
            "input": {
                "messages": [
                    {
                        "role": "user",
                        "content": prompt
                    }
                ]
            },
            "parameters": {
                "max_tokens": 20,
                "temperature": 0.1
            }
        }
        
        # 发送请求
        response = requests.post(QWEN_API_URL, headers=headers, data=json.dumps(payload))
        result = response.json()
        
        # 提取回复内容
        if "output" in result and "text" in result["output"]:
            return result["output"]["text"].strip()
        else:
            return f"API 错误: {result.get('message', '未知错误')}"
            
    except Exception as e:
        return f"请求异常: {str(e)}"
 
print("\n=== 正在生成簇主题... ===")
cluster_themes = {}
for cluster_id, titles in clustered_titles.items():
    print(f"正在分析簇 #{cluster_id}...")
    theme = summarize_cluster_with_qwen(titles)
    cluster_themes[cluster_id] = theme
    print(f"簇 #{cluster_id} 主题: {theme}")
    
    # 添加短暂延迟以避免API限制
    time.sleep(1)
 
# 8. 创建带有主题标签的可视化
plt.figure(figsize=(14, 10))
scatter = plt.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], 
                     c=clusters, cmap='viridis', s=100, alpha=0.7)
plt.colorbar(scatter, label='簇 ID')
plt.title('新闻标题的 K-Means 聚类分析 (带主题标签)', fontsize=16)
 
# 为每个点添加标题索引
for i, title in enumerate(news_titles):
    plt.annotate(f"{i}", (reduced_embeddings[i, 0], reduced_embeddings[i, 1]), 
                fontsize=8, alpha=0.7)
 
# 为每个簇添加主题标签
for cluster_id, center_2d in enumerate(cluster_centers_2d):
    if cluster_id in cluster_themes:
        theme = cluster_themes[cluster_id]
        # 计算标签位置（稍微偏移以避免重叠）
        offset_x = 0.5 if cluster_id % 2 == 0 else -0.5
        offset_y = 0.5 if cluster_id < 3 else -0.5
        
        plt.annotate(
            f"簇 #{cluster_id}: {theme}",
            xy=center_2d,
            xytext=(center_2d[0] + offset_x, center_2d[1] + offset_y),
            fontsize=12,
            weight='bold',
            color='darkred',
            arrowprops=dict(arrowstyle="->", color='red', lw=1.5),
            bbox=dict(boxstyle="round,pad=0.3", facecolor="yellow", alpha=0.3)
        )
 
plt.grid(True, linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()
 
# 9. 打印详细的聚类结果
print("\n=== 详细聚类结果 ===")
for cluster_id, titles in clustered_titles.items():
    theme = cluster_themes.get(cluster_id, "未生成主题")
    print(f"\n--- 簇 #{cluster_id}: {theme} (包含 {len(titles)} 个标题) ---")
    for i, title in enumerate(titles):
        print(f"  {i+1}. {title}")
 
print("\n聚类分析完成!")