基于 Collection 上传文件

package com.tencent.tcvectordb.examples;

import com.tencent.tcvectordb.client.VectorDBClient;
import com.tencent.tcvectordb.model.*;
import com.tencent.tcvectordb.model.Collection;
import com.tencent.tcvectordb.model.param.collection.*;
import com.tencent.tcvectordb.model.param.collectionView.*;
import com.tencent.tcvectordb.model.param.dml.*;
import com.tencent.tcvectordb.model.param.entity.GetImageUrlRes;
import com.tencent.tcvectordb.model.param.enums.EmbeddingModelEnum;
import com.tencent.tcvectordb.model.param.enums.ParsingTypeEnum;
import com.tencent.tcvectordb.utils.JsonUtils;
import com.tencent.tcvectordb.model.param.enums.OrderEnum;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Collectors;
// link database， client 为 VectorDBClient() 创建的客户端对象
Database db = client.database("db-test");
// 初始化 Colleciton 参数
CreateCollectionParam collectionParam = CreateCollectionParam.newBuilder()
        .withName("coll-file-test")
        .withShardNum(2)
        .withReplicaNum(1)
        .withDescription("this is a collection upload file")
        .addField(new FilterIndex("id", FieldType.String, IndexType.PRIMARY_KEY))
        .addField(new VectorIndex("vector", 768, IndexType.HNSW, MetricType.IP, new HNSWParams(16, 200)))
        .addField(new FilterIndex("file_name", FieldType.String, IndexType.FILTER))
        .addField(new FilterIndex("chunk_num", FieldType.Uint64, IndexType.FILTER))
        .addField(new FilterIndex("section_num", FieldType.Uint64, IndexType.FILTER))
        .withEmbedding(Embedding.newBuilder().withModelName(EmbeddingModelEnum.BGE_BASE_ZH.getModelName()).withField("text").withVectorField("vector").build())
        .build();
Collection collection = db.createCollection(collectionParam);

import (
    "github.com/tencent/vectordatabase-sdk-go/tcvectordb"
    "github.com/tencent/vectordatabase-sdk-go/tcvectordb/api"
    "github.com/tencent/vectordatabase-sdk-go/tcvectordb/api/ai_document_set"
)
var (
    ctx        = context.Background()
    database   = "db-test"
    collection = "go-sdk-test"
)
db := client.Database(database)
index := tcvectordb.Indexes{}
index.VectorIndex = append(index.VectorIndex, tcvectordb.VectorIndex{
    FilterIndex: tcvectordb.FilterIndex{
        FieldName: "vector",
        FieldType: tcvectordb.Vector,
        IndexType: tcvectordb.HNSW,
    },
    Dimension:  768,
    MetricType: tcvectordb.IP,
    Params: &tcvectordb.HNSWParam{
        M:              16,
        EfConstruction: 200,
    },
})
index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "id", FieldType: tcvectordb.String, IndexType: tcvectordb.PRIMARY})
index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "file_name", FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER})
index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "chunk_num", FieldType: tcvectordb.Uint64, IndexType: tcvectordb.FILTER})
index.FilterIndex = append(index.FilterIndex, tcvectordb.FilterIndex{FieldName: "section_num", FieldType: tcvectordb.Uint64, IndexType: tcvectordb.FILTER})
// 若需要使用文本信息进行相似性检索文件内容，则创建集合时需要指定 Embedding 模型相关信息。
ebd := &tcvectordb.Embedding{VectorField: "vector", Field: "text", ModelName: "bge-base-zh"}
coll, _ = db.CreateCollectionIfNotExists(ctx, collection, 2, 1, "test collection", index, &tcvectordb.CreateCollectionParams{
    Embedding: ebd,
})
log.Printf("create collection if not exists success: %v: %v", coll.DatabaseName, coll.CollectionName)

// 配置文件 matadata 信息，示例中指定了author 与 tags
Map<String, Object> metaDataMap = new HashMap<>();
metaDataMap.put("author", "Tencent");
metaDataMap.put("tags", Arrays.asList("Embedding", "向量", "AI"));
// 配置文件与数据库中存储的映射字段
Map<String,String> columnMap = new HashMap<>();
columnMap.put("filename", "file_name");
columnMap.put("text", "text");
columnMap.put("imageList", "image_list");
columnMap.put("chunkNum", "chunk_num");
columnMap.put("sectionNum", "section_num");
// 上传文件
// 1. LocalFilePath：指定当前文件在客户端的路径
// 2. FileName:指定文件名
// 3. SplitterProcess：配置文件拆分规则
// 4. ParsingProcess：指定文件解析方式
// 5. metadata：定义文件 meta 信息字段，示例中指定了author 与 tags
// 6. FieldMappings：文件与数据库中存储的映射字段
// 7. EmbeddingModel：Embedding 模型
UploadFileParam param = UploadFileParam.newBuilder()
        .withLocalFilePath("../demo_files/tcvdb.pdf")
        .withSplitterProcess(SplitterPreprocessParams.newBuilder().withAppendKeywordsToChunkEnum(true).Build())
        // parsingProcess is used for parsing pdf file by vision model
        .withParsingProcess(ParsingProcessParam.newBuilder().withParsingType(ParsingTypeEnum.VisionModel).build())
        .withFileName("tcvdb.pdf")
        .withFieldMappings(columnMap)
        .withEmbeddingModel("bge-base-zh")
        .Build();
client.UploadFile("db-test", "coll-file-test", param, metaDataMap);

appendKeywordsToChunk := true
appendTitleToChunk := false
localFilePath := "../demo_files/tcvdb.pdf"
param := tcvectordb.UploadFileParams{
    LocalFilePath: localFilePath,
    SplitterPreprocess: ai_document_set.DocumentSplitterPreprocess{
        AppendKeywordsToChunk: &appendKeywordsToChunk,
        AppendTitleToChunk:    &appendTitleToChunk,
    },
    ParsingProcess: &api.ParsingProcess{
        ParsingType: "AlgorithmParsing",
    },
    EmbeddingModel: "bge-base-zh",
    FieldMappings: map[string]string{
        "filename":  "file_name",
        "text":      "text",
        "imageList": "image_list",
        "chunkNum":   "chunk_num",
        "sectionNum": "section_num",
    },
    MetaData: map[string]interface{}{
        "author": "Tencent",
        "tags":   []string{"向量", "Embedding", "AI"},
    },
}
result, _ := client.UploadFile(ctx, database, collection, param)
log.Printf("UploadFile result: %+v", result)

SearchByEmbeddingItemsParam searchByEmbeddingItemsParam = SearchByEmbeddingItemsParam.newBuilder()
        .withEmbeddingItems(Arrays.asList("商标声明")) 
        // 若使用 HNSW 索引，则需要指定参数 ef，ef 越大，召回率越高，但也会影响检索速度       
        .withParams(new HNSWSearchParams(200))
        // 设置标量字段的 Filter 表达式，过滤所需查询的文档
        .withFilter("file_name=\\"tcvdb.pdf\\"")
        .withRetrieveVector(false)
         // 指定 Top K 的 K 值        
        .withLimit(2)
        .build();
        
List<List<Document>> siDocs = client.searchByEmbeddingItems("db-test", "coll-file-test", searchByEmbeddingItemsParam).getDocuments();
int i = 0;
for (List<Document> docs : siDocs) {
    System.out.println("\\tres: " + i++);
    for (Document doc : docs) {    
        System.out.println(doc.toString());    
    }
}

log.Println("------------------------------ search file chunks by text  ------------------------------")
filter := tcvectordb.NewFilter(`file_name="` + filename + `"`)
searchRes, err := client.SearchByText(ctx, database, collection,
    map[string][]string{"text": {"商标声明"}},
    &tcvectordb.SearchDocumentParams{
        Filter: filter,
        Params:         &tcvectordb.SearchDocParams{Ef: 200}, // 若使用HNSW索引，则需要指定参数ef，ef越大，召回率越高，但也会影响检索速度
        RetrieveVector: false,                                // 是否需要返回向量字段，False：不返回，True：返回
        Limit:          2,                                    // 指定 Top K 的 K 值
})

for i, docs := range searchRes.Documents {
    log.Printf("SearchDocumentResult, index: %d ==================", i)
    for _, doc := range docs {
        log.Printf("SearchDocument: %+v", doc)
    }
}

2025/05/20 15:00:57 SearchDocumentResult, index: 0 ==================
2025/05/20 15:00:57 SearchDocument: {Id:1374280682098630710 Vector:[] SparseVector:[] Score:323.66516 Fields:map[chunk_num:2 file_name:tcvdb.pdf section_num:1 testInt:1024 testStr:v1 text:本文档涉及的第三方主体的商标，依法由权利人所有。未经腾讯云及有关权利人书面许可，任何主体不得以任何方式对前述商标进行使用、复制、修改、传播、抄录等行为，否则将构成对腾讯云及有关权利人商标权的侵犯，腾讯云将依法采取措施追究法律责任。

【服务声明】

本文档意在向您介绍腾讯云全部或部分产品、服务的当时的相关概况，部分产品、服务的内容可能不时有所调整。您所购买的腾讯云产品、服务的种类、服务标准等应由您与腾讯云之间的商业合同约定，除非双方另有约定，否则，腾讯云对本文档内容不做任何明示或默示的承诺或保证。

【联系我们】

我们致力于为您提供个性化的售前购买咨询服务，及相应的技术售后服务，任何问题请联系 4009100100。

]}
2025/05/20 15:00:57 SearchDocument: {Id:1374280682098630709 Vector:[] SparseVector:[] Score:312.57782 Fields:map[chunk_num:1 file_name:tcvdb.pdf image_list:[ef173e04-5503-421a-b4e6-ae414b3e0ce9.png] section_num:1 testInt:1024 testStr:v1 text:# 向量数据库产品简介

【版权声明】

©2013-2023 腾讯云版权所有

本文档（含所有文字、数据、图片等内容）完整的著作权归腾讯云计算（北京）有限责任公司单独所有，未经腾讯云事先明确书面许可，任何主体不得以任何形式复制、修改、使用、抄袭、传播本文档全部或部分内容。前述行为构成对腾讯云著作权的侵犯，腾讯云将依法采取措施追究法律责任。

【商标声明】

![vdb-image](ef173e04-5503-421a-b4e6-ae414b3e0ce9.png)

及其它腾讯云服务相关的商标均为腾讯云计算（北京）有限责任公司及其关联公司所有。]}

System.out.println("---------------------- query ----------------------");
QueryParam queryParam = QueryParam.newBuilder()
        .withFilter("file_name=\\"tcvdb.pdf\\"")
        // limit 限制返回行数，1 到 16384 之间
        .withLimit(20)
        // 偏移
        .withOffset(0)
        // 是否返回 vector 数据
        .withRetrieveVector(false)
        .build();
List<Document> qdos = client.query("db-test", "coll-file-test", queryParam);
for (Document doc : qdos) {
    System.out.println("\\tres: " + doc.toString());
}

log.Println("------------------------------ Query after waiting 15s to parse file ------------------------------")
result, _ := client.Query(ctx, database, collection, []string{}, &tcvectordb.QueryDocumentParams{
    Filter: tcvectordb.NewFilter(`file_name="` + filename + `"`),
    Limit:  2,
})
ids := []string{}
for _, doc := range result.Documents {
    log.Printf("File chunk: %+v", doc)
    ids = append(ids, doc.Id)
}

System.out.println("---------------------- get image url ----------------------");
GetImageUrlRes getImageUrlRes = client.GetImageUrl("db-test", "coll-file-test",
        GetImageUrlParam.newBuilder().setFileName("tcvdb.pdf")
                .setDocumentIds(qdos.stream().map(doc->doc.getId()).collect(Collectors.toList()))
                .build());
System.out.println("get image url res:");
System.out.println(JsonUtils.toJsonString(getImageUrlRes.getImages()));

log.Println("------------------------------ Get file chunks' imageUrls ------------------------------")
res, err := d.client.GetImageUrl(ctx, database, collection, tcvectordb.GetImageUrlParams{
    FileName:    filename,
    DocumentIds: ids,
})
if err != nil {
    return err
}
for _, docImages := range res.Images {
    for _, docImage := range docImages {
        log.Printf("docImage: %+v", docImage)
    }
}

System.out.println("---------------------- get chunk text by chunk_num ----------------------");
Long chunkNum = (Long) qdos.get(1).getObject("chunk_num");
Long sectionNum = (Long) qdos.get(1).getObject("section_num");
if (chunkNum==null || sectionNum==null){
    return;
}
Long startChunkNum = chunkNum-2;
if(startChunkNum < 0){
    startChunkNum = 0L;
}

QueryParam queryParam = QueryParam.newBuilder()
        .withFilter("file_name=\\"tcvdb.pdf\\"  and chunk_num >= "+ startChunkNum +"  and chunk_num <=" + (chunkNum+2))
        // limit 限制返回行数，1 到 16384 之间
        .withLimit(10)
        // 输出按照 chunk_num 排序
        .withSort(OrderRule.newBuilder().withFieldName("chunk_num").withDirection(OrderEnum.DESC).build())
        // 偏移
        .withOffset(0)
        // 是否返回 vector 数据
        .withRetrieveVector(false)
        .build();
// 输出相似性检索结果，检索结果为二维数组，每一位为一组返回结果，分别对应 search 时指定的多个向量
List<Document> docs = client.query("db-test", "coll-file-test", queryParam);
for (Document doc : docs) {
    System.out.println("\\tres: " + doc.toString());
}