.NET 5.0 快速开发框架 千万级数据处理 解决方案

put /tenant_1_books/_settings
{ 
    "index.max_result_window" :"1000000"
}

"ElasticSearchSetting": {

  ///elasticSearch节点集群
  "Nodes": [
    { "node": "http://127.0.0.1:9200" },
    { "node": "http://127.0.0.1:9201" },
    { "node": "http://127.0.0.1:9202" }

  ]
}

// elasticSearch 注入
builder.RegisterModule(new ElasticSearchAutofacModule());

private IElasticSearchRepository<Book> _elasticSearchRepository;
public BookAppService(
 IHttpContextAccessor httpContextAccessor, ICacheManager cacheManager, IMapper mapper, IElasticSearchRepository<Book> elasticSearchRepository) : base(httpContextAccessor, cacheManager)
 {
     _cacheManager = cacheManager;
     _mapper = mapper;
     _elasticSearchRepository = elasticSearchRepository;
 }

 /// <summary>
 /// 查查默认1页10条
 /// </summary>
 /// <returns>返回数据集合</returns>
 public async Task<ApiResult<List<BookAddOrEditDto>>> GetAllAsync()
 {
     var res = new ApiResult<List<BookAddOrEditDto>>();
     var data = await _elasticSearchRepository.GetAllAsync();

     var entityDtoList = _mapper.Map<List<BookAddOrEditDto>>(data);
     return res.Ok(entityDtoList);
 }

//创建表 对应的分片，需要表还没创建时候设置
put /tenant_1_books_0
{
  "settings":{
    
    "number_of_shards":2
  }
}

//创建新的索引数据库，并指定字段映射类型，tenant_1_books_0 中的bookName 类型改为keyword
PUT tenant_1_books_0
 {
  "mappings": {
    "properties": {
      
      "bookName":{
        "type":"keyword"
      }
    }
    
  }
  
 }

  //迁移数据,将tenant_1_books 数据迁移到tenant_1_books_1
 POST _reindex
 {
  "source": {
    "index": "tenant_1_books"
    
  },
  "dest": {
     "index": "tenant_1_books_1"
    
  }
 }

//查看tenant_1_books所有数据【默认会分页】
GET tenant_1_books/_search
{
  "track_total_hits": true,
  "query": {
    "match_all": {}
  }
}
//深度分页方案1 扩大分页限制
//允许深度分页，限制在10w
put /tenant_1_books/_settings
{
     "index.max_result_window" :"1000000"
}
//查询数据 分页，track_total_hits=真实的总数
GET tenant_1_books/_search
{
  "track_total_hits": true,
  "from" : 99000, "size" : 100,
  "query": {
    "match_all": {}
  }
}

 /// <summary>
 /// 深度分页查询 searchAfter
 /// </summary>
 /// <returns></returns>
 [Fact]
 public async Task GetPageByQuerySearchAfterTest()
 {
     int size = 100;
     // "bookName" : {
     //"type" : "keyword"
     //},
     //1、BookName 修改为keyword 所有必须完整匹配，不分词
     Func<QueryContainerDescriptor<Book>, QueryContainer> query1 = q => q.Term(t => t.BookName, "吞噬星空");
     Func<QueryContainerDescriptor<Book>, QueryContainer> query2 = q => q.Match(mq =>
       mq.Field(f => f.BookName).Query("哈利波特").Operator(Operator.And)
       );//由于类型为 keyword，所以Match 查找不出来，只能使用Term 精确查询

     //2.全字匹配+ 分词查询
     Func<QueryContainerDescriptor<Book>, QueryContainer> query3 = q => q
                    .Term(t => t.BookName, "吞噬星空")
                    || q.Match(mq =>
        mq.Field(f => f.BookContent).Query("哈利波特").Operator(Operator.And)
       );
     //排序
     Func<SortDescriptor<Book>, IPromise<IList<ISort>>> sort = s => s.Ascending(a => a.CreateDate).Descending(d=>d.Price);

   var result1=await  _elasticSearchRepository.GetPageByQuerySearchAfterAsync(query3, sort, 100, null);
     //使用上一次查询得到SearchAfter 作为下一次查询的游标
     var result2 = await _elasticSearchRepository.GetPageByQuerySearchAfterAsync(query3, sort, 100, result1.SearchAfter);

     Assert.NotNull(result2.List);
 }

//深度分页方案3 scroll，者每次查询大量的文档，但是对实时性要求并不高，
//后面的每次滚屏（或者叫翻页）都是基于这个快照的结果，也就是即使有新的数据进来也不会别查询到。
//1. 查询
POST tenant_1_books/_search?scroll=1m
{
    "size": 1000,
    "query": {
        "match_all" : {
        }
    }
   
  }
//2. 上一次查询所得到结果，作为游标
POST _search/scroll
{
    "scroll" : "1m",
    "scroll_id" : "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFmJpVWRZVWQ3UkItejk2UUx5bC15bFEAAAAAAAMv2xZyZURRWkowelF6S0NnRjMzWjhfQTh3"
  
}

//聚合获取该字段的所有统计
get /tenant_1_books/_search
{
   "aggs":{
      "extended_stats_price":{"extended_stats":{"field":"price"}}
   }
}
//聚合 总和统计
get /tenant_1_books/_search
{
   "aggs":{
      "total_price":{"sum":{"field":"price"}}
   }
}