干货 | Elasticsearch 索引生命周期管理 ILM 实战指南

- node.attr.box_type: hot
- node.attr.box_type: warm
- node.attr.box_type: cold

# 1、创建基于日期的索引
PUT %3Cmy-index-%7Bnow%2Fd%7D-000001%3E
{
  "aliases": {
    "my-alias": {
      "is_write_index": true
    }
  }
}

# 2、批量导入数据
PUT my-alias/_bulk
{"index":{"_id":1}}
{"title":"testing 01"}
{"index":{"_id":2}}
{"title":"testing 02"}
{"index":{"_id":3}}
{"title":"testing 03"}
{"index":{"_id":4}}
{"title":"testing 04"}
{"index":{"_id":5}}
{"title":"testing 05"}

# 3、rollover 滚动索引
POST my-alias/_rollover
{
  "conditions": {
    "max_age": "7d",
    "max_docs": 5,
    "max_primary_shard_size": "50gb"
  }
}

GET my-alias/_count

# 4、在满足滚动条件的前提下滚动索引
PUT my-alias/_bulk
{"index":{"_id":6}}
{"title":"testing 06"}

# 5、检索数据，验证滚动是否生效
GET my-alias/_search

{
        "_index" : "my-index-2021.05.30-000001",
        "_type" : "_doc",
        "_id" : "5",
        "_score" : 1.0,
        "_source" : {
          "title" : "testing 05"
        }
      },
      {
        "_index" : "my-index-2021.05.30-000002",
        "_type" : "_doc",
        "_id" : "6",
        "_score" : 1.0,
        "_source" : {
          "title" : "testing 06"
        }
      }

# 设置待压缩的索引，分片设置为5个。
PUT kibana_sample_data_logs_ext
{
  "settings": {
    "number_of_shards":5
  }
}

# 准备索引数据
POST _reindex
{
  "source":{
    "index":"kibana_sample_data_logs"
  },
  "dest":{
    "index":"kibana_sample_data_logs_ext"
  }
}

# shrink 压缩之前的三个必要条件
PUT kibana_sample_data_logs_ext/_settings
{
  "settings": {
    "index.number_of_replicas": 0,
    "index.routing.allocation.require._name": "node-024", 
    "index.blocks.write": true                                    
  }
}

# 实施压缩
POST kibana_sample_data_logs_ext/_shrink/kibana_sample_data_logs_shrink
{
  "settings": {
    "index.number_of_replicas": 0,
    "index.number_of_shards": 1, 
    "index.codec": "best_compression" 
  },
  "aliases": {
    "kibana_sample_data_logs_alias": {}
  }
}

# 冷冻索引
POST kibana_sample_data_logs_shrink/_freeze

# 冷冻后，不能写入
POST kibana_sample_data_logs_shrink/_doc/1
{
  "test":"12111"
}

# 冷冻后，能检索，但不返回具体数据，只返回0。
POST kibana_sample_data_logs_shrink/_search

# 解除冷冻
POST kibana_sample_data_logs_shrink/_unfreeze

# 解除冷冻后，可以检索和写入了
POST kibana_sample_data_logs_shrink/_search

# step1: 前提：演示刷新需要
PUT _cluster/settings
{
  "persistent": {
    "indices.lifecycle.poll_interval": "1s"
  }
}

# step2:测试需要，值调的很小
PUT _ilm/policy/my_custom_policy_filter
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "3d",
            "max_docs": 5,
            "max_size": "50gb"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "15s",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          },
          "allocate": {
            "require": {
              "box_type": "warm"
            },
            "number_of_replicas": 0
          },
          "set_priority": {
            "priority": 50
          }
        }
      },
      "cold": {
        "min_age": "30s",
        "actions": {
          "allocate": {
            "require": {
              "box_type": "cold"
            }
          },
          "freeze": {}
        }
      },
      "delete": {
        "min_age": "45s",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

# step3:创建模板，关联配置的ilm_policy
PUT _index_template/timeseries_template
{
  "index_patterns": ["timeseries-*"],                 
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 0,
      "index.lifecycle.name": "my_custom_policy_filter",      
      "index.lifecycle.rollover_alias": "timeseries",
      "index.routing.allocation.require.box_type": "hot"
    }
  }
}

# step4:创建起始索引（便于滚动）
PUT timeseries-000001
{
  "aliases": {
    "timeseries": {
      "is_write_index": true
    }
  }
}

# step5：插入数据
PUT timeseries/_bulk
{"index":{"_id":1}}
{"title":"testing 01"}
{"index":{"_id":2}}
{"title":"testing 02"}
{"index":{"_id":3}}
{"title":"testing 03"}
{"index":{"_id":4}}
{"title":"testing 04"}

# step6：临界值（会滚动）
PUT timeseries/_bulk
{"index":{"_id":5}}
{"title":"testing 05"}

# 下一个索引数据写入
PUT timeseries/_bulk
{"index":{"_id":6}}
{"title":"testing 06"}

PUT _index_template/timebase_template 
{ "index_patterns": ["time_base-*"] }

PUT time_base-000001 { "aliases": { "timebase_alias": { "is_write_index": true } } }