Kubernetes 日志解决方案 Grafana Loki 「Helm 部署案例」

## 添加加速镜像仓库
helm repo add grafana "https://helm-charts.itboon.top/grafana" --force-update
helm repo update grafana

## 添加官方仓库
# helm repo add grafana "https://grafana.github.io/helm-charts" --force-update
# helm repo update grafana

## values-loki.yaml


fullnameOverride: loki

rbac:
  namespaced: true

loki:
  auth_enabled: false
  image:
    # registry: docker.io
    # repository: grafana/loki
    tag: 2.9.6

  readinessProbe:
    initialDelaySeconds: 10
    periodSeconds: 5
    timeoutSeconds: 1

  ## loki.config
  commonConfig:
    ## set to 1, otherwise more replicas are needed to connect to grafana
    replication_factor: 1
  storage:
    bucketNames:
      chunks: Your_Loki_Bucket
      ruler: Your_Loki_Bucket
      admin: Your_Loki_Bucket
    type: s3
    s3:
      ## s3 access, AWS S3 或者兼容 S3 API 的对象存储都可以
      endpoint: cos.ap-guangzhou.myqcloud.com
      accessKeyId: Your_Access_Key_ID
      secretAccessKey: Your_Secret_Access_Key

  compactor:
    working_directory: /var/loki/retention
    shared_store: s3
    compaction_interval: 10m
    retention_enabled: true
    retention_delete_delay: 2h
    retention_delete_worker_count: 150

  ingester:
    max_transfer_retries: 0
    # max_chunk_age: 2h    # default = 2h
    # chunk_idle_period: 30m    # default = 30m
    # chunk_target_size: 1572864
    chunk_encoding: zstd
    autoforget_unhealthy: true
  storage_config:
    tsdb_shipper:
      active_index_directory: /var/loki/tsdb-index
      cache_location: /var/loki/tsdb-cache
      cache_ttl: 72h
      shared_store: s3
  schemaConfig:
    configs:
      - from: 2024-01-01
        store: tsdb
        object_store: s3
        schema: v12
        index:
          prefix: loki_tsdb_index_
          period: 24h
  limits_config:
    ingestion_rate_strategy: local    # default = "global"
    # retention_period: 240h
    ingestion_rate_mb: 20    # default = 4
    ingestion_burst_size_mb: 100    # default = 6
    per_stream_rate_limit: 6MB    # default = 3MB
    per_stream_rate_limit_burst: 50MB    # default = 15MB
    max_concurrent_tail_requests: 20
    max_cache_freshness_per_query: 10m
    max_query_length: 72h
    max_line_size: 256kb
    max_line_size_truncate: true
    ## [max_streams_matchers_per_query: <int> | default = 1000]
    ## [max_query_bytes_read: <int> | default = 0B]
    shard_streams:
      enabled: true
      desired_rate: 2097152    #2MiB
  
  server:
    grpc_server_max_recv_msg_size: 32000100    # default = 4194304
    grpc_server_max_send_msg_size: 32000100    # default = 4194304


test:
  enabled: false

monitoring:
  lokiCanary:
    enabled: false
  dashboards:
    enabled: false
  selfMonitoring:
    enabled: false
    grafanaAgent:
      installOperator: false
  serviceMonitor:
    enabled: false

#sidecar:
#  resources:
#    limits:
#      cpu: 200m
#      memory: 500Mi
#    requests:
#      cpu: 10m
#      memory: 50Mi

gateway:
  replicas: 1
  #resources:
  #  limits:
  #    cpu: 1
  #    memory: 2Gi
  #  requests:
  #    cpu: 50m
  #    memory: 200Mi
  #service:
  #  type: LoadBalancer

backend:
  replicas: 1
  persistence:
    size: 20Gi
    enableStatefulSetAutoDeletePVC: false
    #storageClass: null
  #resources:
  #  limits:
  #    cpu: 2
  #    memory: 8Gi
  #  requests:
  #    cpu: 100m
  #    memory: 500Mi


write:
  replicas: 1
  persistence:
    size: 20Gi
    #storageClass: null
  #resources:
  #  limits:
  #    cpu: 2
  #    memory: 8Gi
  #  requests:
  #    cpu: 500m
  #    memory: 4Gi

read:
  replicas: 3
  resources:
    limits:
      cpu: 1
      memory: 2Gi
    requests:
      cpu: 50m
      memory: 100Mi
  autoscaling:
    enabled: true
    minReplicas: 3
    maxReplicas: 15
    targetCPUUtilizationPercentage: 90

## 部署 Loki
helm upgrade --install loki  \
  --namespace loki \
  --create-namespace \
  -f values-loki.yaml \
  --version 5.47.1 \
  grafana/loki

## values-promtail.yaml


image:
  tag: 2.9.6

#resources:
#  limits:
#    cpu: 2
#    memory: 2Gi
#  requests:
#    cpu: 100m
#    memory: 512Mi

config:
  clients:
    - url: http://loki-gateway/loki/api/v1/push
  snippets:
    extraLimitsConfig: |
      max_line_size: 256kb
      max_line_size_truncate: true
      readline_rate_enabled: true
      readline_rate: 110000    # default = 10000
      readline_burst: 810000    # default = 10000
    pipelineStages:
      - cri: {}
      # - match:
      #     ## 过滤某些 namespace
      #     selector: '{namespace=~"Your_Namespace"}'
      #     action: drop
      # - match:
      #     ## 过滤某些 container
      #     selector: '{container=~"Your_Container_Name"}'
      #     action: drop
      - json:
          expressions:
            log: log
      - output:
          source: log
      - multiline:
          ## 跨行合并
          firstline: '^\S'
          max_wait_time: 3s
          max_lines: 2000
      #- match:
      #    ## 匹配标签的日志将进行额外管道处理
      #    selector: '{container=~"java.*"}'
      #    stages:
      #    - multiline:
      #        firstline: '^[A-Z]{3,9} +\['
      #        max_wait_time: 3s
      #        max_lines: 2000
      #    - regex:
      #        expression: '^(?P<level>[A-Z]{3,9}) +\[.*'
      #        #source: log
      #    - labels:
      #        level:

## 部署 Promtail
helm upgrade --install promtail  \
  --namespace loki \
  --create-namespace \
  -f values-promtail.yaml \
  --version 6.15.5 \
  grafana/promtail

## values-grafana.yaml


datasources:
  datasources.yaml:
    apiVersion: 1
    datasources:
    - name: Loki
      type: loki
      url: http://loki-gateway
      access: proxy
      jsonData:
        maxLines: 500

# ingress:
#   enabled: true
#   ingressClassName: nginx
#   annotations: {}
#     # kubernetes.io/ingress.class: nginx
#     # kubernetes.io/tls-acme: "true"
#   hosts:
#     - grafana.example.com

resources:
  limits:
    cpu: 100m
    memory: 1Gi
  requests:
    cpu: 20m
    memory: 128Mi

## 部署 Grafana
helm upgrade --install grafana  \
  --namespace loki \
  --create-namespace \
  -f values-grafana.yaml \
  --version 7.3.7 \
  grafana/grafana

## 获取 Grafana 初始 admin 密码
kubectl get secret -n loki grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo

loki:
  ingester:
    chunk_encoding: zstd

loki:
  storage_config:
    tsdb_shipper:

loki:
  limits_config:
    ingestion_rate_strategy: local    # default = "global"
    # retention_period: 240h
    ingestion_rate_mb: 20    # default = 4
    ingestion_burst_size_mb: 100    # default = 6
    per_stream_rate_limit: 6MB    # default = 3MB
    per_stream_rate_limit_burst: 50MB    # default = 15MB
    max_concurrent_tail_requests: 20
    max_cache_freshness_per_query: 10m
    max_query_length: 72h
    max_line_size: 256kb
    max_line_size_truncate: true
    ## [max_streams_matchers_per_query: <int> | default = 1000]
    ## [max_query_bytes_read: <int> | default = 0B]
    shard_streams:
      enabled: true
      desired_rate: 2097152    #2MiB
  
  server:
    grpc_server_max_recv_msg_size: 32000100    # default = 4194304
    grpc_server_max_send_msg_size: 32000100    # default = 4194304

[INFO] Reloading
[INFO] plugin/reload: Running configuration SHA512 = 637429c29e6628b9c013e5f8bd5211d4564943dd9f2dd6eeb506a5a2993177f61da856d85e9ed3c8020ebde2b4abbc232f057d9ba5d9df8247d706893feb8754
[INFO] Reloading complete
[ERROR] plugin/errors: 2 cm-cn-central-00001.albatross.10086.cn. HTTPS: read tcp 10.4.63.250:47580->100.100.2.136:53: i/o timeout

- match:
    ## 匹配容器名以 java 开头的容器
    selector: '{container=~"java.*"}'
    stages:
    - multiline:
        ## 跨行合并
        firstline: '^\[[A-Z]{3,9}\] '
        max_wait_time: 3s
        max_lines: 2000
    - regex:
        expression: '^\[(?P<level>[A-Z]{3,9})\] '
        #source: log
    - labels:
        level: