使用 CRD 配置自建 K8S 集群日志采集

2019-12-15 17:13:06,043 [main] ERROR com.test.logging.FooFactory:
java.lang.NullPointerException
    at com.test.logging.FooFactory.createFoo(FooFactory.java:15)
    at com.test.logging.FooFactoryTest.test(FooFactoryTest.java:11)

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  clsDetail:
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
    # 多行日志
    logType: multiline_log
    extractRule:
      # 只有以日期时间开头的行才被认为是新一条日志的开头，否则就添加换行符\\n并追加到当前日志的尾部
      beginningRegex: \\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}\\s.+

__CONTENT__:2019-12-15 17:13:06,043 [main] ERROR com.test.logging.FooFactory:\\njava.lang.NullPointerException\\n    at com.test.logging.FooFactory.createFoo(FooFactory.java:15)\\n    at com.test.logging.FooFactoryTest.test(FooFactoryTest.java:11)

10.135.46.111 - - [22/Jan/2019:19:19:30 +0800] "GET /my/course/1 HTTP/1.1" 127.0.0.1 200 782 9703 "http://127.0.0.1/course/explore?filter%5Btype%5D=all&filter%5Bprice%5D=all&filter%5BcurrentLevelId%5D=all&orderBy=studentNum" "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0"  0.354 0.354

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  clsDetail:
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
    # 完全正则格式
    logType: fullregex_log
    extractRule:
      # 正则表达式，会根据()捕获组提取对应的value
      logRegex: (\\S+)[^\\[]+(\\[[^:]+:\\d+:\\d+:\\d+\\s\\S+)\\s"(\\w+)\\s(\\S+)\\s([^"]+)"\\s(\\S+)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s"([^"]+)"\\s"([^"]+)"\\s+(\\S+)\\s(\\S+).*
      beginningRegex: (\\S+)[^\\[]+(\\[[^:]+:\\d+:\\d+:\\d+\\s\\S+)\\s"(\\w+)\\s(\\S+)\\s([^"]+)"\\s(\\S+)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s"([^"]+)"\\s"([^"]+)"\\s+(\\S+)\\s(\\S+).*
      # 提取的key列表，与提取的value的一一对应
      keys:  ['remote_addr','time_local','request_method','request_url','http_protocol','http_host','status','request_length','body_bytes_sent','http_referer','http_user_agent','request_time','upstream_response_time']

body_bytes_sent: 9703
http_host: 127.0.0.1
http_protocol: HTTP/1.1
http_referer: http://127.0.0.1/course/explore?filter%5Btype%5D=all&filter%5Bprice%5D=all&filter%5BcurrentLevelId%5D=all&orderBy=studentNum
http_user_agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0
remote_addr: 10.135.46.111
request_length: 782
request_method: GET
request_time: 0.354
request_url: /my/course/1
status: 200
time_local: [22/Jan/2019:19:19:30 +0800]
upstream_response_time: 0.354

[2018-10-01T10:30:01,000] [INFO] java.lang.Exception: exception happened
   at TestPrintStackTrace.f(TestPrintStackTrace.java:3)
   at TestPrintStackTrace.g(TestPrintStackTrace.java:7)
   at TestPrintStackTrace.main(TestPrintStackTrace.java:16)

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec: 
  clsDetail: 
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
        #多行-完全正则格式
        logType: multiline_fullregex_log
        extractRule: 
          #行首完全正则表达式，只有以日期时间开头的行才被认为是新一条日志的开头，否则就添加换行符\\n并追加到当前日志的尾部
          beginningRegex: \\[\\d+-\\d+-\\w+:\\d+:\\d+,\\d+\\]\\s\\[\\w+\\]\\s.*
          #正则表达式，会根据()捕获组提取对应的value
          logRegex: \\[(\\d+-\\d+-\\w+:\\d+:\\d+,\\d+)\\]\\s\\[(\\w+)\\]\\s(.*)
          # 提取的 key 列表，与提取的 value 的一一对应
          keys: ['time','level','msg']

time： 2018-10-01T10:30:01,000`
level： INFO`
msg：java.lang.Exception: exception happened
   at TestPrintStackTrace.f(TestPrintStackTrace.java:3)
   at TestPrintStackTrace.g(TestPrintStackTrace.java:7)
   at TestPrintStackTrace.main(TestPrintStackTrace.java:16)

{"remote_ip":"10.135.46.111","time_local":"22/Jan/2019:19:19:34 +0800","body_sent":23,"responsetime":0.232,"upstreamtime":"0.232","upstreamhost":"unix:/tmp/php-cgi.sock","http_host":"127.0.0.1","method":"POST","url":"/event/dispatch","request":"POST /event/dispatch HTTP/1.1","xff":"-","referer":"http://127.0.0.1/my/course/4","agent":"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0","response_code":"200"}

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  clsDetail:
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
    # JSON格式日志
    logType: json_log

agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0
body_sent: 23
http_host: 127.0.0.1
method: POST
referer: http://127.0.0.1/my/course/4
remote_ip: 10.135.46.111
request: POST /event/dispatch HTTP/1.1
response_code: 200
responsetime: 0.232
time_local: 22/Jan/2019:19:19:34 +0800
upstreamhost: unix:/tmp/php-cgi.sock
upstreamtime: 0.232
url: /event/dispatch
xff: -

10.20.20.10 ::: [Tue Jan 22 14:49:45 CST 2019 +0800] ::: GET /online/sample HTTP/1.1 ::: 127.0.0.1 ::: 200 ::: 647 ::: 35 ::: http://127.0.0.1/

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  clsDetail:
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
    # 分隔符日志
    logType: delimiter_log
    extractRule:
      # 分隔符
      delimiter: ':::'
      # 提取的key列表，与被分割的字段一一对应
      keys: ['IP','time','request','host','status','length','bytes','referer']

IP: 10.20.20.10
bytes: 35
host: 127.0.0.1
length: 647
referer: http://127.0.0.1/
request: GET /online/sample HTTP/1.1
status: 200
time: [Tue Jan 22 14:49:45 CST 2019 +0800]

1571394459，http://127.0.0.1/my/course/4|10.135.46.111|200，status:DEAD,

{
 "processors": [
   {
     "type": "processor_split_delimiter",
     "detail": {
       "Delimiter": ",",
       "ExtractKeys": [ "time", "msg1","msg2"]
     },
     "processors": [
       {
         "type": "processor_timeformat",
         "detail": {
           "KeepSource": true,
           "TimeFormat": "%s",
           "SourceKey": "time"
          }
       },
       {
         "type": "processor_split_delimiter",
         "detail": {
           "KeepSource": false,
           "Delimiter": "|",
           "SourceKey": "msg1",
           "ExtractKeys": [ "submsg1","submsg2","submsg3"]
          },
          "processors": []
       },
       {
         "type": "processor_split_key_value",
         "detail": {
           "KeepSource": false,
           "Delimiter": "：",
           "SourceKey": "msg2"
         }
       }
     ]
   }
 ]
}

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  clsDetail:
    topicId: xxxxxx-xx-xx-xx-xxxxxxxx
    # 分隔符日志
    logType: user_define_log
    userDefineRule："{ \\"processors\\": [ { \\"type\\": \\"processor_split_delimiter\\", \\"detail\\": { \\"Delimiter\\": \\",\\", \\"ExtractKeys\\": [ \\"time\\", \\"msg1\\",\\"msg2\\"] }, \\"processors\\": [ { \\"type\\": \\"processor_timeformat\\", \\"detail\\": { \\"KeepSource\\": true, \\"TimeFormat\\": \\"%s\\", \\"SourceKey\\": \\"time\\" } }, { \\"type\\": \\"processor_split_delimiter\\", \\"detail\\": { \\"KeepSource\\": false, \\"Delimiter\\": \\"|\\", \\"SourceKey\\": \\"msg1\\", \\"ExtractKeys\\": [ \\"submsg1\\",\\"submsg2\\",\\"submsg3\\"] }, \\"processors\\": [] }, { \\"type\\": \\"processor_split_key_value\\", \\"detail\\": { \\"KeepSource\\": false, \\"Delimiter\\": \\"：\\", \\"SourceKey\\": \\"msg2\\" } } ] } ] }"

time: 1571394459
submsg1: http://127.0.0.1/my/course/4
submsg2: 10.135.46.111
submsg3: 200
status: DEAD

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  inputDetail:
    type: container_file
    containerFile:
      namespace: production
      workload:
        name: ingress-gateway
        kind: deployment
      container: nginx
      filePaths:                                
      - file: access.log
        path: /data/nginx/log
  ...

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig
spec:
  inputDetail:
    type: container_file
    containerFile:
      namespace: production
      includeLabels:
        k8s-app: ingress-gateway
      container: nginx
      filePaths:                                
      - file: access.log
        path: /data/nginx/log
  ...

apiVersion: cls.cloud.tencent.com/v1
kind: LogConfig                   
metadata:
  name: test                                  ## 采集配置名称
spec:
  inputDetail:
    type: host_file
    hostFile:
      logPath: /data
      filePattern: *.log
  ...