prometheus 告警

groups:
- name: example
  rules:
  - alert: HighErrorRate
    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
    for: 10m
    labels:
      severity: page
    annotations:
      summary: High request latency
      description: description info

rule_files:
  [ - <filepath_glob> ... ]

global:
  [ evaluation_interval: <duration> | default = 1m ]

docker pull prom/alertmanager
mkdir -p /usr/local/prometheus/alertmanager
mkdir -p /usr/local/prometheus/alertmanager/conf
docker run -id --name alertmanager prom/alertmanager
docker cp alertmanager:/etc/alertmanager/alertmanager.yml /usr/local/prometheus/alertmanager/conf/
docker rm -f alertmanager
docker run -id --name alertmanager -p 9093:9093 -v /usr/local/prometheus/alertmanager/:/alertmanager -v /usr/local/prometheus/alertmanager/conf/alertmanager.yml:/etc/alertmanager/alertmanager.yml prom/alertmanager

global:
  # 默认的 SMTP 头部.
  [ smtp_from: <tmpl_string> ]
  # 用于发送邮件的主机地址和端口(SMTP 的地址)
  [ smtp_smarthost: <string> ]
  # SMTP 服务器默认的主机名
  [ smtp_hello: <string> | default = "localhost" ]
  # 使用CRAM-MD5，LOGIN和PLAIN进行SMTP验证。如果为空，则Alertmanager不会对SMTP服务器进行身份验证。
  [ smtp_auth_username: <string> ]
  # 使用 Login 身份验证的密码
  [ smtp_auth_password: <secret> ]
  # 使用 PLAIN 的密码验证
  [ smtp_auth_identity: <string> ]
  # 使用CRAM-MD5的密码 
  [ smtp_auth_secret: <secret> ]
  # 是否开启 TLS 验证
  [ smtp_require_tls: <bool> | default = true ]

  # 用于 slack 通知的 API url
  [ slack_api_url: <secret> ]
  [ victorops_api_key: <secret> ]
  [ victorops_api_url: <string> | default = "https://alert.victorops.com/integrations/generic/20131114/alert/" ]
  [ pagerduty_url: <string> | default = "https://events.pagerduty.com/v2/enqueue" ]
  [ opsgenie_api_key: <secret> ]
  [ opsgenie_api_url: <string> | default = "https://api.opsgenie.com/" ]
  [ hipchat_api_url: <string> | default = "https://api.hipchat.com/" ]
  [ hipchat_auth_token: <secret> ]
  [ wechat_api_url: <string> | default = "https://qyapi.weixin.qq.com/cgi-bin/" ]
  [ wechat_api_secret: <secret> ]
  [ wechat_api_corp_id: <string> ]

  # 默认 http 客户端的配置
  [ http_config: <http_config> ]

  #定义了当Alertmanager持续多长时间未接收到告警后标记告警状态为resolved（已解决）。
  # 该参数的定义可能会影响到告警恢复通知的接收时间，读者可根据自己的实际场景进行定义，其默认值为5分钟。
  [ resolve_timeout: <duration> | default = 5m ]

# 定义了告警通知时的模板 
# 可以采用通配符的形式去包含模板 'templates/*.tmpl'.
templates:
  [ - <filepath> ... ]

# 根据标签匹配，确定当前告警应该如何处理；
route: <route>

# 接收人
receivers:
  - <receiver> ...

# 抑制规则
inhibit_rules:
  [ - <inhibit_rule> ... ]

[ receiver: <string> ]
# 通过什么来进行分组,可以支持多个标签,这个为 prometheus 里面的 labels
[ group_by: '[' <labelname>, ... ']' ]

# 警报是否应继续匹配后续的同级节点。
[ continue: <boolean> | default = false ]

# 警报必须满足的一组相等匹配器才能匹配节点。
match:
  [ <labelname>: <labelvalue>, ... ]

# 警报必须满足以匹配节点的一组正则表达式匹配器。
match_re:
  [ <labelname>: <regex>, ... ]

# 等待时间,如果在等待时间内当前 group 接收到了新的告警,
# 这些告警将会合并为一个通知想 receiver 发送
[ group_wait: <duration> | default = 30s ]

# 相同的 group 之间发送告警通知的时间间隔
[ group_interval: <duration> | default = 5m ]

# How long to wait before sending a notification again if it has already
# been sent successfully for an alert. (Usually ~3h or more).
[ repeat_interval: <duration> | default = 4h ]

# 子路由配置.
routes:
  [ - <route> ... ]

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
       - 192.168.111.65:8999

  - job_name: alertmanager
    metrics_path: /metrics
    static_configs:
    - targets: ['192.168.111.65:9093']

docker run -d -p 8060:8060 --name webhook timonwong/prometheus-webhook-dingtalk:latest --ding.profile="test=https://oapi.dingtalk.com/robot/send?access_token=XXXXXXXXXXXXXXX" 

route:
  receiver: webhook
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 5m
  group_by: ['alertname']
  routes:
  - receiver: webhook
    group_wait: 10s
receivers:
- name: 'webhook'
  webhook_configs:
  - send_resolved: false
    url: http://192.168.111.65:8060/dingtalk/test/send

global:
  resolve_timeout: 5m
  smtp_smarthost: smtp.qq.com:465
  smtp_from: 18163201@qq.com
  smtp_auth_username: 18163201
  smtp_auth_identity: 18163201
  smtp_auth_password: oitsncuccbjcbjca
  smtp_require_tls: false

templates:
  - '/alertmanager/templates/*.tmpl'

route:
  - receiver: email
    group_wait: 10s
    match:
      level: serious

receivers:
- name: 'email'
  email_configs:
  - to: zsf18163201@163.com
    send_resolved: true
    html: '{{  template "alert.html" . }}'
    headers: { Subject: "Prometheus 严重报警邮件"}

mkdir -p /alertmanager/templates/
# cat templates/email.tmpl
{{ define "alert.html" }}
{{ range .Alerts }}
=========start==========<br>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.level }} 级 <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Format "2019-08-04 16:58:15" }} <br>
=========end==========<br>
{{ end }}
{{ end }}

- name: 'wechat'
  wechat_configs:
  - send_resolved: false  #恢复是否发送消息
    corp_id: 'wwde5d94e8b21c9678'  #'企业id，在企业的配置页面可以看到'
    to_user: '@all'
    message: '{{ template "wechat.default.message" . }}'
    agent_id: 1000002 #'应用的AgentId，在应用的配置页面可以看到'
    api_secret: rdoP88Oz7ZSickdxDpaMcQQ-7QnpiedtHNUfshOoH2I #'应用的secret，在应用的配置页面可以看到'

# cat wechat.tmpl
{{ define "wechat.default.message" }}
{{- if gt (len .Alerts.Firing) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 -}}
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}

=====================
{{- end }}
===告警详情===
告警详情: {{ $alert.Annotations.message }}
故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
===参考信息===
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例ip: {{ $alert.Labels.instance }};{{- end -}}
{{- if gt (len $alert.Labels.namespace) 0 -}}故障实例所在namespace: {{ $alert.Labels.namespace }};{{- end -}}
{{- if gt (len $alert.Labels.node) 0 -}}故障物理机ip: {{ $alert.Labels.node }};{{- end -}}
{{- if gt (len $alert.Labels.pod_name) 0 -}}故障pod名称: {{ $alert.Labels.pod_name }}{{- end }}
=====================
{{- end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 -}}
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}

=====================
{{- end }}
===告警详情===
告警详情: {{ $alert.Annotations.message }}
故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
恢复时间: {{ $alert.EndsAt.Format "2006-01-02 15:04:05" }}
===参考信息===
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例ip: {{ $alert.Labels.instance }};{{- end -}}
{{- if gt (len $alert.Labels.namespace) 0 -}}故障实例所在namespace: {{ $alert.Labels.namespace }};{{- end -}}
{{- if gt (len $alert.Labels.node) 0 -}}故障物理机ip: {{ $alert.Labels.node }};{{- end -}}
{{- if gt (len $alert.Labels.pod_name) 0 -}}故障pod名称: {{ $alert.Labels.pod_name }};{{- end }}
=====================
{{- end }}
{{- end }}
{{- end }}

[root@zhangsf conf]# cat docker.rules
groups:
- name: example
  rules:
  - alert: dockerNumber
    expr: count(container_last_seen{name!=""}) >=8    #当容器数量大于等于 8 的时候告警
    for: 1m
    labels:
      docker: number
    annotations:
      summary: check docker runing number
      description: description info

rule_files:
  - /etc/prometheus/*.rules #映射到 docker 的这个目录下了

docker run -id --name test nginx:1.14.2