prometheus-简介及安装

scrape_configs:
    - job_name: 'prometheus'
      static_configs:
      - targets: ['localhost:9090']
   - job_name: 'node'
      static_configs:
      - targets: ['192.168.1.10:9090']

tar xvfz prometheus-*.tar.gz
cd prometheus-*
mv prometheus-* /usr/local/prometheus

[Unit]
Description=https://prometheus.io

[Service]
Restart=on-failure
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml

[Install]
WantedBy=multi-user.target

启动prometheus
./prometheus --config.file=prometheus.yml

检测配置文件
./promtool check config prometheus.yml

重新加载配置文件
kill -hup PID

prometheus.yml通过运行以下命令将您从主机绑定：
docker run -p 9090:9090 -v /tmp/prometheus.yml:/etc/prometheus/prometheus.yml \
       prom/prometheus

或者为配置使用额外的卷：
docker run -p 9090:9090 -v /prometheus-data \
       prom/prometheus --config.file=/prometheus-data/prometheus.yml

global:
  scrape_interval:     15s # 默认情况下，每15s拉取一次目标采样点数据。

  # 我们可以附加一些指定标签到采样点度量标签列表中, 用于和第三方系统进行通信, 包括：federation, remote storage, Alertmanager
  external_labels:
    monitor: 'codelab-monitor'

# 下面就是拉取自身服务采样点数据配置
scrape_configs:
  # job名称会增加到拉取到的所有采样点上，同时还有一个instance目标服务的host：port标签也会增加到采样点上
  - job_name: 'prometheus'

    # 覆盖global的采样点，拉取时间间隔5s
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:9090']

global:
  # 默认抓取周期，可用单位ms、smhdwy #设置每15s采集数据一次，默认1分钟
  [ scrape_interval: <duration> | default = 1m ]
  # 默认抓取超时
  [ scrape_timeout: <duration> | default = 10s ]
  # 估算规则的默认周期# 每15秒计算一次规则。默认1分钟
  [ evaluation_interval: <duration> | default = 1m ]
  # 和外部系统（例如AlertManager）通信时为时间序列或者警情（Alert）强制添加的标签列表
  external_labels:
    [ <labelname>: <labelvalue> ... ]
 
# 规则文件列表
rule_files:
  [ - <filepath_glob> ... ]
 
# 抓取配置列表
scrape_configs:
  [ - <scrape_config> ... ]
 
# Alertmanager相关配置
alerting:
  alert_relabel_configs:
    [ - <relabel_config> ... ]
  alertmanagers:
    [ - <alertmanager_config> ... ]
 
# 远程读写特性相关的配置
remote_write:
  [ - <remote_write> ... ]
remote_read:
  [ - <remote_read> ... ]

# 任务名称，自动作为抓取到的指标的一个标签
job_name: <job_name>
 
# 抓取周期
[ scrape_interval: <duration> | default = <global_config.scrape_interval> ]
# 每次抓取的超时
[ scrape_timeout: <duration> | default = <global_config.scrape_timeout> ]
# 从目标抓取指标的URL路径
[ metrics_path: <path> | default = /metrics ]
# 当添加标签发现指标已经有同名标签时，是否保留原有标签不覆盖
[ honor_labels: <boolean> | default = false ]
# 抓取协议
[ scheme: <scheme> | default = http ]
# HTTP请求参数
params:
  [ <string>: [<string>, ...] ]
 
# 身份验证信息
basic_auth:
  [ username: <string> ]
  [ password: <secret> ]
  [ password_file: <string> ]
# Authorization请求头取值
[ bearer_token: <secret> ]
# 从文件读取Authorization请求头
[ bearer_token_file: /path/to/bearer/token/file ]
 
# TLS配置
tls_config:
  [ <tls_config> ]
 
# 代理配置
[ proxy_url: <string> ]
 
# DNS服务发现配置
dns_sd_configs:
  [ - <dns_sd_config> ... ]
# 文件服务发现配置
file_sd_configs:
  [ - <file_sd_config> ... ]
# K8S服务发现配置
kubernetes_sd_configs:
  [ - <kubernetes_sd_config> ... ]
 
# 此Job的静态配置的目标列表
static_configs:
  [ - <static_config> ... ]
 
# 目标重打标签配置
relabel_configs:
  [ - <relabel_config> ... ]
# 指标重打标签配置
metric_relabel_configs:
  [ - <relabel_config> ... ]
 
# 每次抓取允许的最大样本数量，如果在指标重打标签后，样本数量仍然超过限制，则整个抓取认为失败
# 0表示不限制
[ sample_limit: <int> | default = 0 

CPU使用率：
100 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) * 100)
内存使用率：
100 - (node_memory_MemFree_bytes+node_memory_Cached_bytes+node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100
磁盘使用率：
100 - (node_filesystem_free_bytes{mountpoint="/",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{mountpoint="/",fstype=~"ext4|xfs"} * 100)

/usr/bin/node_exporter --collector.systemd --collector.systemd.unit-whitelist=(docker|sshd|nginx).service

docker run \
  --volume=/:/rootfs:ro \
  --volume=/var/run:/var/run:ro \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --volume=/dev/disk/:/dev/disk:ro \
  --publish=8080:8080 \
  --detach=true \
  --name=cadvisor \
  google/cadvisor:latest

登录mysql为exporter创建账号：
mysql>CREATE USER 'exporter'@'localhost' IDENTIFIED BY 'XXXXXXXX' WITH MAX_USER_CONNECTIONS 3;
mysql>GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'localhost';

# cat .my.cnf 
[client]
user=exporter
password=exporter123

# ./mysqld_exporter --config.my-cnf=.my.cnf
  - job_name: mysql
    static_configs:
    - targets:
      - 192.168.31.66:9104

# vi prometheus.yml
alerting:
  alertmanagers:
  - static_configs:
    - targets:
       - 127.0.0.1:9093

# vi prometheus.yml
rule_files:
   - "rules/*.yml"

# vi rules/node.yml
groups:
- name: example# 报警规则组名称
  rules:
# 任何实例5分钟内无法访问发出告警
  - alert: InstanceDown
    expr: up == 0
    for: 5m#持续时间 ， 表示持续一分钟获取不到信息，则触发报警
    labels:
      severity: page# 自定义标签
    annotations:
      summary: "Instance {{ $labels.instance }} down"# 自定义摘要
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."# 自定义具体描述

route:
  receiver: 'default-receiver'  
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  group_by: [cluster, alertname]
# 所有不匹配以下子路由的告警都将保留在根节点，并发送到“default-receiver”
  routes:
# 所有service=mysql或者service=cassandra的告警分配到数据库接收端
  - receiver: 'database-pager'
    group_wait: 10s
    match_re:
      service: mysql|cassandra
# 所有带有team=frontend标签的告警都与此子路由匹配
# 它们是按产品和环境分组的，而不是集群
  - receiver: 'frontend-pager'
    group_by: [product, environment]
    match:
      team: frontend 

# cat rules/general.yml 
groups:
- name: general.rules
  rules:
  - alert: InstanceDown
    expr: up == 0
    for: 2m
    labels:
      severity: error
    annotations:
      summary: "Instance {{ $labels.instance }} 停止工作"
      description: "{{ $labels.instance }}: job {{ $labels.job }} 已经停止5分钟以上."


# cat rules/node.yml
groups:
- name: node.rules
  rules:
  - alert: NodeFilesystemUsage
    expr: 100 - (node_filesystem_free_bytes{fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"} * 100) > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: {{$labels.mountpoint }} 分区使用过高"
      description: "{{$labels.instance}}: {{$labels.mountpoint }} 分区使用大于 80% (当前值: {{ $value }})"
  - alert: NodeMemoryUsage
    expr: 100 - (node_memory_MemFree_bytes+node_memory_Cached_bytes+node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 内存使用过高"
      description: "{{$labels.instance}}: 内存使用大于 80% (当前值: {{ $value }})"
  - alert: NodeCPUUsage
    expr: 100 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) * 100) > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: CPU使用过高"
      description: "{{$labels.instance}}: CPU使用大于 80% (当前值: {{ $value }})"

# cat rules/reload.yml 
groups:
- name: prometheus.rules
  rules:
  - alert: AlertmanagerReloadFailed
    expr: alertmanager_config_last_reload_successful == 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: Alertmanager配置重新加载失败"
      description: "{{$labels.instance}}: Alertmanager配置重新加载失败"
  - alert: PrometheusReloadFailed
    expr: prometheus_config_last_reload_successful == 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: Prometheus配置重新加载失败"
      description: "{{$labels.instance}}: Prometheus配置重新加载失败"