1.Prometheus监控入门之介绍整体架构及安装

# 数据模型 : metric  @ timestamp  =>  value
http_requests_total{status="200",method="GET"}@1434417560938  => 94355

# 数据格式 : Metric_Name{key="value"}
go_info{instance="localhost:9090", job="prometheus", version="go1.16.2"}

所有事件产生值的大小的总和: basename_sum
事件发生的总次数: basename_count
事件产生的值分布: basename_bucket

# prometheus_server
$ mkdir -p $GOPATH/src/github.com/prometheus
$ cd $GOPATH/src/github.com/prometheus
$ git clone https://github.com/prometheus/prometheus.git
$ cd prometheus
$ make build
$ ./prometheus -config.file=your_config.yml

# prometheus_alertmanager
$ git clone https://github.com/prometheus/alertmanager.git
$ cd alertmanager
$ make build
$ ./alertmanager -config.file=<your_file>

# (1) 此为NFS共享存储便于后面多台主机的
/nfsdisk-31/monitor# mkdir -vp /nfsdisk-31/monitor/prometheus/{conf,data} grafana/data
  # mkdir: created directory '/nfsdisk-31/monitor/prometheus'
  # mkdir: created directory 'prometheus/conf'
  # mkdir: created directory 'prometheus/data'
  # mkdir: created directory 'grafana'
  # mkdir: created directory 'grafana/data'

# (2) grafana 服务数据存储目录所属者设置(否则报错:GF_PATHS_DATA='/var/lib/grafana' is not writable.)
chown 472:root -R /nfsdisk-31/monitor/grafana/data
chmod 777 /nfsdisk-31/monitor/prometheus/data

tree -L 3
.
├── docker-compose.yml
├── grafana
│   └── data
└── prometheus
    ├── conf
    │   ├── alertmanager.yaml
    │   ├── alert.rules
    │   └── prometheus.yml
    └── data

docker network create -d bridge monitor
  # 527a8953a9d57c64e2d728d817f46ea735b104e6f0f2c342f1c7844eebe8ad87
docker network ls
  # NETWORK ID     NAME      DRIVER    SCOPE
  # 4fe5982b8449   bridge    bridge    local
  # 484bfd6f8556   host      host      local
  # 527a8953a9d5   monitor   bridge    local
  # 0b83b3436078   none      null      local

tee /nfsdisk-31/monitor/prometheus/conf/docker-compose.yml <<'END'
version: '3.2'
services:
  prometheus:
    image: prom/prometheus:v2.26.0
    container_name: prometheus_server
    environment:
      TZ: Asia/Shanghai
    volumes:
      - /nfsdisk-31/monitor/prometheus/conf/prometheus.yml:/etc/prometheus/prometheus.yml
      - /nfsdisk-31/monitor/prometheus/conf/alert.rules:/etc/prometheus/alert.rules
      - /nfsdisk-31/monitor/prometheus:/prometheus
      - /etc/localtime:/etc/localtime
    command:
      - '--config.file=/etc/prometheus/prometheus.yaml'
      - '--storage.tsdb.path=/prometheus/data'
      - '--web.enable-admin-api'
      - '--web.enable-lifecycle'
    ports:
      - '30090:9090'
    restart: always
    networks:
      - monitor
    
  pushgateway:
    image: prom/pushgateway
    container_name: prometheus_pushgateway
    environment:
      TZ: Asia/Shanghai
    volumes:
      - /etc/localtime:/etc/localtime
    ports:
      - '30091:9091'
    restart: always
    networks:
      - monitor

  alertmanager:
    image: prom/alertmanager:v0.21.0
    container_name: prometheus_alertmanager
    environment:
      TZ: Asia/Shanghai
    volumes:
      - /nfsdisk-31/monitor/prometheus/conf/alertmanager.yaml:/etc/alertmanager.yaml
      - /nfsdisk-31/monitor/prometheus/alertmanager:/alertmanager
      - /etc/localtime:/etc/localtime
    command:
      - '--config.file=/etc/alertmanager.yaml'
      - '--storage.path=/alertmanager'
    ports:
      - '30093:9093'
    restart: always
    networks:
      - monitor
  
  grafana:
    image: grafana/grafana:7.5.5
    container_name: grafana
    user: "472"
    environment:
      - TZ=Asia/Shanghai
      - GF_SECURITY_ADMIN_PASSWORD=weiyigeek
    volumes:
      - /nfsdisk-31/monitor/grafana/data:/var/lib/grafana
    ports:
      - '30000:3000'
    restart: always
    networks:
      - monitor
    dns:
      - 223.6.6.6
      - 192.168.12.254

networks:
  monitor:
    external: true
END

cd /nfsdisk-31/monitor/prometheus/conf/
# prometheus 监控服务
tee prometheus.yml <<'EOF'
global:
    # pull 频率
    scrape_interval: 120s
    # 超时时间
    scrape_timeout: 15s
    # 外部标签
    external_labels:
      monitor: 'current-monitor'
scrape_configs:
    # 工作组名称
    - job_name: 'prometheus'
      # 监控的主机地址与端口信息
      static_configs:
        - targets: ['localhost:9090']

# 报警规则
rule_files:
  - 'alert.rules'
EOF


# 报警服务
tee alertmanager.yaml <<'EOF'
# 默认采用哪种方式进行预警
route:
  group_by: ['alertname']
  receiver: 'default-receiver'
receivers:
  - name: 'default-receiver'
    #接收者为webhook类型
    webhook_configs:
    #webhook的接收地址
    - url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=4e1165c3-55c1-4bc4-8493-ecc5ccda9275'
EOF


# 报警规则
tee alert.rules <<'EOF'
groups:
- name: node-alert
  rules:
  - alert: service_down
    expr: up == 0
    for: 3m
  - alert: high_load
    expr: node_load1 > 1.0
    for: 5m
EOF

docker-compose ps
  # Name                        Command               State                 Ports
  # -----------------------------------------------------------------------------------------------------
  # grafana                   /run.sh                          Up      0.0.0.0:30000->3000/tcp
  # prometheus_alertmanager   /bin/alertmanager --config ...   Up      0.0.0.0:30093->9093/tcp
  # prometheus_server         /bin/prometheus --config.f ...   Up      0.0.0.0:30090->9090/tcp

# prometheus 监控主服务 - 提供简陋的管理页面
http://192.168.12.107:30090/

# prometheus Alertmanager 服务 - 提供简陋的管理页面
http://192.168.12.107:30093/

# grafana UI 服务 - 提供数据专业化展示 (默认账户密码: admin/weiyigeek)
http://192.168.12.107:30000/login

# 1.解压二进制文件
cd /opt
wget https://github.com/prometheus/node_exporter/releases/download/v1.1.2/node_exporter-1.1.2.linux-amd64.tar.gz
tar -xvf node_exporter-1.1.2.linux-amd64.tar.gz
sudo cp -a node_exporter /usr/local/bin/node_exporter
# sudo ln -s /usr/local/node_exporter-1.2.2.linux-amd64/node_exporter /usr/local/bin/node_exporter

# 2.创建配置文件
sudo mkdir /etc/node_exporter/
sudo tee /etc/node_exporter/exporter.conf <<'EOF'
# 官方建议添加环境变量  `/etc/node_exporter/exporter.confr` ，内容如下：
# OPTIONS="--collector.textfile.directory /var/lib/node_exporter/textfile_collector"
EOF

# > systemd 可用于监控系统服务，需要在 node_exporter 启动时添加 --collector.systemd 参数开启，默认是关闭的。
sudo tee /usr/lib/systemd/system/node_exporter.service <<'EOF'
[Unit]
Description=Node Exporter Clinet
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
StandardError=journal
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=on-failure
RestartSec=3s

[Install]
WantedBy=multi-user.target
EOF
# EnvironmentFile=/etc/node_exporter/exporter.conf

sudo chmod 754 /usr/lib/systemd/system/node_exporter.service

# 重载&开机自启
sudo systemctl daemon-reload
sudo systemctl enable node_exporter.service
sudo systemctl start node_exporter.service

# 日志查看
sudo systemctl status node_exporter.service
● node_exporter.service - Node Exporter Clinet
     Loaded: loaded (/lib/systemd/system/node_exporter.service; disabled; vendor preset: enabled)
     Active: active (running) since Fri 2021-04-30 22:49:33 CST; 10s ago
   Main PID: 1238901 (node_exporter)
      Tasks: 12 (limit: 19111)
     Memory: 11.4M
     CGroup: /system.slice/node_exporter.service
             └─1238901 /usr/local/bin/node_exporter --collector.systemd

Apr 30 22:49:33 weiyigeek-107 node_exporter[1238901]: level=info ts=2021-04-30T14:49:33.154Z caller=node_exporter.go:113 collector=thermal_zone
......
Apr 30 22:49:33 weiyigeek-107 node_exporter[1238901]: level=info ts=2021-04-30T14:49:33.154Z caller=node_exporter.go:113 collector=zfs
Apr 30 22:49:33 weiyigeek-107 node_exporter[1238901]: level=info ts=2021-04-30T14:49:33.154Z caller=node_exporter.go:195 msg="Listening on" address=:9100
Apr 30 22:49:33 weiyigeek-107 node_exporter[1238901]: level=info ts=2021-04-30T14:49:33.155Z caller=tls_config.go:191 msg="TLS is disabled." http2=false

netstat -tlnp | grep "9100"
(Not all processes could be identified, non-owned process info
 will not be shown, you would have to be root to see it all.)
tcp6       0      0 :::9100                 :::*                    LISTEN      -

curl -s http://192.168.12.107:9100/metrics | head -n 20
  # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
  # TYPE go_gc_duration_seconds summary
  go_gc_duration_seconds{quantile="0"} 2.4814e-05
  go_gc_duration_seconds{quantile="0.25"} 5.1128e-05
  go_gc_duration_seconds{quantile="0.5"} 9.3675e-05
  go_gc_duration_seconds{quantile="0.75"} 0.000131435
  go_gc_duration_seconds{quantile="1"} 0.000767665
  go_gc_duration_seconds_sum 0.015132517
  go_gc_duration_seconds_count 135
  # HELP go_goroutines Number of goroutines that currently exist.
  # TYPE go_goroutines gauge
  go_goroutines 8
  # HELP go_info Information about the Go environment.
  # TYPE go_info gauge
  go_info{version="go1.15.8"} 1
  # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
  # TYPE go_memstats_alloc_bytes gauge
  go_memstats_alloc_bytes 3.019424e+06
  # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
  # TYPE go_memstats_alloc_bytes_total counter

vi /nfsdisk-31/monitor/prometheus/conf/prometheus.yml
  - job_name: 'node-resources'
    scrape_interval: 120s
    static_configs:
      - targets: ['192.168.12.107:9100']   # 指定安装和启动后 node_exporter 的主机

/nfsdisk-31/monitor# docker-compose restart prometheus

# PromQL 表达式: 获取系统空闲磁盘的空间大小(单位 GB)
node_filesystem_free_bytes{fstype!="tmpfs"} / (1024*1024*1024)

# 结果
{device="/dev/mapper/ubuntu--vg-lv--0", fstype="ext4", instance="192.168.12.107:9100", job="node-resources", mountpoint="/"}
  76.21196365356445
{device="/dev/sda2", fstype="ext4", instance="192.168.12.107:9100", job="node-resources", mountpoint="/boot"}
  0.6655235290527344
{device="192.168.10.30:/nask8sapp", fstype="nfs", instance="192.168.12.107:9100", job="node-resources", mountpoint="/nfsdisk-31"}
  2275.9747009277344

下载二进制：https://github.com/google/cadvisor/releases/latest
本地运行：./cadvisor  -port=8080 &>>/var/log/cadvisor.log

docker run \
  --volume=/var/run:/var/run:rw \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --volume=/dev/disk/:/dev/disk:ro \
  --publish=9100:8080\
  --detach=true \
  --name=cadvisor \
  google/cadvisor:latest
# 8fa36cd105ebce5c9cc334f2bfaae781833ddce396bfb96bd775a18706c9e1f3

# 注意:在Ret Hat,CentOS, Fedora 等发行版上需要传递如下参数，因为 SELinux 加强了安全策略：
--privileged=true

# 可选参数
--volume=/:/rootfs:ro \

# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision.
# TYPE cadvisor_version_info gauge
cadvisor_version_info{cadvisorRevision="8949c822",cadvisorVersion="v0.32.0",dockerVersion="19.03.14",kernelVersion="5.4.0-60-generic",osVersion="Alpine Linux v3.7"} 1
# HELP container_cpu_load_average_10s Value of container cpu load average over the last 10 seconds.
# TYPE container_cpu_load_average_10s gauge
container_cpu_load_average_10s{container_label_annotation_io_kubernetes_container_hash="",container_label_annotation_io_kubernetes_container_ports="",container_label_annotation_io_kubernetes_container_restartCount="",container_label_annotation_io_kubernetes_container_terminationMessagePath="",container_label_annotation_io_kubernetes_container_terminationMessagePolicy="",
.......

vim /nfsdisk-31/monitor/prometheus/conf/prometheus.yml
scrape_configs:
- job_name: 'node-resources'
    scrape_interval: 120s
    static_configs:
      - targets: ['192.168.12.107:9100','192.168.12.108:9100']

# 将 cAdvisor 添加到监控数据采集任务目标当中
# - job_name: cadvisor
#   static_configs:
#   - targets:
#     - 10.10.107.234:9060

# 重启容器
docker restart prometheus_server
  # prometheus_server

# PS : 设置当前代理为 http://127.0.0.1:1080 或 socket5://127.0.0.1:1080 (此处为了加快拉取速度我采用自己的VPS搭建的梯子)
git config --global http.proxy 'socks5://127.0.0.1:1080'
git config --global https.proxy 'socks5://127.0.0.1:1080'
git clone https://github.com/coreos/kube-prometheus.git

~/K8s/Day10/kube-prometheus$ ls
build.sh            DCO   example.jsonnet  experimental  go.sum  jsonnet           jsonnetfile.lock.json  LICENSE   manifests  OWNERS     scripts                            tests
code-of-conduct.md  docs  examples         go.mod        hack    jsonnetfile.json  kustomization.yaml     Makefile  NOTICE     README.md  sync-to-internal-registry.jsonnet  test.sh

~/K8s/Day10/kube-prometheus/manifests/setup$ grep "quay.io/" *
  # prometheus-operator-deployment.yaml:        - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.44.0
  # prometheus-operator-deployment.yaml:        image: quay.io/prometheus-operator/prometheus-operator:v0.44.0
  # prometheus-operator-deployment.yaml:        image: quay.io/brancz/kube-rbac-proxy:v0.8.0
~/K8s/Day10/kube-prometheus/manifests/setup$ sed -i "s#quay.io/#quay.mirrors.ustc.edu.cn/#g" *
~/K8s/Day10/kube-prometheus/manifests/$ sed -i "s#quay.io/#quay.mirrors.ustc.edu.cn/#g" *

# 修改访问方式为`type: NodePort`以及NodePort暴露的端口
# ---~/K8s/Day10/kube-prometheus/manifests$ cat  alertmanager-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    alertmanager: main
  name: alertmanager-main
  namespace: monitoring
spec:
  type: NodePort   # 修改点1
  ports:
  - name: web
    port: 9093
    targetPort: web
    nodePort: 30070  # 修改点2
  selector:
    alertmanager: main
    app: alertmanager
  sessionAffinity: ClientIP

# ---~/K8s/Day10/kube-prometheus/manifests$ cat prometheus-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    prometheus: k8s
  name: prometheus-k8s
  namespace: monitoring
spec:
  type: NodePort    # 修改点1
  ports:
  - name: web
    port: 9090
    targetPort: web
    nodePort: 30090  # 修改点2
  selector:
    app: prometheus
    prometheus: k8s
  sessionAffinity: ClientIP

# ---~/K8s/Day10/kube-prometheus/manifests$ cat grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    app: grafana
  name: grafana
  namespace: monitoring
spec:
  ports:
  - name: http
    port: 3000
    targetPort: http
    nodePort: 30080  # 修改点2
  selector:
    app: grafana
  type: NodePort   # 修改点1

~/K8s/Day10/kube-prometheus/manifests/setup$ kubectl create -f .
  # namespace/monitoring created
  # customresourcedefinition.apiextensions.k8s.io/alertmanagerconfigs.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/alertmanagers.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/podmonitors.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/probes.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/prometheuses.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/prometheusrules.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/servicemonitors.monitoring.coreos.com created
  # customresourcedefinition.apiextensions.k8s.io/thanosrulers.monitoring.coreos.com created
  # clusterrole.rbac.authorization.k8s.io/prometheus-operator created
  # clusterrolebinding.rbac.authorization.k8s.io/prometheus-operator created
  # deployment.apps/prometheus-operator created
  # service/prometheus-operator created
  # serviceaccount/prometheus-operator created

~/K8s/Day10/kube-prometheus/manifests/setup$ kubectl get pod -n monitoring -o wide --show-labels
# NAME                                   READY   STATUS    RESTARTS   AGE   IP            NODE       LABELS
# prometheus-operator-5bff4cfb76-dtvjj   2/2     Running   0          50m   10.244.2.59   k8s-node-5 app.kubernetes.io/component=controller,app.kubern   etes.io/name=prometheus-operator,app.kubernetes.io/version=v0.44.0,pod-template-hash=5bff4cfb76

~/K8s/Day10/kube-prometheus$ kubectl create -f manifests/
  # alertmanager.monitoring.coreos.com/main created
  # secret/alertmanager-main created
  ...... # 此处略去
  # servicemonitor.monitoring.coreos.com/kubelet created

~/K8s/Day10/kube-prometheus$ kubectl get pod -n monitoring -o wide
  # NAME                                   READY   STATUS    RESTARTS   AGE     IP              NODE          NOMINATED NODE   READINESS GATES
  # alertmanager-main-0                    2/2     Running   0          105m    10.244.2.60     k8s-node-5    <none>           <none>
  # alertmanager-main-1                    2/2     Running   0          105m    10.244.1.157    k8s-node-4    <none>           <none>
  # alertmanager-main-2                    2/2     Running   0          105m    10.244.2.61     k8s-node-5    <none>           <none>
  # grafana-79f4b5649f-6sjg5               1/1     Running   0          105m    10.244.1.159    k8s-node-4    <none>           <none>
  # kube-state-metrics-58c88f48b7-frqb2    3/3     Running   0          105m    10.244.1.160    k8s-node-4    <none>           <none>
  # node-exporter-4nmsz                    2/2     Running   0          105m    10.10.107.214   k8s-node-4    <none>           <none>
  # node-exporter-7x6bm                    2/2     Running   0          105m    10.10.107.215   k8s-node-5    <none>           <none>
  # node-exporter-98crd                    2/2     Running   0          105m    10.10.107.202   ubuntu   <none>           <none>
  # prometheus-adapter-69b8496df6-7mqcv    1/1     Running   0          105m    10.244.2.63     k8s-node-5    <none>           <none>
  # prometheus-k8s-0                       2/2     Running   1          105m    10.244.1.158    k8s-node-4    <none>           <none>
  # prometheus-k8s-1                       2/2     Running   0          105m    10.244.2.62     k8s-node-5    <none>           <none>
  # prometheus-operator-5bff4cfb76-dtvjj   2/2     Running   0          5h26m   10.244.2.59     k8s-node-5    <none>           <none>

~/K8s/Day10/kube-prometheus/manifests$ kubectl get svc -n monitoring -o wide
NAME                    TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE     SELECTOR
prometheus-adapter      ClusterIP   10.100.45.247    <none>        443/TCP                      122m    name=prometheus-adapter
prometheus-operator     ClusterIP   None             <none>        8443/TCP                     5h42m   app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
kube-state-metrics      ClusterIP   None             <none>        8443/TCP,9443/TCP            122m    app.kubernetes.io/name=kube-state-metrics
prometheus-operated     ClusterIP   None             <none>        9090/TCP                     122m    app=prometheus
alertmanager-operated   ClusterIP   None             <none>        9093/TCP,9094/TCP,9094/UDP   122m    app=alertmanager
node-exporter           ClusterIP   None             <none>        9100/TCP                     122m    app.kubernetes.io/name=node-exporter
alertmanager-main       NodePort    10.100.233.132   <none>        9093:30070/TCP               122m    alertmanager=main,app=alertmanager
prometheus-k8s          NodePort    10.103.33.255    <none>        9090:30090/TCP               122m    app=prometheus,prometheus=k8s
grafana                 NodePort    10.104.211.111   <none>        3000:30080/TCP               122m    app=grafana

# Prometheus 自己的指标prometheus 的 WEB 界面上提供了基本的查询 K8S 集群中每个 POD 的 CPU 使用情况，查询条件如下
sum(rate(container_cpu_usage_seconds_total{image!=""}[1m])) by (pod_name, namespace)  # PS:内置函数后面细讲