基于prometheus的监控方案

./prometheus

http://127.0.0.1:9090打开新世界

# my global config
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
    - targets: ['localhost:9090']

/prometheus # ps -ef | grep prometheus
    1 root       13:33 /bin/prometheus --config.file=/etc/prometheus/prometheus.yml ......
/prometheus # kill -HUP 1
#prometheus会有单独协程接收HUP信号进行reload配置

curl -XPOST http://prometheusaddr/-/reload

http://prometheusIp:port/graph

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
 
rule_files:
  - "rules/node.rules"
 
scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']
 #service级别服务发现，发现endpoints里每一个service
  job_name: kubernetes-service-endpoints
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_service_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_service_name
        target_label: kubernetes_name

log_metric{ip="xxx.xxx.xxx.xxx",instance="xxx.xxx.xxx.xxx:xxx",code="access1",metrics_storage="m3db_remote",server="test"}

// 指标结构体
type Metrics struct {
           metrics map[string]*prometheus.Desc
           mutex   sync.Mutex
}

func newGlobalMetric(namespace string, metricName string, docString string, labels []string) *prometheus.Desc {
           return prometheus.NewDesc(namespace+"_"+metricName, docString, labels, nil)
}

func NewMetrics(namespace string) *Metrics {
           return &Metrics{
                      metrics: map[string]*prometheus.Desc{
                                 "log_metric": newGlobalMetric(namespace,
                                            "log_metric",
                                            "The description of log_metric",
                                            []string{"server", "code"},
                                 ),
                      },
           }
}

func (c *Metrics) Describe(ch chan<- *prometheus.Desc) {
           for _, m := range c.metrics {
                      ch <- m
           }
}

func (c *Metrics) Collect(ch chan<- prometheus.Metric) {
           defer Lock.Unlock()
           Lock.Lock()
           ch <- prometheus.MustNewConstMetric(c.metrics["xxxxx"], prometheus.CounterValue, testVal, label1, label2...)
           return
}

metrics := collector.NewMetrics("your_Metrics")
registry := prometheus.NewRegistry()
registry.MustRegister(metrics)
router.Get("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))

type ServiceDiscoveryConfig struct {
           // List of labeled target groups for this job.
           StaticConfigs []*targetgroup.Group `yaml:"static_configs,omitempty"`
           // List of DNS service discovery configurations.
           DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
           // List of file service discovery configurations.
           FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
           // List of Consul service discovery configurations.
           ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
           // List of DigitalOcean service discovery configurations.
           DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
           // List of Docker Swarm service discovery configurations.
           DockerSwarmSDConfigs []*dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
           // List of Serverset service discovery configurations.
           ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
           // NerveSDConfigs is a list of Nerve service discovery configurations.
           NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"`
           // MarathonSDConfigs is a list of Marathon service discovery configurations.
           MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"`
           // List of Kubernetes service discovery configurations.
           KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
           // List of GCE service discovery configurations.
           GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
           // List of EC2 service discovery configurations.
           EC2SDConfigs []*ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
           // List of OpenStack service discovery configurations.
           OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
           // List of Azure service discovery configurations.
           AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"`
           // List of Triton service discovery configurations.
           TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"`
}