SpringBoot集成ELK实现日志收集实践

gang_luo

发布于 2020-08-17 16:57:40

1.3K0

发布于 2020-08-17 16:57:40

文章被收录于专栏：gang_luo

一、Docker Compose搭建ELK

Elasticsearch默认使用mmapfs目录来存储索引。操作系统默认的mmap计数太低可能导致内存不足，我们可以使用下面这条命令来增加内存：

sysctl -w vm.max_map_count=262144

创建Elasticsearch数据挂载路径：

mkdir -p /home/elk/elasticsearch/data

对该路径授予777权限：

chmod 777 /home/elk/elasticsearch/data

创建Elasticsearch插件挂载路径：

mkdir -p /home/elk/elasticsearch/plugins

创建Logstash配置文件存储路径：

mkdir -p /home/elk/logstash

在该路径下创建logstash-febs.conf配置文件

vim /home/elk/logstash/logstash-febs.conf

内容如下所示：

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4560
    codec => json_lines
  }
}
output {
  elasticsearch {
    hosts => "es:9200"
    index => "boottest-logstash-%{+YYYY.MM.dd}"
  }
}

在/home/elk目录下创建docker-compose.yml文件：

vim /home/elk/docker-compose.yml

内容如下：

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.8.1
    container_name: elasticsearch
    environment:
      - "cluster.name=elasticsearch" #集群名称为elasticsearch
      - "discovery.type=single-node" #单节点启动
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m" #jvm内存分配为512MB
    volumes:
      - /home/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
      - /home/elk/elasticsearch/data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
  kibana:
    image: kibana:7.8.1
    container_name: kibana
    links:
      - elasticsearch:es #配置elasticsearch域名为es
    depends_on:
      - elasticsearch
    environment:
      - "elasticsearch.hosts=http://es:9200" #因为上面配置了域名，所以这里可以简写为http://es:9200
    ports:
      - 5601:5601
  logstash:
    image: logstash:7.8.1
    container_name: logstash
    volumes:
      - /home/elk/logstash/logstash-febs.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch
    links:
      - elasticsearch:es
    ports:
      - 4560:4560

使用如下命令启动：

docker-compose up -d

docker ps 观察容器运行情况

Logstash中安装json_lines插件，使用如下命令进入到Logstash容器中并安装插件：

#进入容器
docker exec -it logstash /bin/bash
#安装插件
logstash-plugin install logstash-codec-josn_lines
#安装成功后exit退出容器
exit

汉化Kibana服务

#进入容器
docker exec -it kibana /bin/bash
#编辑配置文件
vi /opt/kibana/config/kibana.yml
#加上一行配置
i18n.locale: zh-CN

重启kibana容器，使用浏览器访问http://192.168.33.10:5601便可以看到Kibana管理界面：

二、SpringBoot项目创建

使用idea创建一个初始化项目

pom文件内容如下：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.3.2.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.elk.lg</groupId>
    <artifactId>boot_elk</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>boot_elk</name>
    <description>Demo project for Spring Boot</description>

    <properties>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <groupId>org.junit.vintage</groupId>
                    <artifactId>junit-vintage-engine</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <!--集成logstash-->
        <dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>5.3</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.4</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

</project>

添加logback.xml配置文件，内容如下：

<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
    <springProperty scope="context" name="springAppName" source="spring.application.name"/>
    <property name="log.path" value="log/boot_elk"/>
    <property name="log.maxHistory" value="15"/>
    <property name="log.colorPattern"
              value="%magenta(%d{yyyy-MM-dd HH:mm:ss}) %highlight(%-5level) %boldCyan(${springAppName:-}) %yellow(%thread) %green(%logger) %msg%n"/>
    <property name="log.pattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5level ${springAppName:-} %thread %logger %msg%n"/>

    <!--输出到控制台-->
    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${log.colorPattern}</pattern>
        </encoder>
    </appender>

    <!--输出到文件-->
    <appender name="file_info" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/info/info.%d{yyyy-MM-dd}.log</fileNamePattern>
            <MaxHistory>${log.maxHistory}</MaxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>${log.pattern}</pattern>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>INFO</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="file_error" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/error/error.%d{yyyy-MM-dd}.log</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>${log.pattern}</pattern>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <!--输出到 logstash的 appender-->
    <appender name="logstash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>192.168.2.2:4560</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder"/>
    </appender>

    <root level="debug">
        <appender-ref ref="console"/>
    </root>

    <root level="info">
        <appender-ref ref="file_info"/>
        <appender-ref ref="file_error"/>
        <appender-ref ref="logstash" />
    </root>
</configuration>

创建一个测试接口

package com.elk.lg.controller;

import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RequestMapping
@RestController
@Slf4j
public class TestController {

    @GetMapping("/testLog")
    public void testLog(){
        log.debug("message0");
        log.info("message1");
        log.warn("message2");
        log.error("message3");
    }

    @GetMapping("/errorLog")
    public void errorLog(){
        int rs = 1/0;
    }

}

调用测试接口输出相关日志：