Go微服务，第10部分：集中式日志记录

> git clone https://github.com/callistaenterprise/goblog.git
> git checkout P10

> go get github.com/sirupsen/logrus

go get golang.org/x/tools/cmd/goimports

cd $GOPATH/src/github.com/callistaenterprise/goblog/accountservice

$GOPATH/bin/goimports -w **/*.go

go build

logrus.Infof("Starting our service...")

INFO[0000] Starting our service...

func init() {
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: "2006-01-02T15:04:05.000",
FullTimestamp: true,
})
}

INFO[2017-07-17T13:22:49.164] Starting our service...

func init() {
    profile := flag.String("profile", "test", "Environment profile")
if *profile == "dev" {
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: "2006-01-02T15:04:05.000",
FullTimestamp: true,
})
} else {
logrus.SetFormatter(&logrus.JSONFormatter{})
}
}

{"level":"info","msg":"Starting our service...","time":"2017-07-17T16:03:35+02:00"}

var LOGGER = logrus.Logger{}   // <-- Create logger instance
func init() {
// Some other init code...
// Example 1 - using global logrus API
logrus.Infof("Successfully initialized")
// Example 2 - using logger instance
LOGGER.Infof("Successfully initialized")
}

{
      "version":"1.1",
      "host":"swarm-manager-0",
      "short_message":"Starting HTTP service at 6868",
      "timestamp":1.487625824614e+09,
      "level":6,
      "_command":"./vipservice-linux-amd64 -profile=test",
      "_container_id":"894edfe2faed131d417eebf77306a0386b43027e0bdf75269e7f9dcca0ac5608",
      "_container_name":"vipservice.1.jgaludcy21iriskcu1fx9nx2p",
      "_created":"2017-02-20T21:23:38.877748337Z",
      "_image_id":"sha256:1df84e91e0931ec14c6fb4e559b5aca5afff7abd63f0dc8445a4e1dc9e31cfe1",
      "_image_name":"someprefix/vipservice:latest",
      "_tag":"894edfe2faed"
}

docker service create \
--log-driver=gelf \
--log-opt gelf-address=udp://192.168.99.100:12202 \
--log-opt gelf-compression-type=none \
--name=accountservice --replicas=1 --network=my_network -p=6767:6767 someprefix/accountservice

mdkir $GOPATH/src/github.com/callistaenterprise/goblog/gelftail
mdkir $GOPATH/src/github.com/callistaenterprise/goblog/gelftail/transformer
mdkir $GOPATH/src/github.com/callistaenterprise/goblog/gelftail/aggregator
cd $GOPATH/src/github.com/callistaenterprise/goblog/gelftail
touch gelftail.go
touch transformer/transform.go
touch aggregator/aggregator.go

package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"net"
"net/http"
"os"
    "io/ioutil"
"github.com/Sirupsen/logrus"
)

var authToken = ""
var port *string
func init() {
data, err := ioutil.ReadFile("token.txt")
    if err != nil {
        msg := "Cannot find token.txt that should contain our Loggly token"
        logrus.Errorln(msg)
        panic(msg)
    }
    authToken = string(data)
    port = flag.String("port", "12202", "UDP port for the gelftail")
    flag.Parse()
}

func main() {
logrus.Println("Starting Gelf-tail server...")
ServerConn := startUDPServer(*port)   // Remember to dereference the pointer for our "port" flag
defer ServerConn.Close()
var bulkQueue = make(chan []byte, 1)  // Buffered channel to put log statements ready for LaaS upload into
go aggregator.Start(bulkQueue, authToken)          // Start goroutine that'll collect and then upload batches of log statements
go listenForLogStatements(ServerConn, bulkQueue)   // Start listening for UDP traffic
logrus.Infoln("Started Gelf-tail server")
wg := sync.WaitGroup{}
wg.Add(1)
wg.Wait()              // Block indefinitely
}

func listenForLogStatements(ServerConn *net.UDPConn, bulkQueue chan[]byte) {
buf := make([]byte, 8192)                        // Buffer to store UDP payload into. 8kb should be enough for everyone, right Bill? :D
var item map[string]interface{}                  // Map to put unmarshalled GELF json log message into
for {
            n, _, err := ServerConn.ReadFromUDP(buf)     // Blocks until data becomes available, which is put into the buffer.
            if err != nil {
                logrus.Errorf("Problem reading UDP message into buffer: %v\n", err.Error())
                continue                                 // Log and continue if there are problms
            }
            err = json.Unmarshal(buf[0:n], &item)        // Try to unmarshal the GELF JSON log statement into the map
            if err != nil {                              // If unmarshalling fails, log and continue. (E.g. filter)
                logrus.Errorln("Problem unmarshalling log message into JSON: " + err.Error())
                item = nil
                continue
            }
            // Send the map into the transform function
            processedLogMessage, err := transformer.ProcessLogStatement(item)    
            if err != nil {
                logrus.Printf("Problem parsing message: %v", string(buf[0:n]))
            } else {
                bulkQueue <- processedLogMessage          // If processing went well, send on channel to aggregator
            }
            item = nil
    }
}

var client = &http.Client{}
var logglyBaseUrl = "https://logs-01.loggly.com/inputs/%s/tag/http/"
var url string
func Start(bulkQueue chan []byte, authToken string) {
        url = fmt.Sprintf(logglyBaseUrl, authToken) // Assemble the final loggly bulk upload URL using the authToken  
        buf := new(bytes.Buffer)
        for {
                msg := <-bulkQueue                 // Blocks here until a message arrives on the channel.
                buf.Write(msg)
                buf.WriteString("\n")              // Loggly needs newline to separate log statements properly.
                size := buf.Len()
                if size > 1024 {                   // If buffer has more than 1024 bytes of data...
                        sendBulk(*buf)  // Upload!
                        buf.Reset()
                }
        }
}

FROM iron/base
EXPOSE 12202/udp
ADD gelftail-linux-amd64 /
ADD token.txt /
ENTRYPOINT ["./gelftail-linux-amd64", "-port=12202"]

#!/bin/bash
export GOOS=linux
export CGO_ENABLED=0
cd gelftail;go get;go build -o gelftail-linux-amd64;echo built `pwd`;cd ..
export GOOS=darwin
docker build -t someprefix/gelftail gelftail/
docker service rm gelftail
docker service create --name=gelftail -p=12202:12202/udp --replicas=1 --network=my_network someprefix/gelftail

> docker logs -f e69dff960cec
time="2017-08-01T20:33:00Z" level=info msg="Starting Gelf-tail server..." 
time="2017-08-01T20:33:00Z" level=info msg="Started Gelf-tail server"

> docker service scale accountservice=2

time="2017-08-01T20:36:08Z" level=info msg="Starting accountservice" 
time="2017-08-01T20:36:08Z" level=info msg="Loading config from http://configserver:8888/accountservice/test/P10\n" 
time="2017-08-01T20:36:08Z" level=info msg="Getting config from http://configserver:8888/accountservice/test/P10\n"