文章/答案/技术大牛

发布

社区首页 >专栏 >golang源码分析之：go-mitmproxy

golang源码分析之：go-mitmproxy

golangLeetcode

发布于 2024-07-06 00:21:32

41100

代码可运行

文章被收录于专栏：golang算法架构leetcode技术phpgolang算法架构leetcode技术php

运行总次数：0

代码可运行

日常开发中，我们除了使用charles、finder抓包外，也可以使用mitmproxy抓包，并且它还提供二次开发能力。在学习mitmproxy之前，我们先学习下他的go版本的精简实现github.com/lqqyt2423/go-mitmproxy/cmd/go-mitmproxy

目前只支持http代理和https代理，不支持websocket和透明代理，这是比mitmproxy弱的地方，它提供了11个hook点，我们可以根据需求，通过addon的方式来定义和添加自己的hook点实现自定义抓包，它的11个hook点定义如下：

type Addon interface {
  // 一个客户端已经连接到了mitmproxy。请注意，一个连接可能对应多个HTTP请求。
  ClientConnected(*ClientConn)

  // 一个客户端连接已关闭（由我们或客户端关闭）。
  ClientDisconnected(*ClientConn)

  // mitmproxy 已连接到服务器。
  ServerConnected(*ConnContext)

  // 服务器连接已关闭（由我们或服务器关闭）。
  ServerDisconnected(*ConnContext)

  // 与服务器的TLS握手已成功完成。
  TlsEstablishedServer(*ConnContext)

  // HTTP请求头已成功读取。此时，请求体为空。
  Requestheaders(*Flow)

  // 完整的HTTP请求已被读取。
  Request(*Flow)

  // HTTP响应头已成功读取。此时，响应体为空。
  Responseheaders(*Flow)

  // 完整的HTTP响应已被读取。
  Response(*Flow)

  // 流式请求体修改器
  StreamRequestModifier(*Flow, io.Reader) io.Reader

  // 流式响应体修改器
  StreamResponseModifier(*Flow, io.Reader) io.Reader
}

下面我们开始分析下它的源码，入口函数位于cmd/go-mitmproxy/main.go，里面定义了Config结构体，每一个field代表了一个抓包工具的可选项：

type Config struct {
version bool // show go-mitmproxy version


Addr         string   // proxy listen addr
WebAddr      string   // web interface listen addr
SslInsecure  bool     // not verify upstream server SSL/TLS certificates.
IgnoreHosts  []string // a list of ignore hosts
AllowHosts   []string // a list of allow hosts
CertPath     string   // path of generate cert files
Debug        int      // debug mode: 1 - print debug log, 2 - show debug from
Dump         string   // dump filename
DumpLevel    int      // dump level: 0 - header, 1 - header + body
Upstream     string   // upstream proxy
UpstreamCert bool     // Connect to upstream server to look up certificate details. Default: True
MapRemote    string   // map remote config filename
MapLocal     string   // map local config filename


filename string // read config from the filename
}

它的main函数里核心包括了下面四步：

1，初始化proxy

p, err := proxy.NewProxy(opts)

代理结构体的定义位于proxy/proxy.go

type Proxy struct {
Opts    *Options
Version string
Addons  []Addon


entry           *entry
attacker        *attacker
shouldIntercept func(req *http.Request) bool              // req is received by proxy.server
upstreamProxy   func(req *http.Request) (*url.URL, error) // req is received by proxy.server, not client request
}

初始化proxy的时候，会设置对应的entry和attacker

2，设置本地和远程拦截策略，也就是根据规则匹配对应的host，并做出是否代理的决策

p.SetShouldInterceptRule(func(req *http.Request) bool {
return !helper.MatchHost(req.Host, config.IgnoreHosts)
})
p.SetShouldInterceptRule(func(req *http.Request) bool {
return helper.MatchHost(req.Host, config.AllowHosts)
})

在proxy上绑定这个拦截规则

func (proxy *Proxy) SetShouldInterceptRule(rule func(req *http.Request) bool) {
proxy.shouldIntercept = rule
}

mapremote和maplocal的具体实现如下：

addon/mapremote.go

if err := helper.NewStructFromFile(filename, &mapRemote); err != nil {
if err := mapRemote.validate(); err != nil {

type MapRemote struct {
proxy.BaseAddon
Items  []*mapRemoteItem
Enable bool
}

addon/maplocal.go

if err := helper.NewStructFromFile(filename, &mapLocal); err != nil {
if err := mapLocal.validate(); err != nil {

type MapLocal struct {
proxy.BaseAddon
Items  []*mapLocalItem
Enable bool
}

3，添加hook点，也就是addon

p.AddAddon(proxy.NewUpstreamCertAddon(false))
p.AddAddon(&proxy.LogAddon{})
p.AddAddon(web.NewWebAddon(config.WebAddr))
mapRemote, err := addon.NewMapRemoteFromFile(config.MapRemote)
p.AddAddon(mapRemote)
mapLocal, err := addon.NewMapLocalFromFile(config.MapLocal)
p.AddAddon(mapLocal)
dumper := addon.NewDumperWithFilename(config.Dump, config.DumpLevel)
p.AddAddon(dumper)

添加hook点就是把拦截点append到数组里就行了

func (proxy *Proxy) AddAddon(addon Addon) {

proxy.Addons = append(proxy.Addons, addon)

}

proxy/addon.go中对hook点进行了约定，也就是说用户可以在这11个hook点上随意定义hook插件

type Addon interface {
// A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests.
ClientConnected(*ClientConn)


// A client connection has been closed (either by us or the client).
ClientDisconnected(*ClientConn)


// Mitmproxy has connected to a server.
ServerConnected(*ConnContext)


// A server connection has been closed (either by us or the server).
ServerDisconnected(*ConnContext)


// The TLS handshake with the server has been completed successfully.
TlsEstablishedServer(*ConnContext)


// HTTP request headers were successfully read. At this point, the body is empty.
Requestheaders(*Flow)


// The full HTTP request has been read.
Request(*Flow)


// HTTP response headers were successfully read. At this point, the body is empty.
Responseheaders(*Flow)


// The full HTTP response has been read.
Response(*Flow)


// Stream request body modifier
StreamRequestModifier(*Flow, io.Reader) io.Reader


// Stream response body modifier
StreamResponseModifier(*Flow, io.Reader) io.Reader


// onAccessProxyServer
AccessProxyServer(req *http.Request, res http.ResponseWriter)
}

同时也定义了一个基础插件和两个最常用的插件实现

type BaseAddon struct{}

type LogAddon struct {
BaseAddon
}

type UpstreamCertAddon struct {
BaseAddon
UpstreamCert bool // Connect to upstream server to look up certificate details.
}

func NewUpstreamCertAddon(upstreamCert bool) *UpstreamCertAddon {
return &UpstreamCertAddon{UpstreamCert: upstreamCert}
}

一些其他的addon定义如下：

web/web.go里定义了webAddOn

type WebAddon struct {
proxy.BaseAddon


server   *http.Server
upgrader *websocket.Upgrader


conns   []*concurrentConn
connsMu sync.RWMutex


flowMessageState map[*proxy.Flow]messageType
flowMu           sync.Mutex
}

实现了一个简单的httpserver，通过websocket将抓包结果发送到web前端，我们可以通过网页查看抓包结果

web := &WebAddon{
flowMessageState: make(map[*proxy.Flow]messageType),
}
web.upgrader = &websocket.Upgrader{
serverMux := new(http.ServeMux)
serverMux.HandleFunc("/echo", web.echo)


fsys, err := fs.Sub(assets, "client/build")
serverMux.Handle("/", http.FileServer(http.FS(fsys)))


web.server = &http.Server{Addr: addr, Handler: serverMux}
web.conns = make([]*concurrentConn, 0)
err := web.server.ListenAndServe()

addon/dumper.go 定义了Dumper将抓包结果保存到文件里

out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
return NewDumper(out, level)

type Dumper struct {
proxy.BaseAddon
out   io.Writer
level int // 0: header 1: header + body
}

4，启动代理

log.Fatal(p.Start())

启动proxy的时候分别会启动两个协程，一个是entry，一个是attacker

func (proxy *Proxy) Start() error {
  go func() {
  if err := proxy.attacker.start(); err != nil {
  return proxy.entry.start()

4.1首先看下entry协程

它的定义位于 proxy/entry.go，它代表了代理server，本质上就是一个http/https服务器

type entry struct {
proxy  *Proxy
server *http.Server
}

func newEntry(proxy *Proxy) *entry {
e := &entry{proxy: proxy}
e.server = &http.Server{

func (e *entry) start() error {
ln, err := net.Listen("tcp", addr)
pln := &wrapListener{
Listener: ln,
proxy:    e.proxy,
}
return e.server.Serve(pln)

重点看下ServeHTTP函数，每当有拦截到来的时候先处理hook，然后初始化attacker的连接，并调用对应的attack方法

func (e *entry) ServeHTTP(res http.ResponseWriter, req *http.Request) {
if req.Method == "CONNECT" {
e.handleConnect(res, req)
// trigger addon event Responseheaders
for _, addon := range e.proxy.Addons {
addon.Responseheaders(f)
}
for _, addon := range proxy.Addons {
addon.AccessProxyServer(req, res)
}
proxy.attacker.initHttpDialFn(req)
proxy.attacker.attack(res, req)

处理连接的时候，会进行请求的拷贝，也就是entry收到的请求到accacker的请求，以及attacker响应到entry的响应

func (e *entry) handleConnect(res http.ResponseWriter, req *http.Request) {
e.directTransfer(res, req, f)
e.httpsDialFirstAttack(res, req, f)
e.httpsDialLazyAttack(res, req, f)

具体拷贝方法定义位于

proxy/helper.go

func transfer(log *log.Entry, server, client io.ReadWriteCloser) {
go func() {
_, err := io.Copy(server, client)
go func() {
_, err := io.Copy(client, server)

拷贝完请求之后就需要向真是上游发起请求

func (e *entry) httpsDialLazyAttack(res http.ResponseWriter, req *http.Request, f *Flow) {
  cconn, err := e.establishConnection(res, f)
  peek, err := cconn.(*wrapClientConn).Peek(3)
  conn, err := proxy.attacker.httpsDial(req.Context(), req)
  transfer(log, conn, cconn)

func (e *entry) httpsDialFirstAttack(res http.ResponseWriter, req *http.Request, f *Flow) {
conn, err := proxy.attacker.httpsDial(req.Context(), req)
cconn, err := e.establishConnection(res, f)
peek, err := cconn.(*wrapClientConn).Peek(3)
if !helper.IsTls(peek) {
// todo: http, ws
transfer(log, conn, cconn)
// is tls
f.ConnContext.ClientConn.Tls = true
proxy.attacker.httpsTlsDial(req.Context(), cconn, conn)

每一次代理请求被定义为一个flow，其代码实现位于：proxy/flow.go

func newFlow() *Flow {
return &Flow{Id:   uuid.NewV4(),
done: make(chan struct{}),}
}

4.2 接下来看下attacker的协程

其代码位于proxy/attacker.go ，创建attacker的时候需要创建一个ca也就是一个自签名的证书，方便被代理的服务使用（代理https时候）

ca, err := newCa(proxy.Opts)
a := &attacker{
proxy: proxy,
ca:    ca,
client: &http.Client{
Transport: &http.Transport
a.server = &http.Server{
Handler: a,
ConnContext:
a.h2Server = &http2.Server{
MaxConcurrentStreams: 100, // todo: wait for remote server setting
NewWriteScheduler:    func() http2.WriteScheduler { return http2.NewPriorityWriteScheduler(nil) },
}

func newCa(opts *Options) (cert.CA, error) {
newCaFunc := opts.NewCaFunc
if newCaFunc != nil {
return newCaFunc()
}
return cert.NewSelfSignCA(opts.CaRootPath)
}

在它的start方法里启动了代理服务

func (a *attacker) start() error {
return a.server.Serve(a.listener)
}

代理连接是在初始化函数里定义的

func (a *attacker) initHttpDialFn(req *http.Request) {
connCtx.dialFn = func(ctx context.Context) error {
addr := helper.CanonicalAddr(req.URL)
c, err := a.proxy.getUpstreamConn(ctx, req)
if err := a.serverTlsHandshake(ctx, connCtx); err != nil {
serverConn := newServerConn()
serverConn.Conn = cw
serverConn.Address = addr
serverConn.client = &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return cw, nil
},

attack方法是代理逻辑的核心

func (a *attacker) attack(res http.ResponseWriter, req *http.Request) {
reply := func(response *Response, body io.Reader) {
res.Header().Add(key, v)
_, err := io.Copy(res, body)
_, err := io.Copy(res, response.BodyReader)
_, err := res.Write(response.Body)
proxyReq, err := http.NewRequestWithContext(proxyReqCtx, f.Request.Method, f.Request.URL.String(), reqBody)

proxyReq.Header.Add(key, v)
proxyRes, err = a.client.Do(proxyReq)
if err := f.ConnContext.dialFn(req.Context()); err != nil {
proxyRes, err = f.ConnContext.ServerConn.client.Do(proxyReq)
reply(f.Response, resBody)

对于https代理，需要发起tls连接

func (a *attacker) httpsTlsDial(ctx context.Context, cconn net.Conn, conn net.Conn) {
clientTlsConn := tls.Server(cconn, &tls.Config{
SessionTicketsDisabled: true, // 设置此值为 true ，确保每次都会调用下面的 GetConfigForClient 方法
GetConfigForClient: func(chi *tls.ClientHelloInfo) (*tls.Config, error) {
c, err := a.ca.GetCert(chi.ServerName)
go func() {
if err := clientTlsConn.HandshakeContext(ctx); err != nil {
if err := a.serverTlsHandshake(ctx, connCtx); err != nil {
// will go to attacker.ServeHTTP
a.serveConn(clientTlsConn, connCtx)
func newCa(opts *Options) (cert.CA, error) {
newCaFunc := opts.NewCaFunc
if newCaFunc != nil {
return newCaFunc()
}
return cert.NewSelfSignCA(opts.CaRootPath)
}
// send clientHello to server, server handshake

func (a *attacker) serverTlsHandshake(ctx context.Context, connCtx *ConnContext) error {
serverTlsConfig := &tls.Config{
serverTlsConn := tls.Client(serverConn.Conn, serverTlsConfig)
if err := serverTlsConn.HandshakeContext(ctx); err != nil {
serverTlsState := serverTlsConn.ConnectionState()
serverConn.tlsState = &serverTlsState
serverConn.client = &http.Client{
Transport: &http.Transport{

自认证证书逻辑位于cert/self_sign_ca.go，创建证书的逻辑如下

storePath, err := getStorePath(path)
ca := &SelfSignCA{
StorePath: storePath,
cache:     lru.New(100),
group:     new(singleflight.Group),
}
if err := ca.load(); err != nil {
key, err := x509.ParsePKCS8PrivateKey(keyDERBlock.Bytes)
x509Cert, err := x509.ParseCertificate(certDERBlock.Bytes)
if err := ca.create(); err != nil {

证书的名字

func (ca *SelfSignCA) caFile() string {
return filepath.Join(ca.StorePath, "mitmproxy-ca.pem")
}

func (ca *SelfSignCA) create() error {
key, cert, err := createCert()
key, err := rsa.GenerateKey(rand.Reader, 2048)
template := &x509.Certificate{
certBytes, err := x509.CreateCertificate
cert, err := x509.ParseCertificate
return ca.saveCert()