扩展 Kubernetes 之 CNI

{
  "cniVersion": "0.4.0",
  "interfaces": [                                            (this key omitted by IPAM plugins)
      {
          "name": "<name>",
          "mac": "<MAC address>",                            (required if L2 addresses are meaningful)
          "sandbox": "<netns path or hypervisor identifier>" (required for container/hypervisor interfaces, empty/omitted for host interfaces)
      }
  ],
  "ips": [
      {
          "version": "<4-or-6>",
          "address": "<ip-and-prefix-in-CIDR>",
          "gateway": "<ip-address-of-the-gateway>",          (optional)
          "interface": <numeric index into 'interfaces' list>
      },
      ...
  ],
  "routes": [                                                (optional)
      {
          "dst": "<ip-and-prefix-in-cidr>",
          "gw": "<ip-of-next-hop>"                           (optional)
      },
      ...
  ],
  "dns": {                                                   (optional)
    "nameservers": <list-of-nameservers>                     (optional)
    "domain": <name-of-local-domain>                         (optional)
    "search": <list-of-additional-search-domains>            (optional)
    "options": <list-of-options>                             (optional)
  }
}

// ------------------------------------
{
  "cniVersion": "0.4.0",
  "name": "dbnet",
  "type": "bridge",
  // type (plugin) specific
  "bridge": "cni0",
  "ipam": {
    "type": "host-local",
    // ipam specific
    "subnet": "10.1.0.0/16",
    "gateway": "10.1.0.1"
  },
  "dns": {
    "nameservers": [ "10.1.0.1" ]
  }
}

// ------------------------------------
{
  "cniVersion": "0.4.0",
  "name": "pci",
  "type": "ovs",
  // type (plugin) specific
  "bridge": "ovs0",
  "vxlanID": 42,
  "ipam": {
    "type": "dhcp",
    "routes": [ { "dst": "10.3.0.0/16" }, { "dst": "10.4.0.0/16" } ]
  },
  // args may be ignored by plugins
  "args": {
    "labels" : {
        "appVersion" : "1.0"
    }
  }
}

{
  "cniVersion": "0.4.0",
  "ips": [
      {
          "version": "<4-or-6>",
          "address": "<ip-and-prefix-in-CIDR>",
          "gateway": "<ip-address-of-the-gateway>"  (optional)
      },
      ...
  ],
  "routes": [                                       (optional)
      {
          "dst": "<ip-and-prefix-in-cidr>",
          "gw": "<ip-of-next-hop>"                  (optional)
      },
      ...
  ]
  "dns": {                                          (optional)
    "nameservers": <list-of-nameservers>            (optional)
    "domain": <name-of-local-domain>                (optional)
    "search": <list-of-search-domains>              (optional)
    "options": <list-of-options>                    (optional)
  }
}

root@VM-4-10-ubuntu:/etc/cni/net.d/multus# cat bridge.conf
{
  "cniVersion": "0.1.0",
  "name": "bridge",
  "type": "bridge",
  "bridge": "cbr0",
  "mtu": 1500,
  "addIf": "eth0",
  "isGateway": true,
  "forceAddress": true,
  "ipMasq": false,
  "hairpinMode": false,
  "promiscMode": true,
  "ipam": {
    "type": "host-local",
    "subnet": "10.4.10.0/24",
    "gateway": "10.4.10.1",
    "routes": [
      { "dst": "0.0.0.0/0" }
    ]
  }
}

graph TD
ControllerManager配置subnet --> NodeSpec中生成子subnet 
NodeSpec中生成子subnet --> CNIAgent在各个node上生成配置文件,写入子subnet 
CNIAgent在各个node上生成配置文件,写入子subnet --> CNIplugin在子subnet中分配Ip

#!/bin/bash -e

if [[ ${DEBUG} -gt 0 ]]; then set -x; fi

exec 3>&1 # make stdout available as fd 3 for the result
exec &>> /var/log/bash-cni-plugin.log

IP_STORE=/tmp/reserved_ips # all reserved ips will be stored there

echo "CNI command: $CNI_COMMAND" 

stdin=`cat /dev/stdin`
echo "stdin: $stdin"

# 分配 ip，从所有 ip 中选择没有 reserved 的, 同时更新到 store 的reserved ip 列表
allocate_ip(){
	for ip in "${all_ips[@]}"
	do
		reserved=false
		for reserved_ip in "${reserved_ips[@]}"
		do
			if [ "$ip" = "$reserved_ip" ]; then
				reserved=true
				break
			fi
		done
		if [ "$reserved" = false ] ; then
			echo "$ip" >> $IP_STORE
			echo "$ip"
			return
		fi
	done
}

# 实现 cni plugin 的4个命令
case $CNI_COMMAND in
ADD)
	network=$(echo "$stdin" | jq -r ".network") # network 配置
	subnet=$(echo "$stdin" | jq -r ".subnet")   # 子网配置
	subnet_mask_size=$(echo $subnet | awk -F  "/" '{print $2}') 

	all_ips=$(nmap -sL $subnet | grep "Nmap scan report" | awk '{print $NF}') # 所有ip
	all_ips=(${all_ips[@]})
	skip_ip=${all_ips[0]}
	gw_ip=${all_ips[1]}
	reserved_ips=$(cat $IP_STORE 2> /dev/null || printf "$skip_ip\n$gw_ip\n") # reserving 10.244.0.0 and 10.244.0.1 预留 ip
	reserved_ips=(${reserved_ips[@]})
	printf '%s\n' "${reserved_ips[@]}" > $IP_STORE # 预留 ip 存储到文件
	container_ip=$(allocate_ip)
	
	# ---- 以上是 ip 分配流程

	mkdir -p /var/run/netns/
	ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID

	rand=$(tr -dc 'A-F0-9' < /dev/urandom | head -c4)
	host_if_name="veth$rand"
	ip link add $CNI_IFNAME type veth peer name $host_if_name 

	ip link set $host_if_name up 
	ip link set $host_if_name master cni0 

	ip link set $CNI_IFNAME netns $CNI_CONTAINERID
	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME up
	ip netns exec $CNI_CONTAINERID ip addr add $container_ip/$subnet_mask_size dev $CNI_IFNAME
	ip netns exec $CNI_CONTAINERID ip route add default via $gw_ip dev $CNI_IFNAME
	
	# ------ 以上是创建 veth, 绑定到 网桥，设置 容器端的 ip, route 的过程

	mac=$(ip netns exec $CNI_CONTAINERID ip link show eth0 | awk '/ether/ {print $2}')
echo "{
  \"cniVersion\": \"0.3.1\",
  \"interfaces\": [                                            
      {
          \"name\": \"eth0\",
          \"mac\": \"$mac\",                            
          \"sandbox\": \"$CNI_NETNS\" 
      }
  ],
  \"ips\": [
      {
          \"version\": \"4\",
          \"address\": \"$container_ip/$subnet_mask_size\",
          \"gateway\": \"$gw_ip\",          
          \"interface\": 0 
      }
  ]
}" >&3

;;

DEL)
	ip=$(ip netns exec $CNI_CONTAINERID ip addr show eth0 | awk '/inet / {print $2}' | sed  s%/.*%% || echo "")
	if [ ! -z "$ip" ]
	then
		sed -i "/$ip/d" $IP_STORE
	fi
;;

GET)
	echo "GET not supported"
	exit 1
;;

VERSION)
echo '{
  "cniVersion": "0.3.1", 
  "supportedVersions": [ "0.3.0", "0.3.1", "0.4.0" ] 
}' >&3
;;

*)
  echo "Unknown cni commandn: $CNI_COMMAND" 
  exit 1
;;

esac

$ ip netns add testing
$ CNI_PATH=/opt/cni/bin/ CNI_IFNAME=test CNI_CONTAINERID=testing cnitool add mynet /var/run/netns/testing
{
    "cniVersion": "0.3.1",
    "interfaces": [
        {
            "name": "eth0",
            "sandbox": "/var/run/netns/testing"
        }
    ],
    "ips": [
        {
            "version": "4",
            "interface": 0,
            "address": "10.244.149.16/24",
            "gateway": "10.244.149.1"
        }
    ],
    "dns": {}
}

$ kubectl apply -f https://raw.githubusercontent.com/s-matyukevich/bash-cni-plugin/master/01_gcp/test-deployment.yml