CNI插件实践

plugins
├── ipam
│   ├── dhcp
│   ├── host-local
│   └── static
├── main
│   ├── bridge
│   ├── host-device
│   ├── ipvlan
│   ├── loopback
│   ├── macvlan
│   ├── ptp 
│   ├── vlan
│   └── windows
└── meta
    ├── bandwidth
    ├── firewall
    ├── flannel
    ├── portmap
    ├── sbr
    └── tuning

type CNI interface {
	AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
	DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error

	AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
	DelNetwork(net *NetworkConfig, rt *RuntimeConf) error
}

// PluginMain is the core "main" for a plugin which includes automatic error handling.
//
// The caller must also specify what CNI spec versions the plugin supports.
//
// When an error occurs in either cmdAdd or cmdDel, PluginMain will print the error
// as JSON to stdout and call os.Exit(1).
//
// To have more control over error handling, use PluginMainWithError() instead.
func PluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error, versionInfo version.PluginInfo) {
	if e := PluginMainWithError(cmdAdd, cmdDel, versionInfo); e != nil {
		if err := e.Print(); err != nil {
			log.Print("Error writing error JSON to stdout: ", err)
		}
		os.Exit(1)
    }
}

type CmdArgs struct {
	ContainerID string  //需要设置网络的容器ID，来自环境变量CNI_CONTAINERID
	Netns       string  //容器的网络命名空间，来自环境变量CNI_NETNS
	IfName      string  //容器内网卡的名称，来自环境变量CNI_IFNAME
	Args        string  //CNI参数，来自环境变量CNI_ARGS
	Path        string  //CNI二进制工具目录，来自环境变量CNI_PATH
	StdinData   []byte  //CNI插件Json配置，需要自行解析内容
}

func cmdAdd(args *skel.CmdArgs) error {
    //step1：通过CmdArgs.StdinData解析自定义的CNI json配置
	conf := NetConf{}
	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
		return fmt.Errorf("failed to load netconf: %v", err)
	}
    //step2：借助CNI库封装，调用JSON文件中指定的ipam工具，申请IP地址
	r, err := ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
	if err != nil {
		return err
	}
	//step3：格式化ipam工具的返回结果，避免不同版本之间的结果不兼容的问题
	result, err := current.NewResultFromResult(r)
	if err != nil {
		return err
	}
	if len(result.IPs) == 0 {
		return errors.New("IPAM plugin returned missing IP config")
	}

	if err := ip.EnableForward(result.IPs); err != nil {
		return fmt.Errorf("Could not enable IP forwarding: %v", err)
	}
	netns, err := ns.GetNS(args.Netns)
	if err != nil {
		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
	}
	defer netns.Close()
    //step4：创建veth pair，一端设置到容器中，并配置申请到的IP地址
	hostInterface, containerInterface, err := setupContainerVeth(netns, args.IfName, conf.MTU, result)
	if err != nil {
		return err
	}
    //step5，在主机节点设置容器IP的路由规则
	if err = setupHostVeth(hostInterface.Name, result); err != nil {
		return err
	}
    //step6，如果开启了ipMasq特性，设置SNAT规则
	if conf.IPMasq {
		chain := utils.FormatChainName(conf.Name, args.ContainerID)
		comment := utils.FormatComment(conf.Name, args.ContainerID)
		for _, ipc := range result.IPs {
			if err = ip.SetupIPMasq(&ipc.Address, chain, comment); err != nil {
				return err
			}
		}
	}

	result.DNS = conf.DNS
	result.Interfaces = []*current.Interface{hostInterface, containerInterface}
    //step7，将结果格式化，并打印到输出流
	return types.PrintResult(result, conf.CNIVersion)
}

func cmdDel(args *skel.CmdArgs) error {
    //step1：通过CmdArgs.StdinData解析自定义的CNI json配置
	conf := NetConf{}
	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
		return fmt.Errorf("failed to load netconf: %v", err)
	}
    //step2：调用json配置中指定的ipam工具，释放IP地址资源
	if err := ipam.ExecDel(conf.IPAM.Type, args.StdinData); err != nil {
		return err
	}

	if args.Netns == "" {
		return nil
	}

	//step3：切换到容器网络命名空间中，清理网卡
	var ipn *net.IPNet
	err := ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
		var err error
		ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
		if err != nil && err == ip.ErrLinkNotFound {
			return nil
		}
		return err
	})

	if err != nil {
		return err
	}
    //如果开启了IPMasq特性，则删掉在add阶段增加SNAT规则
	if ipn != nil && conf.IPMasq {
		chain := utils.FormatChainName(conf.Name, args.ContainerID)
		comment := utils.FormatComment(conf.Name, args.ContainerID)
		err = ip.TeardownIPMasq(ipn, chain, comment)
	}
	return err
}