1.查看flannel的网络rootubuntu0:~# kubectl get pods -n kube-flannelNAME READY STATUS RESTARTS AGE kube-flannel-ds-9wqdk1/1 Running12(10d ago)116d kube-flannel-ds-lp7nk1/1 Running12(10d ago)116d kube-flannel-ds-t7ch41/1 Running19(10d ago)116d rootubuntu0:~# kubectl get pods -n kube-systemNAME READY STATUS RESTARTS AGE coredns-6d8c4cb4d-8d8wn1/1 Running17(10d ago)116d coredns-6d8c4cb4d-rrtjc1/1 Running17(10d ago)116d etcd-ubuntu01/1 Running121(10d ago)116d kube-apiserver-ubuntu01/1 Running98(10d ago)116d kube-controller-manager-ubuntu01/1 Running47(10d ago)116d kube-proxy-2pl6v1/1 Running2(10d ago)46d kube-proxy-hwsck1/1 Running2(10d ago)46d kube-proxy-vt67q1/1 Running2(10d ago)46d kube-scheduler-ubuntu01/1 Running50(10d ago)116d 查看网络配置 rootubuntu0:~# cat /etc/cni/net.d/10-flannel.conflist{name:cbr0,cniVersion:0.3.1,plugins:[{type:flannel,delegate:{hairpinMode:true,isDefaultGateway:true}},{type:portmap,capabilities:{portMappings:true}}]}rootubuntu0:~# ip routedefault via192.168.23.2 dev ens33 proto static10.100.0.0/24 dev cni0 proto kernel scopelinksrc10.100.0.110.100.1.0/24 via10.100.1.0 dev flannel.1 onlink10.100.2.0/24 via10.100.2.0 dev flannel.1 onlink172.17.0.0/16 dev docker0 proto kernel scopelinksrc172.17.0.1 linkdown172.18.0.0/16 dev br-23e5e68d8326 proto kernel scopelinksrc172.18.0.1 linkdown172.20.0.0/16 dev br-5adcfddabf95 proto kernel scopelinksrc172.20.0.1192.168.23.0/24 dev ens33 proto kernel scopelinksrc192.168.23.99 下载calico网络 rootubuntu0:~# wget https://calico-v3-25.netlify.app/archive/v3.25/manifests/calico.yamlrootubuntu0:~# vim calico.yamlrootubuntu0:~# sed -i s/CALICO_IPV4POOL_CIDR: 192.168.0.0\/16/CALICO_IPV4POOL_CIDR: 172.16.0.0\/16/g calico.yamlrootubuntu0:~#rootubuntu0:~#rootubuntu0:~# kubectl apply -f calico.yamlpoddisruptionbudget.policy/calico-kube-controllers created serviceaccount/calico-kube-controllers created serviceaccount/calico-node created configmap/calico-config created customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/caliconodestatuses.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipreservations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created clusterrole.rbac.authorization.k8s.io/calico-node created clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created clusterrolebinding.rbac.authorization.k8s.io/calico-node created daemonset.apps/calico-node created deployment.apps/calico-kube-controllers created rootubuntu0:~# cat /etc/cni/net.d/10-calico.conflist{name:k8s-pod-network,cniVersion:0.3.1,plugins:[{type:calico,log_level:info,log_file_path:/var/log/calico/cni/cni.log,datastore_type:kubernetes,nodename:ubuntu0,mtu:0,ipam:{type:calico-ipam},policy:{type:k8s},kubernetes:{kubeconfig:/etc/cni/net.d/calico-kubeconfig}},{type:portmap,snat:true,capabilities:{portMappings:true}},{type:bandwidth,capabilities:{bandwidth:true}}]}删除flannel网络 rootubuntu0:~# kubectl delete -f flannel/kube-flannel.ymlnamespacekube-flanneldeleted serviceaccountflanneldeleted clusterrole.rbac.authorization.k8s.ioflanneldeleted clusterrolebinding.rbac.authorization.k8s.ioflanneldeleted configmapkube-flannel-cfgdeleted daemonset.appskube-flannel-dsdeleted 为什么为什么flannel网卡还在 网络接口是持久化的Flannel 创建的虚拟网络接口如 flannel.1、cni0 和 veth*是 Linux 内核中的对象删除 Kubernetes 资源不会自动清理这些接口。这些接口需要手动删除或通过重启节点自动清理。CNI 的残留配置Flannel 的 CNI 配置文件可能仍存在于 /etc/cni/net.d/ 目录中这会影响新 CNI 插件如 Calico的安装。 rootubuntu0:~# ip a|tail -n 30inet192.168.0.10/32 scope global kube-ipvs0 valid_lft forever preferred_lft forever inet192.168.20.201/32 scope global kube-ipvs0 valid_lft forever preferred_lft forever inet192.168.0.1/32 scope global kube-ipvs0 valid_lft forever preferred_lft forever31: flannel.1:BROADCAST,MULTICAST,UP,LOWER_UPmtu1450qdisc noqueue state UNKNOWN group default link/ether c2:42:84:db:a1:93 brd ff:ff:ff:ff:ff:ff inet10.100.0.0/32 scope global flannel.1 valid_lft forever preferred_lft forever inet6 fe80::c042:84ff:fedb:a193/64 scopelinkvalid_lft forever preferred_lft forever32: cni0:BROADCAST,MULTICAST,UP,LOWER_UPmtu1450qdisc noqueue state UP group default qlen1000link/ether a2:cc:78:01:99:07 brd ff:ff:ff:ff:ff:ff inet10.100.0.1/24 brd10.100.0.255 scope global cni0 valid_lft forever preferred_lft forever inet6 fe80::a0cc:78ff:fe01:9907/64 scopelinkvalid_lft forever preferred_lft forever1.手动删除网络接口 rootubuntu0:~# ip link delete flannel.1rootubuntu0:~# ip link delete cni0rootubuntu0:~# ansible all -m shell -a ip link delete flannel.1;ip link delete cni0192.168.23.97|CHANGED|rc0192.168.23.98|CHANGED|rc02.删除 Flannel 的 CNI 配置文件 rootubuntu0:~# rm -rf /etc/cni/net.d/10-flannel.conflistrootubuntu0:~# ls /var/lib/cni/cache/ flannel/ networks/ rootubuntu0:~# rm -rf /var/lib/cni/*3.重启kubelet rootubuntu0:~# systemctl restart kubelet4.重启节点部署calico官网 https://docs.tigera.io/calico/3.29/getting-started/kubernetes/requirements rootubuntu0:~# kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.4/manifests/tigera-operator.yamlrootubuntu0:~# wget https://raw.githubusercontent.com/projectcalico/calico/v3.29.4/manifests/custom-resources.yamlrootubuntu0:~# grep cidr custom-resources.yamlcidr:10.100.1.0/16#修改和kubeadm初始化一样的网段rootubuntu0:~# kubectl apply -f custom-resources.yamlinstallation.operator.tigera.io/default created apiserver.operator.tigera.io/default created 下载calicoctl工具 https://archive-os-3-25.netlify.app/archive rootubuntu0:~# curl -L https://github.com/projectcalico/calico/releases/download/v3.27.5/calicoctl-linux-amd64 -o calicoctl移动Calicoctl到PATH变量[rootmaster231 ~]# mv calicoctl /usr/local/bin/[rootmaster231 ~]#添加执行权限[rootmaster231 ~]# chmod x /usr/local/bin/calicoctl[rootmaster231 ~]#[rootmaster231 ~]# ll /usr/local/bin/calicoctl-rwxr-xr-x1root root63805704Feb2810:15 /usr/local/bin/calicoctl*[rootmaster231 ~]#rootubuntu0:~# kubectl -n calico-system get pods -o wide | grep -E ubuntu1|ubuntu2calico-kube-controllers-9d99b9966-fbp5p0/1 CrashLoopBackOff8(2m50s ago)26m10.100.2.101 ubuntu1nonenonecalico-typha-7656d5444d-jmcqt0/1 CrashLoopBackOff7(2m56s ago)26m192.168.23.98 ubuntu1nonenonecsi-node-driver-tjggq2/2 Running05h43m10.100.2.100 ubuntu1nonenonecsi-node-driver-xcdrk2/2 Running06h11m10.100.1.32 ubuntu2nonenone节点未完整注册到 Calico rootubuntu0:~# calicoctl patch node ubuntu1 -p {spec: {bgp: {ipv4Address: 192.168.23.98/24}}}Successfully patched1Noderesource rootubuntu0:~# calicoctl patch node ubuntu2 -p {spec: {bgp: {ipv4Address: 192.168.23.97/24}}}Successfully patched1Noderesource rootubuntu0:~# calicoctl get nodes -o wideNAME ASN IPV4 IPV6 ubuntu0(64512)192.168.23.99/24 ubuntu1(64512)192.168.23.98/24 ubuntu2(64512)192.168.23.97/24 rootubuntu0:~# kubectl -n calico-system get pods -o wide | grep -E ubuntu1|ubuntu2calico-kube-controllers-9d99b9966-fbp5p0/1 Running9(5m38s ago)29m10.100.2.101 ubuntu1nonenonecalico-typha-7656d5444d-jmcqt0/1 Running8(5m44s ago)29m192.168.23.98 ubuntu1nonenonecsi-node-driver-tjggq2/2 Running05h45m10.100.2.100 ubuntu1nonenonecsi-node-driver-xcdrk2/2 Running06h14m10.100.1.32 ubuntu2nonenone