专注网站基础优化,校内 实训网站 建设,百度推广软件,施工企业领导带班记录calico ipam使用 前面的文章pod获取ip地址的过程中提到过calico使用的IP地址的管理模块是其自己开发的模块calico-ipam,本篇文章来讲述下其具体用法。
一、环境信息
版本信息
本环境使用版本是k8s 1.25.3
[rootnode1 ~]# kubectl get node
NAME STATUS ROLES …calico ipam使用 前面的文章pod获取ip地址的过程中提到过calico使用的IP地址的管理模块是其自己开发的模块calico-ipam,本篇文章来讲述下其具体用法。
一、环境信息
版本信息
本环境使用版本是k8s 1.25.3
[rootnode1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
node1 Ready control-plane,worker 206d v1.25.3
node2 Ready worker 206d v1.25.3
node3 Ready worker 206d v1.25.3###集群已经部署了calico cni
[rootnode1 ~]# kubectl get po -n kube-system | grep calico
calico-kube-controllers-75c594996d-x49mw 1/1 Running 5 (12d ago) 206d
calico-node-htq5b 1/1 Running 1 (12d ago) 206d
calico-node-x6xwl 1/1 Running 1 (12d ago) 206d
calico-node-xdx46 1/1 Running 1 (12d ago) 206d
[rootnode1 ~]# ####查看calico的默认配置
[rootnode1 ~]# cat /etc/cni/net.d/10-calico.conflist
{name: k8s-pod-network,cniVersion: 0.3.1,plugins: [{type: calico, ###插件类型log_level: info,log_file_path: /var/log/calico/cni/cni.log,datastore_type: kubernetes,nodename: node1,mtu: 0,ipam: {type: calico-ipam ####ipam类型是calico-ipam},policy: {type: k8s},kubernetes: {kubeconfig: /etc/cni/net.d/calico-kubeconfig}},{type: portmap,snat: true,capabilities: {portMappings: true}},{type: bandwidth,capabilities: {bandwidth: true}}]
}[rootnode1 ~]#
网络模式
###查看目前使用的IP地址池
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never false false all() ###查看网路详细信息
[rootnode1 ~]# calicoctl get ippool default-ipv4-ippool -o yaml
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:creationTimestamp: 2023-04-08T17:18:59Zname: default-ipv4-ippoolresourceVersion: 647uid: 7b9d84e1-ac69-4660-b298-c52e2267ab08
spec:allowedUses:- Workload- TunnelblockSize: 24 ###每个block的大小是24为掩码cidr: 10.233.64.0/18ipipMode: Always ###网络模式是ipipnatOutgoing: truenodeSelector: all() ###所有节点可用vxlanMode: Never
二、环境物理拓扑
每个物理机位于不同的机架上每个物理节点分配不同的ippool -------------------| router |-------------------| |
--------------- --------------- ---------------
| rack-1 | | rack-2 | | rack-3 |
--------------- --------------- ---------------
| node-1 | | node-2 | | node-3 |
- - - - - - - - - - - - - - - - - - - - - - - -三、为节点分配网络
为node打label
[rootnode1 ~]# kubectl label node node1 rack1
node/node1 labeled
[rootnode1 ~]# kubectl label node node2 rack2
node/node2 labeled
[rootnode1 ~]# kubectl label node node3 rack3
node/node3 labeled
[rootnode1 ~]# 为node创建ippool
1首先要禁用环境中默认的ippool因为我环境中有使用默认ippool的pod不做删除操作
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never false false all() ########################
2使用patch命令修改disabledtrue
[rootnode1 ~]# calicoctl patch ipPool default-ipv4-ippool --patch {spec:{disabled: true}}
Successfully patched 1 IPPool resource
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never true false all()
[rootnode1 ~]# ########################
3为三个node创建ippool注意不要和其他网路冲突
[rootnode1 ~]# calicoctl create -f -EOFapiVersion: projectcalico.org/v3kind: IPPoolmetadata:name: rack-1-ippoolspec:cidr: 172.16.1.0/24ipipMode: AlwaysnatOutgoing: truenodeSelector: rack 1 #####此处标签与之前为node打的label 对应EOF
Successfully created 1 IPPool resource(s)
[rootnode1 ~]# calicoctl create -f -EOFapiVersion: projectcalico.org/v3kind: IPPoolmetadata:name: rack-2-ippoolspec:cidr: 172.16.2.0/24ipipMode: AlwaysnatOutgoing: truenodeSelector: rack 2EOF
Successfully created 1 IPPool resource(s)
[rootnode1 ~]# calicoctl create -f -EOFapiVersion: projectcalico.org/v3kind: IPPoolmetadata:name: rack-3-ippoolspec:cidr: 172.16.3.0/24ipipMode: AlwaysnatOutgoing: truenodeSelector: rack 3EOF
Successfully created 1 IPPool resource(s)
[rootnode1 ~]# ###########
4查看创建好的ippool
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never true false all()
rack-1-ippool 172.16.1.0/24 true Always Never false false rack 1
rack-2-ippool 172.16.2.0/24 true Always Never false false rack 2
rack-3-ippool 172.16.3.0/24 true Always Never false false rack 3 四、验证网络
1编辑yaml文件
---
apiVersion: apps/v1
kind: Deployment
metadata:name: nginxlabels:app: nginx
spec:replicas: 10selector:matchLabels:app: nginxtemplate:metadata:labels:app: nginxspec:containers:- name: nginximage: docker.io/library/nginx:latestimagePullPolicy: IfNotPresentports:- containerPort: 80#############################
2启动pod查看pod获取ip情况
[rootnode1 ~]# kubectl apply -f yaml/nginx.yaml
deployment.apps/nginx created
[rootnode1 ~]# kubectl get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-5977dc5756-22kfl 1/1 Running 0 7s 172.16.1.131 node1 none none
nginx-5977dc5756-4lvpq 1/1 Running 0 7s 172.16.3.129 node3 none none
nginx-5977dc5756-59jkh 1/1 Running 0 7s 172.16.1.129 node1 none none
nginx-5977dc5756-9lm7p 1/1 Running 0 7s 172.16.3.132 node3 none none
nginx-5977dc5756-jdcqf 1/1 Running 0 7s 172.16.1.130 node1 none none
nginx-5977dc5756-jvwkf 1/1 Running 0 7s 172.16.2.1 node2 none none
nginx-5977dc5756-nq46g 1/1 Running 0 7s 172.16.2.3 node2 none none
nginx-5977dc5756-tsjf7 1/1 Running 0 7s 172.16.3.131 node3 none none
nginx-5977dc5756-xqmwz 1/1 Running 0 7s 172.16.2.2 node2 none none
nginx-5977dc5756-xt648 1/1 Running 0 7s 172.16.3.130 node3 none none
[rootnode1 ~]#
以上可以看到每个pod在对应的节点获取到的ip和ippool对应##############################
3测试网络联通性在node1可以ping通其他两个节点的pod ip
[rootnode1 ~]# ping 172.16.1.131
PING 172.16.1.131 (172.16.1.131) 56(84) bytes of data.
64 bytes from 172.16.1.131: icmp_seq1 ttl64 time0.306 ms
^C
--- 172.16.1.131 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev 0.306/0.306/0.306/0.000 ms
[rootnode1 ~]# ping 172.16.2.1
PING 172.16.2.1 (172.16.2.1) 56(84) bytes of data.
64 bytes from 172.16.2.1: icmp_seq1 ttl63 time1.25 ms
64 bytes from 172.16.2.1: icmp_seq2 ttl63 time0.906 ms
^C
--- 172.16.2.1 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev 0.906/1.080/1.255/0.177 ms
[rootnode1 ~]# ping 172.16.3.131
PING 172.16.3.131 (172.16.3.131) 56(84) bytes of data.
64 bytes from 172.16.3.131: icmp_seq1 ttl63 time2.26 ms
64 bytes from 172.16.3.131: icmp_seq2 ttl63 time1.52 ms
^C
--- 172.16.3.131 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev 1.528/1.898/2.269/0.373 ms
[rootnode1 ~]# ######################################
注意Calico IPAM不会将IP地址重新分配给已经运行的pod。若要使用新配置的IP池中的IP地址需要更新正在运行的pod需要重建它们。五、迁移ip到新ippool
1 使用旧的ip新建pod以便后续测试
[rootnode1 ~]# kubectl apply -f yaml/nginx.yaml
deployment.apps/nginx created[rootnode1 ~]# kubectl get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-5977dc5756-8jm8g 1/1 Running 0 48s 10.233.90.36 node1 none none
nginx-5977dc5756-8vz6r 1/1 Running 0 48s 10.233.96.68 node2 none none
nginx-5977dc5756-c6ltc 1/1 Running 0 48s 10.233.92.61 node3 none none
nginx-5977dc5756-gmr27 1/1 Running 0 48s 10.233.96.69 node2 none none
nginx-5977dc5756-h7tz5 1/1 Running 0 48s 10.233.92.60 node3 none none
nginx-5977dc5756-k7jpx 1/1 Running 0 48s 10.233.92.59 node3 none none
nginx-5977dc5756-kzfpm 1/1 Running 0 48s 10.233.92.62 node3 none none
nginx-5977dc5756-nnzxt 1/1 Running 0 48s 10.233.90.34 node1 none none
nginx-5977dc5756-ppcxz 1/1 Running 0 48s 10.233.90.35 node1 none none
nginx-5977dc5756-rk9nk 1/1 Running 0 48s 10.233.96.70 node2 none none###########################
2新建ippool
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:name: new-ipv4-ippool
spec:allowedUses:- Workload- TunnelblockSize: 24cidr: 172.16.0.0/16ipipMode: AlwaysnatOutgoing: truenodeSelector: all()vxlanMode: Never[rootnode1 ~]# calicoctl apply -f ippool.yaml
Successfully applied 1 IPPool resource(s)
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never false false all()
new-ipv4-ippool 172.16.0.0/16 true Always Never false false all() ###############################
3禁用旧的ippool不会影响现有 pod 的网络
[rootnode1 ~]# calicoctl patch ipPool default-ipv4-ippool --patch {spec:{disabled: true}}
Successfully patched 1 IPPool resource查看默认的IPPOOL DISABLED 为 true
[rootnode1 ~]# calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED DISABLEBGPEXPORT SELECTOR
default-ipv4-ippool 10.233.64.0/18 true Always Never true false all()
new-ipv4-ippool 172.16.0.0/16 true Always Never false false all() [rootnode1 ~]# ###################################
4:重启新建好的pod看获取ip情况
[rootnode1 ~]# kubectl rollout restart deploy nginx
deployment.apps/nginx restarted
[rootnode1 ~]# kubectl get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-8499ccc976-6q5d8 1/1 Running 0 8s 172.16.154.1 node1 none none
nginx-8499ccc976-8mw42 1/1 Running 0 8s 172.16.44.1 node2 none none
nginx-8499ccc976-9x84p 1/1 Running 0 8s 172.16.28.2 node3 none none
nginx-8499ccc976-f8n28 1/1 Running 0 8s 172.16.44.2 node2 none none
nginx-8499ccc976-fxfft 1/1 Running 0 6s 172.16.28.3 node3 none none
nginx-8499ccc976-jj8hg 1/1 Running 0 6s 172.16.44.3 node2 none none
nginx-8499ccc976-kjf75 1/1 Running 0 8s 172.16.28.1 node3 none none
nginx-8499ccc976-rms74 1/1 Running 0 6s 172.16.154.2 node1 none none
nginx-8499ccc976-trcn8 1/1 Running 0 5s 172.16.28.4 node3 none none
nginx-8499ccc976-z28fw 1/1 Running 0 5s 172.16.154.3 node1 none none
可以看到pod重启后获取到了新ippool的ip################################
5测试网络连通性
[rootnode1 ~]# ping 172.16.44.1
PING 172.16.44.1 (172.16.44.1) 56(84) bytes of data.
64 bytes from 172.16.44.1: icmp_seq1 ttl63 time1.32 ms
^C
--- 172.16.44.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev 1.327/1.327/1.327/0.000 ms
[rootnode1 ~]# ping 172.16.28.2
PING 172.16.28.2 (172.16.28.2) 56(84) bytes of data.
64 bytes from 172.16.28.2: icmp_seq1 ttl63 time2.66 ms
64 bytes from 172.16.28.2: icmp_seq2 ttl63 time1.07 ms
^C
--- 172.16.28.2 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1002ms
rtt min/avg/max/mdev 1.076/1.868/2.660/0.792 ms
[rootnode1 ~]# ping 172.16.154.2
PING 172.16.154.2 (172.16.154.2) 56(84) bytes of data.
64 bytes from 172.16.154.2: icmp_seq1 ttl64 time0.263 ms
64 bytes from 172.16.154.2: icmp_seq2 ttl64 time0.125 ms
^C
--- 172.16.154.2 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 999ms
rtt min/avg/max/mdev 0.125/0.194/0.263/0.069 ms
