CentOS ARM 部署 kubernetes v1.24.6

1.背景

之前安装的kubernetes版本为v1.19.0 树莓派使用(CentOS7.9 armv71 Kubernetes1.19.0), 由于版本过低，一些HPA相关的功能支持不是特别好，因此需要将版本升级，本次会将版本升级为v1.24.6.

2. 如何upgrade

2.1. 优雅升级

kubeadm自带支持优雅升级,详情见 kubeadm upgrade

2.2. 卸载升级

鉴于本次版本升级跨度比较大，kubernetes 1.19使用的还是docker作为CRI，但是到了kubernetes 1.24.6，建议使用的为containerd 作为CRI，因此需要额外安装一些组件；鉴于集群上没有什么重要的数据，因此我采用直接卸载掉旧集群做upgrade；

# 卸载命令
kubeadm reset
1
2

2.2.1. 安装containerd

从 containerd release 中下载对应系统架构的最新版本
对于arm架构，我们需要在 containerd arm release中找到我们需要的版本

// containerd 1.7.0 arm
// https://github.com/alexellis/containerd-arm/releases/download/v1.7.0/containerd-1.7.0-linux-armhf.tar.gz
// 由于官网下载速度过慢，因此我们使用github代理: https://ghproxy.com/
// containerd 1.7.0 arm github proxy (recommend)
mkdir -p ~/kubernetes/containerd-1.7.0 
wget https://ghproxy.com/?q=https%3A%2F%2Fgithub.com%2Falexellis%2Fcontainerd-arm%2Freleases%2Fdownload%2Fv1.7.0%2Fcontainerd-1.7.0-linux-armhf.tar.gz -O ~/kubernetes/containerd-1.7.0-linux-armhf.tar.gz 
tar -zxvf ~/kubernetes/containerd-1.7.0-linux-armhf.tar.gz -C ~/kubernetes/containerd-1.7.0

// 注意好路径 将该可执行文件链接到/usr/bin, /usr/local/bin 路径下
ln -s /root/kubernetes/containerd-1.7.0/containerd /usr/bin/containerd 
ln -s /root/kubernetes/containerd-1.7.0/containerd-shim /usr/bin/containerd-shim
ln -s /root/kubernetes/containerd-1.7.0/containerd-shim-runc-v1 /usr/bin/containerd-shim-runc-v1 
ln -s /root/kubernetes/containerd-1.7.0/containerd-shim-runc-v2 /usr/bin/containerd-shim-runc-v2 
ln -s /root/kubernetes/containerd-1.7.0/containerd-stress /usr/bin/containerd-stress 
ln -s /root/kubernetes/containerd-1.7.0/ctr /usr/bin/ctr 
......

// 导出containerd配置文件到指定目录
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

// 修改containerd默认的sandbox_image地址
sed -i 's#registry.k8s.io/pause:3.8#registry.aliyuncs.com/google_containers/pause:3.8#g' /etc/containerd/config.toml
systemctl daemon-reload
systemctl restart containerd
systemctl enable --now containerd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

2.2.2. 安装 runc

在 runc release找到符合自己系统架构的runc， 通过 github proxy 下载该版本

# arm
 wget https://ghproxy.com/https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.armhf

install -m 755 runc.armhf /usr/bin/runc
runc -v
1
2
3
4
5

2.2.3. 安装critical

mkdir -p /root/kubernetes/critical-1.28.0
wget https://ghproxy.com/?q=https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2Fv1.28.0%2Fcrictl-v1.28.0-linux-arm.tar.gz -O /root/kubernetes/critical-1.28.0/critical.tar.gz
cd /root/kubernetes/critical-1.28.0 && tar -zxvf /root/kubernetes/critical-1.28.0/critical.tar.gz

ln -s /root/kubernetes/critical-1.28.0/crictl /usr/local/bin/crictl

ln -s /root/kubernetes/critical-1.28.0/crictl /usr/bin/crictl
1
2
3
4
5
6
7

2.2.4.安装kubernetes

# 检查该kubelet service 是否存在
cat /usr/lib/systemd/system/kubelet.service
cat /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf 
1
2
3

# kubelet.service
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/kubelet
Restart=on-failure


[Install]
WantedBy=multi-user.target

1
2
3
4
5
6
7
8
9
10
11
12
13

# 10-kubeadm.conf 
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
[root@container1 kubelet.service.d]# 

1
2
3
4
5
6
7
8
9
10
11
12
13
14

kubeadm init --kubernetes-version=v1.24.6 --service-cidr=10.1.0.0/16 --pod-network-cidr=10.244.0.0/16 --v=10 --image-repository="registry.aliyuncs.com/google_containers"

kubectl get po -n kube-system

1
2
3
4
5

Reference

Kubernetes v1.24
Getting Started with Containerd