简要记录一下离线环境下 K3S 的搭建,版本为 v1.23.17+k3s1
,使用外部数据库 MySQL
作元数据存储,禁用默认组件(coredns
、servicelb
、traefik
、local-storage
、metrics-server
)并使用 Helm
单独安装(coredns
、metrics-server
、traefik
、longhorn
)。
需要一台联网主机(虚拟机),和多台未联网主机(服务器)。
curl -fsSL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 bash -s - server \
--data-dir /data/k3s/var/lib/rancher/k3s \
--cluster-cidr 10.8.0.0/16 \
--service-cidr 10.16.0.0/16 \
--cluster-dns 10.16.0.10 \
--service-node-port-range 1-65535 \
--kube-proxy-arg proxy-mode=ipvs \
--disable coredns \
--disable servicelb \
--disable traefik \
--disable local-storage \
--disable metrics-server
# yum
yum install iscsi-initiator-utils nfs-utils
# ubuntu
apt install open-iscsi nfs-common
# 启动
systemctl enable iscsid --now
### coredns
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: coredns
namespace: kube-system
labels:
app: coredns
spec:
repo: https://coredns.github.io/helm
chart: coredns
targetNamespace: kube-system
bootstrap: true
valuesContent: |-
fullnameOverride: coredns
serviceType: ClusterIP
service:
clusterIP: 10.16.0.10
name: coredns
servers:
- zones:
- zone: .
port: 53
plugins:
- name: errors
- name: health
configBlock: |-
lameduck 5s
- name: ready
- name: kubernetes
parameters: cluster.local in-addr.arpa ip6.arpa
configBlock: |-
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
- name: prometheus
parameters: 0.0.0.0:9153
- name: forward
parameters: . /etc/resolv.conf
- name: cache
parameters: 30
- name: loop
- name: reload
- name: loadbalance
### metrics-server
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: metrics-server
namespace: kube-system
labels:
app: metrics-server
spec:
repo: https://charts.bitnami.com/bitnami
chart: metrics-server
targetNamespace: kube-system
bootstrap: true
valuesContent: |
apiService:
create: true
extraArgs:
- --kubelet-insecure-tls
- --kubelet-use-node-status-port
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --metric-resolution=15s
### traefik
---
apiVersion: v1
kind: Namespace
metadata:
name: traefik-system
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: traefik
namespace: traefik-system
labels:
app: traefik
spec:
repo: https://traefik.github.io/charts
chart: traefik
targetNamespace: traefik-system
bootstrap: true
valuesContent: |-
deployment:
kind: Deployment
ingressClass:
enabled: true
isDefaultClass: true
providers:
kubernetesCRD:
enabled: true
allowCrossNamespace: true
allowExternalNameServices: true
allowEmptyServices: true
kubernetesIngress:
enabled: true
allowExternalNameServices: true
allowEmptyServices: true
publishedService:
enabled: true
ports:
traefik:
port: 9000
protocol: TCP
expose: false
exposedPort: 9000
metrics:
port: 9100
protocol: TCP
expose: false
exposedPort: 9100
web:
port: 80
protocol: TCP
expose: true
exposedPort: 80
nodePort: 30080
websecure:
port: 443
protocol: TCP
expose: true
exposedPort: 443
ndoePort: 30443
tls:
enabled: true
service:
type: NodePort
securityContext:
capabilities:
drop: []
add: [ALL]
readOnlyRootFilesystem: false
podSecurityContext:
runAsGroup: 0
runAsNonRoot: false
runAsUser: 0
### longhorn
---
apiVersion: v1
kind: Namespace
metadata:
name: longhorn-system
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: longhorn
namespace: longhorn-system
labels:
app: longhorn
spec:
repo: https://charts.longhorn.io
chart: longhorn
targetNamespace: longhorn-system
bootstrap: true
valuesContent: |-
persistence:
defaultClassReplicaCount: 1
csi:
attacherReplicaCount: 1
provisionerReplicaCount: 1
resizerReplicaCount: 1
snapshotterReplicaCount: 1
defaultSettings:
defaultDataPath: /data/longhorn
defaultReplicaCount: 1
deletingConfirmationFlag: true
longhornUI:
replicas: 1
longhornConversionWebhook:
replicas: 1
longhornAdmissionWebhook:
replicas: 1
longhornRecoveryBackend:
replicas: 1
ingress:
enabled: true
host: longhorn.example.org
kubectl apply -f charts.yaml
查看服务器 glibc 版本
ldd --version
os | os version | glibc version |
---|---|---|
centos | 7.9 | 2.17 |
centos | 8.4 | 2.28 |
ubuntu | 18.04 | 2.27 |
ubuntu | 20.04 | 2.31 |
ubuntu | 22.04 | 2.35 |
创建对应 glibc 版本的容器
# centos 7
kubectl run centos --image=centos:7.9.2009 --command -- /bin/sleep infinity
kubectl exec -it pod/centos -- /bin/bash
# ubuntu 22
kubectl run ubuntu --image=ubuntu:22.04 --command -- /bin/sleep infinity
kubectl exec -it pod/ubuntu -- /bin/bash
下载依赖
# yum
yum install iscsi-initiator-utils nfs-utils --downloadonly --downloaddir=rpm -y
tar -czvf ./rpm.tar.gz ./rpm
# apt
apt update && apt install open-iscsi nfs-common --download-only -y && mkdir -p deb && cp /var/cache/apt/archives/*.deb deb
tar -czvf ./deb.tar.gz ./deb
复制出依赖
# yum
kubectl cp centos:/rpm.tar.gz ./rpm.tar.gz
# apt
kubectl cp ubuntu:/deb.tar.gz ./deb.tar.gz
参考文档:离线安装
wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s-airgap-images-amd64.tar.gz
wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s
wget https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
wget https://get.k3s.io -O install.sh
# 下载 helm chart 包
helm repo add coredns https://coredns.github.io/helm && helm pull coredns/coredns --version 1.26.0
helm repo add bitnami https://charts.bitnami.com/bitnami && helm pull bitnami/metrics-server --version 6.5.2
helm repo add traefik https://traefik.github.io/charts && helm pull traefik/traefik --version 24.0.0
helm repo add longhorn https://charts.longhorn.io && helm pull longhorn/longhorn --version 1.5.1
# 导出镜像
k3s ctr image ls -q | grep -v 'sha256' | sort -u | xargs k3s ctr image export image.tar
# 准备 k3s 镜像
mkdir -p /data/k3s/var/lib/rancher/k3s/agent/images
cp ./k3s-airgap-images-amd64.tar.gz /data/k3s/var/lib/rancher/k3s/agent/images
# 准备 k3s 二进制可执行文件
install ./k3s /usr/local/bin
# 准备 helm 二进制可执行文件
tar -zxvf ./helm-v3.12.2-linux-amd64.tar.gz
install ./linux-amd64/helm /usr/local/bin
# 准备 k3s 安装脚本
chmod +x ./install.sh
# 引导 Server
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \
--data-dir /data/k3s/var/lib/rancher/k3s \
--cluster-cidr 10.8.0.0/16 \
--service-cidr 10.16.0.0/16 \
--cluster-dns 10.16.0.10 \
--service-node-port-range 1-65535 \
--kube-proxy-arg proxy-mode=ipvs \
--disable coredns \
--disable servicelb \
--disable traefik \
--disable local-storage \
--disable metrics-server \
--datastore-endpoint="mysql://:@tcp(:3306)/"
# 查看 Token
cat /data/k3s/var/lib/rancher/k3s/server/token
配置标识在所有 Server 节点必须是相同的。
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \
--data-dir /data/k3s/var/lib/rancher/k3s \
--cluster-cidr 10.8.0.0/16 \
--service-cidr 10.16.0.0/16 \
--cluster-dns 10.16.0.10 \
--service-node-port-range 1-65535 \
--kube-proxy-arg proxy-mode=ipvs \
--disable coredns \
--disable servicelb \
--disable traefik \
--disable local-storage \
--disable metrics-server \
--datastore-endpoint="mysql://:@tcp(:3306)/" \
--token <TOKEN>
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \
--data-dir /data/k3s/var/lib/rancher/k3s \
--datastore-endpoint="mysql://:@tcp(:3306)/" \
--token <TOKEN>
# yum
tar -zxvf rpm.tar.gz
rpm -ivh ./rpm/*.rpm
# apt
tar -zxvf deb.tar.gz
apt install ./deb/*.deb
# 导出镜像
k3s ctr image import ./image.tar
# coredns
helm install coredns coredns-1.26.0.tgz --namespace kube-system --values <VALUES_YAML_FILE>