• k8s-v1.25.4+containerd部署

    k8s-v1.25.4 1master&1node

    实验环境

    主机网络信息和组件信息
    K8S集群角色IP主机名安装的组件网络模式
    master192.168.0.10k8s-master-1apiserver、controller-manager、scheduler、etcd、docker、kubectl、kubelet、kube-proxy、flannel、coredns、metric-serveripvs
    node192.168.0.11k8s-node-1kubelet、kube-proxy、docker、flannelipvs

    注：正常情况下master节点只负责调度，不负责运行kube-proxy、calico、coredns、metric-server，处于节约资源考虑，这里让master也负责工作

    软件版本
    kernelkernel-ml-5.17.6
    centos7.9
    kube-apiserver、kube-controller-manager、kube-scheduler、kubelet、kube-proxy1.25.4
    etcd
    containerdcri-containerd-cni-1.6.9(带crictl/cni/ctr)
    cfssl
    # 网络
	service: 10.96.0.0/16
	pod: 172.16.0.0/16

    • 1
    • 2
    • 3
    主机证书信息

    ​ CA机构三套：apiserver一套，etcd一套，api聚合层一套(由于和apiserver共用一套CA会发生冲突这里单独使用一个CA)，颁发机构分别为：ca-apiserver，ca-etcd，front-proxy-ca

    主机初始化

    配置主机名
    # master-1
	hostnamectl set-hostname k8s-master-1 && bash

# node-1
	hostnamectl set-hostname k8s-node-1 && bash

    • 1
    • 2
    • 3
    • 4
    • 5
    配置hosts文件
    # master,node
cat >> /etc/hosts <<EOF
192.168.0.10 k8s-master-1
192.168.0.11 k8s-node-1
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    免密登陆
    # master	
	ssh-keygen -t rsa
	ssh-copy-id -i .ssh/id_rsa.pub root@k8s-node-1

    • 1
    • 2
    • 3
    关闭防火墙
    # master,node

# 关闭防火墙
	systemctl disable firewalld --now

# 关闭selinux
	sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
	sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
	setenforce 0

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    关闭交换分区
    # master,node
	swapoff -a && sysctl -w vm.swappiness=0
	sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

    • 1
    • 2
    • 3
    配置yum源
    # master,node
	curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
	
 	yum -y install wget jq psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git network-scripts tar curl ntpdate 

    • 1
    • 2
    • 3
    • 4
    配置同步时间
    # master,node
# 同步时间
    ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
    echo 'Asia/Shanghai' >/etc/timezone
    ntpdate time2.aliyun.com
    
# 加入到crontab
	*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    升级内核
    # master,node

# 更新系统
	yum update -y --exclude=kernel* 
	
#导入ELRepo仓库的公共密钥
	rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

# 安装ELRepo仓库的yum源
	rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

# 查看可用内核
[root@docker ~]# yum --disablerepo="*" --enablerepo="elrepo-kernel" list available

# 安装最新版本内核（--enablerepo 选项开启 CentOS 系统上的指定仓库。默认开启的是 elrepo，这里用 elrepo-kernel 替换）
	yum --enablerepo=elrepo-kernel install kernel-lt*
	

# 查看当前内核版本
[root@k8s-master-1 ~]# rpm -qa | grep kernel
kernel-lt-5.4.224-1.el7.elrepo.x86_64
kernel-lt-headers-5.4.224-1.el7.elrepo.x86_64
kernel-lt-devel-5.4.224-1.el7.elrepo.x86_64
kernel-lt-tools-libs-5.4.224-1.el7.elrepo.x86_64
kernel-lt-tools-libs-devel-5.4.224-1.el7.elrepo.x86_64
kernel-3.10.0-1160.el7.x86_64
kernel-lt-doc-5.4.224-1.el7.elrepo.noarch
kernel-lt-tools-5.4.224-1.el7.elrepo.x86_64

# 查看默认内核
[root@k8s-master-1 ~]# grubby --default-kernel
/boot/vmlinuz-3.10.0-1160.el7.x86_64

# 所有节点更改内核启动顺序
	grub2-set-default /boot/vmlinuz-5.4.224-1.el7.elrepo.x86_64

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    修改内核参数
    原版
    # master,node,# 末尾添加如下内容
cat >> /etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited　
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    # 安装ipvs
	yum install ipvsadm ipset sysstat conntrack libseccomp -y

# 设置内核参数
cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

# 重启模块加载服务
	systemctl restart systemd-modules-load.service

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    # master,node

# 开启k8s内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system

# 查看k8s参数是否生效
reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    安装containerd
    # master与node节点

# contained下载链接
	wget https://github.com/containerd/containerd/releases/download/v1.6.9/cri-containerd-cni-1.6.9-linux-amd64.tar.gz

# cni插件
	wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz

# 解压containerd
	tar -zxvf cri-containerd-cni-1.6.9-linux-amd64.tar.gz -C /
	rm -rf /etc/cni/net.d/10-containerd-net.conflist

# 创建服务启动文件
cat > /etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

# 配置Containerd所需的模块
cat > /etc/modules-load.d/containerd.conf <<EOF
overlay
br_netfilter
EOF

# 加载模块
	systemctl restart systemd-modules-load.service

# 配置Containerd所需的内核
cat >> /etc/sysctl.d/99-kubernetes-cri.conf << EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 加载内核
	sysctl --system

# 创建Containerd的配置文件
	mkdir -p /etc/containerd
	containerd config default | tee /etc/containerd/config.toml

# 修改Containerd的配置文件
	sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
	cat /etc/containerd/config.toml | grep SystemdCgroup

	sed -i "s#registry.k8s.io/pause:3.6#registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.7#g" /etc/containerd/config.toml
	cat /etc/containerd/config.toml | grep sandbox_image

# 启动并设置为开机启动
	systemctl daemon-reload
	systemctl enable --now containerd

# 修改crictl配置
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

#测试
	systemctl restart  containerd
	crictl info

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    • 80
    • 81
    • 82
    安装cfssl
    # master节点安装
# 过程略

    • 1
    • 2

    CA初始化

    注意点：

    1. 所有证书均在master节点生成，然后下发给其他node节点
    2. etcd、apiserver、apiaggregation这里分别使用了三套CA机构来颁发证书，通常情况下etcd、apiserver和与apiserver通信的其他组件可以共用一套CA机构，apiaggregation一套CA机构
    # 创建CA配置文件
cat > ca-config.json <<EOF
{
    "signing": {
        "default": {
            "expiry": "87600h"
        },
        "profiles": {
            "kubernetes": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21

    注解：

    字段解释
    signing表示该证书可用于签名其它证书，生成的 ca.pem 证书中CA=TRUE
    server auth表示 client 可以用该该证书对 server 提供的证书进行验证
    client auth表示 server 可以用该该证书对 client 提供的证书进行验证；
    config.json可以定义多个profiles，分别指定不同的过期时间、使用场景等参数；后续在签名证书时使用某个profile
    etcd-ca
    # 创建CA请求文件
cat > etcd-ca-csr.json <<EOF
{     
    "CN": "etcd",
    "key": {
        "algo": "rsa",
        "size": 2048
    },  
    "names": [{   
        "C": "CN",
        "L": "hunan",
        "ST": "changsha",
        "O": "etcd",
        "OU": "system"
    }],
    "ca":{
    	"expiry": "876000h"
    }
}
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20

    注解：

    字段解释
    hosts这里为空，任意主机都能使用etcd-ca.pem这个证书
    CNCommon Name，kube-apiserver 从证书中提取该字段作为请求的用户名(User Name)，浏览器使用该字段验证网站是否合法，申请 SSL 证书的具体网站域名
    C申请单位所属国家，只能是两个字母的国家码。例如，中国填写为 CN
    LLocality，地区，城市
    STState，州，省
    OOrganization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)，公司名称
    OU部门名称
    # 生成CA证书
[root@k8s-master-1 certificate]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
2022/11/19 10:03:56 [INFO] generating a new CA key and certificate from CSR
2022/11/19 10:03:56 [INFO] generate received request
2022/11/19 10:03:56 [INFO] received CSR
2022/11/19 10:03:56 [INFO] generating key: rsa-2048
2022/11/19 10:03:56 [INFO] encoded CSR
2022/11/19 10:03:56 [INFO] signed certificate with serial number 141927445926371701278371932147140106424236751726

# 查看生成内容
[root@k8s-master-1 ssl]# ls etcd*
etcd-ca.csr  etcd-ca-csr.json  etcd-ca-key.pem  etcd-ca.pem

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12

    注解：

    1. etcd-ca-key.pem 生成的私钥
    2. etcd-ca.pem 生成的证书，后续将使用这个去颁发证书
    kube-apiserver-ca
    # 创建CA请求文件
cat > kube-apiserver-ca-csr.json <<EOF 
{
  "CN": "kubernetes",
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [{
      "C": "CN",
      "ST": "hunan",
      "L": "changsha",
      "O": "k8s",
      "OU": "system"
    }]
}
EOF

# 生成CA证书
	cfssl gencert -initca kube-apiserver-ca-csr.json | cfssljson -bare kube-apiserver-ca

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    apiaggregation-ca
    # 创建CA请求文件
cat > front-proxy-ca-csr.json <<EOF 
{
  "CN": "kubernetes",
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [{
      "C": "CN",
      "ST": "hunan",
      "L": "changsha",
      "O": "k8s",
      "OU": "system"
    }]
}
EOF

# 生成CA证书
	cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare front-proxy-ca

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20

    部署etcd

    ​ Kubernetes使用Etcd进行数据存储，所以先准备一个Etcd数据库，为解决Etcd单点故障，应采用集群方式部署，如果使用3台作为集群可以容忍1台故障，如果5台作为集群可以容忍2台故障，由于本次实验室单节点，这里采用单节点etcd

    创建etcd证书
    # 创建etcd请求文件
cat > etcd-csr.json<<EOF 
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.0.10"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "CN",
    "ST": "hunan",
    "L": "changsha",
    "O": "k8s",
    "OU": "system"
  }]
}
EOF

# 生成证书
	cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson  -bare etcd

# 注意：
	hosts需要填上运行etcd的机器的IP地址

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    创建etcd配置文件
    # 下载etcd
	wget https://github.com/etcd-io/etcd/releases/download/v3.5.5/etcd-v3.5.5-linux-amd64.tar.gz

# 上传etcd压缩包
	mv etcd etcdctl etcdutl /usr/local/bin

# 创建相应文件夹
	mkdir -p /etc/etcd/pki
	mkdir -p /var/lib/etcd/

# 创建etcd配置文件
cat > /etc/etcd/etcd.config.yml<<EOF
name: 'k8s-master-1'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.10:2380'
listen-client-urls: 'https://192.168.0.10:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.10:2380'
advertise-client-urls: 'https://192.168.0.10:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-1=https://192.168.0.10:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/etcd/pki/etcd.pem'
  key-file: '/etc/etcd/pki/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/etcd/pki/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/etcd/pki/etcd.pem'
  key-file: '/etc/etcd/pki/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/etcd/pki/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

# 创建启动服务文件
cat > /usr/lib/systemd/system/etcd.service <<"EOF"
[Unit]
Description=Etcd Server
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF

# 将证书移动到相应位置
	cp etcd.pem etcd-key.pem etcd-ca.pem /etc/etcd/pki/
	
# 启动etcd
	systemctl enable etcd --now
	
# 查看etcd集群状态
[root@k8s-master-1 certificate]# etcdctl --write-out=table --cacert=/etc/etcd/pki/etcd-ca.pem --cert=/etc/etcd/pki/etcd.pem --key=/etc/etcd/pki/etcd-key.pem --endpoints=https://192.168.0.10:2379  endpoint health
+---------------------------+--------+------------+-------+
|         ENDPOINT          | HEALTH |    TOOK    | ERROR |
+---------------------------+--------+------------+-------+
| https://192.168.0.10:2379 |   true | 5.592532ms |       |
+---------------------------+--------+------------+-------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    • 80
    • 81
    • 82
    • 83
    • 84
    • 85
    • 86
    • 87
    • 88
    • 89
    • 90
    • 91
    • 92

    部署apiserver

    上传k8s组件
    # 上传kubernetes-server二进制包(master)
	cp kube-apiserver kube-scheduler kube-controller-manager kubectl kube-proxy kubelet  /usr/local/bin/
	scp kubectl kube-proxy kubelet root@k8s-node-1:/usr/local/bin

# 创建相关目录(master,node)
	mkdir -p /etc/kubernetes/pki
	mkdir -p /var/log/kubernetes/{kube-apiserver,kube-controller-manager,kube-scheduler,kube-proxy,kubelet}

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    创建token.csv文件
    # 格式：token，用户名，UID，用户组,kubelet-bootstrap这个用户要被api-server所信任
cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

# system:kubelet-bootstrap 这个组内置

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6

    注：token.csv后边用于给kubelet自动颁发证书所使用的

    创建apiserver证书
    # 创建apiserver请求文件
cat > kube-apiserver-csr.json <<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "192.168.0.10",
    "10.96.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "hunan",
      "L": "changsha",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF

# host
	host内填写运行apiserver的主机IP/VIP,service的第一个IP，其余按照上面填写即可，node节点由于是使用bootstrap机制自动颁发证书，不用将其IP填写进来
	一般情况下hosts字段中IP为所有Master/LB/VIP IP
	
# 生成证书
	cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson  -bare kube-apiserver

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    创建apiaggregation证书

    这里host目前预留，后期加上

    # 创建apiaggregation证书请求文件
cat > front-proxy-client-csr.json <<EOF 
{
  "CN": "front-proxy-client",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "CN",
    "ST": "hunan",
    "L": "changsha",
    "O": "k8s",
    "OU": "system"
  }]
}
EOF

# 生成证书
	cfssl gencert -ca=front-proxy-ca.pem -ca-key=front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssljson  -bare front-proxy-client

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    创建service公/私钥
    # 生成私钥
	openssl genrsa -out ./service.key 2048

# 生成公钥
	openssl rsa -in ./service.key -pubout -out ./service.pub

    • 1
    • 2
    • 3
    • 4
    • 5

    注：这对公私钥主要用于service account

    创建apiserver配置文件
    #  创建apiserver配置文件
cat > /usr/lib/systemd/system/kube-apiserver.service <<"EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
 
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
    --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \
    --anonymous-auth=false \
    --allow-privileged=true \
    --bind-address=192.168.0.10 \
    --secure-port=6443 \
    --advertise-address=192.168.0.10 \
    --authorization-mode=Node,RBAC \
    --enable-bootstrap-token-auth \
    --enable-aggregator-routing=true \
    --feature-gates=EphemeralContainers=true \
    --token-auth-file=/etc/kubernetes/token.csv \
    --service-cluster-ip-range=10.96.0.0/16 \
    --service-node-port-range=30000-32767 \
    --service-account-key-file=/etc/kubernetes/pki/service.pub \
    --service-account-signing-key-file=/etc/kubernetes/pki/service.key \
    --service-account-issuer=https://kubernetes.default.svc.cluster.local \
    --tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem  \
    --tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem \
    --client-ca-file=/etc/kubernetes/pki/kube-apiserver-ca.pem \
    --kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem \
    --kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem \
    --etcd-cafile=/etc/etcd/pki/etcd-ca.pem \
    --etcd-certfile=/etc/etcd/pki/etcd.pem \
    --etcd-keyfile=/etc/etcd/pki/etcd-key.pem \
    --etcd-servers=https://192.168.0.10:2379 \
    --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
    --requestheader-allowed-names=front-proxy-client   \
    --requestheader-extra-headers-prefix=X-Remote-Extra-  \v
    --requestheader-group-headers=X-Remote-Group     \
    --requestheader-username-headers=X-Remote-User   \
    --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
    --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem    \
    --audit-log-maxage=30 \
    --audit-log-maxbackup=3 \
    --audit-log-maxsize=100 \
    --audit-log-path=/var/log/kubernetes/kube-apiserver/kube-apiserver-audit.log \
    --event-ttl=1h \
    --logtostderr=true \
    --log-dir=/var/log/kubernetes \
    --v=2
    
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
 
[Install]
WantedBy=multi-user.target

EOF

# 复制证书到相应目录
	cp service.pub service.key kube-apiserver.pem kube-apiserver-key.pem kube-apiserver-ca.pem kube-apiserver-ca-key.pem front-proxy-client.pem front-proxy-client-key.pem front-proxy-ca.pem /etc/kubernetes/pki/
	cp token.csv /etc/kubernetes


# 启动
	systemctl enable kube-apiserver.service --now

# 检查是否正常运行
	systemctl status kube-apiserver

# 不携带证书访问
[root@k8s-master-1 ~]# curl -k https://192.168.0.10:6443
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {
    
  },
  "status": "Failure",
  "message": "Unauthorized",
  "reason": "Unauthorized",
  "code": 401  #正常

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    • 80
    • 81
    • 82
    • 83
    • 84

    部署kubectl

    创建kubctl证书
    # 创建kubectl证书请求文件
cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "hunan",
      "L": "changsha",
      "O": "system:masters",             
      "OU": "system"
    }
  ]
}
EOF

# 生成证书
	cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

# 将证书放入相应位置
	cp admin*.pem /etc/kubernetes/pki/

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26

    注解：

    1. cluster-admin(内置角色，权限最大) 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予了调用kube-apiserver 的所有 API的权限
    2. O指定该证书的 Group 为 system:masters，必须是system:masters，否则后面kubectl create clusterrolebinding报错
    创建kubectl的kubeconfig配置文件
    # 设置集群参数
	kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.10:6443 --kubeconfig=kube.config

# 设置客户端认证参数
	kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config

# 设置上下文参数
	kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config
	
# 设置默认上下文
	kubectl config use-context kubernetes --kubeconfig=kube.config

# 拷贝到指定目录
    mkdir -p ~/.kube
    cp -i kube.config ~/.kube/config
	
# 查看svc
[root@k8s-master-1 ssl]# kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP   52m


# 授权apiserver用户访问kubelet，这个用户在apiserver证书的CN字段声明了，后续apiserver需要与kubelet通信
	kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24

    部署kube-controller-manager

    创建kube-controller-manager证书
    # 创建kube-controller-manager证书请求文件
cat > kube-controller-manager-csr.json <<EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "192.168.0.10"
    ],
    "names": [
      {
        "C": "CN",
        "ST": "hunan",
        "L": "changsha",
        "O": "system:kube-controller-manager",
        "OU": "system"
      }
    ]
}
EOF

# 生成证书
	cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26

    注解：

    1. system:kube-controller-manager，kubernetes 内置的 ClusterRoleBindings system:kube-controller-manager 赋予 kube-controller-manager 工作所需的权限
    创建kube-controller-manager的kubeconfig
    # 设置集群参数
	kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.10:6443 --kubeconfig=kube-controller-manager.kubeconfig

# 设置客户端认证参数
	kubectl config set-credentials system:kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
	
# 设置上下文参数
	kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig

# 设置默认上下文
	kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    创建kube-controller-manager的配置文件
    # 创建kube-controller-manager启动配置文件
cat > /usr/lib/systemd/system/kube-controller-manager.service <<"EOF"
[Unit]                                                                     
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]      
ExecStart=/usr/local/bin/kube-controller-manager \
    --secure-port=10257 \
    --bind-address=127.0.0.1 \
    --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
    --service-cluster-ip-range=10.96.0.0/16 \
    --cluster-cidr=172.16.0.0/16 \
    --cluster-name=kubernetes \
    --cluster-signing-cert-file=/etc/kubernetes/pki/kube-apiserver-ca.pem \
    --cluster-signing-key-file=/etc/kubernetes/pki/kube-apiserver-ca-key.pem \
    --cluster-signing-duration=87600h \
    --allocate-node-cidrs=true \
    --node-cidr-mask-size=24 \
    --root-ca-file=/etc/kubernetes/pki/kube-apiserver-ca.pem \
    --service-account-private-key-file=/etc/kubernetes/pki/service.key \
    --use-service-account-credentials=true \
    --leader-elect=true \
    --controllers=*,bootstrapsigner,tokencleaner \
    --tls-cert-file=/etc/kubernetes/pki/kube-controller-manager.pem \
    --tls-private-key-file=/etc/kubernetes/pki/kube-controller-manager-key.pem \
    --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
    --requestheader-allowed-names=front-proxy-client   \
    --requestheader-extra-headers-prefix=X-Remote-Extra-  \
    --requestheader-group-headers=X-Remote-Group     \
    --requestheader-username-headers=X-Remote-User   \
    --logtostderr=true \
    --log-dir=/var/log/kubernetes/kube-controller-manager \
    --v=2     
Restart=on-failure
RestartSec=5   
[Install]      
WantedBy=multi-user.target

EOF

# 复制文件
	cp kube-controller-manager*.pem /etc/kubernetes/pki/
	cp kube-controller-manager.kubeconfig /etc/kubernetes/

# 启动服务
	systemctl enable kube-controller-manager --now

# 检查kube-controller-manager运行状态
[root@k8s-master-1 certificate]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                        ERROR
scheduler            Unhealthy   Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused   
controller-manager   Healthy     ok                                                                                         
etcd-0               Healthy     {"health":"true","reason":""} 

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54

    部署kube-scheduler

    创建kube-scheduler证书
    # 创建证书请求文件
cat > kube-scheduler-csr.json <<EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "192.168.0.10"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "hunan",
        "L": "changsha",
        "O": "system:kube-scheduler",
        "OU": "system"
      }
    ]
}
EOF

注：
	O 为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限
	
# 生成证书
	cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    创建kube-scheduler的kubeconfig
    # 设置集群参数
	kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.10:6443 --kubeconfig=kube-scheduler.kubeconfig
	
# 设置客户端认证参数
	kubectl config set-credentials system:kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig

# 设置上下文参数
	kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig

# 设置默认上下文
	kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    创建kube-scheduler的配置文件
    cat > /usr/lib/systemd/system/kube-scheduler.service <<"EOF"
[Unit]                                      
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-scheduler  \
    --bind-address=127.0.0.1  \
    --secure-port=10259 \
    --kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig  \
    --feature-gates=EphemeralContainers=true \
    --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
    --requestheader-allowed-names=front-proxy-client   \
    --requestheader-extra-headers-prefix=X-Remote-Extra-  \
    --requestheader-group-headers=X-Remote-Group     \
    --requestheader-username-headers=X-Remote-User   \
    --leader-elect=true  \
    --logtostderr=true  \
    --log-dir=/var/log/kubernetes/kube-scheduler  \
     --v=2
 
Restart=on-failure
RestartSec=5
 
[Install]
WantedBy=multi-user.target
EOF

# 复制文件
	cp kube-scheduler*.pem /etc/kubernetes/pki/
	cp kube-scheduler.kubeconfig /etc/kubernetes/

# 启动服务
	systemctl enable kube-scheduler.service --now

# 查看服务状态
[root@k8s-master-1 ssl]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE                         ERROR
scheduler            Healthy   ok                              
controller-manager   Healthy   ok                              
etcd-0               Healthy   {"health":"true","reason":""} 

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41

    部署kubelet

    # 截取token
BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)

    • 1
    • 2
    创建kubelet的kubeconfig
    # 设置集群参数
	kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.10:6443 --kubeconfig=kubelet-bootstrap.kubeconfig

# 设置客户端认证参数
	kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig

# 设置上下文参数
	kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig

# 设置默认上下文
	kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    创建kublet的配置文件
    # kubelet.json 
cat <<-EOF > /etc/kubernetes/kubelet-conf.yml
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/kube-apiserver-ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 172.16.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    创建kubelet启动文件
    # 创建文件夹
	mkdir -p /etc/kubernetes/manifests

#创建kubelet启动配置,master节点
cat >/usr/lib/systemd/system/kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service
[Service]
ExecStart=/usr/local/bin/kubelet \
    --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \
    --cert-dir=/etc/kubernetes/pki \
    --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
    --config=/etc/kubernetes/kubelet-conf.yml \
    --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
    --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target                            
EOF

# 移动相关文件
	mkdir -p /var/lib/kubelet
	cp kubelet-bootstrap.kubeconfig /etc/kubernetes/
	mkdir -p /etc/kubernetes/manifests

# 启动服务
	systemctl enable kubelet --now

# node节点配置
	for i in k8s-node-1 ; do ssh root@$i "mkdir -p /etc/kubernetes/pki && mkdir -p /var/lib/kubelet && mkdir -p /var/log/kubernetes/{kubelet,kube-proxy} && mkdir -p /etc/kubernetes/manifests" && scp kubelet-bootstrap.kubeconfig root@$i:/etc/kubernetes && scp kube-apiserver-ca.pem root@$i:/etc/kubernetes/ssl && ssh root@$i "systemctl enable kubelet --now"; done

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    创建RBAC规则自动批复CSR

    apiserver 自动创建了两条 ClusterRole，分别是

    1. system:certificates.k8s.io:certificatesigningrequests:nodeclient
    2. system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
    # 我们再增加一条
cat <<EOF | kubectl apply -f -
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserver
rules:
- apiGroups: ["certificates.k8s.io"]
  resources: ["certificatesigningrequests/selfnodeserver"]
  verbs: ["create"]
EOF


# 将ClusterRole绑定到适当的用户组，以完成自动批准相关CSR请求，此处的system:bootstrappers组与token.csv中的组对应

# token.csv，格式 Token,用户名,UID,用户组
fbecd7fb7d3c75efc7f8bd8c0896addf,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

# 允许 system:bootstrappers 组用户创建 CSR 请求
	kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:kubelet-bootstrap

# 自动批准 system:bootstrappers 组用户 TLS bootstrapping 首次申请证书的 CSR 请求，clusterrolebinding kubelet-bootstrap及node-client-auto-approve-csr 中的--group=system:kubelet-bootstrap 可以替换为--user=kubelet-bootstrap,与token.csv保持一致
	kubectl create clusterrolebinding node-client-auto-approve-csr --clusterrole=system:certificates.k8s.io:certificatesigningrequests:nodeclient --group=system:kubelet-bootstrap

# 自动批准 system:nodes 组用户更新 kubelet 自身与 apiserver 通讯证书的 CSR 请求
	kubectl create clusterrolebinding node-client-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeclient --group=system:nodes

# 自动批准 system:nodes 组用户更新 kubelet 10250 api 端口证书的 CSR 请求
	kubectl create clusterrolebinding node-server-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeserver --group=system:nodes
	

# 查看csr,可以发现master节点加入集群后，自动就签发证书了
[root@k8s-master-1 ~]# kubectl get csr
NAME                                                   AGE     SIGNERNAME                                    REQUESTOR           CONDITION
node-csr-7yhdBfn1JE3dUOPfvRLVkRzlljdgno9X0C_X0gqipzg   2m16s   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   Approved,Issued

# 查看节点状态
[root@k8s-master-1 ~]# kubectl get nodes
NAME           STATUS     ROLES    AGE    VERSION
k8s-master-1   NotReady   <none>   114s   v1.20.10

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40

    部署kube-proxy

    创建kube-proxy证书
    # 创建证书请求文件
cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
      "C": "CN",
      "ST": "hunan",
      "L": "changsha",
      "O": "system:kube-proxy",
      "OU": "system"}]
}
EOF

# 生成证书
	cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19

    注解：

    1. CN：指定该证书的 User 为 system:kube-proxy
    2. 预定义的 RoleBinding system:node-proxier 将User system:kube-proxy 与 Role system:node-proxier 绑定，该 Role 授予了调用 kube-apiserver Proxy 相关 API 的权限
    3. 该证书只会被 kube-proxy 当做 client 证书使用，所以 hosts 字段为空
    创建kube-proxy的kubeconfig
    # 预配置，不确定是否一定需要配置
	kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy

# 设置集群参数
	kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.10:6443 --kubeconfig=kube-proxy.kubeconfig

# 设置客户端认证参数
	kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig

# 设置上下文参数
	kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig

# 设置默认上下文
	kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    创建kube-proxy配置文件

    新版本

    # # clusterCIDR 为POD IP
cat <<-EOF>/etc/kubernetes/kube-proxy.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 172.16.0.0/16 
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
healthzBindAddress: 0.0.0.0:10256
metricsBindAddress: 0.0.0.0:10249
enableProfiling: false
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s 
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s 
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
udpIdleTimeout: 250ms
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    创建kube-proxy启动文件
    # 创建kube-proxy启动文件
cat <<-EOF> /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
 
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/kube-proxy.yaml \
  --feature-gates=EphemeralContainers=true \
  --v=5
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
 
[Install]
WantedBy=multi-user.target

EOF

# 拷贝证书，创建相关文件夹
	mkdir -p /var/lib/kube-proxy
	cp kube-proxy.kubeconfig /etc/kubernetes/

# 启动服务
	systemctl enable kube-proxy.service --now
	
# node节点配置
	for i in root@k8s-node-1 root@k8s-node-2; do ssh $i "mkdir -p /var/lib/kube-proxy" && scp kube-proxy.kubeconfig $i:/etc/kubernetes && ssh $i "systemctl enable kube-proxy.service --now" ; done

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31

    部署Flanel

    Github链接：https://github.com/flannel-io/flannel

    Github yaml库链接：https://github.com/flannel-io/flannel/tree/master/Documentation

    Github yaml部署链接：https://github.com/flannel-io/flannel/blob/master/Documentation/kube-flannel.yml

    # 修改地方
	修改网段即可

    • 1
    • 2

    部署coredns

    apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
rules:
  - apiGroups:
    - ""
    resources:
    - endpoints
    - services
    - pods
    - namespaces
    verbs:
    - list
    - watch
  - apiGroups:
    - discovery.k8s.io
    resources:
    - endpointslices
    verbs:
    - list
    - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "CoreDNS"
    app.kubernetes.io/name: coredns
spec:
  # replicas: not specified here:
  # 1. Default is 1.
  # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
      app.kubernetes.io/name: coredns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
        app.kubernetes.io/name: coredns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      tolerations:
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      nodeSelector:
        kubernetes.io/os: linux
      affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
               matchExpressions:
               - key: k8s-app
                 operator: In
                 values: ["kube-dns"]
             topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: coredns/coredns:1.9.4
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
    app.kubernetes.io/name: coredns
spec:
  selector:
    k8s-app: kube-dns
    app.kubernetes.io/name: coredns
  clusterIP: 10.96.0.10 
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35
    • 36
    • 37
    • 38
    • 39
    • 40
    • 41
    • 42
    • 43
    • 44
    • 45
    • 46
    • 47
    • 48
    • 49
    • 50
    • 51
    • 52
    • 53
    • 54
    • 55
    • 56
    • 57
    • 58
    • 59
    • 60
    • 61
    • 62
    • 63
    • 64
    • 65
    • 66
    • 67
    • 68
    • 69
    • 70
    • 71
    • 72
    • 73
    • 74
    • 75
    • 76
    • 77
    • 78
    • 79
    • 80
    • 81
    • 82
    • 83
    • 84
    • 85
    • 86
    • 87
    • 88
    • 89
    • 90
    • 91
    • 92
    • 93
    • 94
    • 95
    • 96
    • 97
    • 98
    • 99
    • 100
    • 101
    • 102
    • 103
    • 104
    • 105
    • 106
    • 107
    • 108
    • 109
    • 110
    • 111
    • 112
    • 113
    • 114
    • 115
    • 116
    • 117
    • 118
    • 119
    • 120
    • 121
    • 122
    • 123
    • 124
    • 125
    • 126
    • 127
    • 128
    • 129
    • 130
    • 131
    • 132
    • 133
    • 134
    • 135
    • 136
    • 137
    • 138
    • 139
    • 140
    • 141
    • 142
    • 143
    • 144
    • 145
    • 146
    • 147
    • 148
    • 149
    • 150
    • 151
    • 152
    • 153
    • 154
    • 155
    • 156
    • 157
    • 158
    • 159
    • 160
    • 161
    • 162
    • 163
    • 164
    • 165
    • 166
    • 167
    • 168
    • 169
    • 170
    • 171
    • 172
    • 173
    • 174
    • 175
    • 176
    • 177
    • 178
    • 179
    • 180
    • 181
    • 182
    • 183
    • 184
    • 185
    • 186
    • 187
    • 188
    • 189
    • 190
    • 191
    • 192
    • 193
    • 194
    • 195
    • 196
    • 197
    • 198
    • 199
    • 200
    • 201
  • 相关阅读:
    Ruby on Rails 实践：课程导读
    SpringBoot SpringBoot 运维实用篇 1 打包与运行 1.1 程序打包与运行【Windows版】
    对于html和css初学者，你有什么值得推荐的简单的网页适合练习？
    Mybatis-Plus常见注解
    kafka详解一
    C++_pen_静态与常量
    Keras框架常见问题
    用PyPy加速Python程序
    cmake 之add_definitions使用误区
    宏转录组分析揭示不同土壤生境中氮循环基因的表达
  • 原文地址：https://blog.csdn.net/qq_41586875/article/details/127938533