• k8s从入门到精通

    目录

    k8s介绍

    一、kubernetes 部署

    1.0 sealos一键部署

    2.0二进制安装包下载

    二、harbor仓库搭建

    三、k8s流量入口Ingress

    3.1 helm3安装 ingress-nginx（k8s版本要求1.20以上）

    四、k8sHPA 自动水平伸缩pod

    五、k8s存储

    5.1 k8s持久化存储02pv pvc

    5.2 StorageClass

    5.ubuntu20.04系统

    六、k8s架构师课程之有状态服务StatefulSet

    七、k8s一次性和定时任务

    八、k8sRBAC角色访问控制

    九、k8s业务日志收集上节介绍、下节实战

    十、k8s的Prometheus监控实战

    十一、k8s安装kuboard图形化界面

    11.离线安装下载:

    十二、 k8s架构师课程基于gitlab的CICD自动化

    十三、k8s安装kubesphere3.3

    k8s介绍

    k8s架构图

    02ffa04351fb4123890e5732826a5e83.png

    一、kubernetes 部署

    1.0 sealos一键部署

    sealos 离线安装k8s-CSDN博客

    1.环境规划

    1.  
    2. 系统版本 CentOS 7.9.2009
    3. 服务器配置2核4g
    4.  
    5. k8s版本1.20.2
    6. docker版本docker-ce-19.03.14  docker-ce-cli-19.03.14
    7.  
    8. IP规划-3主2子节点
    9. 20.6.100.220 k8s-m01
    10. 20.6.100.221 k8s-m02
    11. 20.6.100.222 k8s-m03
    12. 20.6.100.223 k8s-node01
    13. 20.6.100.224 k8s-node02
    14. 20.6.100.225 k8s-ck

    .系统配置yum

    1.  
    2. #添加访问互联路由
    3. cat > /etc/resolv.conf <<EOF
    4. nameserver 8.8.8.8
    5. nameserver 114.114.114.114
    6. EOF
    7.  
    8.  
    9.  
    10. #设置为阿里云yum源
    11. mkdir -p /etc/yum.repos.d/bak && mv /etc/yum.repos.d/CentOS* /etc/yum.repos.d/bak
    12.  
    13. curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
    14.  
    15.  
    16. yum clean all 
    17.  
    18.  
    19. yum -y install sshpass  wget conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git lrzsz unzip gcc telnet
    20.

    2.0二进制安装包下载

    1. 链接：https://pan.baidu.com/s/1OBT9pxcZiuHx0hLxS2Fd1g?pwd=fhbg 
    2. 提取码：fhbg

    2.1安装脚本来着-博哥爱运维

    cat k8s_install_new .sh

    1. #!/bin/bash
    2. # auther: boge
    3. # descriptions:  the shell scripts will use ansible to deploy K8S at binary for siample
    4. # 传参检测
    5.  
    6. [ $# -ne 6 ] && echo -e "Usage: $0 rootpasswd netnum nethosts cri cni k8s-cluster-name\nExample: bash $0 newpasswd 20.6.100 220\ 221\ 222\ 223\ 224 [containerd|docker] [calico|flannel] test\n" && exit 11 
    7.  
    8. # 变量定义
    9. export release=3.0.0
    10. export k8s_ver=v1.20.2  # v1.20.2, v1.19.7, v1.18.15, v1.17.17
    11. rootpasswd=$1
    12. netnum=$2
    13. nethosts=$3
    14. cri=$4
    15. cni=$5
    16. clustername=$6
    17. if ls -1v ./kubeasz*.tar.gz &>/dev/null;then software_packet="$(ls -1v ./kubeasz*.tar.gz )";else software_packet="";fi
    18. pwd="/etc/kubeasz"
    19.  
    20.  
    21. # deploy机器升级软件库
    22. if cat /etc/redhat-release &>/dev/null;then
    23.     yum update -y
    24. else
    25.     apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y
    26.     [ $? -ne 0 ] && apt-get -yf install
    27. fi
    28.  
    29. # deploy机器检测python环境
    30. python2 -V &>/dev/null
    31. if [ $? -ne 0 ];then
    32.     if cat /etc/redhat-release &>/dev/null;then
    33.         yum install gcc openssl-devel bzip2-devel wget -y
    34.         wget https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tgz
    35.         tar xzf Python-2.7.16.tgz
    36.         cd Python-2.7.16
    37.         ./configure --enable-optimizations
    38.         make altinstall
    39.         ln -s /usr/bin/python2.7 /usr/bin/python
    40.         cd -
    41.     else
    42.         apt-get install -y python2.7 && ln -s /usr/bin/python2.7 /usr/bin/python
    43.     fi
    44. fi
    45.  
    46. # deploy机器设置pip安装加速源
    47. if [[ $clustername != 'aws' ]]; then
    48. mkdir ~/.pip
    49. cat > ~/.pip/pip.conf <<CB
    50. [global]
    51. index-url = https://mirrors.aliyun.com/pypi/simple
    52. [install]
    53. trusted-host=mirrors.aliyun.com
    54. CB
    55. fi
    56.  
    57.  
    58. # deploy机器安装相应软件包
    59. if cat /etc/redhat-release &>/dev/null;then
    60.     yum install git python-pip sshpass wget -y
    61.     [ -f ./get-pip.py ] && python ./get-pip.py || {
    62.     wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python get-pip.py
    63.     }
    64. else
    65.     apt-get install git python-pip sshpass -y
    66.     [ -f ./get-pip.py ] && python ./get-pip.py || {
    67.     wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python get-pip.py
    68.     }
    69. fi
    70. python -m pip install --upgrade "pip < 21.0"
    71.  
    72. pip -V
    73. pip install --no-cache-dir ansible netaddr
    74.  
    75.  
    76. # 在deploy机器做其他node的ssh免密操作
    77. for host in `echo "${nethosts}"`
    78. do
    79.     echo "============ ${netnum}.${host} ===========";
    80.  
    81.     if [[ ${USER} == 'root' ]];then
    82.         [ ! -f /${USER}/.ssh/id_rsa ] &&\
    83.         ssh-keygen -t rsa -P '' -f /${USER}/.ssh/id_rsa
    84.     else
    85.         [ ! -f /home/${USER}/.ssh/id_rsa ] &&\
    86.         ssh-keygen -t rsa -P '' -f /home/${USER}/.ssh/id_rsa
    87.     fi
    88.     sshpass -p ${rootpasswd} ssh-copy-id -o StrictHostKeyChecking=no ${USER}@${netnum}.${host}
    89.  
    90.     if cat /etc/redhat-release &>/dev/null;then
    91.         ssh -o StrictHostKeyChecking=no ${USER}@${netnum}.${host} "yum update -y"
    92.     else
    93.         ssh -o StrictHostKeyChecking=no ${USER}@${netnum}.${host} "apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y"
    94.         [ $? -ne 0 ] && ssh -o StrictHostKeyChecking=no ${USER}@${netnum}.${host} "apt-get -yf install"
    95.     fi
    96. done
    97.  
    98.  
    99. # deploy机器下载k8s二进制安装脚本
    100.  
    101. if [[ ${software_packet} == '' ]];then
    102.     curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
    103.     sed -ri "s+^(K8S_BIN_VER=).*$+\1${k8s_ver}+g" ezdown
    104.     chmod +x ./ezdown
    105.     # 使用工具脚本下载
    106.     ./ezdown -D && ./ezdown -P
    107. else
    108.     tar xvf ${software_packet} -C /etc/
    109.     chmod +x ${pwd}/{ezctl,ezdown}
    110. fi
    111.  
    112. # 初始化一个名为my的k8s集群配置
    113.  
    114. CLUSTER_NAME="$clustername"
    115. ${pwd}/ezctl new ${CLUSTER_NAME}
    116. if [[ $? -ne 0 ]];then
    117.     echo "cluster name [${CLUSTER_NAME}] was exist in ${pwd}/clusters/${CLUSTER_NAME}."
    118.     exit 1
    119. fi
    120.  
    121. if [[ ${software_packet} != '' ]];then
    122.     # 设置参数，启用离线安装
    123.     sed -i 's/^INSTALL_SOURCE.*$/INSTALL_SOURCE: "offline"/g' ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    124. fi
    125.  
    126.  
    127. # to check ansible service
    128. ansible all -m ping
    129.  
    130. #---------------------------------------------------------------------------------------------------
    131.  
    132.  
    133.  
    134.  
    135. #修改二进制安装脚本配置 config.yml
    136.  
    137. sed -ri "s+^(CLUSTER_NAME:).*$+\1 \"${CLUSTER_NAME}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    138.  
    139. ## k8s上日志及容器数据存独立磁盘步骤（参考阿里云的）
    140.  
    141. [ ! -d /var/lib/container ] && mkdir -p /var/lib/container/{kubelet,docker}
    142.  
    143. ## cat /etc/fstab     
    144. # UUID=105fa8ff-bacd-491f-a6d0-f99865afc3d6 /                       ext4    defaults        1 1
    145. # /dev/vdb /var/lib/container/ ext4 defaults 0 0
    146. # /var/lib/container/kubelet /var/lib/kubelet none defaults,bind 0 0
    147. # /var/lib/container/docker /var/lib/docker none defaults,bind 0 0
    148.  
    149. ## tree -L 1 /var/lib/container
    150. # /var/lib/container
    151. # ├── docker
    152. # ├── kubelet
    153. # └── lost+found
    154.  
    155. # docker data dir
    156. DOCKER_STORAGE_DIR="/var/lib/container/docker"
    157. sed -ri "s+^(STORAGE_DIR:).*$+STORAGE_DIR: \"${DOCKER_STORAGE_DIR}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    158. # containerd data dir
    159. CONTAINERD_STORAGE_DIR="/var/lib/container/containerd"
    160. sed -ri "s+^(STORAGE_DIR:).*$+STORAGE_DIR: \"${CONTAINERD_STORAGE_DIR}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    161. # kubelet logs dir
    162. KUBELET_ROOT_DIR="/var/lib/container/kubelet"
    163. sed -ri "s+^(KUBELET_ROOT_DIR:).*$+KUBELET_ROOT_DIR: \"${KUBELET_ROOT_DIR}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    164. if [[ $clustername != 'aws' ]]; then
    165.     # docker aliyun repo
    166.     REG_MIRRORS="https://pqbap4ya.mirror.aliyuncs.com"
    167.     sed -ri "s+^REG_MIRRORS:.*$+REG_MIRRORS: \'[\"${REG_MIRRORS}\"]\'+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    168. fi
    169. # [docker]信任的HTTP仓库
    170. sed -ri "s+127.0.0.1/8+${netnum}.0/24+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    171. # disable dashboard auto install
    172. sed -ri "s+^(dashboard_install:).*$+\1 \"no\"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    173.  
    174.  
    175. # 融合配置准备
    176. CLUSEER_WEBSITE="${CLUSTER_NAME}k8s.gtapp.xyz"
    177. lb_num=$(grep -wn '^MASTER_CERT_HOSTS:' ${pwd}/clusters/${CLUSTER_NAME}/config.yml |awk -F: '{print $1}')
    178. lb_num1=$(expr ${lb_num} + 1)
    179. lb_num2=$(expr ${lb_num} + 2)
    180. sed -ri "${lb_num1}s+.*$+  - "${CLUSEER_WEBSITE}"+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    181. sed -ri "${lb_num2}s+(.*)$+#\1+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    182.  
    183. # node节点最大pod 数
    184. MAX_PODS="120"
    185. sed -ri "s+^(MAX_PODS:).*$+\1 ${MAX_PODS}+g" ${pwd}/clusters/${CLUSTER_NAME}/config.yml
    186.  
    187.  
    188.  
    189. # 修改二进制安装脚本配置 hosts
    190. # clean old ip
    191. sed -ri '/192.168.1.1/d' ${pwd}/clusters/${CLUSTER_NAME}/hosts
    192. sed -ri '/192.168.1.2/d' ${pwd}/clusters/${CLUSTER_NAME}/hosts
    193. sed -ri '/192.168.1.3/d' ${pwd}/clusters/${CLUSTER_NAME}/hosts
    194. sed -ri '/192.168.1.4/d' ${pwd}/clusters/${CLUSTER_NAME}/hosts
    195.  
    196. # 输入准备创建ETCD集群的主机位
    197. echo "enter etcd hosts here (example: 222 221 220) ↓"
    198. read -p "" ipnums
    199. for ipnum in `echo ${ipnums}`
    200. do
    201.     echo $netnum.$ipnum
    202.     sed -i "/\[etcd/a $netnum.$ipnum"  ${pwd}/clusters/${CLUSTER_NAME}/hosts
    203. done
    204.  
    205. # 输入准备创建KUBE-MASTER集群的主机位
    206. echo "enter kube-master hosts here (example: 222 221 220) ↓"
    207. read -p "" ipnums
    208. for ipnum in `echo ${ipnums}`
    209. do
    210.     echo $netnum.$ipnum
    211.     sed -i "/\[kube_master/a $netnum.$ipnum"  ${pwd}/clusters/${CLUSTER_NAME}/hosts
    212. done
    213.  
    214. # 输入准备创建KUBE-NODE集群的主机位
    215. echo "enter kube-node hosts here (example: 224 223) ↓"
    216. read -p "" ipnums
    217. for ipnum in `echo ${ipnums}`
    218. do
    219.     echo $netnum.$ipnum
    220.     sed -i "/\[kube_node/a $netnum.$ipnum"  ${pwd}/clusters/${CLUSTER_NAME}/hosts
    221. done
    222.  
    223. # 配置容器运行时CNI
    224. case ${cni} in
    225.     flannel)
    226.     sed -ri "s+^CLUSTER_NETWORK=.*$+CLUSTER_NETWORK=\"${cni}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/hosts
    227.     ;;
    228.     calico)
    229.     sed -ri "s+^CLUSTER_NETWORK=.*$+CLUSTER_NETWORK=\"${cni}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/hosts
    230.     ;;
    231.     *)
    232.     echo "cni need be flannel or calico."
    233.     exit 11
    234. esac
    235.  
    236. # 配置K8S的ETCD数据备份的定时任务
    237. if cat /etc/redhat-release &>/dev/null;then
    238.     if ! grep -w '94.backup.yml' /var/spool/cron/root &>/dev/null;then echo "00 00 * * * `which ansible-playbook` ${pwd}/playbooks/94.backup.yml &> /dev/null" >> /var/spool/cron/root;else echo exists ;fi
    239.     chown root.crontab /var/spool/cron/root
    240.     chmod 600 /var/spool/cron/root
    241. else
    242.     if ! grep -w '94.backup.yml' /var/spool/cron/crontabs/root &>/dev/null;then echo "00 00 * * * `which ansible-playbook` ${pwd}/playbooks/94.backup.yml &> /dev/null" >> /var/spool/cron/crontabs/root;else echo exists ;fi
    243.     chown root.crontab /var/spool/cron/crontabs/root
    244.     chmod 600 /var/spool/cron/crontabs/root
    245. fi
    246. rm /var/run/cron.reboot
    247. service crond restart 
    248.  
    249.  
    250.  
    251.  
    252. #---------------------------------------------------------------------------------------------------
    253. # 准备开始安装了
    254. rm -rf ${pwd}/{dockerfiles,docs,.gitignore,pics,dockerfiles} &&\
    255. find ${pwd}/ -name '*.md'|xargs rm -f
    256. read -p "Enter to continue deploy k8s to all nodes >>>" YesNobbb
    257.  
    258. # now start deploy k8s cluster 
    259. cd ${pwd}/
    260.  
    261. # to prepare CA/certs & kubeconfig & other system settings 
    262. ${pwd}/ezctl setup ${CLUSTER_NAME} 01
    263. sleep 1
    264. # to setup the etcd cluster
    265. ${pwd}/ezctl setup ${CLUSTER_NAME} 02
    266. sleep 1
    267. # to setup the container runtime(docker or containerd)
    268. case ${cri} in
    269.     containerd)
    270.     sed -ri "s+^CONTAINER_RUNTIME=.*$+CONTAINER_RUNTIME=\"${cri}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/hosts
    271.     ${pwd}/ezctl setup ${CLUSTER_NAME} 03
    272.     ;;
    273.     docker)
    274.     sed -ri "s+^CONTAINER_RUNTIME=.*$+CONTAINER_RUNTIME=\"${cri}\"+g" ${pwd}/clusters/${CLUSTER_NAME}/hosts
    275.     ${pwd}/ezctl setup ${CLUSTER_NAME} 03
    276.     ;;
    277.     *)
    278.     echo "cri need be containerd or docker."
    279.     exit 11
    280. esac
    281. sleep 1
    282. # to setup the master nodes
    283. ${pwd}/ezctl setup ${CLUSTER_NAME} 04
    284. sleep 1
    285. # to setup the worker nodes
    286. ${pwd}/ezctl setup ${CLUSTER_NAME} 05
    287. sleep 1
    288. # to setup the network plugin(flannel、calico...)
    289. ${pwd}/ezctl setup ${CLUSTER_NAME} 06
    290. sleep 1
    291. # to setup other useful plugins(metrics-server、coredns...)
    292. ${pwd}/ezctl setup ${CLUSTER_NAME} 07
    293. sleep 1
    294. # [可选]对集群所有节点进行操作系统层面的安全加固  https://github.com/dev-sec/ansible-os-hardening
    295. #ansible-playbook roles/os-harden/os-harden.yml
    296. #sleep 1
    297. cd `dirname ${software_packet:-/tmp}`
    298.  
    299.  
    300. k8s_bin_path='/opt/kube/bin'
    301.  
    302.  
    303. echo "-------------------------  k8s version list  ---------------------------"
    304. ${k8s_bin_path}/kubectl version
    305. echo
    306. echo "-------------------------  All Healthy status check  -------------------"
    307. ${k8s_bin_path}/kubectl get componentstatus
    308. echo
    309. echo "-------------------------  k8s cluster info list  ----------------------"
    310. ${k8s_bin_path}/kubectl cluster-info
    311. echo
    312. echo "-------------------------  k8s all nodes list  -------------------------"
    313. ${k8s_bin_path}/kubectl get node -o wide
    314. echo
    315. echo "-------------------------  k8s all-namespaces's pods list   ------------"
    316. ${k8s_bin_path}/kubectl get pod --all-namespaces
    317. echo
    318. echo "-------------------------  k8s all-namespaces's service network   ------"
    319. ${k8s_bin_path}/kubectl get svc --all-namespaces
    320. echo
    321. echo "-------------------------  k8s welcome for you   -----------------------"
    322. echo
    323.  
    324. # you can use k alias kubectl to siample
    325. echo "alias k=kubectl && complete -F __start_kubectl k" >> ~/.bashrc
    326.  
    327. # get dashboard url
    328. ${k8s_bin_path}/kubectl cluster-info|grep dashboard|awk '{print $NF}'|tee -a /root/k8s_results
    329.  
    330. # get login token
    331. ${k8s_bin_path}/kubectl -n kube-system describe secret $(${k8s_bin_path}/kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')|grep 'token:'|awk '{print $NF}'|tee -a /root/k8s_results
    332. echo
    333. echo "you can look again dashboard and token info at  >>> /root/k8s_results <<<"
    334. #echo ">>>>>>>>>>>>>>>>> You can excute command [ source ~/.bashrc ] <<<<<<<<<<<<<<<<<<<<"
    335. echo ">>>>>>>>>>>>>>>>> You need to excute command [ reboot ] to restart all nodes <<<<<<<<<<<<<<<<<<<<"
    336. rm -f $0
    337. [ -f ${software_packet} ] && rm -f ${software_packet}
    338. #rm -f ${pwd}/roles/deploy/templates/${USER_NAME}-csr.json.j2
    339. #sed -ri "s+${USER_NAME}+admin+g" ${pwd}/roles/prepare/tasks/main.yml
    340.  
    341.

    2.2运行脚本 并加入参数， newpasswd是服务器密码

    1. bash k8s_install_new .sh newpasswd 20.6.100 220\ 221\ 222\ 223\ 224 docker calico test
    2.

    #脚本运行中途需要输入如下参数

    588fee197dae4b68ab3dcdd86e6462cf.png

    二、harbor仓库搭建

    1.安装

    1.  
    2. #目录/root上传文件docker-compose和harbor-offline-installer-v1.2.0.tgz
    3.  
    4. mv /root/docker-compose /usr/local/bin/
    5. chmod a+x /usr/local/bin/docker-compose
    6.  
    7. ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
    8.  
    9. tar -zxvf harbor-offline-installer-v2.4.1.tgz
    10.  
    11. mv harbor /usr/local/
    12.  
    13. cd /usr/local/harbor/
    14.  
    15. cp harbor.yml.tmpl harbor.yml
    16.  
    17. sed -i 's/hostname: reg.mydomain.com/hostname: 20.6.100.225/g' harbor.yml
    18. sed -i 's/https/#https/g' harbor.yml
    19. sed -i 's/port: 443/#port: 443/g' harbor.yml
    20. sed -i 's/certificate/#certificate/g' harbor.yml
    21. sed -i 's/private_key/#private_key/g' harbor.yml
    22.  
    23.  
    24. mkdir -p /etc/docker/
    25.  
    26. cat >/etc/docker/daemon.json<<EOF
    27. {
    28.     "registry-mirrors": ["https://gv33cz42.mirror.aliyuncs.com", "https://registry.cn-hangzhou.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn"],
    29.     "exec-opts": ["native.cgroupdriver=systemd"],
    30.     "max-concurrent-downloads": 20,
    31.     "live-restore": true,
    32.     "storage-driver": "overlay2",
    33.     "max-concurrent-uploads": 10,
    34.     "debug": true,
    35.     "log-opts": {
    36.     "max-size": "100m",
    37.     "max-file": "10"
    38.     },
    39.     "insecure-registries": ["20.6.100.225:80"]
    40. }
    41. EOF
    42.  
    43.  
    44.  
    45.  
    46. systemctl daemon-reload && systemctl restart docker && systemctl status docker
    47.  
    48.  
    49. #安装
    50. ./install.sh
    51.  
    52.  
    53. ## 重启harbor
    54. cd /usr/local/harbor/
    55. docker-compose down -v
    56. docker-compose up -d
    57. docker ps|grep harbor
    58.

    2.需要访问仓库的其他节点的 daemon.json添加如下内容

    1.  
    2. cat >/etc/docker/daemon.json<<EOF
    3. {
    4.     "registry-mirrors": ["https://gv33cz42.mirror.aliyuncs.com", "https://registry.cn-hangzhou.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn"],
    5.     "exec-opts": ["native.cgroupdriver=systemd"],
    6.     "max-concurrent-downloads": 20,
    7.     "live-restore": true,
    8.     "storage-driver": "overlay2",
    9.     "max-concurrent-uploads": 10,
    10.     "debug": true,
    11.     "log-opts": {
    12.     "max-size": "100m",
    13.     "max-file": "10"
    14.     },
    15.     "insecure-registries": ["20.6.100.225:80"]
    16. }
    17. EOF
    18.  
    19.  
    20.  
    21. systemctl daemon-reload && systemctl restart docker && systemctl status docker
    22.  
    23.  
    24.

    3.节点使用仓库

    1.  
    2. #登入仓库网站
    3.  
    4. docker login -u admin -p Harbor12345 20.6.100.225:80
    5.  
    6. #下载镜像
    7. docker pull daocloud.io/library/nginx:1.9.1
    8.  
    9. #给镜像打上标签
    10. docker tag daocloud.io/library/nginx:1.9.1 20.6.100.225:5000/library/nginx:1.9.1
    11.  
    12. #镜像上传
    13. docker push 20.6.100.225:80/library/nginx:1.9.1
    14.  
    15. #删除镜像
    16. docker rmi 20.6.100.225:80/library/nginx:1.9.1
    17.  
    18. #打包
    19. docker save daocloud.io/library/nginx:1.9.1  > /root/nginx-1.9.1.tar
    20.  
    21. #加载包
    22. docker load -i /root/nginx-1.9.1.tar

    三、k8s流量入口Ingress

    1.逻辑图

    64ab6d7904084f559abce88ed3114559.png

    Ingress是允许入站连接到达集群服务的一组规则。即介于物理网络和群集svc之间的一组转发规则。
    其实就是实现L4 L7的负载均衡:
    注意：这里的Ingress并非将外部流量通过Service来转发到服务pod上，而只是通过Service来找到对应的Endpoint来发现pod进行转发

       
        internet
            |
       [ Ingress ]   ---> [ Services ] ---> [ Endpoint ]
       --|-----|--                                 |
       [ Pod,pod,...... ]<-------------------------|

    aliyun-ingress-controller有一个很重要的修改，就是它支持路由配置的动态更新，
    大家用过Nginx的可以知道，在修改完Nginx的配置，我们是需要进行nginx -s reload来重加载配置才能生效的，
    在K8s上，这个行为也是一样的，但由于K8s运行的服务会非常多，所以它的配置更新是非常频繁的，
    因此，如果不支持配置动态更新，对于在高频率变化的场景下，Nginx频繁Reload会带来较明显的请求访问问题：

    1.造成一定的QPS抖动和访问失败情况
    2.对于长连接服务会被频繁断掉
    3.造成大量的处于shutting down的Nginx Worker进程，进而引起内存膨胀

    详细原理分析见这篇文章： https://developer.aliyun.com/article/692732

    2.准备来部署aliyun-ingress-controller，下面直接是生产中在用的yaml配置，我们保存了aliyun-ingress-nginx.yaml准备开始部署：

    1.  
    2.  
    3. cat > /data/k8s/aliyun-ingress-nginx.yaml <<EOF
    4. apiVersion: v1
    5. kind: Namespace
    6. metadata:
    7.   name: ingress-nginx
    8.   labels:
    9.     app: ingress-nginx
    11. ---
    12. apiVersion: v1
    13. kind: ServiceAccount
    14. metadata:
    15.   name: nginx-ingress-controller
    16.   namespace: ingress-nginx
    17.   labels:
    18.     app: ingress-nginx
    20. ---
    21. apiVersion: rbac.authorization.k8s.io/v1beta1
    22. kind: ClusterRole
    23. metadata:
    24.   name: nginx-ingress-controller
    25.   labels:
    26.     app: ingress-nginx
    27. rules:
    28.   - apiGroups:
    29.       - ""
    30.     resources:
    31.       - configmaps
    32.       - endpoints
    33.       - nodes
    34.       - pods
    35.       - secrets
    36.       - namespaces
    37.       - services
    38.     verbs:
    39.       - get
    40.       - list
    41.       - watch
    42.   - apiGroups:
    43.       - "extensions"
    44.       - "networking.k8s.io"
    45.     resources:
    46.       - ingresses
    47.     verbs:
    48.       - get
    49.       - list
    50.       - watch
    51.   - apiGroups:
    52.       - ""
    53.     resources:
    54.       - events
    55.     verbs:
    56.       - create
    57.       - patch
    58.   - apiGroups:
    59.       - "extensions"
    60.       - "networking.k8s.io"
    61.     resources:
    62.       - ingresses/status
    63.     verbs:
    64.       - update
    65.   - apiGroups:
    66.       - ""
    67.     resources:
    68.       - configmaps
    69.     verbs:
    70.       - create
    71.   - apiGroups:
    72.       - ""
    73.     resources:
    74.       - configmaps
    75.     resourceNames:
    76.       - "ingress-controller-leader-nginx"
    77.     verbs:
    78.       - get
    79.       - update
    81. ---
    82. apiVersion: rbac.authorization.k8s.io/v1beta1
    83. kind: ClusterRoleBinding
    84. metadata:
    85.   name: nginx-ingress-controller
    86.   labels:
    87.     app: ingress-nginx
    88. roleRef:
    89.   apiGroup: rbac.authorization.k8s.io
    90.   kind: ClusterRole
    91.   name: nginx-ingress-controller
    92. subjects:
    93.   - kind: ServiceAccount
    94.     name: nginx-ingress-controller
    95.     namespace: ingress-nginx
    97. ---
    98. apiVersion: v1
    99. kind: Service
    100. metadata:
    101.   labels:
    102.     app: ingress-nginx
    103.   name: nginx-ingress-lb
    104.   namespace: ingress-nginx
    105. spec:
    106.   # DaemonSet need:
    107.   # ----------------
    108.   type: ClusterIP
    109.   # ----------------
    110.   # Deployment need:
    111.   # ----------------
    112. #  type: NodePort
    113.   # ----------------
    114.   ports:
    115.   - name: http
    116.     port: 80
    117.     targetPort: 80
    118.     protocol: TCP
    119.   - name: https
    120.     port: 443
    121.     targetPort: 443
    122.     protocol: TCP
    123.   - name: metrics
    124.     port: 10254
    125.     protocol: TCP
    126.     targetPort: 10254
    127.   selector:
    128.     app: ingress-nginx
    130. ---
    131. kind: ConfigMap
    132. apiVersion: v1
    133. metadata:
    134.   name: nginx-configuration
    135.   namespace: ingress-nginx
    136.   labels:
    137.     app: ingress-nginx
    138. data:
    139.   keep-alive: "75"
    140.   keep-alive-requests: "100"
    141.   upstream-keepalive-connections: "10000"
    142.   upstream-keepalive-requests: "100"
    143.   upstream-keepalive-timeout: "60"
    144.   allow-backend-server-header: "true"
    145.   enable-underscores-in-headers: "true"
    146.   generate-request-id: "true"
    147.   http-redirect-code: "301"
    148.   ignore-invalid-headers: "true"
    149.   log-format-upstream: '{"@timestamp": "$time_iso8601","remote_addr": "$remote_addr","x-forward-for": "$proxy_add_x_forwarded_for","request_id": "$req_id","remote_user": "$remote_user","bytes_sent": $bytes_sent,"request_time": $request_time,"status": $status,"vhost": "$host","request_proto": "$server_protocol","path": "$uri","request_query": "$args","request_length": $request_length,"duration": $request_time,"method": "$request_method","http_referrer": "$http_referer","http_user_agent":  "$http_user_agent","upstream-sever":"$proxy_upstream_name","proxy_alternative_upstream_name":"$proxy_alternative_upstream_name","upstream_addr":"$upstream_addr","upstream_response_length":$upstream_response_length,"upstream_response_time":$upstream_response_time,"upstream_status":$upstream_status}'
    150.   max-worker-connections: "65536"
    151.   worker-processes: "2"
    152.   proxy-body-size: 20m
    153.   proxy-connect-timeout: "10"
    154.   proxy_next_upstream: error timeout http_502
    155.   reuse-port: "true"
    156.   server-tokens: "false"
    157.   ssl-ciphers: ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
    158.   ssl-protocols: TLSv1 TLSv1.1 TLSv1.2
    159.   ssl-redirect: "false"
    160.   worker-cpu-affinity: auto
    162. ---
    163. kind: ConfigMap
    164. apiVersion: v1
    165. metadata:
    166.   name: tcp-services
    167.   namespace: ingress-nginx
    168.   labels:
    169.     app: ingress-nginx
    171. ---
    172. kind: ConfigMap
    173. apiVersion: v1
    174. metadata:
    175.   name: udp-services
    176.   namespace: ingress-nginx
    177.   labels:
    178.     app: ingress-nginx
    180. ---
    181. apiVersion: apps/v1
    182. kind: DaemonSet
    183. metadata:
    184.   name: nginx-ingress-controller
    185.   namespace: ingress-nginx
    186.   labels:
    187.     app: ingress-nginx
    188.   annotations:
    189.     component.version: "v0.30.0"
    190.     component.revision: "v1"
    191. spec:
    192.   # Deployment need:
    193.   # ----------------
    194. #  replicas: 1
    195.   # ----------------
    196.   selector:
    197.     matchLabels:
    198.       app: ingress-nginx
    199.   template:
    200.     metadata:
    201.       labels:
    202.         app: ingress-nginx
    203.       annotations:
    204.         prometheus.io/port: "10254"
    205.         prometheus.io/scrape: "true"
    206.         scheduler.alpha.kubernetes.io/critical-pod: ""
    207.     spec:
    208.       # DaemonSet need:
    209.       # ----------------
    210.       hostNetwork: true
    211.       # ----------------
    212.       serviceAccountName: nginx-ingress-controller
    213.       priorityClassName: system-node-critical
    214.       affinity:
    215.         podAntiAffinity:
    216.           preferredDuringSchedulingIgnoredDuringExecution:
    217.           - podAffinityTerm:
    218.               labelSelector:
    219.                 matchExpressions:
    220.                 - key: app
    221.                   operator: In
    222.                   values:
    223.                   - ingress-nginx
    224.               topologyKey: kubernetes.io/hostname
    225.             weight: 100
    226.         nodeAffinity:
    227.           requiredDuringSchedulingIgnoredDuringExecution:
    228.             nodeSelectorTerms:
    229.             - matchExpressions:
    230.               - key: type
    231.                 operator: NotIn
    232.                 values:
    233.                 - virtual-kubelet
    234.       containers:
    235.         - name: nginx-ingress-controller
    236.           image: registry.cn-beijing.aliyuncs.com/acs/aliyun-ingress-controller:v0.30.0.2-9597b3685-aliyun
    237.           args:
    238.             - /nginx-ingress-controller
    239.             - --configmap=$(POD_NAMESPACE)/nginx-configuration
    240.             - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
    241.             - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
    242.             - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb
    243.             - --annotations-prefix=nginx.ingress.kubernetes.io
    244.             - --enable-dynamic-certificates=true
    245.             - --v=2
    246.           securityContext:
    247.             allowPrivilegeEscalation: true
    248.             capabilities:
    249.               drop:
    250.                 - ALL
    251.               add:
    252.                 - NET_BIND_SERVICE
    253.             runAsUser: 101
    254.           env:
    255.             - name: POD_NAME
    256.               valueFrom:
    257.                 fieldRef:
    258.                   fieldPath: metadata.name
    259.             - name: POD_NAMESPACE
    260.               valueFrom:
    261.                 fieldRef:
    262.                   fieldPath: metadata.namespace
    263.           ports:
    264.             - name: http
    265.               containerPort: 80
    266.             - name: https
    267.               containerPort: 443
    268.           livenessProbe:
    269.             failureThreshold: 3
    270.             httpGet:
    271.               path: /healthz
    272.               port: 10254
    273.               scheme: HTTP
    274.             initialDelaySeconds: 10
    275.             periodSeconds: 10
    276.             successThreshold: 1
    277.             timeoutSeconds: 10
    278.           readinessProbe:
    279.             failureThreshold: 3
    280.             httpGet:
    281.               path: /healthz
    282.               port: 10254
    283.               scheme: HTTP
    284.             periodSeconds: 10
    285.             successThreshold: 1
    286.             timeoutSeconds: 10
    287. #          resources:
    288. #            limits:
    289. #              cpu: "1"
    290. #              memory: 2Gi
    291. #            requests:
    292. #              cpu: "1"
    293. #              memory: 2Gi
    294.           volumeMounts:
    295.           - mountPath: /etc/localtime
    296.             name: localtime
    297.             readOnly: true
    298.       volumes:
    299.       - name: localtime
    300.         hostPath:
    301.           path: /etc/localtime
    302.           type: File
    303.       nodeSelector:
    304.         boge/ingress-controller-ready: "true"
    305.       tolerations:
    306.       - operator: Exists
    308.       initContainers:
    309.       - command:
    310.         - /bin/sh
    311.         - -c
    312.         - |
    313.           mount -o remount rw /proc/sys
    314.           sysctl -w net.core.somaxconn=65535
    315.           sysctl -w net.ipv4.ip_local_port_range="1024 65535"
    316.           sysctl -w fs.file-max=1048576
    317.           sysctl -w fs.inotify.max_user_instances=16384
    318.           sysctl -w fs.inotify.max_user_watches=524288
    319.           sysctl -w fs.inotify.max_queued_events=16384
    320.         image: registry.cn-beijing.aliyuncs.com/acs/busybox:v1.29.2
    321.         imagePullPolicy: Always
    322.         name: init-sysctl
    323.         securityContext:
    324.           privileged: true
    325.           procMount: Default
    327. ---
    328. ## Deployment need for aliyun'k8s:
    329. #apiVersion: v1
    330. #kind: Service
    331. #metadata:
    332. #  annotations:
    333. #    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-id: "lb-xxxxxxxxxxxxxxxxxxx"
    334. #    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-force-override-listeners: "true"
    335. #  labels:
    336. #    app: nginx-ingress-lb
    337. #  name: nginx-ingress-lb-local
    338. #  namespace: ingress-nginx
    339. #spec:
    340. #  externalTrafficPolicy: Local
    341. #  ports:
    342. #  - name: http
    343. #    port: 80
    344. #    protocol: TCP
    345. #    targetPort: 80
    346. #  - name: https
    347. #    port: 443
    348. #    protocol: TCP
    349. #    targetPort: 443
    350. #  selector:
    351. #    app: ingress-nginx
    352. #  type: LoadBalancer
    353. EOF
    354.

    3.#开始部署

    kubectl  apply -f aliyun-ingress-nginx.yaml

    4.我们查看下pod，会发现空空如也，为什么会这样呢？

    kubectl -n ingress-nginx get pod

    注意上面的yaml配置里面，我使用了节点选择配置，只有打了我指定lable标签的node节点，也会被允许调度pod上去运行
    nodeSelector:
    boge/ingress-controller-ready: “true”

    5.给mast 打标签，想运行什么节点上就给什么节点打标签

    kubectl label node 20.6.100.220 boge/ingress-controller-ready=true
    kubectl label node 20.6.100.221 boge/ingress-controller-ready=true
    kubectl label node 20.6.100.222 boge/ingress-controller-ready=true

    6.接着可以看到pod就被调试到这两台node上启动了

    kubectl -n ingress-nginx get pod -o wide

    7 子节点修改haproxy.cfg
    vi /etc/haproxy/haproxy.cfg

    1. listen ingress-http
    2.         bind 0.0.0.0:80
    3.         mode tcp
    4.         option tcplog
    5.         option dontlognull
    6.         option dontlog-normal
    7.         balance roundrobin
    8.         server 20.6.100.220 20.6.100.220:80 check inter 2000 fall 2 rise 2 weight 1
    9.         server 20.6.100.221 20.6.100.221:80 check inter 2000 fall 2 rise 2 weight 1
    10.         server 20.6.100.222 20.6.100.222:80 check inter 2000 fall 2 rise 2 weight 1
    11. 	
    12.  
    13. listen ingress-https
    14.         bind 0.0.0.0:443
    15.         mode tcp
    16.         option tcplog
    17.         option dontlognull
    18.         option dontlog-normal
    19.         balance roundrobin
    20.         server 20.6.100.220 20.6.100.220:443 check inter 2000 fall 2 rise 2 weight 1
    21.         server 20.6.100.221 20.6.100.221:443 check inter 2000 fall 2 rise 2 weight 1
    22.         server 20.6.100.222 20.6.100.222:443 check inter 2000 fall 2 rise 2 weight 1

    8.子节点安装keepalived

    yum install -y keepalived

    9.编辑配置修改为如下：

    这里是node 20.6.100.223

    1. cat > /etc/keepalived/keepalived.conf <<EOF
    2. global_defs {
    3.     router_id lb-master
    4. }
    6. vrrp_script check-haproxy {
    7.     script "killall -0 haproxy"
    8.     interval 5
    9.     weight -60
    10. }
    12. vrrp_instance VI-kube-master {
    13.     state MASTER
    14.     priority 120
    15.     unicast_src_ip 20.6.100.223
    16.     unicast_peer {
    17.         20.6.100.224
    18.     }
    19.     dont_track_primary
    20.     interface ens160  # 注意这里的网卡名称修改成你机器真实的内网网卡名称，可用命令ip addr查看
    21.     virtual_router_id 111
    22.     advert_int 3
    23.     track_script {
    24.         check-haproxy
    25.     }
    26.     virtual_ipaddress {
    27.         20.6.100.226
    28.     }
    29. }
    30. EOF

    10.这里是node 20.6.100.224

    1. cat > /etc/keepalived/keepalived.conf <<EOF
    2. global_defs {
    3.     router_id lb-master
    4. }
    6. vrrp_script check-haproxy {
    7.     script "killall -0 haproxy"
    8.     interval 5
    9.     weight -60
    10. }
    12. vrrp_instance VI-kube-master {
    13.     state MASTER
    14.     priority 120
    15.     unicast_src_ip 20.6.100.224
    16.     unicast_peer {
    17.         20.6.100.223
    18.     }
    19.     dont_track_primary
    20.     interface ens160
    21.     virtual_router_id 111
    22.     advert_int 3
    23.     track_script {
    24.         check-haproxy
    25.     }
    26.     virtual_ipaddress {
    27.         20.6.100.226
    28.     }
    29. }
    30. EOF

    11.启动服务器

    1. # 重启服务
    2. systemctl restart haproxy.service
    3. systemctl restart keepalived.service
    4.  
    5.  
    6. # 查看运行状态
    7. systemctl status haproxy.service 
    8. systemctl status keepalived.service
    9.  
    10. # 添加开机自启动（haproxy默认安装好就添加了自启动）
    11. systemctl enable keepalived.service
    12. # 查看是否添加成功
    13. systemctl is-enabled keepalived.service 
    14. #enabled就代表添加成功了
    15.  
    16. # 同时我可查看下VIP是否已经生成
    17. ip a|grep 226     
    18.  
    19.

    12.然后准备nginx的ingress配置，保留为nginx-ingress.yaml，并执行它

    1. apiVersion: v1
    2. kind: Service
    3. metadata:
    4.   namespace: test
    5.   name: nginx
    6.   labels:
    7.     app: nginx
    8. spec:
    9.   ports:
    10.     - port: 80
    11.       protocol: TCP
    12.       targetPort: 80
    13.   selector:
    14.     app: nginx
    15. ---
    16. apiVersion: apps/v1
    17. kind: Deployment
    18. metadata:
    19.   namespace: test
    20.   name: nginx
    21.   labels:
    22.     app: nginx
    23. spec:
    24.   replicas: 1
    25.   selector:
    26.     matchLabels:
    27.       app: nginx
    28.   template:
    29.     metadata:
    30.       labels:
    31.         app: nginx
    32.     spec:
    33.       containers:
    34.         - name: nginx
    35.           image: nginx
    36.           ports:
    37.             - containerPort: 80
    38. ---
    39. apiVersion: extensions/v1beta1
    40. kind: Ingress
    41. metadata:
    42.   namespace: test
    43.   name: nginx-ingress
    44. spec:
    45.   rules:
    46.     - host: nginx.boge.com
    47.       http:
    48.         paths:
    49.           - backend:
    50.               serviceName: nginx
    51.               servicePort: 80
    52.             path: /

    13.运行

    1.   
    2. kubectl apply -f nginx-ingress.yaml
    3. #查看创建的ingress资源
    4. # kubectl get ingress
    5. NAME            CLASS    HOSTS            ADDRESS   PORTS   AGE
    6. nginx-ingress      nginx.boge.com             80      13s
    7.  
    8. # 我们在其它节点上，加下本地hosts，来测试下效果
    9. 20.6.1.226 nginx.boge.com
    10.  
    11. #测试
    12. curl nginx.boge.com


    14.生产环境正常情况下大部分是一个Ingress对应一个Service服务，但在一些特殊情况，需要复用一个Ingress来访问多个服务的，下面我们来实践下
    再创建一个nginx的deployment和service，注意名称修改下不要冲突了

    1. # kubectl create deployment web --image=nginx
    2. deployment.apps/web created
    3.  
    4. # kubectl expose deployment web --port=80 --target-port=80
    5. service/web exposed
    6.  
    7. # 确认下创建结果
    8. # kubectl  get deployments.apps 
    9. NAME    READY   UP-TO-DATE   AVAILABLE   AGE
    10. nginx   1/1     1            1           16h
    11. web     1/1     1            1           45s
    12. # kubectl  get pod
    13.  
    14. # kubectl  get svc
    15.  
    16. # 接着来修改Ingress
    17. # 注意：这里可以通过两种方式来修改K8s正在运行的资源
    18. # 第一种：直接通过edit修改在线服务的资源来生效，这个通常用在测试环境，在实际生产中不建议这么用
    19. kubectl edit ingress nginx-ingress
    20.  
    21. # 第二种： 通过之前创建ingress的yaml配置，在上面进行修改，再apply更新进K8s，在生产中是建议这么用的，我们这里也用这种方式来修改
    22. # vim nginx-ingress.yaml
    23. apiVersion: extensions/v1beta1
    24. kind: Ingress
    25. metadata:
    26.   annotations:
    27.     nginx.ingress.kubernetes.io/rewrite-target: /    # 注意这里需要把进来到服务的请求重定向到/，这个和传统的nginx配置是一样的，不配会404
    28.   name: nginx-ingress
    29. spec:
    30.   rules:
    31.     - host: nginx.boge.com
    32.       http:
    33.         paths:
    34.           - backend:
    35.               serviceName: nginx
    36.               servicePort: 80
    37.             path: /nginx  # 注意这里的路由名称要是唯一的
    38.           - backend:       # 从这里开始是新增加的
    39.               serviceName: web
    40.               servicePort: 80
    41.             path: /web  # 注意这里的路由名称要是唯一的
    42. # 开始创建
    43. [root@node-1 ~]# kubectl  apply -f nginx-ingress.yaml 
    44. ingress.extensions/nginx-ingress configured
    45.  
    46. # 同时为了更直观的看到效果，我们按前面讲到的方法来修改下nginx默认的展示页面
    47. # kubectl exec -it nginx-f89759699-6vgr8 -- bash
    48. echo "i am nginx" > /usr/share/nginx/html/index.html
    49.  
    50. # kubectl exec -it web-5dcb957ccc-nr2m7 -- bash
    51. echo "i am web" > /usr/share/nginx/html/index.html

    15.因为http属于是明文传输数据不安全，在生产中我们通常会配置https加密通信，现在实战下Ingress的tls配置

    1. # 这里我先自签一个https的证书
    2.  
    3. #1. 先生成私钥key
    4. openssl genrsa -out tls.key 2048
    5.  
    6.  
    7. #2.再基于key生成tls证书(注意：这里我用的*.boge.com，这是生成泛域名的证书，后面所有新增加的三级域名都是可以用这个证书的)
    8. openssl req -new -x509 -key tls.key -out tls.cert -days 360 -subj /CN=*.boge.com
    9.  
    10. # 看下创建结果
    11. # ll
    12.  
    13. -rw-r--r-- 1 root root 1099 Nov 27 11:44 tls.cert
    14. -rw-r--r-- 1 root root 1679 Nov 27 11:43 tls.key
    15.  
    16. # 在K8s上创建tls的secret（注意默认ns是default）
    17. kubectl create secret tls mytls --cert=tls.cert --key=tls.key 
    18.  
    19.  
    20. # 然后修改先的ingress的yaml配置
    21. # cat nginx-ingress.yaml 
    22. apiVersion: extensions/v1beta1
    23. kind: Ingress
    24. metadata:
    25.   annotations:
    26.     nginx.ingress.kubernetes.io/rewrite-target: /    # 注意这里需要把进来到服务的请求重定向到/，这个和传统的nginx配置是一样的，不配会404
    27.   name: nginx-ingress
    28. spec:
    29.   rules:
    30.     - host: nginx.boge.com
    31.       http:
    32.         paths:
    33.           - backend:
    34.               serviceName: nginx
    35.               servicePort: 80
    36.             path: /nginx  # 注意这里的路由名称要是唯一的
    37.           - backend:       # 从这里开始是新增加的
    38.               serviceName: web
    39.               servicePort: 80
    40.             path: /web  # 注意这里的路由名称要是唯一的
    41.   tls:    # 增加下面这段，注意缩进格式
    42.       - hosts:
    43.           - nginx.boge.com   # 这里域名和上面的对应
    44.         secretName: mytls    # 这是我先生成的secret
    45.  
    46. # 进行更新
    47. kubectl  apply -f nginx-ingress.yaml         
    48.

    16.测试现在再来看看https访问的效果：

    https://nginx.boge.com/nginx

    https://nginx.boge.com/web

    注意：这里因为是我自签的证书，所以浏览器地访问时会提示您的连接不是私密连接 ，我这里用的谷歌浏览器，直接点高级，再点击继续前往nginx.boge.com（不安全）

    3.1 helm3安装 ingress-nginx（k8s版本要求1.20以上）

    1.下载 ingress-nginx-4.2.5.tgz

    1. helm fetch ingress-nginx/ingress-nginx --version 4.2.5
    2. #或者curl -LO https://github.com/kubernetes/ingress-nginx/releases/download/helm-chart-4.2.5/ingress-nginx-4.2.5.tgz

    2.解压，修改文件

    sudo tar -xvf ingress-nginx-4.2.5.tgz && sudo cd ingress-nginx

    #下面是已经修改完毕的，可以直接使用

    vim values.yaml

    1. ## nginx configuration
    2. ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md
    3.  
    4. commonLabels: {}
    5. # scmhash: abc123
    6. # myLabel: aakkmd
    7.  
    8. controller:
    9.   name: controller
    10.   image:
    11.     chroot: false
    12.     registry: registry.cn-hangzhou.aliyuncs.com
    13.     image: google_containers/nginx-ingress-controller
    14.     ## repository:
    15.     tag: "v1.3.1"
    16.     #digest: sha256:54f7fe2c6c5a9db9a0ebf1131797109bb7a4d91f56b9b362bde2abd237dd1974
    17.     #digestChroot: sha256:a8466b19c621bd550b1645e27a004a5cc85009c858a9ab19490216735ac432b1
    18.     pullPolicy: IfNotPresent
    19.     # www-data -> uid 101
    20.     runAsUser: 101
    21.     allowPrivilegeEscalation: true
    22.  
    23.   # -- Use an existing PSP instead of creating one
    24.   existingPsp: ""
    25.  
    26.   # -- Configures the controller container name
    27.   containerName: controller
    28.  
    29.   # -- Configures the ports that the nginx-controller listens on
    30.   containerPort:
    31.     http: 80
    32.     https: 443
    33.  
    34.   # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
    35.   config: {}
    36.  
    37.   # -- Annotations to be added to the controller config configuration configmap.
    38.   configAnnotations: {}
    39.  
    40.   # -- Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers
    41.   proxySetHeaders: {}
    42.  
    43.   # -- Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers
    44.   addHeaders: {}
    45.  
    46.   # -- Optionally customize the pod dnsConfig.
    47.   dnsConfig: {}
    48.  
    49.   # -- Optionally customize the pod hostname.
    50.   hostname: {}
    51.  
    52.   # -- Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'.
    53.   # By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller
    54.   # to keep resolving names inside the k8s network, use ClusterFirstWithHostNet.
    55.   dnsPolicy: ClusterFirstWithHostNet
    56.  
    57.   # -- Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network
    58.   # Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply
    59.   reportNodeInternalIp: false
    60.  
    61.   # -- Process Ingress objects without ingressClass annotation/ingressClassName field
    62.   # Overrides value for --watch-ingress-without-class flag of the controller binary
    63.   # Defaults to false
    64.   watchIngressWithoutClass: false
    65.  
    66.   # -- Process IngressClass per name (additionally as per spec.controller).
    67.   ingressClassByName: false
    68.  
    69.   # -- This configuration defines if Ingress Controller should allow users to set
    70.   # their own *-snippet annotations, otherwise this is forbidden / dropped
    71.   # when users add those annotations.
    72.   # Global snippets in ConfigMap are still respected
    73.   allowSnippetAnnotations: true
    74.  
    75.   # -- Required for use with CNI based kubernetes installations (such as ones set up by kubeadm),
    76.   # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920
    77.   # is merged
    78.   hostNetwork: true
    79.  
    80.   ## Use host ports 80 and 443
    81.   ## Disabled by default
    82.   hostPort:
    83.     # -- Enable 'hostPort' or not
    84.     enabled: false
    85.     ports:
    86.       # -- 'hostPort' http port
    87.       http: 80
    88.       # -- 'hostPort' https port
    89.       https: 443
    90.  
    91.   # -- Election ID to use for status update
    92.   electionID: ingress-controller-leader
    93.  
    94.   ## This section refers to the creation of the IngressClass resource
    95.   ## IngressClass resources are supported since k8s >= 1.18 and required since k8s >= 1.19
    96.   ingressClassResource:
    97.     # -- Name of the ingressClass
    98.     name: nginx
    99.     # -- Is this ingressClass enabled or not
    100.     enabled: true
    101.     # -- Is this the default ingressClass for the cluster
    102.     default: false
    103.     # -- Controller-value of the controller that is processing this ingressClass
    104.     controllerValue: "k8s.io/ingress-nginx"
    105.  
    106.     # -- Parameters is a link to a custom resource containing additional
    107.     # configuration for the controller. This is optional if the controller
    108.     # does not require extra parameters.
    109.     parameters: {}
    110.  
    111.   # -- For backwards compatibility with ingress.class annotation, use ingressClass.
    112.   # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation
    113.   ingressClass: nginx
    114.  
    115.   # -- Labels to add to the pod container metadata
    116.   podLabels: {}
    117.   #  key: value
    118.  
    119.   # -- Security Context policies for controller pods
    120.   podSecurityContext: {}
    121.  
    122.   # -- See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls
    123.   sysctls: {}
    124.   # sysctls:
    125.   #   "net.core.somaxconn": "8192"
    126.  
    127.   # -- Allows customization of the source of the IP address or FQDN to report
    128.   # in the ingress status field. By default, it reads the information provided
    129.   # by the service. If disable, the status field reports the IP address of the
    130.   # node or nodes where an ingress controller pod is running.
    131.   publishService:
    132.     # -- Enable 'publishService' or not
    133.     enabled: true
    134.     # -- Allows overriding of the publish service to bind to
    135.     # Must be /
    136.     pathOverride: ""
    137.  
    138.   # Limit the scope of the controller to a specific namespace
    139.   scope:
    140.     # -- Enable 'scope' or not
    141.     enabled: false
    142.     # -- Namespace to limit the controller to; defaults to $(POD_NAMESPACE)
    143.     namespace: ""
    144.     # -- When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels
    145.     # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces.
    146.     namespaceSelector: ""
    147.  
    148.   # -- Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE)
    149.   configMapNamespace: ""
    150.  
    151.   tcp:
    152.     # -- Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE)
    153.     configMapNamespace: ""
    154.     # -- Annotations to be added to the tcp config configmap
    155.     annotations: {}
    156.  
    157.   udp:
    158.     # -- Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE)
    159.     configMapNamespace: ""
    160.     # -- Annotations to be added to the udp config configmap
    161.     annotations: {}
    162.  
    163.   # -- Maxmind license key to download GeoLite2 Databases.
    164.   ## https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases
    165.   maxmindLicenseKey: ""
    166.  
    167.   # -- Additional command line arguments to pass to nginx-ingress-controller
    168.   # E.g. to specify the default SSL certificate you can use
    169.   extraArgs: {}
    170.   ## extraArgs:
    171.   ##   default-ssl-certificate: "/"
    172.  
    173.   # -- Additional environment variables to set
    174.   extraEnvs: []
    175.   # extraEnvs:
    176.   #   - name: FOO
    177.   #     valueFrom:
    178.   #       secretKeyRef:
    179.   #         key: FOO
    180.   #         name: secret-resource
    181.  
    182.   # -- Use a `DaemonSet` or `Deployment`
    183.   kind: DaemonSet
    184.  
    185.   # -- Annotations to be added to the controller Deployment or DaemonSet
    186.   ##
    187.   annotations: {}
    188.   #  keel.sh/pollSchedule: "@every 60m"
    189.  
    190.   # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels
    191.   ##
    192.   labels: {}
    193.   #  keel.sh/policy: patch
    194.   #  keel.sh/trigger: poll
    195.  
    196.  
    197.   # -- The update strategy to apply to the Deployment or DaemonSet
    198.   ##
    199.   updateStrategy: {}
    200.   #  rollingUpdate:
    201.   #    maxUnavailable: 1
    202.   #  type: RollingUpdate
    203.  
    204.   # -- `minReadySeconds` to avoid killing pods before we are ready
    205.   ##
    206.   minReadySeconds: 0
    207.  
    208.  
    209.   # -- Node tolerations for server scheduling to nodes with taints
    210.   ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
    211.   ##
    212.   tolerations: []
    213.   #  - key: "key"
    214.   #    operator: "Equal|Exists"
    215.   #    value: "value"
    216.   #    effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)"
    217.  
    218.   # -- Affinity and anti-affinity rules for server scheduling to nodes
    219.   ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
    220.   ##
    221.   affinity: {}
    222.     # # An example of preferred pod anti-affinity, weight is in the range 1-100
    223.     # podAntiAffinity:
    224.     #   preferredDuringSchedulingIgnoredDuringExecution:
    225.     #   - weight: 100
    226.     #     podAffinityTerm:
    227.     #       labelSelector:
    228.     #         matchExpressions:
    229.     #         - key: app.kubernetes.io/name
    230.     #           operator: In
    231.     #           values:
    232.     #           - ingress-nginx
    233.     #         - key: app.kubernetes.io/instance
    234.     #           operator: In
    235.     #           values:
    236.     #           - ingress-nginx
    237.     #         - key: app.kubernetes.io/component
    238.     #           operator: In
    239.     #           values:
    240.     #           - controller
    241.     #       topologyKey: kubernetes.io/hostname
    242.  
    243.     # # An example of required pod anti-affinity
    244.     # podAntiAffinity:
    245.     #   requiredDuringSchedulingIgnoredDuringExecution:
    246.     #   - labelSelector:
    247.     #       matchExpressions:
    248.     #       - key: app.kubernetes.io/name
    249.     #         operator: In
    250.     #         values:
    251.     #         - ingress-nginx
    252.     #       - key: app.kubernetes.io/instance
    253.     #         operator: In
    254.     #         values:
    255.     #         - ingress-nginx
    256.     #       - key: app.kubernetes.io/component
    257.     #         operator: In
    258.     #         values:
    259.     #         - controller
    260.     #     topologyKey: "kubernetes.io/hostname"
    261.  
    262.   # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
    263.   ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
    264.   ##
    265.   topologySpreadConstraints: []
    266.     # - maxSkew: 1
    267.     #   topologyKey: topology.kubernetes.io/zone
    268.     #   whenUnsatisfiable: DoNotSchedule
    269.     #   labelSelector:
    270.     #     matchLabels:
    271.     #       app.kubernetes.io/instance: ingress-nginx-internal
    272.  
    273.   # -- `terminationGracePeriodSeconds` to avoid killing pods before we are ready
    274.   ## wait up to five minutes for the drain of connections
    275.   ##
    276.   terminationGracePeriodSeconds: 300
    277.  
    278.   # -- Node labels for controller pod assignment
    279.   ## Ref: https://kubernetes.io/docs/user-guide/node-selection/
    280.   ##
    281.   nodeSelector:
    282.     kubernetes.io/os: linux
    283.     ingress: "true"
    284.   ## Liveness and readiness probe values
    285.   ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
    286.   ##
    287.   ## startupProbe:
    288.   ##   httpGet:
    289.   ##     # should match container.healthCheckPath
    290.   ##     path: "/healthz"
    291.   ##     port: 10254
    292.   ##     scheme: HTTP
    293.   ##   initialDelaySeconds: 5
    294.   ##   periodSeconds: 5
    295.   ##   timeoutSeconds: 2
    296.   ##   successThreshold: 1
    297.   ##   failureThreshold: 5
    298.   livenessProbe:
    299.     httpGet:
    300.       # should match container.healthCheckPath
    301.       path: "/healthz"
    302.       port: 10254
    303.       scheme: HTTP
    304.     initialDelaySeconds: 10
    305.     periodSeconds: 10
    306.     timeoutSeconds: 1
    307.     successThreshold: 1
    308.     failureThreshold: 5
    309.   readinessProbe:
    310.     httpGet:
    311.       # should match container.healthCheckPath
    312.       path: "/healthz"
    313.       port: 10254
    314.       scheme: HTTP
    315.     initialDelaySeconds: 10
    316.     periodSeconds: 10
    317.     timeoutSeconds: 1
    318.     successThreshold: 1
    319.     failureThreshold: 3
    320.  
    321.  
    322.   # -- Path of the health check endpoint. All requests received on the port defined by
    323.   # the healthz-port parameter are forwarded internally to this path.
    324.   healthCheckPath: "/healthz"
    325.  
    326.   # -- Address to bind the health check endpoint.
    327.   # It is better to set this option to the internal node address
    328.   # if the ingress nginx controller is running in the `hostNetwork: true` mode.
    329.   healthCheckHost: ""
    330.  
    331.   # -- Annotations to be added to controller pods
    332.   ##
    333.   podAnnotations: {}
    334.  
    335.   replicaCount: 1
    336.  
    337.   minAvailable: 1
    338.  
    339.   ## Define requests resources to avoid probe issues due to CPU utilization in busy nodes
    340.   ## ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903
    341.   ## Ideally, there should be no limits.
    342.   ## https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/
    343.   resources:
    344.   ##  limits:
    345.   ##    cpu: 100m
    346.   ##    memory: 90Mi
    347.     requests:
    348.       cpu: 100m
    349.       memory: 90Mi
    350.  
    351.   # Mutually exclusive with keda autoscaling
    352.   autoscaling:
    353.     enabled: false
    354.     minReplicas: 1
    355.     maxReplicas: 11
    356.     targetCPUUtilizationPercentage: 50
    357.     targetMemoryUtilizationPercentage: 50
    358.     behavior: {}
    359.       # scaleDown:
    360.       #   stabilizationWindowSeconds: 300
    361.       #  policies:
    362.       #   - type: Pods
    363.       #     value: 1
    364.       #     periodSeconds: 180
    365.       # scaleUp:
    366.       #   stabilizationWindowSeconds: 300
    367.       #   policies:
    368.       #   - type: Pods
    369.       #     value: 2
    370.       #     periodSeconds: 60
    371.  
    372.   autoscalingTemplate: []
    373.   # Custom or additional autoscaling metrics
    374.   # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics
    375.   # - type: Pods
    376.   #   pods:
    377.   #     metric:
    378.   #       name: nginx_ingress_controller_nginx_process_requests_total
    379.   #     target:
    380.   #       type: AverageValue
    381.   #       averageValue: 10000m
    382.  
    383.   # Mutually exclusive with hpa autoscaling
    384.   keda:
    385.     apiVersion: "keda.sh/v1alpha1"
    386.     ## apiVersion changes with keda 1.x vs 2.x
    387.     ## 2.x = keda.sh/v1alpha1
    388.     ## 1.x = keda.k8s.io/v1alpha1
    389.     enabled: false
    390.     minReplicas: 1
    391.     maxReplicas: 11
    392.     pollingInterval: 30
    393.     cooldownPeriod: 300
    394.     restoreToOriginalReplicaCount: false
    395.     scaledObject:
    396.       annotations: {}
    397.       # Custom annotations for ScaledObject resource
    398.       #  annotations:
    399.       # key: value
    400.     triggers: []
    401.  #     - type: prometheus
    402.  #       metadata:
    403.  #         serverAddress: http://:9090
    404.  #         metricName: http_requests_total
    405.  #         threshold: '100'
    406.  #         query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
    407.  
    408.     behavior: {}
    409.  #     scaleDown:
    410.  #       stabilizationWindowSeconds: 300
    411.  #       policies:
    412.  #       - type: Pods
    413.  #         value: 1
    414.  #         periodSeconds: 180
    415.  #     scaleUp:
    416.  #       stabilizationWindowSeconds: 300
    417.  #       policies:
    418.  #       - type: Pods
    419.  #         value: 2
    420.  #         periodSeconds: 60
    421.  
    422.   # -- Enable mimalloc as a drop-in replacement for malloc.
    423.   ## ref: https://github.com/microsoft/mimalloc
    424.   ##
    425.   enableMimalloc: true
    426.  
    427.   ## Override NGINX template
    428.   customTemplate:
    429.     configMapName: ""
    430.     configMapKey: ""
    431.  
    432.   service:
    433.     enabled: true
    434.  
    435.     # -- If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were
    436.     # using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
    437.     # It allows choosing the protocol for each backend specified in the Kubernetes service.
    438.     # See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244
    439.     # Will be ignored for Kubernetes versions older than 1.20
    440.     ##
    441.     appProtocol: true
    442.  
    443.     annotations: {}
    444.     labels: {}
    445.     # clusterIP: ""
    446.  
    447.     # -- List of IP addresses at which the controller services are available
    448.     ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
    449.     ##
    450.     externalIPs: []
    451.  
    452.     # -- Used by cloud providers to connect the resulting `LoadBalancer` to a pre-existing static IP according to https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer
    453.     loadBalancerIP: ""
    454.     loadBalancerSourceRanges: []
    455.  
    456.     enableHttp: true
    457.     enableHttps: true
    458.  
    459.     ## Set external traffic policy to: "Local" to preserve source IP on providers supporting it.
    460.     ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer
    461.     # externalTrafficPolicy: ""
    462.  
    463.     ## Must be either "None" or "ClientIP" if set. Kubernetes will default to "None".
    464.     ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies
    465.     # sessionAffinity: ""
    466.  
    467.     ## Specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified,
    468.     ## the service controller allocates a port from your cluster’s NodePort range.
    469.     ## Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
    470.     # healthCheckNodePort: 0
    471.  
    472.     # -- Represents the dual-stack-ness requested or required by this Service. Possible values are
    473.     # SingleStack, PreferDualStack or RequireDualStack.
    474.     # The ipFamilies and clusterIPs fields depend on the value of this field.
    475.     ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/
    476.     ipFamilyPolicy: "SingleStack"
    477.  
    478.     # -- List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically
    479.     # based on cluster configuration and the ipFamilyPolicy field.
    480.     ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/
    481.     ipFamilies:
    482.       - IPv4
    483.  
    484.     ports:
    485.       http: 80
    486.       https: 443
    487.  
    488.     targetPorts:
    489.       http: http
    490.       https: https
    491.  
    492.     type: LoadBalancer
    493.  
    494.     ## type: NodePort
    495.     ## nodePorts:
    496.     ##   http: 32080
    497.     ##   https: 32443
    498.     ##   tcp:
    499.     ##     8080: 32808
    500.     nodePorts:
    501.       http: ""
    502.       https: ""
    503.       tcp: {}
    504.       udp: {}
    505.  
    506.     external:
    507.       enabled: true
    508.  
    509.     internal:
    510.       # -- Enables an additional internal load balancer (besides the external one).
    511.       enabled: false
    512.       # -- Annotations are mandatory for the load balancer to come up. Varies with the cloud service.
    513.       annotations: {}
    514.  
    515.       # loadBalancerIP: ""
    516.  
    517.       # -- Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0.
    518.       loadBalancerSourceRanges: []
    519.  
    520.       ## Set external traffic policy to: "Local" to preserve source IP on
    521.       ## providers supporting it
    522.       ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer
    523.       # externalTrafficPolicy: ""
    524.  
    525.   # shareProcessNamespace enables process namespace sharing within the pod.
    526.   # This can be used for example to signal log rotation using `kill -USR1` from a sidecar.
    527.   shareProcessNamespace: false
    528.  
    529.   # -- Additional containers to be added to the controller pod.
    530.   # See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example.
    531.   extraContainers: []
    532.   #  - name: my-sidecar
    533.   #      - name: POD_NAME
    534.   #        valueFrom:
    535.   #          fieldRef:
    536.   #            fieldPath: metadata.name
    537.   #      - name: POD_NAMESPACE
    538.   #        valueFrom:
    539.   #          fieldRef:
    540.   #            fieldPath: metadata.namespace
    541.   #    volumeMounts:
    542.   #    - name: copy-portal-skins
    543.   #      mountPath: /srv/var/lib/lemonldap-ng/portal/skins
    544.  
    545.   # -- Additional volumeMounts to the controller main container.
    546.   extraVolumeMounts: []
    547.   #  - name: copy-portal-skins
    548.   #   mountPath: /var/lib/lemonldap-ng/portal/skins
    549.  
    550.   # -- Additional volumes to the controller pod.
    551.   extraVolumes: []
    552.   #  - name: copy-portal-skins
    553.   #    emptyDir: {}
    554.  
    555.   # -- Containers, which are run before the app containers are started.
    556.   extraInitContainers: []
    557.   # - name: init-myservice
    558.   #   command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;']
    559.  
    560.   extraModules: []
    561.   ## Modules, which are mounted into the core nginx image
    562.   # - name: opentelemetry
    563.   #
    564.   # The image must contain a `/usr/local/bin/init_module.sh` executable, which
    565.   # will be executed as initContainers, to move its config files within the
    566.   # mounted volume.
    567.  
    568.   admissionWebhooks:
    569.     annotations: {}
    570.     # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem".
    571.  
    572.     ## Additional annotations to the admission webhooks.
    573.     ## These annotations will be added to the ValidatingWebhookConfiguration and
    574.     ## the Jobs Spec of the admission webhooks.
    575.     enabled: true
    576.     # -- Additional environment variables to set
    577.     extraEnvs: []
    578.     # extraEnvs:
    579.     #   - name: FOO
    580.     #     valueFrom:
    581.     #       secretKeyRef:
    582.     #         key: FOO
    583.     #         name: secret-resource
    584.     # -- Admission Webhook failure policy to use
    585.     failurePolicy: Fail
    586.     # timeoutSeconds: 10
    587.     port: 8443
    588.     certificate: "/usr/local/certificates/cert"
    589.     key: "/usr/local/certificates/key"
    590.     namespaceSelector: {}
    591.     objectSelector: {}
    592.     # -- Labels to be added to admission webhooks
    593.     labels: {}
    594.  
    595.     # -- Use an existing PSP instead of creating one
    596.     existingPsp: ""
    597.     networkPolicyEnabled: false
    598.  
    599.     service:
    600.       annotations: {}
    601.       # clusterIP: ""
    602.       externalIPs: []
    603.       # loadBalancerIP: ""
    604.       loadBalancerSourceRanges: []
    605.       servicePort: 443
    606.       type: ClusterIP
    607.  
    608.     createSecretJob:
    609.       resources: {}
    610.         # limits:
    611.         #   cpu: 10m
    612.         #   memory: 20Mi
    613.         # requests:
    614.         #   cpu: 10m
    615.         #   memory: 20Mi
    616.  
    617.     patchWebhookJob:
    618.       resources: {}
    619.  
    620.     patch:
    621.       enabled: true
    622.       image:
    623.         registry: registry.cn-hangzhou.aliyuncs.com
    624.         image: google_containers/kube-webhook-certgen
    625.         ## for backwards compatibility consider setting the full image url via the repository value below
    626.         ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
    627.         ## repository:
    628.         tag: v1.3.0
    629.         #    digest: sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47
    630.         pullPolicy: IfNotPresent
    631.       # -- Provide a priority class name to the webhook patching job
    632.       ##
    633.       priorityClassName: ""
    634.       podAnnotations: {}
    635.       nodeSelector:
    636.         kubernetes.io/os: linux
    637.       tolerations: []
    638.       # -- Labels to be added to patch job resources
    639.       labels: {}
    640.       securityContext:
    641.         runAsNonRoot: true
    642.         runAsUser: 2000
    643.         fsGroup: 2000
    644.  
    645.  
    646.   metrics:
    647.     port: 10254
    648.     # if this port is changed, change healthz-port: in extraArgs: accordingly
    649.     enabled: false
    650.  
    651.     service:
    652.       annotations: {}
    653.       # prometheus.io/scrape: "true"
    654.       # prometheus.io/port: "10254"
    655.  
    656.       # clusterIP: ""
    657.  
    658.       # -- List of IP addresses at which the stats-exporter service is available
    659.       ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
    660.       ##
    661.       externalIPs: []
    662.  
    663.       # loadBalancerIP: ""
    664.       loadBalancerSourceRanges: []
    665.       servicePort: 10254
    666.       type: ClusterIP
    667.       # externalTrafficPolicy: ""
    668.       # nodePort: ""
    669.  
    670.     serviceMonitor:
    671.       enabled: false
    672.       additionalLabels: {}
    673.       ## The label to use to retrieve the job name from.
    674.       ## jobLabel: "app.kubernetes.io/name"
    675.       namespace: ""
    676.       namespaceSelector: {}
    677.       ## Default: scrape .Release.Namespace only
    678.       ## To scrape all, use the following:
    679.       ## namespaceSelector:
    680.       ##   any: true
    681.       scrapeInterval: 30s
    682.       # honorLabels: true
    683.       targetLabels: []
    684.       relabelings: []
    685.       metricRelabelings: []
    686.  
    687.     prometheusRule:
    688.       enabled: false
    689.       additionalLabels: {}
    690.       # namespace: ""
    691.       rules: []
    692.         # # These are just examples rules, please adapt them to your needs
    693.         # - alert: NGINXConfigFailed
    694.         #   expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0
    695.         #   for: 1s
    696.         #   labels:
    697.         #     severity: critical
    698.         #   annotations:
    699.         #     description: bad ingress config - nginx config test failed
    700.         #     summary: uninstall the latest ingress changes to allow config reloads to resume
    701.         # - alert: NGINXCertificateExpiry
    702.         #   expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800
    703.         #   for: 1s
    704.         #   labels:
    705.         #     severity: critical
    706.         #   annotations:
    707.         #     description: ssl certificate(s) will expire in less then a week
    708.         #     summary: renew expiring certificates to avoid downtime
    709.         # - alert: NGINXTooMany500s
    710.         #   expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5
    711.         #   for: 1m
    712.         #   labels:
    713.         #     severity: warning
    714.         #   annotations:
    715.         #     description: Too many 5XXs
    716.         #     summary: More than 5% of all requests returned 5XX, this requires your attention
    717.         # - alert: NGINXTooMany400s
    718.         #   expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5
    719.         #   for: 1m
    720.         #   labels:
    721.         #     severity: warning
    722.         #   annotations:
    723.         #     description: Too many 4XXs
    724.         #     summary: More than 5% of all requests returned 4XX, this requires your attention
    725.  
    726.   # -- Improve connection draining when ingress controller pod is deleted using a lifecycle hook:
    727.   # With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds
    728.   # to 300, allowing the draining of connections up to five minutes.
    729.   # If the active connections end before that, the pod will terminate gracefully at that time.
    730.   # To effectively take advantage of this feature, the Configmap feature
    731.   # worker-shutdown-timeout new value is 240s instead of 10s.
    732.   ##
    733.   lifecycle:
    734.     preStop:
    735.       exec:
    736.         command:
    737.           - /wait-shutdown
    738.  
    739.   priorityClassName: ""
    740.  
    741. # -- Rollback limit
    742. ##
    743. revisionHistoryLimit: 10
    744.  
    745. ## Default 404 backend
    746. ##
    747. defaultBackend:
    748.   ##
    749.   enabled: false
    750.  
    751.   name: defaultbackend
    752.   image:
    753.     registry: k8s.gcr.io
    754.     image: defaultbackend-amd64
    755.     ## for backwards compatibility consider setting the full image url via the repository value below
    756.     ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
    757.     ## repository:
    758.     tag: "1.5"
    759.     pullPolicy: IfNotPresent
    760.     # nobody user -> uid 65534
    761.     runAsUser: 65534
    762.     runAsNonRoot: true
    763.     readOnlyRootFilesystem: true
    764.     allowPrivilegeEscalation: false
    765.  
    766.   # -- Use an existing PSP instead of creating one
    767.   existingPsp: ""
    768.  
    769.   extraArgs: {}
    770.  
    771.   serviceAccount:
    772.     create: true
    773.     name: ""
    774.     automountServiceAccountToken: true
    775.   # -- Additional environment variables to set for defaultBackend pods
    776.   extraEnvs: []
    777.  
    778.   port: 8080
    779.  
    780.   ## Readiness and liveness probes for default backend
    781.   ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
    782.   ##
    783.   livenessProbe:
    784.     failureThreshold: 3
    785.     initialDelaySeconds: 30
    786.     periodSeconds: 10
    787.     successThreshold: 1
    788.     timeoutSeconds: 5
    789.   readinessProbe:
    790.     failureThreshold: 6
    791.     initialDelaySeconds: 0
    792.     periodSeconds: 5
    793.     successThreshold: 1
    794.     timeoutSeconds: 5
    795.  
    796.   # -- Node tolerations for server scheduling to nodes with taints
    797.   ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
    798.   ##
    799.   tolerations: []
    800.   #  - key: "key"
    801.   #    operator: "Equal|Exists"
    802.   #    value: "value"
    803.   #    effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)"
    804.  
    805.   affinity: {}
    806.  
    807.   # -- Security Context policies for controller pods
    808.   # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
    809.   # notes on enabling and using sysctls
    810.   ##
    811.   podSecurityContext: {}
    812.  
    813.   # -- Security Context policies for controller main container.
    814.   # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
    815.   # notes on enabling and using sysctls
    816.   ##
    817.   containerSecurityContext: {}
    818.  
    819.   # -- Labels to add to the pod container metadata
    820.   podLabels: {}
    821.   #  key: value
    822.  
    823.   # -- Node labels for default backend pod assignment
    824.   ## Ref: https://kubernetes.io/docs/user-guide/node-selection/
    825.   ##
    826.   nodeSelector:
    827.     kubernetes.io/os: linux
    828.  
    829.   # -- Annotations to be added to default backend pods
    830.   ##
    831.   podAnnotations: {}
    832.  
    833.   replicaCount: 1
    834.  
    835.   minAvailable: 1
    836.  
    837.   resources: {}
    838.   # limits:
    839.   #   cpu: 10m
    840.   #   memory: 20Mi
    841.   # requests:
    842.   #   cpu: 10m
    843.   #   memory: 20Mi
    844.  
    845.   extraVolumeMounts: []
    846.   ## Additional volumeMounts to the default backend container.
    847.   #  - name: copy-portal-skins
    848.   #   mountPath: /var/lib/lemonldap-ng/portal/skins
    849.  
    850.   extraVolumes: []
    851.   ## Additional volumes to the default backend pod.
    852.   #  - name: copy-portal-skins
    853.   #    emptyDir: {}
    854.  
    855.   autoscaling:
    856.     annotations: {}
    857.     enabled: false
    858.     minReplicas: 1
    859.     maxReplicas: 2
    860.     targetCPUUtilizationPercentage: 50
    861.     targetMemoryUtilizationPercentage: 50
    862.  
    863.   service:
    864.     annotations: {}
    865.  
    866.     # clusterIP: ""
    867.  
    868.     # -- List of IP addresses at which the default backend service is available
    869.     ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
    870.     ##
    871.     externalIPs: []
    872.  
    873.     # loadBalancerIP: ""
    874.     loadBalancerSourceRanges: []
    875.     servicePort: 80
    876.     type: ClusterIP
    877.  
    878.   priorityClassName: ""
    879.   # -- Labels to be added to the default backend resources
    880.   labels: {}
    881.  
    882. ## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266
    883. rbac:
    884.   create: true
    885.   scope: false
    886.  
    887. ## If true, create & use Pod Security Policy resources
    888. ## https://kubernetes.io/docs/concepts/policy/pod-security-policy/
    889. podSecurityPolicy:
    890.   enabled: false
    891.  
    892. serviceAccount:
    893.   create: true
    894.   name: ""
    895.   automountServiceAccountToken: true
    896.   # -- Annotations for the controller service account
    897.   annotations: {}
    898.  
    899. # -- Optional array of imagePullSecrets containing private registry credentials
    900. ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
    901. imagePullSecrets: []
    902. # - name: secretName
    903.  
    904. # -- TCP service key-value pairs
    905. ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md
    906. ##
    907. tcp: {}
    908. #  8080: "default/example-tcp-svc:9000"
    909.  
    910. # -- UDP service key-value pairs
    911. ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md
    912. ##
    913. udp: {}
    914. #  53: "kube-system/kube-dns:53"
    915.  
    916. # -- Prefix for TCP and UDP ports names in ingress controller service
    917. ## Some cloud providers, like Yandex Cloud may have a requirements for a port name regex to support cloud load balancer integration
    918. portNamePrefix: ""
    919.  
    920. # -- (string) A base64-encoded Diffie-Hellman parameter.
    921. # This can be generated with: `openssl dhparam 4096 2> /dev/null | base64`
    922. ## Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param
    923. dhParam:

    3.安装ingress

    1. # 选择节点打label
    2.  kubectl label node k8s-node01 ingress=true  # k8s-node01是自己自定义的node节点名称
    3.  kubectl get node --show-labels
    4.  
    5. #创建命名空间 
    6.  kubectl create ns ingress-nginx
    7.  
    8. # 使用helm进行安装 
    9.  helm install ingress-nginx -f values.yaml -n ingress-nginx .
    10.  
    11.  helm list -n ingress-nginx
    12.  kubectl -n ingress-nginx get pods  -o wide
    13.  kubectl -n ingress-nginx get svc -o wide
    14.  
    15.  
    16. # 删除ingress-nginx
    17.  helm delete ingress-nginx -n ingress-nginx
    18. # 更新ingress-nginx
    19.  helm upgrade ingress-nginx -n -f values.yaml -n ingress-nginx .

    4.测试网页

    vim test-nginx.yaml

    1. ---
    2. apiVersion: apps/v1
    3. kind: Deployment
    4. metadata:
    5.   name: my-deploy-nginx
    6. spec:
    7.   replicas: 2
    8.   selector:
    9.     matchLabels:
    10.       app: mynginx
    11.   template:
    12.     metadata:
    13.       labels:
    14.         app: mynginx
    15.     spec:
    16.       containers:
    17.       - name: mynginx
    18.         image: nginx
    19.         ports:
    20.         - containerPort: 80
    21. ---
    22. kind: Service
    23. apiVersion: v1
    24. metadata:
    25.   name: myservice
    26. spec:
    27.   ports:
    28.     - protocol: TCP
    29.       port: 80
    30.       targetPort: 80
    31.   selector:
    32.     app: mynginx
    33.   type: ClusterIP
    34. ---
    35. apiVersion: networking.k8s.io/v1
    36. kind: Ingress
    37. metadata:
    38.   name: mynginx
    39. spec:
    40.   ingressClassName: "nginx"    #在部署ingress-nginx时，valume.yaml文件中定义的
    41.   rules:
    42.   - host: nginx.rw.com
    43.     http:
    44.       paths:
    45.       - path: /
    46.         pathType: Prefix
    47.         backend:
    48.           service:
    49.             name: myservice
    50.             port:
    51.               number: 80

    kubectl apply -f test-nginx.yaml

    kubectl  get all

    kubectl get ingress

    四、k8sHPA 自动水平伸缩pod

    1.pod内资源分配的配置格式如下：

    默认可以只配置requests，但根据生产中的经验，建议把limits资源限制也加上，因为对K8s来说，只有这两个都配置了且配置的值都要一样，这个pod资源的优先级才是最高的，在node资源不够的情况下，首先是把没有任何资源分配配置的pod资源给干掉，其次是只配置了requests的，最后才是两个都配置的情况，仔细品品
     

          resources:
            limits:   # 限制单个pod最多能使用1核（1000m 毫核）cpu以及2G内存
              cpu: "1"
              memory: 2Gi
            requests: # 保证这个pod初始就能分配这么多资源
              cpu: "1"
              memory: 2Gi

    2.我们现在以上面创建的deployment资源web来实践下hpa的效果，首先用我们学到的方法导出web的yaml配置，并增加资源分配配置增加

    apiVersion: v1
    kind: Service
    metadata:
      labels:
        app: web
      name: web
    spec:
      ports:
      - port: 80
        protocol: TCP
        targetPort: 80
      selector:
        app: web
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: web
      name: web
      namespace: default
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: web
      template:
        metadata:
          labels:
            app: web
        spec:
          containers:
          - image: nginx
            name: web
            resources:
              limits:   # 因为我这里是测试环境，所以这里CPU只分配50毫核（0.05核CPU）和20M的内存
                cpu: "50m"
                memory: 20Mi
              requests: # 保证这个pod初始就能分配这么多资源
                cpu: "50m"
                memory: 20Mi

    3.运行

    kubectl  apply -f web.yaml

    4.第一种：为deployment资源web创建hpa，pod数量上限3个，最低1个，在pod平均CPU达到50%后开始扩容

    kubectl  autoscale deployment web --max=3 --min=1 --cpu-percent=50

    5.第二种创建hpa

    cat hpa-web.yaml

    1. apiVersion: autoscaling/v2beta1                         # v2beta1版本
    2. #apiVersion: apps/v1
    3. kind: HorizontalPodAutoscaler
    4. metadata:
    5.   name: web
    6. spec:
    7.   maxReplicas: 10
    8.   minReplicas: 1                                                        # 1-10个pod范围内扩容与裁剪
    9.   scaleTargetRef:
    10.     apiVersion: apps/v1
    11.     kind: Deployment
    12.     name: web
    13.   metrics:
    14.   - type: Resource
    15.     resource:
    16.       name: memory
    17.       targetAverageUtilization: 50                      # 50%内存利用

    6.执行

    kubectl apply -f hpa-web.yaml

    7.我们启动一个临时pod，来模拟大量请求

    kubectl run -it --rm busybox --image=busybox -- sh
    / # while :;do wget -q -O- http://web;done

    #等待2 ~ 3分钟，注意k8s为了避免频繁增删pod，对副本的增加速度有限制

    kubectl get hpa web -w

    236eee9b4ba2418da17427aceae4a9a9.png

    五、k8s存储

    5.1 k8s持久化存储02pv pvc

    开始部署NFS-SERVER

    1. # 我们这里在10.0.1.201上安装（在生产中，大家要提供作好NFS-SERVER环境的规划）
    2. yum -y install nfs-utils
    3.  
    4. # 创建NFS挂载目录
    5. mkdir /nfs_dir
    6. chown nobody.nobody /nfs_dir
    7.  
    8. # 修改NFS-SERVER配置
    9. echo '/nfs_dir *(rw,sync,no_root_squash)' > /etc/exports
    10.  
    11. # 重启服务
    12. systemctl restart rpcbind.service
    13. systemctl restart nfs-utils.service 
    14. systemctl restart nfs-server.service 
    15.  
    16. # 增加NFS-SERVER开机自启动
    17. systemctl enable  rpcbind.service
    18. systemctl enable  nfs-utils.service 
    19. systemctl enable  nfs-server.service 
    20.  
    21. # 验证NFS-SERVER是否能正常访问
    22. showmount -e 10.0.1.201                 
    23.  
    24.  
    25.  
    26.  
    27. #需要挂载的服务器执行
    28. yum install nfs-utils -y
    29.  
    30.

    接着准备好pv的yaml配置，保存为pv1.yaml

    1. # cat pv1.yaml 
    2. apiVersion: v1
    3. kind: PersistentVolume
    4. metadata:
    5.   name: pv1
    6.   labels:
    7.     type: test-claim    # 这里建议打上一个独有的标签，方便在多个pv的时候方便提供pvc选择挂载
    8. spec:
    9.   capacity:
    10.     storage: 1Gi     # <----------  1
    11.   accessModes:
    12.     - ReadWriteOnce     # <----------  2
    13.   persistentVolumeReclaimPolicy: Recycle     # <----------  3
    14.   storageClassName: nfs     # <----------  4
    15.   nfs:
    16.     path: /nfs_dir/pv1     # <----------  5
    17.     server: 10.0.1.201
    18.

    1. capacity 指定 PV 的容量为 1G。
    2. accessModes 指定访问模式为 ReadWriteOnce，支持的访问模式有： ReadWriteOnce – PV 能以 read-write 模式 mount 到单个节点。 ReadOnlyMany – PV 能以 read-only 模式 mount 到多个节点。 ReadWriteMany – PV 能以 read-write 模式 mount 到多个节点。
    3. persistentVolumeReclaimPolicy 指定当 PV 的回收策略为 Recycle，支持的策略有： Retain – 需要管理员手工回收。 Recycle – 清除 PV 中的数据，效果相当于执行 rm -rf /thevolume/*。 Delete – 删除 Storage Provider 上的对应存储资源，例如 AWS EBS、GCE PD、Azure Disk、OpenStack Cinder Volume 等。
    4. storageClassName 指定 PV 的 class 为 nfs。相当于为 PV 设置了一个分类，PVC 可以指定 class 申请相应 class 的 PV。
    5. 指定 PV 在 NFS 服务器上对应的目录，这里注意，我测试的时候，需要手动先创建好这个目录并授权好，不然后面挂载会提示目录不存在 mkdir /nfsdata/pv1 && chown -R nobody.nogroup /nfsdata 。

    创建这个pv

    1. # kubectl apply -f pv1.yaml 
    2. persistentvolume/pv1 created
    3.  
    4. # STATUS 为 Available，表示 pv1 就绪，可以被 PVC 申请
    5. # kubectl get pv
    6. NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
    7. pv1    1Gi        RWO            Recycle          Available           nfs                     4m45s
    8.

    接着准备PVC的yaml，保存为pvc1.yaml

    cat pvc1.yaml

    1. apiVersion: v1
    2. kind: PersistentVolumeClaim
    3. metadata:
    4.   name: pvc1
    5. spec:
    6.   accessModes:
    7.     - ReadWriteOnce
    8.   resources:
    9.     requests:
    10.       storage: 1Gi
    11.   storageClassName: nfs
    12.   selector:
    13.     matchLabels:
    14.       type: test-claim

    创建这个pvc

    1. # kubectl apply -f pvc1.yaml          
    2. persistentvolumeclaim/pvc1 created
    3.  
    4. # 看下pvc的STATUS为Bound代表成功挂载到pv了
    5. # kubectl get pvc           
    6. NAME   STATUS   VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    7. pvc1   Bound    pv1      1Gi        RWO            nfs            2s
    8.  
    9. # 这个时候再看下pv，STATUS也是Bound了，同时CLAIM提示被default/pvc1消费
    10. # kubectl get pv
    11. NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM          STORAGECLASS   REASON   AGE
    12. pv1    1Gi        RWO            Recycle          Bound    default/pvc1   nfs  
    13.

    下面讲下如何回收PVC以及PV

    1. # 这里删除时会一直卡着，我们按ctrl+c看看怎么回事
    2. # kubectl delete pvc pvc1 
    3. persistentvolumeclaim "pvc1" deleted
    4. ^C
    5.  
    6. # 看下pvc发现STATUS是Terminating删除中的状态，我分析是因为服务pod还在占用这个pvc使用中
    7. # kubectl get pvc
    8. NAME   STATUS        VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    9. pvc1   Terminating   pv1      1Gi        RWO            nfs            21m
    10.  
    11. # 先删除这个pod
    12. # kubectl delete pod nginx-569546db98-99qpq 
    13. pod "nginx-569546db98-99qpq" deleted
    14.  
    15. # 再看先删除的pvc已经没有了
    16. # kubectl get pvc
    17. No resources found in default namespace.
    18.  
    19. # 根据先前创建pv时的数据回收策略为Recycle – 清除 PV 中的数据，这时果然先创建的index.html已经被删除了，在生产中要尤其注意这里的模式，注意及时备份数据，注意及时备份数据，注意及时备份数据
    20. # ll /nfs_dir/pv1/
    21. total 0
    22.  
    23. # 虽然此时pv是可以再次被pvc来消费的，但根据生产的经验，建议在删除pvc时，也同时把它消费的pv一并删除，然后再重启创建都是可以的
    24.

    5.2 StorageClass

    1.k8s持久化存储的第三节，给大家带来 StorageClass动态存储的讲解。

    我们上节课提到了K8s对于存储解耦的设计是，pv交给存储管理员来管理，我们只管用pvc来消费就好，但这里我们实际还是得一起管理pv和pvc，在实际工作中，我们（存储管理员）可以提前配置好pv的动态供给StorageClass，来根据pvc的消费动态生成pv。
    StorageClass

    我这是直接拿生产中用的实例来作演示，利用nfs-client-provisioner来生成一个基于nfs的StorageClass，部署配置yaml配置如下，保持为nfs-sc.yaml：

    1. apiVersion: v1
    2. kind: ServiceAccount
    3. metadata:
    4.   name: nfs-client-provisioner
    5.   namespace: kube-system
    6.  
    7. ---
    8. kind: ClusterRole
    9. apiVersion: rbac.authorization.k8s.io/v1
    10. metadata:
    11.   name: nfs-client-provisioner-runner
    12. rules:
    13.   - apiGroups: [""]
    14.     resources: ["persistentvolumes"]
    15.     verbs: ["get", "list", "watch", "create", "delete"]
    16.   - apiGroups: [""]
    17.     resources: ["persistentvolumeclaims"]
    18.     verbs: ["get", "list", "watch", "update"]
    19.   - apiGroups: ["storage.k8s.io"]
    20.     resources: ["storageclasses"]
    21.     verbs: ["get", "list", "watch"]
    22.   - apiGroups: [""]
    23.     resources: ["events"]
    24.     verbs: ["list", "watch", "create", "update", "patch"]
    25.   - apiGroups: [""]
    26.     resources: ["endpoints"]
    27.     verbs: ["get", "list", "watch", "create", "update", "patch"]
    28.  
    29. ---
    30. kind: ClusterRoleBinding
    31. apiVersion: rbac.authorization.k8s.io/v1
    32. metadata:
    33.   name: run-nfs-client-provisioner
    34. subjects:
    35.   - kind: ServiceAccount
    36.     name: nfs-client-provisioner
    37.     namespace: kube-system 
    38. roleRef:
    39.   kind: ClusterRole
    40.   name: nfs-client-provisioner-runner
    41.   apiGroup: rbac.authorization.k8s.io
    42.  
    43. ---
    44. kind: Deployment
    45. apiVersion: apps/v1
    46. metadata:
    47.   name: nfs-provisioner-01
    48.   namespace: kube-system
    49. spec:
    50.   replicas: 1
    51.   strategy:
    52.     type: Recreate
    53.   selector:
    54.     matchLabels:
    55.       app: nfs-provisioner-01
    56.   template:
    57.     metadata:
    58.       labels:
    59.         app: nfs-provisioner-01
    60.     spec:
    61.       serviceAccountName: nfs-client-provisioner
    62.       containers:
    63.         - name: nfs-client-provisioner
    64. #老版本插件使用jmgao1983/nfs-client-provisioner:latest
    65. #          image: jmgao1983/nfs-client-provisioner:latest
    66.           image: vbouchaud/nfs-client-provisioner:latest
    67.           imagePullPolicy: IfNotPresent
    68.           volumeMounts:
    69.             - name: nfs-client-root
    70.               mountPath: /persistentvolumes
    71.           env:
    72.             - name: PROVISIONER_NAME
    73.               value: nfs-provisioner-01  # 此处供应者名字供storageclass调用
    74.             - name: NFS_SERVER
    75.               value: 10.0.1.201   # 填入NFS的地址
    76.             - name: NFS_PATH
    77.               value: /nfs_dir   # 填入NFS挂载的目录
    78.       volumes:
    79.         - name: nfs-client-root
    80.           nfs:
    81.             server: 10.0.1.201   # 填入NFS的地址
    82.             path: /nfs_dir   # 填入NFS挂载的目录
    83.  
    84. ---
    85. apiVersion: storage.k8s.io/v1
    86. kind: StorageClass
    87. metadata:
    88.   name: nfs-boge
    89. provisioner: nfs-provisioner-01
    90. # Supported policies: Delete、 Retain ， default is Delete
    91. reclaimPolicy: Retain
    92.

    2.开始创建这个StorageClass

    1. # kubectl apply -f nfs-sc.yaml 
    2. serviceaccount/nfs-client-provisioner created
    3. clusterrole.rbac.authorization.k8s.io/nfs-client-provisioner-runner created
    4. clusterrolebinding.rbac.authorization.k8s.io/run-nfs-client-provisioner created
    5. deployment.apps/nfs-provisioner-01 created
    6.   orageclass.storage.k8s.io/nfs-boge created
    7.  
    8. # 注意这个是在放kube-system的namespace下面，这里面放置一些偏系统类的服务
    9. # kubectl -n kube-system get pod -w
    10. NAME                                       READY   STATUS              RESTARTS   AGE
    11. calico-kube-controllers-7fdc86d8ff-dpdm5   1/1     Running             1          24h
    12. calico-node-8jcp5                          1/1     Running             1          24h
    13. calico-node-m92rn                          1/1     Running             1          24h
    14. calico-node-xg5n4                          1/1     Running             1          24h
    15. calico-node-xrfqq                          1/1     Running             1          24h
    16. coredns-d9b6857b5-5zwgf                    1/1     Running             1          24h
    17. metrics-server-869ffc99cd-wfj44            1/1     Running             2          24h
    18. nfs-provisioner-01-5db96d9cc9-qxlgk        0/1     ContainerCreating   0          9s
    19. nfs-provisioner-01-5db96d9cc9-qxlgk        1/1     Running             0          21s
    20.  
    21. # StorageClass已经创建好了
    22. # kubectl get sc
    23. NAME       PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
    24. nfs-boge   nfs-provisioner-01   Retain          Immediate           false                  37s
    25.

    3.我们来基于StorageClass创建一个pvc，看看动态生成的pv是什么效果

    1. # vim pvc-sc.yaml 
    2. kind: PersistentVolumeClaim
    3. apiVersion: v1
    4. metadata:
    5.   name: pvc-sc
    6. spec:
    7.   storageClassName: nfs-boge
    8.   accessModes:
    9.     - ReadWriteMany
    10.   resources:
    11.     requests:
    12.       storage: 1Mi
    13.       
    14. # kubectl  apply -f pvc-sc.yaml 
    15. persistentvolumeclaim/pvc-sc created
    16.  
    17. # kubectl  get pvc
    18. NAME     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    19. pvc-sc   Bound    pvc-63eee4c7-90fd-4c7e-abf9-d803c3204623   1Mi        RWX            nfs-boge       3s
    20. pvc1     Bound    pv1                                        1Gi        RWO            nfs            24m
    21.  
    22. # kubectl  get pv
    23. NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM            STORAGECLASS   REASON   AGE
    24. pv1                                        1Gi        RWO            Recycle          Bound    default/pvc1     nfs                     49m
    25. pvc-63eee4c7-90fd-4c7e-abf9-d803c3204623   1Mi        RWX            Retain           Bound    default/pvc-sc   nfs-boge                7s
    26.

    4.我们修改下nginx的yaml配置，将pvc的名称换成上面的pvc-sc：

    1. # vim nginx.yaml 
    2. ---
    3. apiVersion: apps/v1
    4. kind: Deployment
    5. metadata:
    6.   labels:
    7.     app: nginx
    8.   name: nginx
    9. spec:
    10.   replicas: 1
    11.   selector:
    12.     matchLabels:
    13.       app: nginx
    14.   template:
    15.     metadata:
    16.       labels:
    17.         app: nginx
    18.     spec:
    19.       containers:
    20.       - image: nginx
    21.         name: nginx
    22.         volumeMounts:    # 我们这里将nginx容器默认的页面目录挂载
    23.           - name: html-files
    24.             mountPath: "/usr/share/nginx/html"
    25.       volumes:
    26.         - name: html-files
    27.           persistentVolumeClaim:
    28.             claimName: pvc-sc
    29.             
    30.             
    31. # kubectl apply -f nginx.yaml 
    32. service/nginx unchanged
    33. deployment.apps/nginx configured
    34.  
    35. # 这里注意下，因为是动态生成的pv，所以它的目录基于是一串随机字符串生成的，这时我们直接进到pod内来创建访问页面
    36. # kubectl exec -it nginx-57cdc6d9b4-n497g -- bash
    37. root@nginx-57cdc6d9b4-n497g:/# echo 'storageClass used' > /usr/share/nginx/html/index.html
    38. root@nginx-57cdc6d9b4-n497g:/# exit
    39.  
    40. # curl 10.68.238.54                              
    41. storageClass used
    42.  
    43. # 我们看下NFS挂载的目录
    44. # ll /nfs_dir/
    45. total 0
    46. drwxrwxrwx 2 root root 24 Nov 27 17:52 default-pvc-sc-pvc-63eee4c7-90fd-4c7e-abf9-d803c3204623
    47. drwxr-xr-x 2 root root  6 Nov 27 17:25 pv1
    48.

    5.ubuntu20.04系统

    1. #1安装nfs服务端
    2. sudo apt install nfs-kernel-server -y
    3.  
    4. #2. 创建目录
    5. sudo mkdir -p /nfs_dir/
    6.  
    7. #3. 使任何客户端均可访问
    8. sudo chown nobody:nogroup /data/k8s/　　
    9. #sudo chmod 755 /nfs_dir/
    10. sudo chmod 777 /nfs_dir/
    11.  
    12. #4. 配置/etc/exports文件, 使任何ip均可访问(加入以下语句)
    13. vi /etc/exports
    14. /nfs_dir/ *(rw,sync,no_subtree_check)
    15. 　　
    16. #5. 检查nfs服务的目录
    17. # (重新加载配置)
    18. sudo exportfs -ra
    19. #(查看共享的目录和允许访问的ip段)
    20. sudo showmount -e 
    21.  
    22. #6. 重启nfs服务使以上配置生效
    23. sudo systemctl restart nfs-kernel-server
    24. #sudo /etc/init.d/nfs-kernel-server restart
    25.  
    26. #查看nfs服务的状态是否为active状态:active(exited)或active(runing)
    27. systemctl status nfs-kernel-server
    28.  
    29. #7. 测试nfs服务是否成功启动
    30.  
    31. #安装nfs 客户端
    32. sudo apt-get install nfs-common
    33. #创建挂载目录
    34. sudo mkdir /nfs_dir/
    35.  
    36. #7.4 在主机上的Linux中测试是否正常
    37. sudo mount -t nfs -o nolock -o tcp 192.168.100.11:/nfs_dir/ /nfs_dir/
    38.  
    39. #错误 mount.nfs: access denied by server while mounting

    六、k8s架构师课程之有状态服务StatefulSet

    1.StatefulSet

    前面我们讲到了Deployment、DaemonSet都只适合用来跑无状态的服务pod，那么这里的StatefulSet（简写sts）就是用来跑有状态服务pod的。

    那怎么理解有状态服务和无状态服务呢？简单快速地理解为：无状态服务最典型的是WEB服务器的每次http请求，它的每次请求都是全新的，和之前的没有关系；那么有状态服务用网游服务器来举例比较恰当了，每个用户的登陆请求，服务端都是先根据这个用户之前注册过的帐号密码等信息来判断这次登陆请求是否正常。

    无状态服务因为相互之前都是独立的，很适合用横向扩充来增加服务的资源量

    还有一个很形象的比喻，在K8s的无状态服务的pod有点类似于农村圈养的牲畜，饲养它们的人不会给它们每个都单独取个名字（pod都是随机名称，IP每次发生重启也是变化的），当其中一只病了或被卖了，带来的感观只是数量上的减少，这时再买些相应数量的牲畜回来就可以回到之前的状态了（当一个pod因为某些原来被删除掉的时候，K8s会启动一个新的pod来代替它）；而有状态服务的pod就像养的一只只宠物，主人对待自己喜欢的宠物都会给它们取一个比较有特色的名字（在K8s上运行的有状态服务的pod，都会被给予一个独立的固定名称），并且每只宠物都有它独特的外貌和性格，如果万一这只宠物丢失了，那么需要到宠物店再买一只同样品种同样毛色的宠物来代替了（当有状态服务的pod删除时，K8s会启动一个和先前一模一样名称的pod来代替它）。

    有状态服务sts比较常见的mongo复制集 ，redis cluster，rabbitmq cluster等等，这些服务基本都会用StatefulSet模式来运行，当然除了这个，它们内部集群的关系还需要一系列脚本或controller来维系它们间的状态，这些会在后面进阶课程专门来讲，现在为了让大家先更好的明白StatefulSet，我这里直接还是用nginx服务来实战演示

    1. 1、创建pv
    2. -------------------------------------------
    3.  
    4. root@node1:~# cat web-pv.yaml 
    5. # mkdir -p /nfs_dir/{web-pv0,web-pv1}
    6. apiVersion: v1
    7. kind: PersistentVolume
    8. metadata:
    9.   name: web-pv0
    10.   labels:
    11.     type: web-pv0
    12. spec:
    13.   capacity:
    14.     storage: 1Gi
    15.   accessModes:
    16.     - ReadWriteOnce
    17.   persistentVolumeReclaimPolicy: Retain
    18.   storageClassName: my-storage-class
    19.   nfs:
    20.     path: /nfs_dir/web-pv0
    21.     server: 10.0.1.201
    22. ---
    23. apiVersion: v1
    24. kind: PersistentVolume
    25. metadata:
    26.   name: web-pv1
    27.   labels:
    28.     type: web-pv1
    29. spec:
    30.   capacity:
    31.     storage: 1Gi
    32.   accessModes:
    33.     - ReadWriteOnce
    34.   persistentVolumeReclaimPolicy: Retain
    35.   storageClassName: my-storage-class
    36.   nfs:
    37.     path: /nfs_dir/web-pv1
    38.     server: 10.0.1.201
    39.  
    40.  
    41. 2、创建pvc(这一步可以省去让其自动创建，这里手动创建是为了让大家能更清楚在sts里面pvc的创建过程)
    42. -------------------------------------------
    43. 这一步非常非常的关键，因为如果创建的PVC的名称和StatefulSet中的名称没有对应上，
    44. 那么StatefulSet中的Pod就肯定创建不成功.
    45.  
    46. 我们在这里创建了一个叫做www-web-0和www-web-1的PVC，这个名字是不是很奇怪，
    47. 而且在这个yaml里并没有提到PV的名字，所以PV和PVC是怎么bound起来的呢？
    48. 是通过labels标签下的key:value键值对来进行匹配的，
    49. 我们在创建PV时指定了label的键值对，在PVC里通过selector可以指定label。
    50.  
    51. 然后再回到这个PVC的名称定义：www-web-0，为什么叫这样一个看似有规律的名字呢，
    52. 这里需要看看下面创建StatefulSet中的yaml，
    53. 首先我们看到StatefulSet的name叫web，设置的replicas为2个，
    54. volumeMounts和volumeClaimTemplates的name必须相同，为www，
    55. 所以StatefulSet创建的第一个Pod的name应该为web-0，第二个为web-1。
    56. 这里StatefulSet中的Pod与PVC之间的绑定关系是通过名称来匹配的，即：
    57.  
    58. PVC_name  =  volumeClaimTemplates_name + "-" + pod_name
    59. www-web-0     =       www               + "-" +   web-0
    60. www-web-1     =       www               + "-" +   web-1
    61.  
    62.  
    63. root@node1:~# cat web-pvc.yaml 
    64. kind: PersistentVolumeClaim
    65. apiVersion: v1
    66. metadata:
    67.   name: www-web-0
    68. spec:
    69.   accessModes:
    70.     - ReadWriteOnce
    71.   resources:
    72.     requests:
    73.       storage: 1Gi
    74.   storageClassName: my-storage-class
    75.   selector:
    76.     matchLabels:
    77.       type: web-pv0
    78. ---
    79. kind: PersistentVolumeClaim
    80. apiVersion: v1
    81. metadata:
    82.   name: www-web-1
    83. spec:
    84.   accessModes:
    85.     - ReadWriteOnce
    86.   resources:
    87.     requests:
    88.       storage: 1Gi
    89.   storageClassName: my-storage-class
    90.   selector:
    91.     matchLabels:
    92.       type: web-pv1
    93.  
    94.  
    95. 3、创建Service 和 StatefulSet
    96. -------------------------------------------
    97. 在上一步中我们已经创建了名为www-web-0的PVC了，接下来创建一个service和statefulset，
    98. service的名称可以随意取，但是statefulset的名称已经定死了，为web，
    99. 并且statefulset中的volumeClaimTemplates_name必须为www，volumeMounts_name也必须为www。
    100. 只有这样，statefulset中的pod才能通过命名来匹配到PVC，否则会创建失败。
    101.  
    102. root@node1:~# cat web.yaml 
    103. apiVersion: v1
    104. kind: Service
    105. metadata:
    106.   name: web-headless
    107.   labels:
    108.     app: nginx
    109. spec:
    110.   ports:
    111.   - port: 80
    112.     name: web
    113.   clusterIP: None
    114.   selector:
    115.     app: nginx
    116. ---
    117.  
    118. apiVersion: v1
    119. kind: Service
    120. metadata:
    121.   name: web
    122.   labels:
    123.     app: nginx
    124. spec:
    125.   ports:
    126.   - port: 80
    127.     name: web
    128.   selector:
    129.     app: nginx
    130.  
    131. ---
    132. apiVersion: apps/v1
    133. kind: StatefulSet
    134. metadata:
    135.   name: web
    136. spec:
    137.   selector:
    138.     matchLabels:
    139.       app: nginx # has to match .spec.template.metadata.labels
    140.   serviceName: "web-headless"   #需要第4行的name一致
    141.   replicas: 2 # by default is 1
    142.   template:
    143.     metadata:
    144.       labels:
    145.         app: nginx # has to match .spec.selector.matchLabels
    146.     spec:
    147.       terminationGracePeriodSeconds: 10
    148.       containers:
    149.       - name: nginx
    150.         image: nginx
    151.         imagePullPolicy: IfNotPresent
    152.         ports:
    153.         - containerPort: 80
    154.           name: web
    155.         volumeMounts:
    156.         - name: www
    157.           mountPath: /usr/share/nginx/html
    158.   volumeClaimTemplates:
    159.   - metadata:
    160.       name: www
    161.     spec:
    162.       accessModes: [ "ReadWriteOnce" ]
    163.       storageClassName: "my-storage-class"
    164.       resources:
    165.         requests:
    166.           storage: 1Gi
    167.

    2.动态存储创建sts-web.yaml

    1.  
    2. cat sts-web.yaml
    3.  
    4. apiVersion: v1
    5. kind: Service
    6. metadata:
    7.   name: web-headless
    8.   labels:
    9.     app: nginx
    10. spec:
    11.   ports:
    12.   - port: 80
    13.     name: web
    14.   clusterIP: None
    15.   selector:
    16.     app: nginx
    17. ---
    18.  
    19. apiVersion: v1
    20. kind: Service
    21. metadata:
    22.   name: web
    23.   labels:
    24.     app: nginx
    25. spec:
    26.   ports:
    27.   - port: 80
    28.     name: web
    29.   selector:
    30.     app: nginx
    31.  
    32. ---
    33. apiVersion: apps/v1
    34. kind: StatefulSet
    35. metadata:
    36.   name: web
    37. spec:
    38.   selector:
    39.     matchLabels:
    40.       app: nginx # has to match .spec.template.metadata.labels
    41.   serviceName: "nginx"
    42.   replicas: 2 # by default is 1
    43.   template:
    44.     metadata:
    45.       labels:
    46.         app: nginx # has to match .spec.selector.matchLabels
    47.     spec:
    48.       terminationGracePeriodSeconds: 10
    49.       containers:
    50.       - name: nginx
    51.         image: nginx
    52.         imagePullPolicy: IfNotPresent
    53.         ports:
    54.         - containerPort: 80
    55.           name: web
    56.         volumeMounts:
    57.         - name: www
    58.           mountPath: /usr/share/nginx/html
    59.   volumeClaimTemplates:
    60.   - metadata:
    61.       name: www
    62.     spec:
    63.       accessModes: [ "ReadWriteOnce" ]
    64.       storageClassName: "nfs-boge"  #之前创建的sc
    65.       resources:
    66.         requests:
    67.           storage: 1Gi
    68.  
    69.  
    70.  
    71.  
    72.  
    73. 执行测试
    74. [root@k8s-m01 nfs_dir]# kubectl apply -f sts-web.yaml -n renwei
    75.  
    76. 编辑测试页面
    77. [root@k8s-m01 nfs_dir]# echo 00000 > renwei-www-web-0-pvc-477af122-468d-43f2-b935-5b1bd653fca5/index.html
    78.  
    79. [root@k8s-m01 nfs_dir]# echo 11111 >renwei-www-web-1-pvc-f3e720ec-98bf-4d3e-85bf-44eae02fa1c7/index.html
    80.  
    81.  
    82. 启动临时程序，测试页面
    83. [root@k8s-m01 nfs_dir]# kubectl det pod svc -n renwei
    84.  
    85. [root@k8s-m01 nfs_dir]# curl 0.68.186.15
    86.  
    87.  
    88. [root@k8s-m01 nfs_dir]# kubectl run --rm -it  busybox --image=busybox -- sh
    89. If you don't see a command prompt, try pressing enter.
    90. / #
    91. / #
    92. / # wget www-web-0.web-headless.renwei
    93. / # cat index.html
    94. / # exit
    95. [root@k8s-m01 nfs_dir]# kubectl delete -f sts-web.yaml -n renwei
    97. #删除pv,pvc
    98. [root@k8s-m01 nfs_dir]# kubectl get pvc -n renwei
    99. [root@k8s-m01 nfs_dir]# kubectl -n renwei delete pvc www-web-0
    100. [root@k8s-m01 nfs_dir]# kubectl -n renwei delete pvc www-web-1
    102. [root@k8s-m01 nfs_dir]# kubectl get pv
    103. [root@k8s-m01 nfs_dir]# kubectl delete pv pvc-477af122-468d-43f2-b935-5b1bd653fca5
    104. [root@k8s-m01 nfs_dir]# kubectl delete pv pvc-f3e720ec-98bf-4d3e-85bf-44eae02fa1c7
    106. [root@k8s-m01 nfs_dir]# kubectl get pvc -n renwei
    107. [root@k8s-m01 nfs_dir]# kubectl get pv

    七、k8s一次性和定时任务

    1.首先讲下一次性任务，在K8s中它叫job，直接来实战一番，先准备下yaml配置

    这里我们不知道yaml怎么写，可以直接kubectl create job -h就能看到命令行创建示例了，然后可以根据创建出来的服务资源来导出它的yaml配置为my-job.yaml

    1. apiVersion: batch/v1   # 1. batch/v1 是当前 Job 的 apiVersion
    2. kind: Job        #  2. 指明当前资源的类型为 Job
    3. metadata:
    4.   name: my-job
    5. spec:
    6.   template:
    7.     metadata:
    8.     spec:
    9.       containers:
    10.       - image: busybox
    11.         name: my-job
    12.         command: ["echo","Hello, boge."]
    13.       restartPolicy: Never   # 3. restartPolicy 指定什么情况下需要重启容器。对于 Job，只能设置为 Never 或者 OnFailure
    14.

    2.创建它并查看结果

    1. # kubectl apply -f my-job.yaml 
    2. job.batch/my-job created
    3.  
    4. # kubectl get jobs.batch 
    5. NAME     COMPLETIONS   DURATION   AGE
    6. my-job   1/1           2s         73s
    7. # COMPLETIONS 已完成的
    8. # DURATION  这个job运行所花费的时间
    9. # AGE 这个job资源已经从创建到目前为止的时间
    10.  
    11. # job会生成一个pod,当完成任务后会是Completed的状态
    12. # kubectl get pod
    13. NAME           READY   STATUS      RESTARTS   AGE
    14. my-job-7h6fb   0/1     Completed   0          31s
    15.  
    16. # 看下这个job生成的pod日志
    17. # kubectl logs my-job-7h6fb 
    18. Hello, boge.
    19.

    3. ob失败了会有什么现象出现呢？

    我们编辑这个job的yaml，把执行的命令改成一个不存在的命令看看会发生什么

    1. apiVersion: batch/v1   # 1. batch/v1 是当前 Job 的 apiVersion
    2. kind: Job        #  2. 指明当前资源的类型为 Job
    3. metadata:
    4.   name: my-job
    5. spec:
    6.   template:
    7.     metadata:
    8.     spec:
    9.       containers:
    10.       - image: busybox
    11.         name: my-job
    12.         command: ["echoaaa","Hello, boge."]
    13.       restartPolicy: Never   # 3. restartPolicy 指定什么情况下需要重启容器。对于 Job，只能设置为 Never 或者 OnFailure
    14.

    4.创建它

    1. # kubectl apply -f my-job.yaml 
    2.  
    3. # 可以观察到这个job因为不成功，并且restartPolicy重启模式是Never不会被重启，但它的job状态始终未完成，所以它会一直不停的创建新的pod，直到COMPLETIONS为1/1，对于我们这个示例，它显然永远都不会成功
    4. # kubectl get pod
    5. NAME           READY   STATUS       RESTARTS   AGE
    6. my-job-9fcbm   0/1     StartError   0          47s
    7. my-job-bt2kd   0/1     StartError   0          54s
    8. my-job-mlnzz   0/1     StartError   0          37s
    9. my-job-mntdp   0/1     StartError   0          17s
    10.  
    11. # kubectl get job
    12. NAME     COMPLETIONS   DURATION   AGE
    13. my-job   0/1           15s        15s
    14.  
    15. # 找一个pod看下事件描述，会很清晰地指出命令不存在
    16. # kubectl describe pod my-job-9fcbm 
    17. Name:         my-job-9fcbm
    18. Namespace:    default
    19. ......
    20. Events:
    21.   Type     Reason     Age   From               Message
    22.   ----     ------     ----  ----               -------
    23.   Normal   Scheduled  44s   default-scheduler  Successfully assigned default/my-job-9fcbm to 10.0.0.204
    24.   Normal   Pulling    43s   kubelet            Pulling image "busybox"
    25.   Normal   Pulled     36s   kubelet            Successfully pulled image "busybox" in 7.299038719s
    26.   Normal   Created    36s   kubelet            Created container my-job
    27.   Warning  Failed     36s   kubelet            Error: failed to create containerd task: OCI runtime create failed: container_linux.go:370: starting container process caused: exec: "echoaaa": executable file not found in $PATH: unknown
    28.  
    29. # 删除掉这个job，不然那创建的pod数量可有够多的了
    30. # kubectl  delete job my-job
    31.  
    32. # 试试把restartPolicy重启模式换成OnFailure观察看看
    33. # kubectl get pod
    34. NAME           READY   STATUS             RESTARTS   AGE
    35. my-job-gs95h   0/1     CrashLoopBackOff   3          84s
    36.  
    37. # 可以看到它不会创建新的pod，而是会尝试重启自身，以期望恢复正常，这里看到已经重启了3次，还会持续增加到5，然后会被K8s给删除以尝试，因为这里只是job而不是deployment，它不会自己再启动一个新的pod，所以这个job等于就没有了，这里说明OnFailure是生效的，至少不会有那么多错误的pod出现了
    38.

    5.并行执行job

    准备好yaml配置

    1. apiVersion: batch/v1
    2. kind: Job
    3. metadata:
    4.   name: my-job
    5. spec:
    6.   parallelism: 2  # 并行执行2个job
    7.   template:
    8.     metadata:
    9.       name: my-job
    10.     spec:
    11.       containers:
    12.       - image: busybox
    13.         name: my-job
    14.         command: ["echo","Hello, boge."]
    15.       restartPolicy: OnFailure
    16.

    6.创建并查看结果

    1. # kubectl apply -f my-job.yaml 
    2. job.batch/my-job created
    3.  
    4. # job一共启动了2个pod，并且它们的AGE一样，可见是并行创建的
    5. # kubectl get pod
    6. NAME           READY   STATUS      RESTARTS   AGE
    7. my-job-fwf8l   0/1     Completed   0          7s
    8. my-job-w2fxd   0/1     Completed   0          7s
    9.

    7.再来个组合测试下并行完成定制的总任务数量

    1. apiVersion: batch/v1
    2. kind: Job
    3. metadata:
    4.   name: myjob
    5. spec:
    6.   completions: 6   # 此job完成pod的总数量
    7.   parallelism: 2   # 每次并发跑2个job
    8.   template:
    9.     metadata:
    10.       name: myjob
    11.     spec:
    12.       containers:
    13.       - name: hello
    14.         image: busybox
    15.         command: ["echo"," hello boge! "]
    16.       restartPolicy: OnFailure
    17.

    8.创建并查看结果

    1. # 可以看到是每次并发2个job，完成6个总量即停止
    2. # kubectl get pod
    3. NAME          READY   STATUS      RESTARTS   AGE
    4. myjob-54wmk   0/1     Completed   0          11s
    5. myjob-fgtmj   0/1     Completed   0          15s
    6. myjob-fkj5l   0/1     Completed   0          7s
    7. myjob-hsccm   0/1     Completed   0          7s
    8. myjob-lrpsr   0/1     Completed   0          15s
    9. myjob-ppfns   0/1     Completed   0          11s
    10.  
    11. # 符合预期
    12. # kubectl get job
    13. NAME    COMPLETIONS   DURATION   AGE
    14. myjob   6/6           14s        34s
    15.  
    16. # 测试完成后删掉这个资源
    17. kubectl delete job myjob
    18.

    9 到此，job的内容就讲完了，在生产中，job比较适合用在CI/CD流水线中，作完一次性任务使用，我在生产中基本没怎么用这个资源。
    cronjob

    上面的job是一次性任务，那我们需要定时循环来执行一个任务可以吗？答案肯定是可以的，就像我们在linux系统上面用crontab一样，在K8s上用cronjob的另一个好处就是它是分布式的，执行的pod可以是在集群中的任意一台NODE上面（这点和cronsun有点类似）

    让我们开始实战吧，先准备一下cronjob的yaml配置为my-cronjob.yaml

    1. apiVersion: batch/v1beta1     # <---------  当前 CronJob 的 apiVersion
    2. kind: CronJob                 # <---------  当前资源的类型
    3. metadata:
    4.   name: hello
    5. spec:
    6.   schedule: "* * * * *"      # <---------  schedule 指定什么时候运行 Job，其格式与 Linux crontab 一致,这里 * * * * * 的含义是每一分钟启动一次
    7.   jobTemplate:               # <---------  定义 Job 的模板，格式与前面 Job 一致
    8.     spec:
    9.       template:
    10.         spec:
    11.           containers:
    12.           - name: hello
    13.             image: busybox
    14.             command: ["echo","boge like cronjob."]
    15.           restartPolicy: OnFailure
    16.

    10.正常创建后，我们过几分钟来看看运行结果

    1. # 这里会显示cronjob的综合信息
    2. # kubectl get cronjobs.batch 
    3. NAME    SCHEDULE    SUSPEND   ACTIVE   LAST SCHEDULE   AGE
    4. hello   * * * * *   False     0        66s             2m20s
    5.  
    6. # 可以看到它每隔一分钟就会创建一个pod来执行job任务
    7. # kubectl get pod
    8. NAME                     READY   STATUS              RESTARTS   AGE
    9. hello-1610267460-9b6hp   0/1     Completed           0          2m5s
    10. hello-1610267520-fm427   0/1     Completed           0          65s
    11. hello-1610267580-v8g4h   0/1     ContainerCreating   0          5s
    12.  
    13. # 测试完成后删掉这个资源
    14. # kubectl delete cronjobs.batch hello 
    15. cronjob.batch "hello" deleted
    16.

    cronjob定时任务在生产中的用处很多，这也是为什么上面job我说用得很少的缘故，我们可以把一些需要定时定期运行的任务，在K8s上以cronjob运行，依托K8s强大的资源调度以及服务自愈能力，我们可以放心的把定时任务交给它执行。

    八、k8sRBAC角色访问控制

    1.RBAC

    在k8s上我们如何控制访问权限呢，答案就是Role-based access control (RBAC) - 基于角色（Role）的访问控制，（RBAC）是一种基于组织中用户的角色来调节控制对 计算机或网络资源的访问的方法。

    在早期的K8s版本，RBAC还未出现的时候，整个K8s的安全是较为薄弱的，有了RBAC后，我们可以对K8s集群的访问人员作非常明细化的控制，控制他们能访问什么资源，以只读还是可以读写的形式来访问，目前RBAC是K8s默认的安全授权标准，所以我们非常有必要来掌握RBAC的使用，这样才有更有力的保障我们的K8s集群的安全使用，下面我们将以生产中的实际使用来大家了解及掌握RBAC的生产应用。

    RBAC里面的几种资源关系图，下面将用下面的资源来演示生产中经典的RBAC应用

                      |--- Role --- RoleBinding                只在指定namespace中生效
    ServiceAccount ---|
                      |--- ClusterRole --- ClusterRoleBinding  不受namespace限制，在整个K8s集群中生效

    2.在我看来，RBAC在K8s上的用途主要分为两大类：

    第一类是保证在K8s上运行的pod服务具有相应的集群权限，如gitlab的CI/CD，它需要能访问除自身以外其他pod，比如gitlab-runner的pod的权限，再比如gitlab-runner的pod需要拥有创建新的临时pod的权限，用以来构建CI/CD自动化流水线，这里大家没用过不懂没关系，先简单了解下就可以了，在本课程后面基于K8s及gitlab的生产实战CI/CD内容会给大家作详细实战讲解；

    第二类是创建能访问K8s相应资源、拥有对应权限的kube-config配置给到使用K8s的人员，来作为连接K8s的授权凭证

    第一类的实战这里先暂时以早期的helm2来作下讲解，helm是一个快捷安装K8s各类资源的管理工具，通过之前给大家讲解的，一个较为完整的服务可能会存在deployment，service，configmap，secret，ingress等资源来组合使用，大家在用的过程中可能会觉得配置使用较为麻烦，这时候helm就出现了，它把这些资源都打包封装成它自己能识别的内容，我们在安装一个服务的时候，就只需要作下简单的配置，一条命令即可完成上述众多资源的配置安装，titller相当于helm的服务端，它是需要有权限在K8s中创建各类资源的，在初始安装使用时，如果没有配置RBAC权限，我们会看到如下报错：

    root@node1:~# helm install stable/mysql
    Error: no available release name found

    3.这时，我们可以来快速解决这个问题，创建sa关联K8s自带的最高权限的ClusterRole（生产中建议不要这样做，权限太高有安全隐患，这个就和linux的root管理帐号一样，一般都是建议通过sudo来控制帐号权限）

    1. kubectl create serviceaccount --namespace kube-system tiller
    2. kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
    3. kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'
    4.

    4.第二类，我这里就直接以我在生产中实施的完整脚本来做讲解及实战，相信会给大家带来一个全新的学习感受，并能很快掌握它们：

    1.创建对指定namespace有所有权限的kube-config

    1. #!/bin/bash
    2. #
    3. # This Script based on  https://jeremievallee.com/2018/05/28/kubernetes-rbac-namespace-user.html
    4. # K8s'RBAC doc:         https://kubernetes.io/docs/reference/access-authn-authz/rbac
    5. # Gitlab'CI/CD doc:     hhttps://docs.gitlab.com/ee/user/permissions.html#running-pipelines-on-protected-branches
    6. #
    7. # In honor of the remarkable Windson
    8.  
    9. BASEDIR="$(dirname "$0")"
    10. folder="$BASEDIR/kube_config"
    11.  
    12. echo -e "All namespaces is here: \n$(kubectl get ns|awk 'NR!=1{print $1}')"
    13. echo "endpoint server if local network you can use $(kubectl cluster-info |awk '/Kubernetes/{print $NF}')"
    14.  
    15. namespace=$1
    16. endpoint=$(echo "$2" | sed -e 's,https\?://,,g')
    17.  
    18. if [[ -z "$endpoint" || -z "$namespace" ]]; then
    19.     echo "Use "$(basename "$0")" NAMESPACE ENDPOINT";
    20.     exit 1;
    21. fi
    22.  
    23. if ! kubectl get ns|awk 'NR!=1{print $1}'|grep -w "$namespace";then kubectl create ns "$namespace";else echo "namespace: $namespace was exist."; exit 1 ;fi
    24.  
    25. echo "---
    26. apiVersion: v1
    27. kind: ServiceAccount
    28. metadata:
    29.   name: $namespace-user
    30.   namespace: $namespace
    31. ---
    32. kind: Role
    33. apiVersion: rbac.authorization.k8s.io/v1beta1
    34. metadata:
    35.   name: $namespace-user-full-access
    36.   namespace: $namespace
    37. rules:
    38. - apiGroups: ['', 'extensions', 'apps', 'metrics.k8s.io']
    39.   resources: ['*']
    40.   verbs: ['*']
    41. - apiGroups: ['batch']
    42.   resources:
    43.   - jobs
    44.   - cronjobs
    45.   verbs: ['*']
    46. ---
    47. kind: RoleBinding
    48. apiVersion: rbac.authorization.k8s.io/v1beta1
    49. metadata:
    50.   name: $namespace-user-view
    51.   namespace: $namespace
    52. subjects:
    53. - kind: ServiceAccount
    54.   name: $namespace-user
    55.   namespace: $namespace
    56. roleRef:
    57.   apiGroup: rbac.authorization.k8s.io
    58.   kind: Role
    59.   name: $namespace-user-full-access
    60. ---
    61. # https://kubernetes.io/zh/docs/concepts/policy/resource-quotas/
    62. apiVersion: v1
    63. kind: ResourceQuota
    64. metadata:
    65.   name: $namespace-compute-resources
    66.   namespace: $namespace
    67. spec:
    68.   hard:
    69.     pods: "10"
    70.     services: "10"
    71.     persistentvolumeclaims: "5"
    72.     requests.cpu: "1"
    73.     requests.memory: 2Gi
    74.     limits.cpu: "2"
    75.     limits.memory: 4Gi" | kubectl apply -f -
    76. kubectl -n $namespace describe quota $namespace-compute-resources
    77. mkdir -p $folder
    78. tokenName=$(kubectl get sa $namespace-user -n $namespace -o "jsonpath={.secrets[0].name}")
    79. token=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data.token}" | base64 --decode)
    80. certificate=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data['ca\.crt']}")
    81.  
    82. echo "apiVersion: v1
    83. kind: Config
    84. preferences: {}
    85. clusters:
    86. - cluster:
    87.     certificate-authority-data: $certificate
    88.     server: https://$endpoint
    89.   name: $namespace-cluster
    90. users:
    91. - name: $namespace-user
    92.   user:
    93.     as-user-extra: {}
    94.     client-key-data: $certificate
    95.     token: $token
    96. contexts:
    97. - context:
    98.     cluster: $namespace-cluster
    99.     namespace: $namespace
    100.     user: $namespace-user
    101.   name: $namespace
    102. current-context: $namespace" > $folder/$namespace.kube.conf
    103.

    2.创建对指定namespace有所有权限的kube-config（在已有的namespace中创建）

    1. #!/bin/bash
    2.  
    3.  
    4. BASEDIR="$(dirname "$0")"
    5. folder="$BASEDIR/kube_config"
    6.  
    7. echo -e "All namespaces is here: \n$(kubectl get ns|awk 'NR!=1{print $1}')"
    8. echo "endpoint server if local network you can use $(kubectl cluster-info |awk '/Kubernetes/{print $NF}')"
    9.  
    10. namespace=$1
    11. endpoint=$(echo "$2" | sed -e 's,https\?://,,g')
    12.  
    13. if [[ -z "$endpoint" || -z "$namespace" ]]; then
    14.     echo "Use "$(basename "$0")" NAMESPACE ENDPOINT";
    15.     exit 1;
    16. fi
    17.  
    18.  
    19. echo "---
    20. apiVersion: v1
    21. kind: ServiceAccount
    22. metadata:
    23.   name: $namespace-user
    24.   namespace: $namespace
    25. ---
    26. kind: Role
    27. apiVersion: rbac.authorization.k8s.io/v1beta1
    28. metadata:
    29.   name: $namespace-user-full-access
    30.   namespace: $namespace
    31. rules:
    32. - apiGroups: ['', 'extensions', 'apps', 'metrics.k8s.io']
    33.   resources: ['*']
    34.   verbs: ['*']
    35. - apiGroups: ['batch']
    36.   resources:
    37.   - jobs
    38.   - cronjobs
    39.   verbs: ['*']
    40. ---
    41. kind: RoleBinding
    42. apiVersion: rbac.authorization.k8s.io/v1beta1
    43. metadata:
    44.   name: $namespace-user-view
    45.   namespace: $namespace
    46. subjects:
    47. - kind: ServiceAccount
    48.   name: $namespace-user
    49.   namespace: $namespace
    50. roleRef:
    51.   apiGroup: rbac.authorization.k8s.io
    52.   kind: Role
    53.   name: $namespace-user-full-access" | kubectl apply -f -
    54.  
    55. mkdir -p $folder
    56. tokenName=$(kubectl get sa $namespace-user -n $namespace -o "jsonpath={.secrets[0].name}")
    57. token=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data.token}" | base64 --decode)
    58. certificate=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data['ca\.crt']}")
    59.  
    60. echo "apiVersion: v1
    61. kind: Config
    62. preferences: {}
    63. clusters:
    64. - cluster:
    65.     certificate-authority-data: $certificate
    66.     server: https://$endpoint
    67.   name: $namespace-cluster
    68. users:
    69. - name: $namespace-user
    70.   user:
    71.     as-user-extra: {}
    72.     client-key-data: $certificate
    73.     token: $token
    74. contexts:
    75. - context:
    76.     cluster: $namespace-cluster
    77.     namespace: $namespace
    78.     user: $namespace-user
    79.   name: $namespace
    80. current-context: $namespace" > $folder/$namespace.kube.conf
    81.

    3.同上，创建只读权限的

    1. #!/bin/bash
    2.  
    3.  
    4. BASEDIR="$(dirname "$0")"
    5. folder="$BASEDIR/kube_config"
    6.  
    7. echo -e "All namespaces is here: \n$(kubectl get ns|awk 'NR!=1{print $1}')"
    8. echo "endpoint server if local network you can use $(kubectl cluster-info |awk '/Kubernetes/{print $NF}')"
    9.  
    10. namespace=$1
    11. endpoint=$(echo "$2" | sed -e 's,https\?://,,g')
    12.  
    13. if [[ -z "$endpoint" || -z "$namespace" ]]; then
    14.     echo "Use "$(basename "$0")" NAMESPACE ENDPOINT";
    15.     exit 1;
    16. fi
    17.  
    18.  
    19. echo "---
    20. apiVersion: v1
    21. kind: ServiceAccount
    22. metadata:
    23.   name: $namespace-user-readonly
    24.   namespace: $namespace
    25. ---
    26. kind: Role
    27. apiVersion: rbac.authorization.k8s.io/v1beta1
    28. metadata:
    29.   name: $namespace-user-readonly-access
    30.   namespace: $namespace
    31. rules:
    32. - apiGroups: ['', 'extensions', 'apps', 'metrics.k8s.io']
    33.   resources: ['pods', 'pods/log']
    34.   verbs: ['get', 'list', 'watch']
    35. - apiGroups: ['batch']
    36.   resources: ['jobs', 'cronjobs']
    37.   verbs: ['get', 'list', 'watch']
    38. ---
    39. kind: RoleBinding
    40. apiVersion: rbac.authorization.k8s.io/v1beta1
    41. metadata:
    42.   name: $namespace-user-view-readonly
    43.   namespace: $namespace
    44. subjects:
    45. - kind: ServiceAccount
    46.   name: $namespace-user-readonly
    47.   namespace: $namespace
    48. roleRef:
    49.   apiGroup: rbac.authorization.k8s.io
    50.   kind: Role
    51.   name: $namespace-user-readonly-access" | kubectl apply -f -
    52.  
    53. mkdir -p $folder
    54. tokenName=$(kubectl get sa $namespace-user-readonly -n $namespace -o "jsonpath={.secrets[0].name}")
    55. token=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data.token}" | base64 --decode)
    56. certificate=$(kubectl get secret $tokenName -n $namespace -o "jsonpath={.data['ca\.crt']}")
    57.  
    58. echo "apiVersion: v1
    59. kind: Config
    60. preferences: {}
    61. clusters:
    62. - cluster:
    63.     certificate-authority-data: $certificate
    64.     server: https://$endpoint
    65.   name: $namespace-cluster-readonly
    66. users:
    67. - name: $namespace-user-readonly
    68.   user:
    69.     as-user-extra: {}
    70.     client-key-data: $certificate
    71.     token: $token
    72. contexts:
    73. - context:
    74.     cluster: $namespace-cluster-readonly
    75.     namespace: $namespace
    76.     user: $namespace-user-readonly
    77.   name: $namespace
    78. current-context: $namespace" > $folder/$namespace-readonly.kube.conf
    79.

    4.最后，来一个多个集群配置融合的创建，这个在多集群管理方面非常有用，这里只以创建只读权限配置作为演示

    1. #!/bin/bash
    2. # describe: create k8s cluster all namespaces resources with readonly clusterrole, no exec 、delete ...
    3.  
    4. # look system default to study:
    5. # kubectl describe clusterrole view
    6.  
    7. # restore all change:
    8. #kubectl -n kube-system delete sa all-readonly-${clustername}
    9. #kubectl delete clusterrolebinding all-readonly-${clustername}
    10. #kubectl delete clusterrole all-readonly-${clustername}
    11.  
    12.  
    13. clustername=$1
    14.  
    15. Help(){
    16.     echo "Use "$(basename "$0")" ClusterName(example: k8s1|k8s2|k8s3|delk8s1|delk8s2|delk8s3|3in1)";
    17.     exit 1;
    18. }
    19.  
    20. if [[ -z "${clustername}" ]]; then
    21.     Help
    22. fi
    23.  
    24. case ${clustername} in
    25.     k8s1)
    26.     endpoint="https://x.x.x.x:123456"
    27.     ;;
    28.     k8s2)
    29.     endpoint="https://x.x.x.x:123456"
    30.     ;;
    31.     k8s3)
    32.     endpoint="https://x.x.x.x:123456"
    33.     ;;
    34.     delk8s1)
    35.     kubectl -n kube-system delete sa all-readonly-k8s1
    36.     kubectl delete clusterrolebinding all-readonly-k8s1
    37.     kubectl delete clusterrole all-readonly-k8s1
    38.     echo "${clustername} successful."
    39.     exit 0
    40.     ;;
    41.     delk8s2)
    42.     kubectl -n kube-system delete sa all-readonly-k8s2
    43.     kubectl delete clusterrolebinding all-readonly-k8s2
    44.     kubectl delete clusterrole all-readonly-k8s2
    45.     echo "${clustername} successful."
    46.     exit 0
    47.     ;;
    48.     delk8s3)
    49.     kubectl -n kube-system delete sa all-readonly-k8s3
    50.     kubectl delete clusterrolebinding all-readonly-k8s3
    51.     kubectl delete clusterrole all-readonly-k8s3
    52.     echo "${clustername} successful."
    53.     exit 0
    54.     ;;
    55.     3in1)
    56.     KUBECONFIG=./all-readonly-k8s1.conf:all-readonly-k8s2.conf:all-readonly-k8s3.conf kubectl config view --flatten > ./all-readonly-3in1.conf
    57.     kubectl --kubeconfig=./all-readonly-3in1.conf config use-context "k8s3"
    58.     kubectl --kubeconfig=./all-readonly-3in1.conf config set-context "k8s3" --namespace="default"
    59.     kubectl --kubeconfig=./all-readonly-3in1.conf config get-contexts
    60.     echo -e "\n\n\n"
    61.     cat ./all-readonly-3in1.conf |base64 -w 0
    62.     exit 0
    63.     ;;
    64.     *)
    65.     Help
    66. esac
    67.  
    68. echo "---
    69. apiVersion: rbac.authorization.k8s.io/v1
    70. kind: ClusterRole
    71. metadata:
    72.   name: all-readonly-${clustername}
    73. rules:
    74. - apiGroups:
    75.   - ''
    76.   resources:
    77.   - configmaps
    78.   - endpoints
    79.   - persistentvolumes
    80.   - persistentvolumeclaims
    81.   - pods
    82.   - replicationcontrollers
    83.   - replicationcontrollers/scale
    84.   - serviceaccounts
    85.   - services
    86.   - nodes
    87.   verbs:
    88.   - get
    89.   - list
    90.   - watch
    91. - apiGroups:
    92.   - ''
    93.   resources:
    94.   - bindings
    95.   - events
    96.   - limitranges
    97.   - namespaces/status
    98.   - pods/log
    99.   - pods/status
    100.   - replicationcontrollers/status
    101.   - resourcequotas
    102.   - resourcequotas/status
    103.   verbs:
    104.   - get
    105.   - list
    106.   - watch
    107. - apiGroups:
    108.   - ''
    109.   resources:
    110.   - namespaces
    111.   verbs:
    112.   - get
    113.   - list
    114.   - watch
    115. - apiGroups:
    116.   - apps
    117.   resources:
    118.   - controllerrevisions
    119.   - daemonsets
    120.   - deployments
    121.   - deployments/scale
    122.   - replicasets
    123.   - replicasets/scale
    124.   - statefulsets
    125.   - statefulsets/scale
    126.   verbs:
    127.   - get
    128.   - list
    129.   - watch
    130. - apiGroups:
    131.   - autoscaling
    132.   resources:
    133.   - horizontalpodautoscalers
    134.   verbs:
    135.   - get
    136.   - list
    137.   - watch
    138. - apiGroups:
    139.   - batch
    140.   resources:
    141.   - cronjobs
    142.   - jobs
    143.   verbs:
    144.   - get
    145.   - list
    146.   - watch
    147. - apiGroups:
    148.   - extensions
    149.   resources:
    150.   - daemonsets
    151.   - deployments
    152.   - deployments/scale
    153.   - ingresses
    154.   - networkpolicies
    155.   - replicasets
    156.   - replicasets/scale
    157.   - replicationcontrollers/scale
    158.   verbs:
    159.   - get
    160.   - list
    161.   - watch
    162. - apiGroups:
    163.   - policy
    164.   resources:
    165.   - poddisruptionbudgets
    166.   verbs:
    167.   - get
    168.   - list
    169.   - watch
    170. - apiGroups:
    171.   - networking.k8s.io
    172.   resources:
    173.   - networkpolicies
    174.   verbs:
    175.   - get
    176.   - list
    177.   - watch
    178. - apiGroups:
    179.   - metrics.k8s.io
    180.   resources:
    181.   - pods
    182.   verbs:
    183.   - get
    184.   - list
    185.   - watch" | kubectl apply -f -
    186.  
    187. kubectl -n kube-system create sa all-readonly-${clustername}
    188. kubectl create clusterrolebinding all-readonly-${clustername} --clusterrole=all-readonly-${clustername} --serviceaccount=kube-system:all-readonly-${clustername}
    189.  
    190. tokenName=$(kubectl -n kube-system get sa all-readonly-${clustername} -o "jsonpath={.secrets[0].name}")
    191. token=$(kubectl -n kube-system get secret $tokenName -o "jsonpath={.data.token}" | base64 --decode)
    192. certificate=$(kubectl -n kube-system get secret $tokenName -o "jsonpath={.data['ca\.crt']}")
    193.  
    194. echo "apiVersion: v1
    195. kind: Config
    196. preferences: {}
    197. clusters:
    198. - cluster:
    199.     certificate-authority-data: $certificate
    200.     server: $endpoint
    201.   name: all-readonly-${clustername}-cluster
    202. users:
    203. - name: all-readonly-${clustername}
    204.   user:
    205.     as-user-extra: {}
    206.     client-key-data: $certificate
    207.     token: $token
    208. contexts:
    209. - context:
    210.     cluster: all-readonly-${clustername}-cluster
    211.     user: all-readonly-${clustername}
    212.   name: ${clustername}
    213. current-context: ${clustername}" > ./all-readonly-${clustername}.conf
    214.

    5.命令测试

    1.  
    2. [root@k8s-m01 rbac]# kubectl --kubeconfig=./kube_config/web.kube.conf get pod
    3. No resources found in web namespace.
    4. [root@k8s-m01 rbac]#
    5. [root@k8s-m01 rbac]#
    6. [root@k8s-m01 rbac]# kubectl --kubeconfig=./kube_config/web.kube.conf get pod -A
    7. Error from server (Forbidden): pods is forbidden: User "system:serviceaccount:web:web-user" cannot list resource "pods" in API group "" at the cluster scope
    8.  
    9. [root@k8s-m01 rbac]# kubectl -n web get sa
    10. NAME       SECRETS   AGE
    11. default    1         119m
    12. web-user   1         42m
    13. [root@k8s-m01 rbac]#
    14. [root@k8s-m01 rbac]# kubectl -n web get role
    15. NAME                   CREATED AT
    16. web-user-full-access   2022-07-28T15:09:05Z
    17. [root@k8s-m01 rbac]#
    18. [root@k8s-m01 rbac]# kubectl -n web get rolebindings.rbac.authorization.k8s.io
    19. NAME            ROLE                        AGE
    20. web-user-view   Role/web-user-full-access   42m
    21. [root@k8s-m01 rbac]#
    22.  
    23.  
    24.

    6.rules 中的参数说明如下：


    apiGroups: 支持的API组列表，例如”apiVersion: batch/v1”、”apiVersion: extensions”、”apiVersion: apps”
    resources: 支持的资源对象列表，例如 pods、deployments、secrets、jobs、configmaps、endpoints、persistentvolumeclaims、replicationcontrollers、statefulsets、namespaces等。
    verbs: 对资源对象 的操作方法列表, 例如get获取, list列表, watch监视, patch修补, delete删除, update更新, create创建 ,deletecollection级联删除, redirect重定向,replace代替...等。

    九、k8s业务日志收集上节介绍、下节实战

    日志收集

    现在市面上大多数课程都是以EFK来作来K8s项目的日志解决方案，它包括三个组件：Elasticsearch, Fluentd（filebeat）, Kibana；Elasticsearch 是日志存储和日志搜索引擎，Fluentd 负责把k8s集群的日志发送给 Elasticsearch, Kibana 则是可视化界面查看和检索存储在 Elasticsearch 的数据。

    但根据生产中实际使用情况来看，它有以下弊端：

    1、日志收集系统 EFK是在每个kubernetes的NODE节点以daemonset的形式启动一个fluentd的pod，来收集NODE节点上的日志，如容器日志（/var/log/containers/*.log），但里面无法作细分，想要的和不想要的都收集进来了，带来的后面就是磁盘IO压力会比较大，日志过滤麻烦。

    2、无法收集对应POD里面的业务日志 上面第1点只能收集pod的stdout日志，但是pod内如有需要收集的业务日志，像pod内的/tmp/datalog/*.log，那EFK是无能为力的，只能是在pod内启动多个容器（filebeat）去收集容器内日志，但这又会带来的是pod多容器性能的损耗，这个接下来会详细讲到。

    3、fluentd的采集速率性能较低，只能不到filebeat的1/10的性能。

    基于此，我通过调研发现了阿里开源的智能容器采集工具 Log-Pilot，github地址：
    https://github.com/AliyunContainerService/log-pilot

    下面以sidecar 模式和log-pilot这两种方式的日志收集形式做个详细对比说明：

    第一种模式是 sidecar 模式，这种需要我们在每个 Pod 中都附带一个 logging 容器来进行本 Pod 内部容器的日志采集，一般采用共享卷的方式，但是对于这一种模式来说，很明显的一个问题就是占用的资源比较多，尤其是在集群规模比较大的情况下，或者说单个节点上容器特别多的情况下，它会占用过多的系统资源，同时也对日志存储后端占用过多的连接数。当我们的集群规模越大，这种部署模式引发的潜在问题就越大。

    92a2c4d4a5d641a89ef349eb1385e82e.png

    另一种模式是 Node 模式，这种模式是我们在每个 Node 节点上仅需部署一个 logging 容器来进行本 Node 所有容器的日志采集。这样跟前面的模式相比最明显的优势就是占用资源比较少，同样在集群规模比较大的情况下表现出的优势越明显，同时这也是社区推荐的一种模式

    495965e2bcb94165ba0a7ad9e879c2e7.png

    经过多方面测试，log-pilot对现有业务pod侵入性很小，只需要在原有pod的内传入几行env环境变量，即可对此pod相关的日志进行收集，已经测试了后端接收的工具有logstash、elasticsearch、kafka、redis、file，均OK，下面开始部署整个日志收集环境。

    我们这里用一个tomcat服务来模拟业务服务，用log-pilot分别收集它的stdout以及容器内的业务数据日志文件到指定后端存储（这里分别以elasticsearch、kafka的这两种企业常用的接收工具来做示例）

    准备好相应的yaml配置

    vi tomcat-test.yaml

    1. apiVersion: apps/v1
    2. kind: Deployment
    3. metadata:
    4.   labels:
    5.     app: tomcat
    6.   name: tomcat
    7. spec:
    8.   replicas: 1
    9.   selector:
    10.     matchLabels:
    11.       app: tomcat
    12.   template:
    13.     metadata:
    14.       labels:
    15.         app: tomcat
    16.     spec:
    17.       tolerations:
    18.       - key: "node-role.kubernetes.io/master"
    19.         effect: "NoSchedule"
    20.       containers:
    21.       - name: tomcat
    22.         image: "tomcat:7.0"
    23.         env:      # 注意点一，添加相应的环境变量（下面收集了两块日志1、stdout 2、/usr/local/tomcat/logs/catalina.*.log）
    24.         - name: aliyun_logs_tomcat-syslog   # 如日志发送到es，那index名称为 tomcat-syslog
    25.           value: "stdout"
    26.         - name: aliyun_logs_tomcat-access   # 如日志发送到es，那index名称为 tomcat-access
    27.           value: "/usr/local/tomcat/logs/catalina.*.log"
    28.         volumeMounts:   # 注意点二，对pod内要收集的业务日志目录需要进行共享，可以收集多个目录下的日志文件
    29.           - name: tomcat-log
    30.             mountPath: /usr/local/tomcat/logs
    31.       volumes:
    32.         - name: tomcat-log
    33.           emptyDir: {}
    34.

    #运行ym
    kubectl apply -f tomcat-test.yaml

    #查看pod
    kubectl get pod

    #查看日志
    kubectl logs tomcat-7cd6957667-bjx9r

    #进入容器看看日志文件
    kubectl exec -it tomcat-7cd6957667-bjx9r /bin/bash

    ls -l /usr/local/tomcat/logs/
    exit

    vi elasticsearch.6.8.13-statefulset.yaml

    1. apiVersion: apps/v1
    2. kind: StatefulSet
    3. metadata:
    4.   labels:
    5.     addonmanager.kubernetes.io/mode: Reconcile
    6.     k8s-app: elasticsearch-logging
    7.     version: v6.8.13
    8.   name: elasticsearch-logging
    9.   # namespace: logging
    10. spec:
    11.   replicas: 1
    12.   revisionHistoryLimit: 10
    13.   selector:
    14.     matchLabels:
    15.       k8s-app: elasticsearch-logging
    16.       version: v6.8.13
    17.   serviceName: elasticsearch-logging
    18.   template:
    19.     metadata:
    20.       labels:
    21.         k8s-app: elasticsearch-logging
    22.         version: v6.8.13
    23.     spec:
    24. #      nodeSelector:
    25. #        esnode: "true"  ## 注意给想要运行到的node打上相应labels
    26.       containers:
    27.       - env:
    28.         - name: NAMESPACE
    29.           valueFrom:
    30.             fieldRef:
    31.               apiVersion: v1
    32.               fieldPath: metadata.namespace
    33.         - name: cluster.name
    34.           value: elasticsearch-logging-0
    35.         - name: ES_JAVA_OPTS
    36.           value: "-Xms512m -Xmx512m"
    37.         image: elastic/elasticsearch:6.8.13
    38.         name: elasticsearch-logging
    39.         ports:
    40.         - containerPort: 9200
    41.           name: db
    42.           protocol: TCP
    43.         - containerPort: 9300
    44.           name: transport
    45.           protocol: TCP
    46.         volumeMounts:
    47.         - mountPath: /usr/share/elasticsearch/data
    48.           name: elasticsearch-logging
    49.       dnsConfig:
    50.         options:
    51.         - name: single-request-reopen
    52.       initContainers:
    53.       - command:
    54.         - /bin/sysctl
    55.         - -w
    56.         - vm.max_map_count=262144
    57.         image: busybox
    58.         imagePullPolicy: IfNotPresent
    59.         name: elasticsearch-logging-init
    60.         resources: {}
    61.         securityContext:
    62.           privileged: true
    63.       - name: fix-permissions
    64.         image: busybox
    65.         command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
    66.         securityContext:
    67.           privileged: true
    68.         volumeMounts:
    69.         - name: elasticsearch-logging
    70.           mountPath: /usr/share/elasticsearch/data
    71.       volumes:
    72.       - name: elasticsearch-logging
    73.         hostPath:
    74.           path: /esdata
    75. ---
    76. apiVersion: v1
    77. kind: Service
    78. metadata:
    79.   labels:
    80.     k8s-app: elasticsearch-logging
    81.   name: elasticsearch
    82.   # namespace: logging
    83. spec:
    84.   ports:
    85.   - port: 9200
    86.     protocol: TCP
    87.     targetPort: db
    88.   selector:
    89.     k8s-app: elasticsearch-logging
    90.   type: ClusterIP
    91.  
    92.

    kubectl apply -f elasticsearch.6.8.13-statefulset.yaml

    kubectl get pod

    vim kibana.6.8.13.yaml

    1. apiVersion: apps/v1
    2. kind: Deployment
    3. metadata:
    4.   name: kibana
    5.   # namespace: logging
    6.   labels:
    7.     app: kibana
    8. spec:
    9.   selector:
    10.     matchLabels:
    11.       app: kibana
    12.   template:
    13.     metadata:
    14.       labels:
    15.         app: kibana
    16.     spec:
    17.       containers:
    18.       - name: kibana
    19.         image: elastic/kibana:6.8.13
    20.         resources:
    21.           limits:
    22.             cpu: 1000m
    23.           requests:
    24.             cpu: 100m
    25.         env:
    26.           - name: ELASTICSEARCH_URL
    27.             value: http://elasticsearch:9200
    28.         ports:
    29.         - containerPort: 5601
    30. ---
    31. apiVersion: v1
    32. kind: Service
    33. metadata:
    34.   name: kibana
    35.   # namespace: logging
    36.   labels:
    37.     app: kibana
    38. spec:
    39.   ports:
    40.   - port: 5601
    41.     protocol: TCP
    42.     targetPort: 5601
    43.   type: ClusterIP
    44.   selector:
    45.     app: kibana
    46. ---
    47. apiVersion: extensions/v1beta1
    48. kind: Ingress
    49. metadata:
    50.   name: kibana
    51.   # namespace: logging
    52. spec:
    53.   rules:
    54.   - host: kibana.boge.com
    55.     http:
    56.       paths:
    57.       - path: /
    58.         backend:
    59.           serviceName: kibana
    60.           servicePort: 5601

    vi log-pilot.yml        # 后端输出的elasticsearch

    1. apiVersion: apps/v1
    2. kind: DaemonSet
    3. metadata:
    4.   name: log-pilot
    5.   labels:
    6.     app: log-pilot
    7.   # 设置期望部署的namespace
    8. #  namespace: ns-elastic
    9. spec:
    10.   selector:
    11.     matchLabels:
    12.       app: log-pilot
    13.   updateStrategy:
    14.     type: RollingUpdate
    15.   template:
    16.     metadata:
    17.       labels:
    18.         app: log-pilot
    19.       annotations:
    20.         scheduler.alpha.kubernetes.io/critical-pod: ''
    21.     spec:
    22.       # 是否允许部署到Master节点上
    23.       #tolerations:
    24.       #- key: node-role.kubernetes.io/master
    25.       #  effect: NoSchedule
    26.       containers:
    27.       - name: log-pilot
    28.         # 版本请参考https://github.com/AliyunContainerService/log-pilot/releases
    29.         image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
    30.         resources:
    31.           limits:
    32.             memory: 500Mi
    33.           requests:
    34.             cpu: 200m
    35.             memory: 200Mi
    36.         env:
    37.           - name: "NODE_NAME"
    38.             valueFrom:
    39.               fieldRef:
    40.                 fieldPath: spec.nodeName
    41.           ##--------------------------------
    42. #          - name: "LOGGING_OUTPUT"
    43. #            value: "logstash"
    44. #          - name: "LOGSTASH_HOST"
    45. #            value: "logstash-g1"
    46. #          - name: "LOGSTASH_PORT"
    47. #            value: "5044"
    48.           ##--------------------------------
    49.           - name: "LOGGING_OUTPUT"
    50.             value: "elasticsearch"
    51.           ## 请确保集群到ES网络可达
    52.           - name: "ELASTICSEARCH_HOSTS"
    53.             value: "elasticsearch:9200"
    54.           ## 配置ES访问权限
    55.           #- name: "ELASTICSEARCH_USER"
    56.           #  value: "{es_username}"
    57.           #- name: "ELASTICSEARCH_PASSWORD"
    58.           #  value: "{es_password}"
    59.           ##--------------------------------
    60.           ## https://github.com/AliyunContainerService/log-pilot/blob/master/docs/filebeat/docs.md
    61.           ## to file need configure 1
    62. #          - name: LOGGING_OUTPUT
    63. #            value: file
    64. #          - name: FILE_PATH
    65. #            value: /tmp
    66. #          - name: FILE_NAME
    67. #            value: filebeat.log
    68.         volumeMounts:
    69.         - name: sock
    70.           mountPath: /var/run/docker.sock
    71.         - name: root
    72.           mountPath: /host
    73.           readOnly: true
    74.         - name: varlib
    75.           mountPath: /var/lib/filebeat
    76.         - name: varlog
    77.           mountPath: /var/log/filebeat
    78.         - name: localtime
    79.           mountPath: /etc/localtime
    80.           readOnly: true
    81.          ## to file need configure 2
    82. #        - mountPath: /tmp
    83. #          name: mylog
    84.         livenessProbe:
    85.           failureThreshold: 3
    86.           exec:
    87.             command:
    88.             - /pilot/healthz
    89.           initialDelaySeconds: 10
    90.           periodSeconds: 10
    91.           successThreshold: 1
    92.           timeoutSeconds: 2
    93.         securityContext:
    94.           capabilities:
    95.             add:
    96.             - SYS_ADMIN
    97.       terminationGracePeriodSeconds: 30
    98.       volumes:
    99.       - name: sock
    100.         hostPath:
    101.           path: /var/run/docker.sock
    102.       - name: root
    103.         hostPath:
    104.           path: /
    105.       - name: varlib
    106.         hostPath:
    107.           path: /var/lib/filebeat
    108.           type: DirectoryOrCreate
    109.       - name: varlog
    110.         hostPath:
    111.           path: /var/log/filebeat
    112.           type: DirectoryOrCreate
    113.       - name: localtime
    114.         hostPath:
    115.           path: /etc/localtime
    116.        ## to file need configure 3
    117. #      - hostPath:
    118. #          path: /tmp/mylog
    119. #          type: ""
    120. #        name: mylog
    121.

    kubectl apply -f log-pilot.yml

    kubectl get pod

    节点加入hosts解析
    echo “20.6.100.226 kibana.boge.com” >> /etc/hosts

    eb3c20a1545f4a0ea0d95b4a63c87d2b.png

    4.进入页面

    f1a429ddb6754b138b2396535b816094.png

    5.定义日志名称

    9e9cb9383e234dd5abee33f4edca3784.png

    6.创建日期

    85634154b59142358f80e3bc81b82b23.png

     7.查看日志展示
    d0a1b35756604c8181f39c232d5e3793.png

    删除测试环境
    kubectl delete -f log-pilot.yml
    kubectl delete -f kibana.6.8.13.yaml
    kubectl delete -f elasticsearch.6.8.13-statefulset.yaml
    kubectl delete -f tomcat-test.yaml

    8.第2种：#后端输出到kafka

    kubectl apply -f tomcat-test.yaml

    vi log-pilot2-kafka.yaml

    1. ---
    2. apiVersion: v1
    3. kind: ConfigMap
    4. metadata:
    5.   name: log-pilot2-configuration
    6.   #namespace: ns-elastic
    7. data:
    8.   logging_output: "kafka"
    9.   kafka_brokers: "10.0.1.204:9092"
    10.   kafka_version: "0.10.0"
    11.   # configure all valid topics in kafka
    12.   # when disable auto-create topic
    13.   kafka_topics: "tomcat-syslog,tomcat-access"
    14. ---
    15. apiVersion: apps/v1
    16. kind: DaemonSet
    17. metadata:
    18.   name: log-pilot2
    19.   #namespace: ns-elastic
    20.   labels:
    21.     k8s-app: log-pilot2
    22. spec:
    23.   selector:
    24.     matchLabels:
    25.       k8s-app: log-pilot2
    26.   updateStrategy:
    27.     type: RollingUpdate
    28.   template:
    29.     metadata:
    30.       labels:
    31.         k8s-app: log-pilot2
    32.     spec:
    33.       tolerations:
    34.       - key: node-role.kubernetes.io/master
    35.         effect: NoSchedule
    36.       containers:
    37.       - name: log-pilot2
    38. #
    39. #        wget https://github.com/AliyunContainerService/log-pilot/archive/v0.9.7.zip
    40. #        unzip log-pilot-0.9.7.zip
    41. #        vim ./log-pilot-0.9.7/assets/filebeat/config.filebeat
    42. #        ...
    43. #        output.kafka:
    44. #            hosts: [$KAFKA_BROKERS]
    45. #            topic: '%{[topic]}'
    46. #            codec.format:
    47. #                string: '%{[message]}'
    48. #        ...
    49.         image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
    50.         env:
    51.           - name: "LOGGING_OUTPUT"
    52.             valueFrom:
    53.               configMapKeyRef:
    54.                 name: log-pilot2-configuration
    55.                 key: logging_output
    56.           - name: "KAFKA_BROKERS"
    57.             valueFrom:
    58.               configMapKeyRef:
    59.                 name: log-pilot2-configuration
    60.                 key: kafka_brokers
    61.           - name: "KAFKA_VERSION"
    62.             valueFrom:
    63.               configMapKeyRef:
    64.                 name: log-pilot2-configuration
    65.                 key: kafka_version
    66.           - name: "NODE_NAME"
    67.             valueFrom:
    68.               fieldRee:
    69.                 fieldPath: spec.nodeName
    70.         volumeMounts:
    71.         - name: sock
    72.           mountPath: /var/run/docker.sock
    73.         - name: logs
    74.           mountPath: /var/log/filebeat
    75.         - name: state
    76.           mountPath: /var/lib/filebeat
    77.         - name: root
    78.           mountPath: /host
    79.           readOnly: true
    80.         - name: localtime
    81.           mountPath: /etc/localtime
    82.         # configure all valid topics in kafka
    83.         # when disable auto-create topic
    84.         - name: config-volume
    85.           mountPath: /etc/filebeat/config
    86.         securityContext:
    87.           capabilities:
    88.             add:
    89.             - SYS_ADMIN
    90.       terminationGracePeriodSeconds: 30
    91.       volumes:
    92.       - name: sock
    93.         hostPath:
    94.           path: /var/run/docker.sock
    95.           type: Socket
    96.       - name: logs
    97.         hostPath:
    98.           path: /var/log/filebeat
    99.           type: DirectoryOrCreate
    100.       - name: state
    101.         hostPath:
    102.           path: /var/lib/filebeat
    103.           type: DirectoryOrCreate
    104.       - name: root
    105.         hostPath:
    106.           path: /
    107.           type: Directory
    108.       - name: localtime
    109.         hostPath:
    110.           path: /etc/localtime
    111.           type: File
    112.       # kubelet sync period
    113.       - name: config-volume
    114.         configMap:
    115.           name: log-pilot2-configuration
    116.           items:
    117.           - key: kafka_topics
    118.             path: kafka_topics
    119.

    9.准备一个测试用的kafka服务

    1. # 部署前准备
    2. # 0. 先把代码pull到本地
    3. # https://github.com/wurstmeister/kafka-docker
    4. # 修改docker-compose.yml为：
    5. #——------------------------------
    6. version: '2'
    7. services:
    8.   zookeeper:
    9.     image: wurstmeister/zookeeper
    10.     ports:
    11.       - "2181:2181"
    12.   kafka:
    13.     #build: .
    14.     image: wurstmeister/kafka
    15.     ports:
    16.       - "9092:9092"
    17.     environment:
    18.       KAFKA_ADVERTISED_HOST_NAMe: 10.0.1.204  # docker运行的机器IP
    19.       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    20.     volumes:
    21.       - /var/run/docker.sock:/var/run/docker.sock
    22.       - /nfs_storageclass/kafka:/kafka
    23. #——------------------------------
    24.  
    25. #启动
    26. # docker-compose up -d
    27. #删除
    28. #docker-compose down -v
    29.  
    30. #重启
    31. #docker-compose up -d
    32.  
    33.  
    34. # 2. result look:
    35. #查看启动进程
    36. # docker ps |grep zook
    37.  
    38. #docker ps |grep ka
    39.  
    40.  
    41. #进入kafka
    42. bash-4.4# docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=10.0.1.204 -e ZK=10.0.1.204:2181 -i -t wurstmeister/kafka /bin/bash
    43.  
    44. # 4. list topic
    45. bash-4.4# kafka-topics.sh --zookeeper 10.0.1.204:2181 --list
    46. tomcat-access
    47. tomcat-syslog
    48.  
    49. # 5. consumer topic data:
    50. bash-4.4# kafka-console-consumer.sh --bootstrap-server 10.0.1.204:9092 --topic tomcat-access --from-beginning
    51.

    十、k8s的Prometheus监控实战

    1. 链接：https://pan.baidu.com/s/1O-RQd3n2HJrOdRbElidnEA?pwd=hoqv 
    2. 提取码：hoqv

    1.导入上传tar包

    sudo docker load -i alertmanager-v0.21.0.tar
    sudo docker load -i grafana-7.3.4.tar
    sudo docker load -i k8s-prometheus-adapter-v0.8.2.tar
    sudo docker load -i kube-rbac-proxy-v0.8.0.tar
    sudo docker load -i kube-state-metrics-v1.9.7.tar
    sudo docker load -i node-exporter-v1.0.1.tar
    sudo docker load -i prometheus-config-reloader-v0.43.2.tar
    sudo docker load -i prometheus_demo_service.tar
    sudo docker load -i prometheus-operator-v0.43.2.tar
    sudo docker load -i prometheus-v2.22.1.tar

    2.解压下载的代码包

    sudo unzip kube-prometheus-master.zip
    sudo rm -f kube-prometheus-master.zip && cd kube-prometheus-master

    3.这里建议先看下有哪些镜像，便于在下载镜像快的节点上先收集好所有需要的离线docker镜像

    1. find ./ -type f |xargs grep 'image: '|sort|uniq|awk '{print $3}'|grep ^[a-zA-Z]|grep -Evw 'error|kubeRbacProxy'|sort -rn|uniq
    2.

    4.开始创建所有服务

    1. kubectl create -f manifests/setup
    2. kubectl create -f manifests/
    3. #过一会查看创建结果：
    4. kubectl -n monitoring get all
    5.  
    6.  
    7.  
    8. # 附：清空上面部署的prometheus所有服务：
    9. # kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
    10.  
    11.  
    12.

    5.访问下prometheus的UI

    1. # 修改下prometheus UI的service模式，便于我们访问
    2. # kubectl -n monitoring patch svc prometheus-k8s -p '{"spec":{"type":"NodePort"}}'
    3. service/prometheus-k8s patched
    4.  
    5. # kubectl -n monitoring get svc prometheus-k8s 
    6. NAME             TYPE       CLUSTER-IP    EXTERNAL-IP   PORT(S)          AGE
    7. prometheus-k8s   NodePort   10.68.23.79           9090:22129/TCP   7m43s
    8.

    6.然后因为K8s的这两上核心组件我们是以二进制形式部署的，为了能让K8s上的prometheus能发现，我们还需要来创建相应的service和endpoints来将其关联起来
    注意：我们需要将endpoints里面的mast IP换成我们实际情况的

    1. apiVersion: v1
    2. kind: Service
    3. metadata:
    4.   namespace: kube-system
    5.   name: kube-controller-manager
    6.   labels:
    7.     k8s-app: kube-controller-manager
    8. spec:
    9.   type: ClusterIP
    10.   clusterIP: None
    11.   ports:
    12.   - name: http-metrics
    13.     port: 10252
    14.     targetPort: 10252
    15.     protocol: TCP
    16.  
    17. ---
    18. apiVersion: v1
    19. kind: Endpoints
    20. metadata:
    21.   labels:
    22.     k8s-app: kube-controller-manager
    23.   name: kube-controller-manager
    24.   namespace: kube-system
    25. subsets:
    26. - addresses:
    27.   - ip: 10.0.1.201
    28.   - ip: 10.0.1.202
    29.   ports:
    30.   - name: http-metrics
    31.     port: 10252
    32.     protocol: TCP
    33.  
    34. ---
    35.  
    36. apiVersion: v1
    37. kind: Service
    38. metadata:
    39.   namespace: kube-system
    40.   name: kube-scheduler
    41.   labels:
    42.     k8s-app: kube-scheduler
    43. spec:
    44.   type: ClusterIP
    45.   clusterIP: None
    46.   ports:
    47.   - name: http-metrics
    48.     port: 10251
    49.     targetPort: 10251
    50.     protocol: TCP
    51.  
    52. ---
    53. apiVersion: v1
    54. kind: Endpoints
    55. metadata:
    56.   labels:
    57.     k8s-app: kube-scheduler
    58.   name: kube-scheduler
    59.   namespace: kube-system
    60. subsets:
    61. - addresses:
    62.   - ip: 10.0.1.201
    63.   - ip: 10.0.1.202
    64.   ports:
    65.   - name: http-metrics
    66.     port: 10251
    67.     protocol: TCP
    68.

    7.将上面的yaml配置保存为repair-prometheus.yaml，然后创建它

    kubectl apply -f repair-prometheus.yaml

    8.记得还要修改一个地方

    # kubectl -n monitoring edit servicemonitors.monitoring.coreos.com kube-scheduler
    # 将下面两个地方的https换成http
        port: https-metrics
        scheme: https

    # kubectl -n monitoring edit servicemonitors.monitoring.coreos.com kube-controller-manager
    # 将下面两个地方的https换成http
        port: https-metrics
        scheme: https

    9.然后再返回prometheus UI处，耐心等待几分钟，就能看到已经被发现了

    monitoring/kube-controller-manager/0 (2/2 up)
    monitoring/kube-scheduler/0 (2/2 up)

    10.使用prometheus来监控ingress-nginx

    我们前面部署过ingress-nginx，这个是整个K8s上所有服务的流量入口组件很关键，因此把它的metrics指标收集到prometheus来做好相关监控至关重要，因为前面ingress-nginx服务是以daemonset形式部署的，并且映射了自己的端口到宿主机上，那么我可以直接用pod运行NODE上的IP来看下metrics

    curl 10.0.1.201:10254/metrics

    11.创建 servicemonitor配置让prometheus能发现ingress-nginx的metrics

    1. # vim servicemonitor.yaml
    2. apiVersion: monitoring.coreos.com/v1
    3. kind: ServiceMonitor
    4. metadata:
    5.   labels:
    6.     app: ingress-nginx
    7.   name: nginx-ingress-scraping
    8.   namespace: ingress-nginx
    9. spec:
    10.   endpoints:
    11.   - interval: 30s
    12.     path: /metrics
    13.     port: metrics
    14.   jobLabel: app
    15.   namespaceSelector:
    16.     matchNames:
    17.     - ingress-nginx
    18.   selector:
    19.     matchLabels:
    20.       app: ingress-nginx
    21.

    12.创建它

    # kubectl apply -f servicemonitor.yaml
    servicemonitor.monitoring.coreos.com/nginx-ingress-scraping created
    # kubectl -n ingress-nginx get servicemonitors.monitoring.coreos.com
    NAME                     AGE
    nginx-ingress-scraping   8s

    13.指标一直没收集上来，看看proemtheus服务的日志，发现报错如下：

    # kubectl -n monitoring logs prometheus-k8s-0 -c prometheus |grep error

    level=error ts=2020-12-13T09:52:35.565Z caller=klog.go:96 component=k8s_client_runtime func=ErrorDepth msg="/app/discovery/kubernetes/kubernetes.go:426: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"ingress-nginx\""

    14.需要修改prometheus的clusterrole

    #   kubectl edit clusterrole prometheus-k8s
    #------ 原始的rules -------
    rules:
    - apiGroups:
      - ""
      resources:
      - nodes/metrics
      verbs:
      - get
    - nonResourceURLs:
      - /metrics
      verbs:
      - get
    #---------------------------

    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      name: prometheus-k8s
    rules:
    - apiGroups:
      - ""
      resources:
      - nodes
      - services
      - endpoints
      - pods
      - nodes/proxy
      verbs:
      - get
      - list
      - watch
    - apiGroups:
      - ""
      resources:
      - configmaps
      - nodes/metrics
      verbs:
      - get
    - nonResourceURLs:
      - /metrics
      verbs:
      - get
     

    15.再到prometheus UI上看下，发现已经有了

    ingress-nginx/nginx-ingress-scraping/0 (1/1 up)

    16.使用Prometheus来监控二进制部署的ETCD集群

    作为K8s所有资源存储的关键服务ETCD，我们也有必要把它给监控起来，正好借这个机会，完整的演示一次利用Prometheus来监控非K8s集群服务的步骤

    在前面部署K8s集群的时候，我们是用二进制的方式部署的ETCD集群，并且利用自签证书来配置访问ETCD，正如前面所说，现在关键的服务基本都会留有指标metrics接口支持prometheus的监控，利用下面命令，我们可以看到ETCD都暴露出了哪些监控指标出来

    curl --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubeasz/clusters/test/ssl/etcd.pem  --key /etc/kubeasz/clusters/test/ssl/etcd-key.pem https://10.0.1.201:2379/metrics

    17.上面查看没问题后，接下来我们开始进行配置使ETCD能被prometheus发现并监控

    # 首先把ETCD的证书创建为secret
    kubectl -n monitoring create secret generic etcd-certs --from-file=/etc/kubeasz/clusters/test/ssl/etcd.pem   --from-file=/etc/kubeasz/clusters/test/ssl/etcd-key.pem   --from-file=/etc/kubeasz/clusters/test/ssl/ca.pem

    # 接着在prometheus里面引用这个secrets
    kubectl -n monitoring edit prometheus k8s

    spec:
    ...
      secrets:
      - etcd-certs

    # 保存退出后，prometheus会自动重启服务pod以加载这个secret配置，过一会，我们进pod来查看下是不是已经加载到ETCD的证书了
    # kubectl -n monitoring exec -it prometheus-k8s-0 -c prometheus  -- sh
    /prometheus $ ls /etc/prometheus/secrets/etcd-certs/
    ca.pem        etcd-key.pem  etcd.pem

    18.接下来准备创建service、endpoints以及ServiceMonitor的yaml配置

    注意替换下面的NODE节点IP为实际ETCD所在NODE内网IP

    # vim prometheus-etcd.yaml
    apiVersion: v1
    kind: Service
    metadata:
      name: etcd-k8s
      namespace: monitoring
      labels:
        k8s-app: etcd
    spec:
      type: ClusterIP
      clusterIP: None
      ports:
      - name: api
        port: 2379
        protocol: TCP
    ---
    apiVersion: v1
    kind: Endpoints
    metadata:
      name: etcd-k8s
      namespace: monitoring
      labels:
        k8s-app: etcd
    subsets:
    - addresses:
      - ip: 10.0.1.201
      - ip: 10.0.1.202
      - ip: 10.0.1.203
      ports:
      - name: api
        port: 2379
        protocol: TCP
    ---
    apiVersion: monitoring.coreos.com/v1
    kind: ServiceMonitor
    metadata:
      name: etcd-k8s
      namespace: monitoring
      labels:
        k8s-app: etcd-k8s
    spec:
      jobLabel: k8s-app
      endpoints:
      - port: api
        interval: 30s
        scheme: https
        tlsConfig:
          caFile: /etc/prometheus/secrets/etcd-certs/ca.pem
          certFile: /etc/prometheus/secrets/etcd-certs/etcd.pem
          keyFile: /etc/prometheus/secrets/etcd-certs/etcd-key.pem
          #use insecureSkipVerify only if you cannot use a Subject Alternative Name
          insecureSkipVerify: true
      selector:
        matchLabels:
          k8s-app: etcd
      namespaceSelector:
        matchNames:
        - monitoring

    19.开始创建上面的资源

    kubectl apply -f prometheus-etcd.yaml

    #过一会，就可以在prometheus UI上面看到ETCD集群被监控了

    monitoring/etcd-k8s/0 (3/3 up)

    20.接下来我们用grafana来展示被监控的ETCD指标


    查看grafana端口
    kubectl -n monitoring get svc |grep grafana

    1. 在grafana官网模板中心搜索etcd，下载这个json格式的模板文件
    https://grafana.com/dashboards/3070

    2.然后打开自己先部署的grafana首页，
    点击左边菜单栏四个小正方形方块HOME --- Manage
    再点击右边 Import dashboard ---
    点击Upload .json File 按钮，上传上面下载好的json文件 etcd_rev3.json，
    然后在prometheus选择数据来源
    点击Import，即可显示etcd集群的图形监控信息

    21.prometheus监控数据以及grafana配置持久化存储配置

    这节实战课给大家讲解下如果配置prometheus以及grafana的数据持久化。prometheus数据持久化配置

    # 注意这下面的statefulset服务就是我们需要做数据持久化的地方
    # kubectl -n monitoring get statefulset,pod|grep prometheus-k8s
    statefulset.apps/prometheus-k8s      2/2     5h41m
    pod/prometheus-k8s-0                       2/2     Running   1          19m
    pod/prometheus-k8s-1                       2/2     Running   1          19m

    # 看下我们之前准备的StorageClass动态存储
    # kubectl get sc
    NAME       PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
    nfs-boge   nfs-provisioner-01   Retain          Immediate           false                  4d

    # 准备prometheus持久化的pvc配置
    # kubectl -n monitoring edit prometheus k8s

    spec:
    ......
      storage:
        volumeClaimTemplate:
          spec:
            accessModes: [ "ReadWriteOnce" ]
            storageClassName: "nfs-boge"
            resources:
              requests:
                storage: 1Gi

    # 上面修改保存退出后，过一会我们查看下pvc创建情况，以及pod内的数据挂载情况
    # kubectl -n monitoring get pvc
    NAME                                 STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    prometheus-k8s-db-prometheus-k8s-0   Bound    pvc-055e6b11-31b7-4503-ba2b-4f292ba7bd06   1Gi        RWO            nfs-boge       17s
    prometheus-k8s-db-prometheus-k8s-1   Bound    pvc-249c344b-3ef8-4a5d-8003-b8ce8e282d32   1Gi        RWO            nfs-boge       17s


    # kubectl -n monitoring exec -it prometheus-k8s-0 -c prometheus -- sh
    /prometheus $ df -Th
    ......
    10.0.1.201:/nfs_dir/monitoring-prometheus-k8s-db-prometheus-k8s-0-pvc-055e6b11-31b7-4503-ba2b-4f292ba7bd06/prometheus-db
                         nfs4           97.7G      9.4G     88.2G  10% /prometheus

    22.grafana配置持久化存储配置

    # 保存pvc为grafana-pvc.yaml
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: grafana
      namespace: monitoring
    spec:
      storageClassName: nfs-boge
      accessModes:
        - ReadWriteMany
      resources:
        requests:
          storage: 1Gi

    # 开始创建pvc
    # kubectl apply -f grafana-pvc.yaml

    # 看下创建的pvc
    # kubectl -n monitoring get pvc
    NAME                                 STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    grafana                              Bound    pvc-394a26e1-3274-4458-906e-e601a3cde50d   1Gi        RWX            nfs-boge       3s
    prometheus-k8s-db-prometheus-k8s-0   Bound    pvc-055e6b11-31b7-4503-ba2b-4f292ba7bd06   1Gi        RWO            nfs-boge       6m46s
    prometheus-k8s-db-prometheus-k8s-1   Bound    pvc-249c344b-3ef8-4a5d-8003-b8ce8e282d32   1Gi        RWO            nfs-boge       6m46s


    # 编辑grafana的deployment资源配置
    # kubectl -n monitoring edit deployments.apps grafana

    # 旧的配置
          volumes:
          - emptyDir: {}
            name: grafana-storage
    # 替换成新的配置
          volumes:
          - name: grafana-storage
            persistentVolumeClaim:
              claimName: grafana

    # 同时加入下面的env环境变量，将登陆密码进行固定修改
        spec:
          containers:
            - containerPort: 3000
              name: http
              protocol: TCP
          ......
            env:
            - name: GF_SECURITY_ADMIN_USER
              value: admin
            - name: GF_SECURITY_ADMIN_PASSWORD
              value: admin321

    # 过一会，等grafana重启完成后，用上面的新密码进行登陆
    # kubectl -n monitoring get pod -w|grep grafana
    grafana-5698bf94f4-prbr2               0/1     Running   0          3s
    grafana-5698bf94f4-prbr2               1/1     Running   0          4s

    # 因为先前的数据并未持久化，所以会发现先导入的ETCD模板已消失，这时重新再导入一次，后面重启也不会丢了

    22 prometheus发送报警

    早期我们经常用邮箱接收报警邮件，但是报警不及时，而且目前各云平台对邮件发送限制还比较严格，所以目前在生产中用得更为多的是基于webhook来转发报警内容到企业中用的聊天工具中，比如钉钉、企业微信、飞书等。

    prometheus的报警组件是Alertmanager，它支持自定义webhook的方式来接受它发出的报警，它发出的日志json字段比较多，我们需要根据需要接收的app来做相应的日志清洗转发

    这里博哥将用golang结合Gin网络框架来编写一个日志清洗转发工具，分别对这几种常用的报警方式作详细地说明及实战

    下载boge-webhook.zip
    https://cloud.189.cn/t/B3EFZvnuMvuu (访问码:h1wx)

    首先看下报警规则及报警发送配置是什么样的

    prometheus-operator的规则非常齐全，基本属于开箱即用类型，大家可以根据日常收到的报警，对里面的rules报警规则作针对性的调整，比如把报警观察时长缩短一点等

    #进入压缩包
    cd /data/k8s/prometheus/webhook

    cat  Dockerfile
    #打包镜像
    docker build -t dockerck.e21.cn/product/alertmanaer-webhook:1.0 .

    #推送镜像
    docker push dockerck.e21.cn/product/alertmanaer-webhook:1.0

    kubectl apply -f alertmanaer-webhook.yaml

    find / -name "prometheus-rules.yaml"

    监控报警规划修改   vim /data/k8s/prometheus/kube-prometheus-master/manifests/prometheus-rules.yaml
    修改完成记得更新   kubectl apply -f /data/k8s/prometheus/kube-prometheus-master/manifests/prometheus-rules.yaml
    # 通过这里可以获取需要创建的报警配置secret名称
    # kubectl -n monitoring edit statefulsets.apps alertmanager-main
    ...
          volumes:
          - name: config-volume
            secret:
              defaultMode: 420
              secretName: alertmanager-main
    ...

    # 注意事先在配置文件 alertmanager.yaml 里面编辑好收件人等信息 ，再执行下面的命令


    kubectl -n monitoring get secrets alertmanager-main
    kubectl -n monitoring delete secrets alertmanager-main
    kubectl create secret generic  alertmanager-main --from-file=alertmanager.yaml -n monitoring


    kubectl -n monitoring delete pod alertmanager-main-0
    kubectl -n monitoring delete pod alertmanager-main-1
    kubectl -n monitoring delete pod alertmanager-main-2

    #查看svc
    kubectl get svc |grep alertmanaer-dingtalk-svc

    #测试串  
    curl -X POST -H 'Content-type: application/json' -d '{"name": "boge","titlea": "'"$(id)"'", "texta": "'"$(whoami)--$(hostname)"'"}' 10.68.64.119/5e00fc1a/prometheus/weixin

    十一、k8s安装kuboard图形化界面

    1.安装-获取部署 Kuboard 所需的 YAML 文件：

    curl -o kuboard-v3.yaml https://addons.kuboard.cn/kuboard/kuboard-v3-storage-class.yaml

    编辑 kuboard-v3.yaml 文件中的配置，该部署文件中，有1处配置必须修改：

    storageClassName

      volumeClaimTemplates:
      - metadata:
          name: data
        spec:
          # 请填写一个有效的 StorageClass name
          storageClassName: please-provide-a-valid-StorageClass-name-here
          accessModes: [ "ReadWriteMany" ]
          resources:
            requests:
              storage: 5Gi

    2.部署到 Kubernetes 集群

    kubectl create -f kuboard-v3.yaml

    访问 Kuboard

    在浏览器中打开链接 http://your-node-ip-address:30080

    输入初始用户名和密码，并登录
        用户名： admin
        密码： Kuboard123

    3.卸载

    执行 Kuboard v3 的卸载

    kubectl delete -f https://addons.kuboard.cn/kuboard/kuboard-v3-storage-class.yaml

    4.进入页面，导入集群
    4.1、名称和描述，自己定义
    4.2、将 cat /root/.kube/config 内容复制 kubeconfig中

    151c1b3b4e49472f9fa00aa10580aa9e.png

    11.离线安装下载:

    https://cloud.189.cn/t/qiaMzyvU3Aj2 (访问码:6ngs)

    十二、 k8s架构师课程基于gitlab的CICD自动化

    1.这节课我们先来部署gitlab私有代码仓库所需要的数据库postgresql和redis。

    需要注意的是，如果大家的nfs-server的地址和挂载目录不是按博哥前面课程讲得来定义的话，那么下面的yaml配置中需要记得替换。

    mkdir -p /nfs_dir/{gitlab_etc_ver130806,gitlab_log_ver130806,gitlab_opt_ver130806,gitlab_postgresql_data_ver130806}

    #创建表空间
    kubectl create namespace gitlab-ver130806

    kubectl get ns

    2.部署postgresql

    vi 3postgres.yaml

    1. # pv 3postgres.yaml
    2. ---
    3. apiVersion: v1
    4. kind: PersistentVolume
    5. metadata:
    6.   name: gitlab-postgresql-data-ver130806
    7.   labels:
    8.     type: gitlab-postgresql-data-ver130806
    9. spec:
    10.   capacity:
    11.     storage: 10Gi
    12.   accessModes:
    13.     - ReadWriteOnce
    14.   persistentVolumeReclaimPolicy: Retain
    15.   storageClassName: nfs
    16.   nfs:
    17.     path: /nfs_dir/gitlab_postgresql_data_ver130806
    18.     server: 10.0.1.201
    19.  
    20. # pvc
    21. ---
    22. kind: PersistentVolumeClaim
    23. apiVersion: v1
    24. metadata:
    25.   name: gitlab-postgresql-data-ver130806-pvc
    26. spec:
    27.   accessModes:
    28.     - ReadWriteOnce
    29.   resources:
    30.     requests:
    31.       storage: 10Gi
    32.   storageClassName: nfs
    33.   selector:
    34.     matchLabels:
    35.       type: gitlab-postgresql-data-ver130806
    36. ---
    37. apiVersion: v1
    38. kind: Service
    39. metadata:
    40.   name: postgresql
    41.   labels:
    42.     app: gitlab
    43.     tier: postgreSQL
    44. spec:
    45.   ports:
    46.     - port: 5432
    47.   selector:
    48.     app: gitlab
    49.     tier: postgreSQL
    50.  
    51. ---
    52.  
    53. apiVersion: apps/v1
    54. kind: Deployment
    55. metadata:
    56.   name: postgresql
    57.   labels:
    58.     app: gitlab
    59.     tier: postgreSQL
    60. spec:
    61.   replicas: 1
    62.   selector:
    63.     matchLabels:
    64.       app: gitlab
    65.       tier: postgreSQL
    66.   strategy:
    67.     type: Recreate
    68.   template:
    69.     metadata:
    70.       labels:
    71.         app: gitlab
    72.         tier: postgreSQL
    73.     spec:
    74.       #nodeSelector:
    75.       #  gee/disk: "500g"
    76.       containers:
    77.         - image: postgres:12.6-alpine
    78.         #- image: harbor.boge.com/library/postgres:12.6-alpine
    79.           name: postgresql
    80.           env:
    81.             - name: POSTGRES_USER
    82.               value: gitlab
    83.             - name: POSTGRES_DB
    84.               value: gitlabhq_production
    85.             - name: POSTGRES_PASSWORD
    86.               value: bogeusepg
    87.             - name: TZ
    88.               value: Asia/Shanghai
    89.           ports:
    90.             - containerPort: 5432
    91.               name: postgresql
    92.           livenessProbe:
    93.             exec:
    94.               command:
    95.               - sh
    96.               - -c
    97.               - exec pg_isready -U gitlab -h 127.0.0.1 -p 5432 -d gitlabhq_production
    98.             initialDelaySeconds: 110
    99.             timeoutSeconds: 5
    100.             failureThreshold: 6
    101.           readinessProbe:
    102.             exec:
    103.               command:
    104.               - sh
    105.               - -c
    106.               - exec pg_isready -U gitlab -h 127.0.0.1 -p 5432 -d gitlabhq_production
    107.             initialDelaySeconds: 20
    108.             timeoutSeconds: 3
    109.             periodSeconds: 5
    110. #          resources:
    111. #            requests:
    112. #              cpu: 100m
    113. #              memory: 512Mi
    114. #            limits:
    115. #              cpu: "1"
    116. #              memory: 1Gi
    117.           volumeMounts:
    118.             - name: postgresql
    119.               mountPath: /var/lib/postgresql/data
    120.       volumes:
    121.         - name: postgresql
    122.           persistentVolumeClaim:
    123.             claimName: gitlab-postgresql-data-ver130806-pvc

    3,创建pod

    kubectl -n gitlab-ver130806 apply -f 3postgres.yaml


    #检查pod
    kubectl -n gitlab-ver130806 get pod

    4.部署redis

    vi 4redis.yaml

    1. ---
    2. apiVersion: v1
    3. kind: Service
    4. metadata:
    5.   name: redis
    6.   labels:
    7.     app: gitlab
    8.     tier: backend
    9. spec:
    10.   ports:
    11.     - port: 6379
    12.       targetPort: 6379
    13.   selector:
    14.     app: gitlab
    15.     tier: backend
    16. ---
    17.  
    18. apiVersion: apps/v1
    19. kind: Deployment
    20. metadata:
    21.   name: redis
    22.   labels:
    23.     app: gitlab
    24.     tier: backend
    25. spec:
    26.   replicas: 1
    27.   selector:
    28.     matchLabels:
    29.       app: gitlab
    30.       tier: backend
    31.   strategy:
    32.     type: Recreate
    33.   template:
    34.     metadata:
    35.       labels:
    36.         app: gitlab
    37.         tier: backend
    38.     spec:
    39.       #nodeSelector:
    40.       #  gee/disk: "500g"
    41.       containers:
    42.         - image: redis:6.2.0-alpine3.13
    43.         #- image: harbor.boge.com/library/redis:6.2.0-alpine3.13
    44.           name: redis
    45.           command:
    46.             - "redis-server"
    47.           args:
    48.             - "--requirepass"
    49.             - "bogeuseredis"
    50. #          resources:
    51. #            requests:
    52. #              cpu: "1"
    53. #              memory: 2Gi
    54. #            limits:
    55. #              cpu: "1"
    56. #              memory: 2Gi
    57.           ports:
    58.             - containerPort: 6379
    59.               name: redis
    60.           livenessProbe:
    61.             exec:
    62.               command:
    63.               - sh
    64.               - -c
    65.               - "redis-cli ping"
    66.             initialDelaySeconds: 30
    67.             periodSeconds: 10
    68.             timeoutSeconds: 5
    69.             successThreshold: 1
    70.             failureThreshold: 3
    71.           readinessProbe:
    72.             exec:
    73.               command:
    74.               - sh
    75.               - -c
    76.               - "redis-cli ping"
    77.             initialDelaySeconds: 5
    78.             periodSeconds: 10
    79.             timeoutSeconds: 1
    80.             successThreshold: 1
    81.             failureThreshold: 3
    82.       initContainers:
    83.       - command:
    84.         - /bin/sh
    85.         - -c
    86.         - |
    87.           ulimit -n 65536
    88.           mount -o remount rw /sys
    89.           echo never > /sys/kernel/mm/transparent_hugepage/enabled
    90.           mount -o remount rw /proc/sys
    91.           echo 2000 > /proc/sys/net/core/somaxconn
    92.           echo 1 > /proc/sys/vm/overcommit_memory
    93.         image: registry.cn-beijing.aliyuncs.com/acs/busybox:v1.29.2
    94.         imagePullPolicy: IfNotPresent
    95.         name: init-redis
    96.         resources: {}
    97.         securityContext:
    98.           privileged: true
    99.           procMount: Default
    100.

    #创建pod

    kubectl -n gitlab-ver130806 apply -f 4redis.yaml
    #检查pod
    kubectl -n gitlab-ver130806 get pod

    5.开始部署gitlab服务。

    先定制一下镜像

    Dockerfile

    1. FROM gitlab/gitlab-ce:13.8.6-ce.0
    2.  
    3. RUN rm /etc/apt/sources.list \
    4.     && echo 'deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main' > /etc/apt/sources.list.d/pgdg.list \
    5.     && wget --no-check-certificate -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
    6. COPY sources.list /etc/apt/sources.list
    7.  
    8. RUN apt-get update -yq && \
    9.     apt-get install -y vim iproute2 net-tools iputils-ping curl wget software-properties-common unzip postgresql-client-12 && \
    10.     rm -rf /var/cache/apt/archives/*
    11.  
    12. RUN ln -svf /usr/bin/pg_dump /opt/gitlab/embedded/bin/pg_dump
    13.  
    14. #---------------------------------------------------------------
    15. # docker build -t gitlab/gitlab-ce:13.8.6-ce.1 .

    sources.list

    1. deb http://mirrors.aliyun.com/ubuntu/ xenial main
    2. deb-src http://mirrors.aliyun.com/ubuntu/ xenial main
    3. deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main
    4. deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main
    5. deb http://mirrors.aliyun.com/ubuntu/ xenial universe
    6. deb-src http://mirrors.aliyun.com/ubuntu/ xenial universe
    7. deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
    8. deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
    9. deb http://mirrors.aliyun.com/ubuntu xenial-security main
    10. deb-src http://mirrors.aliyun.com/ubuntu xenial-security main
    11. deb http://mirrors.aliyun.com/ubuntu xenial-security universe
    12. deb-src http://mirrors.aliyun.com/ubuntu xenial-security universe
    13.

    6.打包推送镜像

    docker tag gitlab/gitlab-ce:13.8.6-ce.1 dockerck.e21.cn/library/gitlab-ce:13.8.6-ce.1
    docker push dockerck.e21.cn/library/gitlab-ce:13.8.6-ce.1

    7.开始部署5gitlab.yaml

    vi 5gitlab.yaml

    1. # restore gitlab data command example:
    2. #   kubectl -n gitlab-ver130806 exec -it $(kubectl -n gitlab-ver130806 get pod|grep -v runner|grep gitlab|awk '{print $1}') -- gitlab-rake gitlab:backup:restore BACKUP=1602889879_2020_10_17_12.9.2
    3. #   kubectl -n gitlab-ver130806 exec -it $(kubectl -n gitlab-ver130806 get pod|grep -v runner|grep gitlab|awk '{print $1}') -- gitlab-ctl reconfigure
    4. #   kubectl -n gitlab-ver130806 exec -it $(kubectl -n gitlab-ver130806 get pod|grep -v runner|grep gitlab|awk '{print $1}') -- gitlab-ctl status
    5.  
    6. # pv
    7. ---
    8. apiVersion: v1
    9. kind: PersistentVolume
    10. metadata:
    11.   name: gitlab-etc-ver130806
    12.   labels:
    13.     type: gitlab-etc-ver130806
    14. spec:
    15.   capacity:
    16.     storage: 1Gi
    17.   accessModes:
    18.     - ReadWriteOnce
    19.   persistentVolumeReclaimPolicy: Retain
    20.   storageClassName: nfs
    21.   nfs:
    22.     path: /nfs_dir/gitlab_etc_ver130806
    23.     server: 10.0.1.201
    24.  
    25. # pvc
    26. ---
    27. kind: PersistentVolumeClaim
    28. apiVersion: v1
    29. metadata:
    30.   name: gitlab-etc-ver130806-pvc
    31. spec:
    32.   accessModes:
    33.     - ReadWriteOnce
    34.   resources:
    35.     requests:
    36.       storage: 1Gi
    37.   storageClassName: nfs
    38.   selector:
    39.     matchLabels:
    40.       type: gitlab-etc-ver130806
    41. # pv
    42. ---
    43. apiVersion: v1
    44. kind: PersistentVolume
    45. metadata:
    46.   name: gitlab-log-ver130806
    47.   labels:
    48.     type: gitlab-log-ver130806
    49. spec:
    50.   capacity:
    51.     storage: 1Gi
    52.   accessModes:
    53.     - ReadWriteOnce
    54.   persistentVolumeReclaimPolicy: Retain
    55.   storageClassName: nfs
    56.   nfs:
    57.     path: /nfs_dir/gitlab_log_ver130806
    58.     server: 10.0.1.201
    59.  
    60. # pvc
    61. ---
    62. kind: PersistentVolumeClaim
    63. apiVersion: v1
    64. metadata:
    65.   name: gitlab-log-ver130806-pvc
    66. spec:
    67.   accessModes:
    68.     - ReadWriteOnce
    69.   resources:
    70.     requests:
    71.       storage: 1Gi
    72.   storageClassName: nfs
    73.   selector:
    74.     matchLabels:
    75.       type: gitlab-log-ver130806
    76.       
    77. # pv
    78. ---
    79. apiVersion: v1
    80. kind: PersistentVolume
    81. metadata:
    82.   name: gitlab-opt-ver130806
    83.   labels:
    84.     type: gitlab-opt-ver130806
    85. spec:
    86.   capacity:
    87.     storage: 1Gi
    88.   accessModes:
    89.     - ReadWriteOnce
    90.   persistentVolumeReclaimPolicy: Retain
    91.   storageClassName: nfs
    92.   nfs:
    93.     path: /nfs_dir/gitlab_opt_ver130806
    94.     server: 10.0.1.201
    95.  
    96. # pvc
    97. ---
    98. kind: PersistentVolumeClaim
    99. apiVersion: v1
    100. metadata:
    101.   name: gitlab-opt-ver130806-pvc
    102. spec:
    103.   accessModes:
    104.     - ReadWriteOnce
    105.   resources:
    106.     requests:
    107.       storage: 1Gi
    108.   storageClassName: nfs
    109.   selector:
    110.     matchLabels:
    111.       type: gitlab-opt-ver130806
    112. ---
    113. apiVersion: v1
    114. kind: Service
    115. metadata:
    116.   name: gitlab
    117.   labels:
    118.     app: gitlab
    119.     tier: frontend
    120. spec:
    121.   ports:
    122.     - name: gitlab-ui
    123.       port: 80
    124.       protocol: TCP
    125.       targetPort: 80
    126.     - name: gitlab-ssh
    127.       port: 22
    128.       protocol: TCP
    129.       targetPort: 22
    130.       nodePort: 32155
    131.   selector:
    132.     app: gitlab
    133.     tier: frontend
    134.   type: NodePort
    135. ---
    136. apiVersion: v1
    137. kind: ServiceAccount
    138. metadata:
    139.   name: gitlab
    140. ---
    141. apiVersion: rbac.authorization.k8s.io/v1
    142. kind: ClusterRoleBinding
    143. metadata:
    144.   name: gitlab-cb-ver130806
    145. roleRef:
    146.   apiGroup: rbac.authorization.k8s.io
    147.   kind: ClusterRole
    148.   name: cluster-admin
    149. subjects:
    150.   - kind: ServiceAccount
    151.     name: gitlab
    152.     namespace: gitlab-ver130806
    153. ---
    154.  
    155. apiVersion: apps/v1
    156. kind: Deployment
    157. metadata:
    158.   name: gitlab
    159.   labels:
    160.     app: gitlab
    161.     tier: frontend
    162. spec:
    163.   replicas: 1
    164.   selector:
    165.     matchLabels:
    166.       app: gitlab
    167.       tier: frontend
    168.   strategy:
    169.     type: Recreate
    170.   template:
    171.     metadata:
    172.       labels:
    173.         app: gitlab
    174.         tier: frontend
    175.     spec:
    176.       serviceAccountName: gitlab
    177.       containers:
    178.         - image: harbor.boge.com/library/gitlab-ce:13.8.6-ce.1
    179.           name: gitlab
    180. #          resources:
    181. #            requests:
    182. #              cpu: 400m
    183. #              memory: 4Gi
    184. #            limits:
    185. #              cpu: "800m"
    186. #              memory: 8Gi
    187.           securityContext:
    188.             privileged: true
    189.           env:
    190.             - name: TZ
    191.               value: Asia/Shanghai
    192.             - name: GITLAB_OMNIBUS_CONFIG
    193.               value: |
    194.                 postgresql['enable'] = false
    195.                 gitlab_rails['db_username'] = "gitlab"
    196.                 gitlab_rails['db_password'] = "bogeusepg"
    197.                 gitlab_rails['db_host'] = "postgresql"
    198.                 gitlab_rails['db_port'] = "5432"
    199.                 gitlab_rails['db_database'] = "gitlabhq_production"
    200.                 gitlab_rails['db_adapter'] = 'postgresql'
    201.                 gitlab_rails['db_encoding'] = 'utf8'
    202.                 redis['enable'] = false
    203.                 gitlab_rails['redis_host'] = 'redis'
    204.                 gitlab_rails['redis_port'] = '6379'
    205.                 gitlab_rails['redis_password'] = 'bogeuseredis'
    206.                 gitlab_rails['gitlab_shell_ssh_port'] = 22
    207.                 external_url 'http://git.boge.com/'
    208.                 nginx['listen_port'] = 80
    209.                 nginx['listen_https'] = false
    210.                 #-------------------------------------------
    211.                 gitlab_rails['gitlab_email_enabled'] = true
    212.                 gitlab_rails['gitlab_email_from'] = 'admin@boge.com'
    213.                 gitlab_rails['gitlab_email_display_name'] = 'boge'
    214.                 gitlab_rails['gitlab_email_reply_to'] = 'gitlab@boge.com'
    215.                 gitlab_rails['gitlab_default_can_create_group'] = true
    216.                 gitlab_rails['gitlab_username_changing_enabled'] = true
    217.                 gitlab_rails['smtp_enable'] = true
    218.                 gitlab_rails['smtp_address'] = "smtp.exmail.qq.com"
    219.                 gitlab_rails['smtp_port'] = 465
    220.                 gitlab_rails['smtp_user_name'] = "gitlab@boge.com"
    221.                 gitlab_rails['smtp_password'] = "bogesendmail"
    222.                 gitlab_rails['smtp_domain'] = "exmail.qq.com"
    223.                 gitlab_rails['smtp_authentication'] = "login"
    224.                 gitlab_rails['smtp_enable_starttls_auto'] = true
    225.                 gitlab_rails['smtp_tls'] = true
    226.                 #-------------------------------------------
    227.                 # 关闭 promethues
    228.                 prometheus['enable'] = false
    229.                 # 关闭 grafana
    230.                 grafana['enable'] = false
    231.                 # 减少内存占用
    232.                 unicorn['worker_memory_limit_min'] = "200 * 1 << 20"
    233.                 unicorn['worker_memory_limit_max'] = "300 * 1 << 20"
    234.                 # 减少 sidekiq 的并发数
    235.                 sidekiq['concurrency'] = 16
    236.                 # 减少 postgresql 数据库缓存
    237.                 postgresql['shared_buffers'] = "256MB"
    238.                 # 减少 postgresql 数据库并发数量
    239.                 postgresql['max_connections'] = 8
    240.                 # 减少进程数   worker=CPU核数+1
    241.                 unicorn['worker_processes'] = 2
    242.                 nginx['worker_processes'] = 2
    243.                 puma['worker_processes'] = 2
    244.                 # puma['per_worker_max_memory_mb'] = 850
    245.                 # 保留3天备份的数据文件
    246.                 gitlab_rails['backup_keep_time'] = 259200
    247.                 #-------------------------------------------
    248.           ports:
    249.             - containerPort: 80
    250.               name: gitlab
    251.           livenessProbe:
    252.             exec:
    253.               command:
    254.               - sh
    255.               - -c
    256.               - "curl -s http://127.0.0.1/-/health|grep -w 'GitLab OK'"
    257.             initialDelaySeconds: 120
    258.             periodSeconds: 10
    259.             timeoutSeconds: 5
    260.             successThreshold: 1
    261.             failureThreshold: 3
    262.           readinessProbe:
    263.             exec:
    264.               command:
    265.               - sh
    266.               - -c
    267.               - "curl -s http://127.0.0.1/-/health|grep -w 'GitLab OK'"
    268.             initialDelaySeconds: 120
    269.             periodSeconds: 10
    270.             timeoutSeconds: 5
    271.             successThreshold: 1
    272.             failureThreshold: 3
    273.           volumeMounts:
    274.             - mountPath: /etc/gitlab
    275.               name: gitlab1
    276.             - mountPath: /var/log/gitlab
    277.               name: gitlab2
    278.             - mountPath: /var/opt/gitlab
    279.               name: gitlab3
    280.             - mountPath: /etc/localtime
    281.               name: tz-config
    282.  
    283.       volumes:
    284.         - name: gitlab1
    285.           persistentVolumeClaim:
    286.             claimName: gitlab-etc-ver130806-pvc
    287.         - name: gitlab2
    288.           persistentVolumeClaim:
    289.             claimName: gitlab-log-ver130806-pvc
    290.         - name: gitlab3
    291.           persistentVolumeClaim:
    292.             claimName: gitlab-opt-ver130806-pvc
    293.         - name: tz-config
    294.           hostPath:
    295.             path: /usr/share/zoneinfo/Asia/Shanghai
    296.  
    297.       securityContext:
    298.         runAsUser: 0
    299.         fsGroup: 0
    300.

    8.创建pod
     

    kubectl -n gitlab-ver130806 apply -f 5gitlab.yaml

    kubectl -n gitlab-ver130806 get pod

    9.部署gitlab-tls

    vi 6gitlab-tls

    1. # old version
    2.  
    3. #apiVersion: extensions/v1beta1
    4. #kind: Ingress
    5. #metadata:
    6. #  name: gitlab
    7. #  annotations:
    8. #    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    9. #    nginx.ingress.kubernetes.io/proxy-body-size: "20m"
    10. #spec:
    11. #  tls:
    12. #  - hosts:
    13. #    - git.boge.com
    14. #    secretName: mytls
    15. #  rules:
    16. #  - host: git.boge.com
    17. #    http:
    18. #      paths:
    19. #      - path: /
    20. #        backend:
    21. #          serviceName: gitlab
    22. #          servicePort: 80
    23.  
    24. # Add tls
    25. # openssl genrsa -out tls.key 2048
    26. # openssl req -new -x509 -key tls.key -out tls.cert -days 360 -subj /CN=*.boge.com
    27. # kubectl -n gitlab-ver130806 create secret tls mytls --cert=tls.cert --key=tls.key 
    28.  
    29. # new version
    30.  
    31. ## https://kubernetes.io/docs/concepts/services-networking/ingress/
    32. apiVersion: networking.k8s.io/v1
    33. kind: Ingress
    34. metadata:
    35.   name: gitlab
    36.   annotations:
    37.     nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
    38.     nginx.ingress.kubernetes.io/proxy-body-size: "20m"
    39. spec:
    40.   tls:
    41.   - hosts:
    42.     - git.boge.com
    43.     secretName: mytls
    44.   rules:
    45.   - host: git.boge.com
    46.     http:
    47.       paths:
    48.       - path: /
    49.         pathType: Prefix
    50.         backend:
    51.           service:
    52.             name: gitlab
    53.             port:
    54.               number: 80
    55.  
    56. ---
    57.

    10.#创建pod


    kubectl -n gitlab-ver130806 apply -f 6gitlab-tls.yaml

    kubectl -n gitlab-ver130806 get pod

    11.在安装服务器上增加ssh端口转发

    我们要保持所有开发人员能使用默认的22端口来通过ssh拉取代码，那么就需要做如下端口转发配置

    vim /etc/ssh/sshd_config

    Port 10022


    systemctl restart sshd

    # 注意配置此转发前，需要将对应NODE的本身ssh连接端口作一下修改，以防后面登陆不了该机器
    #kubectl get svc -n gitlab-ver130806 |grep git
    iptables -t nat -A PREROUTING -d 10.0.1.204 -p tcp --dport 22 -j DNAT --to-destination 10.0.1.204:31755

    #↑ 删除上面创建的这一条规则，将-A换成-D即可
    #iptables -t nat -D PREROUTING -d 10.0.1.204 -p tcp --dport 22 -j DNAT --to-destination 10.0.1.204:31755


    iptables -t nat  -nvL PREROUTING


    #将需要拉去仓库代码的服务器密钥（cat /root/.ssh/id_rsa.pub） 加入git页面（SSH Keys）

    12.这节课我们来讲gitlab里面的runner，gitlab的CI/CD自动化，都是由gitlab下发指令，依靠runner这个组件去执行的，我们这里也是把runner运行在k8s上面。

    runner按字面意思就是奔跑者的意思，它在整个自动化流程里面的角色也相当于一个外卖小哥，它接收gitlab下发的自动化指令，来去做相应的操作，从而实现整个CI/CD的效果。
    部署gitlab-runner

    1.  
    2.  
    3. mkdir -p /nfs_dir/{gitlab-runner1-ver130806-docker,gitlab-runner2-ver130806-share}
    4.  
    5. # gitlab-ci-multi-runner register
    6.  
    7. #                   Active  √ Paused Runners don't accept new jobs
    8. #                Protected     This runner will only run on pipelines triggered on protected branches
    9. #        Run untagged jobs     Indicates whether this runner can pick jobs without tags
    10. # Lock to current projects     When a runner is locked, it cannot be assigned to other projects
    11.  
    12. # pv
    13. ---
    14. apiVersion: v1
    15. kind: PersistentVolume
    16. metadata:
    17.   name: gitlab-runner1-ver130806-docker
    18.   labels:
    19.     type: gitlab-runner1-ver130806-docker
    20. spec:
    21.   capacity:
    22.     storage: 0.1Gi
    23.   accessModes:
    24.     - ReadWriteMany
    25.   persistentVolumeReclaimPolicy: Retain
    26.   storageClassName: nfs
    27.   nfs:
    28.     path: /nfs_dir/gitlab-runner1-ver130806-docker
    29.     server: 10.0.1.201
    30.  
    31. # pvc
    32. ---
    33. kind: PersistentVolumeClaim
    34. apiVersion: v1
    35. metadata:
    36.   name: gitlab-runner1-ver130806-docker
    37.   namespace: gitlab-ver130806
    38. spec:
    39.   accessModes:
    40.     - ReadWriteMany
    41.   resources:
    42.     requests:
    43.       storage: 0.1Gi
    44.   storageClassName: nfs
    45.   selector:
    46.     matchLabels:
    47.       type: gitlab-runner1-ver130806-docker
    48.  
    49.  
    50. ---
    51. # https://docs.gitlab.com/runner/executors
    52.  
    53. #concurrent = 30
    54. #check_interval = 0
    55.  
    56. #[session_server]
    57. #  session_timeout = 1800
    58.  
    59. #[[runners]]
    60. #  name = "gitlab-runner1-ver130806-docker"
    61. #  url = "http://git.boge.com"
    62. #  token = "xxxxxxxxxxxxxxxxxxxxxx"
    63. #  executor = "kubernetes"
    64. #  [runners.kubernetes]
    65. #    namespace = "gitlab-ver130806"
    66. #    image = "docker:stable"
    67. #    helper_image = "gitlab/gitlab-runner-helper:x86_64-9fc34d48-pwsh"
    68. #    privileged = true
    69. #    [[runners.kubernetes.volumes.pvc]]
    70. #      name = "gitlab-runner1-ver130806-docker"
    71. #      mount_path = "/mnt"
    72.  
    73. ---
    74. apiVersion: apps/v1
    75. kind: Deployment
    76. metadata:
    77.   name: gitlab-runner1-ver130806-docker
    78.   namespace: gitlab-ver130806
    79. spec:
    80.   replicas: 1
    81.   selector:
    82.     matchLabels:
    83.       name: gitlab-runner1-ver130806-docker
    84.   template:
    85.     metadata:
    86.       labels:
    87.         name: gitlab-runner1-ver130806-docker
    88.     spec:
    89.       hostAliases:
    90.      #kubectl -n gitlab-ver130806 get svc |grep git
    91.       - ip: "10.68.140.109"
    92.         hostnames:
    93.         - "git.boge.com"
    94.       serviceAccountName: gitlab
    95.       containers:
    96.       - args:
    97.         - run
    98.         image: gitlab/gitlab-runner:v13.10.0
    99.         name: gitlab-runner1-ver130806-docker
    100.         volumeMounts:
    101.         - mountPath: /etc/gitlab-runner
    102.           name: config
    103.         - mountPath: /etc/ssl/certs
    104.           name: cacerts
    105.           readOnly: true
    106.       restartPolicy: Always
    107.       volumes:
    108.       - persistentVolumeClaim:
    109.           claimName: gitlab-runner1-ver130806-docker
    110.         name: config
    111.       - hostPath:
    112.           path: /usr/share/ca-certificates/mozilla
    113.         name: cacerts
    114.

    8aa8696ef17f4709907a1d306aec557a.png

    13.部署

    kubectl -n gitlab-ver130806 apply -f 7gitlab-runner-docker.yaml

    kubectl -n gitlab-ver130806 exec -it gitlab-runner1-ver130806-docker-7cc8c bash

    # 进入git 注册
    root@gitlab-runner1-ver130806-docker-7cc8cc7595-nqcmc:/# gitlab-ci-multi-runner register

    07a79a982aa04d2d8c4b72f45c8ca2ee.png

    14        vi /nfs_dir/gitlab-runner1-ver130806-docker/config.toml

    1. concurrent = 30
    2. check_interval = 0
    3.  
    4. [session_server]
    5.   session_timeout = 1800
    6.  
    7. [[runners]]
    8.   name = "gitlab-runner1-ver130806-docker"
    9.   url = "http://git.boge.com/"
    10.   token = "2sU_GyKpbgVisPNmp-Fb"     #这个是系统自动生成的不动
    11.   executor = "kubernetes"
    12.   [runners.kubernetes]
    13.     namespace = "gitlab-ver130806"
    14.     image = "docker:stable"
    15.     helper_image = "gitlab/gitlab-runner-helper:x86_64-9fc34d48-pwsh"
    16.     privileged = true
    17.     [[runners.kubernetes.volumes.pvc]]
    18.       name = "gitlab-runner1-ver130806-docker"
    19.       mount_path = "/mnt"

    15.#查看pod

    1. kubectl -n gitlab-ver130806 get pod
    2.  
    3. #删除runne的pod ，等它重启
    4. kubectl -n gitlab-ver130806 delete pod gitlab-runner1-ver130806-docker-7cc8cc7595-nqcmc

    进入gti页面 ，去掉第4个√，保留第一个√，保存退出

    0aede3a26f5e4464b107a78b21356924.png

    share 

    1. # gitlab-ci-multi-runner register
    2.  
    3. #                   Active  √ Paused Runners don't accept new jobs
    4. #                Protected     This runner will only run on pipelines triggered on protected branches
    5. #        Run untagged jobs  √ Indicates whether this runner can pick jobs without tags
    6. # Lock to current projects     When a runner is locked, it cannot be assigned to other projects
    7.  
    8. # pv
    9. ---
    10. apiVersion: v1
    11. kind: PersistentVolume
    12. metadata:
    13.   name: gitlab-runner2-ver130806-share
    14.   labels:
    15.     type: gitlab-runner2-ver130806-share
    16. spec:
    17.   capacity:
    18.     storage: 0.1Gi
    19.   accessModes:
    20.     - ReadWriteMany
    21.   persistentVolumeReclaimPolicy: Retain
    22.   storageClassName: nfs
    23.   nfs:
    24.     path: /nfs_dir/gitlab-runner2-ver130806-share
    25.     server: 10.0.1.201
    26.  
    27. # pvc
    28. ---
    29. kind: PersistentVolumeClaim
    30. apiVersion: v1
    31. metadata:
    32.   name: gitlab-runner2-ver130806-share
    33.   namespace: gitlab-ver130806
    34. spec:
    35.   accessModes:
    36.     - ReadWriteMany
    37.   resources:
    38.     requests:
    39.       storage: 0.1Gi
    40.   storageClassName: nfs
    41.   selector:
    42.     matchLabels:
    43.       type: gitlab-runner2-ver130806-share
    44.  
    45.  
    46. ---
    47. # https://docs.gitlab.com/runner/executors
    48.  
    49. #concurrent = 30
    50. #check_interval = 0
    51.  
    52. #[session_server]
    53. #  session_timeout = 1800
    54.  
    55. #[[runners]]
    56. #  name = "gitlab-runner2-ver130806-share"
    57. #  url = "http://git.boge.com"
    58. #  token = "xxxxxxxxxxxxxxxx"
    59. #  executor = "kubernetes"
    60. #  [runners.kubernetes]
    61. #    namespace = "gitlab-ver130806"
    62. #    image = "registry.cn-beijing.aliyuncs.com/acs/busybox/busybox:v1.29.2"
    63. #    helper_image = "gitlab/gitlab-runner-helper:x86_64-9fc34d48-pwsh"
    64. #    privileged = false
    65. #    [[runners.kubernetes.volumes.pvc]]
    66. #      name = "gitlab-runner2-ver130806-share"
    67. #      mount_path = "/mnt"
    68.  
    69. ---
    70. apiVersion: apps/v1
    71. kind: Deployment
    72. metadata:
    73.   name: gitlab-runner2-ver130806-share
    74.   namespace: gitlab-ver130806
    75. spec:
    76.   replicas: 1
    77.   selector:
    78.     matchLabels:
    79.       name: gitlab-runner2-ver130806-share
    80.   template:
    81.     metadata:
    82.       labels:
    83.         name: gitlab-runner2-ver130806-share
    84.     spec:
    85.       hostAliases:
    86.       #kubectl -n gitlab-ver130806 get svc |grep git
    87.       - ip: "10.68.140.109"
    88.         hostnames:
    89.         - "git.boge.com"
    90.       serviceAccountName: gitlab
    91.       containers:
    92.       - args:
    93.         - run
    94.         image: gitlab/gitlab-runner:v13.10.0
    95.         name: gitlab-runner2-ver130806-share
    96.         volumeMounts:
    97.         - mountPath: /etc/gitlab-runner
    98.           name: config
    99.         - mountPath: /etc/ssl/certs
    100.           name: cacerts
    101.           readOnly: true
    102.       restartPolicy: Always
    103.       volumes:
    104.       - persistentVolumeClaim:
    105.           claimName: gitlab-runner2-ver130806-share
    106.         name: config
    107.       - hostPath:
    108.           path: /usr/share/ca-certificates/mozilla
    109.         name: cacerts
    110.

    16.执行yaml

    kubectl -n gitlab-ver130806 apply -f 8gitlab-runner-share.yaml

    kubectl -n gitlab-ver130806 get pod

    #进入pod
    kubectl -n gitlab-ver130806 exec -it gitlab-runner2-ver130806-share-555695cf9 bash


    #gitlab-ci-multi-runner register

    f4ef152fa8fc424d990f93bd329b51f4.png

    17#修改配置
    vi /nfs_dir/gitlab-runner2-ver130806-share/config.toml

    1. concurrent = 30
    2. check_interval = 0
    3.  
    4. [session_server]
    5.   session_timeout = 1800
    6.  
    7. [[runners]]
    8.   name = "gitlab-runner2-ver130806-share"
    9.   url = "http://git.boge.com/"
    10.   token = "yj5tWzuaAB8xjP4kfbKQ"#使用系统自动生成的，不修改
    11.   executor = "kubernetes"
    12.   [runners.kubernetes]
    13.     namespace = "gitlab-ver130806"
    14.     image = "registry.cn-beijing.aliyuncs.com/acs/busybox/busybox:v1.29.2"
    15.     helper_image = "gitlab/gitlab-runner-helper:x86_64-9fc34d48-pwsh"
    16.     privileged = false
    17.     [[runners.kubernetes.volumes.pvc]]
    18.       name = "gitlab-runner2-ver130806-share"
    19.       mount_path = "/mnt"

    18.执行

    kubectl -n gitlab-ver130806 get pod

    kubectl -n gitlab-ver130806 delete pod gitlab-runner2-ver130806-share-555695cf99-7nhp2

    进入gti页面 ，保留第一个√，第三个√保存退出

    19.这节课我们继续来配置gitlab相关的服务。
    增加gitlab在k8s的内部解析

    为什么这么做呢，博哥这里总结了两点原因：

    优化gitlab网络通信，对于runner要调用gitlab服务来说，直接走内部地址速度更快
    如果是在用阿里云的同学，采用在k8s上部署gitlab的话，那么k8s内部服务比如runner是不能通过同集群前面的公网入口SLB来请求访问的，这里阿里云自身网络架构原因，这个时候我们只需要做如下配置即可完美解决

    #编辑
    kubectl -n kube-system edit configmaps coredns

    # kubectl -n kube-system get configmaps coredns  -o yaml
    apiVersion: v1
    data:
      Corefile: |
        .:53 {
            errors
            health
            ready
            log
            rewrite stop {
              name regex git.boge.com gitlab.gitlab-ver130806.svc.cluster.local
              answer name gitlab.gitlab-ver130806.svc.cluster.local git.boge.com
            }

            kubernetes cluster.local in-addr.arpa ip6.arpa {

              pods verified
              fallthrough in-addr.arpa ip6.arpa
            }
            autopath @kubernetes
            prometheus :9153
            forward . /etc/resolv.conf
            cache 30
            loop
            reload
            loadbalance
        }
    kind: ConfigMap
    metadata:
      name: coredns
      namespace: kube-system

    20.#查看pod

    #查看pod
    kubectl -n kube-system get pod |grep  coredns

    #删除pod
    kubectl -n kube-system delete pod coredns-5787695b7f-vzfm6

    21. 我们现在在k8s来部署dind服务，提供整个CI（持续集成）的功能。

    1.  
    2.  
    3. 我们现在在k8s来部署dind服务，提供整个CI（持续集成）的功能。
    4.  
    5. 我们看看docker version列出的结果 Docker采取的是C/S架构 Docker进程默认不监听任何端口，它会生成一个socket（/var/run/docker.sock）文件来进行本地进程通信 Docker C/S 之间采取Rest API作为通信协议，我们可以让Docker daemon进程监听一个端口，这就为我们用docker client调用远程调用docker daemon进程执行镜像构建提供了可行性
    6. ![在这里插入图片描述](https://img-blog.csdnimg.cn/8a6d8ec893c344fb99c6cc72e69e5b88.png#pic_center)
    7.  
    8. > docker in docker
    9.  
    10. ```bash
    11. # dind pip instll staus : kill -9  code 137(128+9) ,may be limits(cpu,memory) resources need change
    12.  
    13. # only have docker client ,use dind can be use normal
    14. #dindSvc=$(kubectl -n kube-system get svc dind |awk 'NR==2{print $3}')
    15. #export DOCKER_HOST="tcp://${dindSvc}:2375/"
    16. #export DOCKER_DRIVER=overlay2
    17. #export DOCKER_TLS_CERTDIR=""
    18.  
    19.  
    20. ---
    21. # SVC
    22. kind: Service
    23. apiVersion: v1
    24. metadata:
    25.   name: dind
    26.   namespace: kube-system
    27. spec:
    28.   selector:
    29.     app: dind
    30.   ports:
    31.     - name: tcp-port
    32.       port: 2375
    33.       protocol: TCP
    34.       targetPort: 2375
    35.  
    36. ---
    37. # Deployment
    38. apiVersion: apps/v1
    39. kind: Deployment
    40. metadata:
    41.   name: dind
    42.   namespace: kube-system
    43.   labels:
    44.     app: dind
    45. spec:
    46.   replicas: 1
    47.   selector:
    48.     matchLabels:
    49.       app: dind
    50.   template:
    51.     metadata:
    52.       labels:
    53.         app: dind
    54.     spec:
    55.       hostNetwork: true
    56.       containers:
    57.       - name: dind
    58.         #image: docker:19-dind
    59.         image: harbor.boge.com/library/docker:19-dind
    60.         lifecycle:
    61.           postStart:
    62.             exec:
    63.               command: ["/bin/sh", "-c", "docker login harbor.boge.com -u 'admin' -p 'boge666'"]
    64.            # 3. when delete this pod , use this keep kube-proxy to flush role done
    65.           preStop:
    66.             exec:
    67.               command: ["/bin/sh", "-c", "sleep 5"]
    68.         ports:
    69.         - containerPort: 2375
    70. #        resources:
    71. #          requests:
    72. #            cpu: 200m
    73. #            memory: 256Mi
    74. #          limits:
    75. #            cpu: 0.5
    76. #            memory: 1Gi
    77.         readinessProbe:
    78.           tcpSocket:
    79.             port: 2375
    80.           initialDelaySeconds: 10
    81.           periodSeconds: 30
    82.         livenessProbe:
    83.           tcpSocket:
    84.             port: 2375
    85.           initialDelaySeconds: 10
    86.           periodSeconds: 30
    87.         securityContext: 
    88.             privileged: true
    89.         env: 
    90.           - name: DOCKER_HOST 
    91.             value: tcp://localhost:2375
    92.           - name: DOCKER_DRIVER 
    93.             value: overlay2
    94.           - name: DOCKER_TLS_CERTDIR 
    95.             value: ''
    96.         volumeMounts: 
    97.           - name: docker-graph-storage
    98.             mountPath: /var/lib/docker
    99.           - name: tz-config
    100.             mountPath: /etc/localtime
    101.            # kubectl -n kube-system create secret generic harbor-ca --from-file=harbor-ca=/data/harbor/ssl/tls.cert
    102.           - name: harbor-ca
    103.             mountPath: /etc/docker/certs.d/harbor.boge.com/ca.crt
    104.             subPath: harbor-ca
    105.        # kubectl create secret docker-registry boge-secret --docker-server=harbor.boge.com --docker-username=admin --docker-password=boge666 --docker-email=admin@boge.com
    106.       hostAliases:
    107.       - hostnames:
    108.         - harbor.boge.com
    109.         ip: 10.0.1.204
    110.       imagePullSecrets:
    111.       - name: bogeharbor
    112.       volumes:
    113. #      - emptyDir:
    114. #          medium: ""
    115. #          sizeLimit: 10Gi
    116.       - hostPath:
    117.           path: /var/lib/container/docker
    118.         name: docker-graph-storage
    119.       - hostPath:
    120.           path: /usr/share/zoneinfo/Asia/Shanghai
    121.         name: tz-config
    122.       - name: harbor-ca
    123.         secret:
    124.           secretName: harbor-ca
    125.           defaultMode: 0600
    126. #
    127. #        kubectl taint node 10.0.1.201 Ingress=:NoExecute
    128. #        kubectl describe node 10.0.1.201 |grep -i taint
    129. #        kubectl taint node 10.0.1.201 Ingress:NoExecute-
    130.       nodeSelector:
    131.         kubernetes.io/hostname: "10.0.1.201"
    132.       tolerations:
    133.       - operator: Exists
    134.

    22。#创建证书


    kubectl -n kube-system create secret generic harbor-ca --from-file=harbor-ca=/data/harbor/ssl/tls.cert

    #执行yaml
    kubectl apply -f 10bind.yaml

    #git 添加ssh
    cat /root/.ssh/id_rsa.pub

    024983237bc148b7aafc309a5ec11c6c.png

    1.  
    2. #Git global setup Git全局设置
    3. git config --global user.name "Administrator"
    4. git config --global user.email "admin@example.com"
    5.  
    6. #Create a new repository  创建新存储库 
    7. git clone git@git.boge.com:root/test.git
    8. cd test
    9. touch README.md
    10. #添加上传文件
    11. git add README.md
    12. #添加描述
    13. git commit -m "add README"
    14.  
    15. #推送
    16. git push -u origin master
    17.  
    18. #Push an existing folder 推送现有文件夹
    19. cd existing_folder
    20. git init
    21. git remote add origin git@git.boge.com:root/test.git
    22. git add .
    23. git commit -m "Initial commit"
    24. git push -u origin master
    25.  
    26. #Push an existing Git repository 推送现有Git存储库
    27. cd existing_repo
    28. git remote rename origin old-origin
    29. git remote add origin git@git.boge.com:root/test.git
    30. git push -u origin --all
    31. git push -u origin --tags

    接着我们找一台机器，这里我们选取10.0.1.201这台机器，加一条本地hosts 10.0.1.204 git.boge.com，来试下推送gitlab代码仓库有无问题，详细操作见本节同名视频课程，希望大家能对着视频自己动手操作一遍，理解上面这些配置的含义，后面可以举一反三，在k8s的其他服务也可以这么来做，达到访问更优的效果。

    添加ssh key：

     在git客户端执行（我是在linux服务器上）：

    # ssh-keygen -t ed25519 -C "123123@qq.com"   一直回车
    # cd ~/.ssh/
    # cat id_ed25519.pub

    在git客户端克隆，注意端口号。不需要输入密码：

    #git clone ssh://git@git.boge.com:32155/zoujiaojiao/it.git

    十三、k8s安装kubesphere3.3

    1.部署kubesphere时需要默认 StorageClass

    kubectl edit sc nfs-boge

      metadata:
        annotations:
          storageclass.beta.kubernetes.io/is-default-class: "true"

    2.下载yaml

    wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/kubesphere-installer.yaml

    wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/cluster-configuration.yaml
     

    #将ectd下的 endpointIps改为你的master节点的私有IP地址。

    endpointIps: 20.120.100.220

    #运行yaml

    kubectl apply -f kubesphere-installer.yaml

    kubectl apply -f cluster-configuration.yaml

    3.查看日志

    kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f

    #访问任意机器的 30880端口
    #账号 ： admin
    #密码 ： P@88w0rd

    4.解决etcd监控证书找不到问题

    kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs  --from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt  --from-file=etcd-client.crt=/etc/kubernetes/pki/apiserver-etcd-client.crt  --from-file=etcd-client.key=/etc/kubernetes/pki/apiserver-etcd-client.key
     

    5.文件未删除干净

    failed: [localhost] (item={'ns': 'kubesphere-system', 'kind': 'users.iam.kubesphere.io', 'resource': 'admin', 'release': 'ks-core'}) => {"ansible_loop_var": "item", "changed": true, "cmd": "/usr/local/bin/kubectl -n kubesphere-system annotate --overwrite users.iam.kubesphere.io admin meta.helm.sh/release-name=ks-core && /usr/local/bin/kubectl -n kubesphere-system annotate --overwrite users.iam.kubesphere.io admin meta.helm.sh/release-namespace=kubesphere-system && /usr/local/bin/kubectl -n kubesphere-system label --overwrite users.iam.kubesphere.io admin app.kubernetes.io/managed-by=Helm\n", "delta": "0:00:00.675675", "end": "2022-02-10 04:53:09.022419", "failed_when_result": true, "item": {"kind": "users.iam.kubesphere.io", "ns": "kubesphere-system", "release": "ks-core", "resource": "admin"}, "msg": "non-zero return code", "rc": 1, "start": "2022-02-10 04:53:08.346744", "stderr": "Error from server (InternalError): Internal error occurred: failed calling webhook \"users.iam.kubesphere.io\": Post \"https://ks-controller-manager.kubesphere-system.svc:443/validate-email-iam-kubesphere-io-v1alpha2?timeout=30s\": service \"ks-controller-manager\" not found", "stderr_lines": ["Error from server (InternalError): Internal error occurred: failed calling webhook \"users.iam.kubesphere.io\": Post \"https://ks-controller-manager.kubesphere-system.svc:443/validate-email-iam-kubesphere-io-v1alpha2?timeout=30s\": service \"ks-controller-manager\" not found"], "stdout": "", "stdout_lines": []}

    参考   https://github.com/kubesphere/ks-installer/blob/master/scripts/kubesphere-delete.sh 将sh文件下载到master节点，然后删除后重新安装

    del.sh

    1. #!/usr/bin/env bash
    2.  
    3. function delete_sure(){
    4.   cat << eof
    5. $(echo -e "\033[1;36mNote:\033[0m")
    6. Delete the KubeSphere cluster, including the module kubesphere-system kubesphere-devops-system kubesphere-devops-worker kubesphere-monitoring-system kubesphere-logging-system openpitrix-system.
    7. eof
    8.  
    9. read -p "Please reconfirm that you want to delete the KubeSphere cluster.  (yes/no) " ans
    10. while [[ "x"$ans != "xyes" && "x"$ans != "xno" ]]; do
    11.     read -p "Please reconfirm that you want to delete the KubeSphere cluster.  (yes/no) " ans
    12. done
    13.  
    14. if [[ "x"$ans == "xno" ]]; then
    15.     exit
    16. fi
    17. }
    18.  
    19.  
    20. delete_sure
    21.  
    22. # delete ks-installer
    23. kubectl delete deploy ks-installer -n kubesphere-system 2>/dev/null
    24.  
    25. # delete helm
    26. for namespaces in kubesphere-system kubesphere-devops-system kubesphere-monitoring-system kubesphere-logging-system openpitrix-system kubesphere-monitoring-federated
    27. do
    28.   helm list -n $namespaces | grep -v NAME | awk '{print $1}' | sort -u | xargs -r -L1 helm uninstall -n $namespaces 2>/dev/null
    29. done
    30.  
    31. # delete kubefed
    32. kubectl get cc -n kubesphere-system ks-installer -o jsonpath="{.status.multicluster}" | grep enable
    33. if [[ $? -eq 0 ]]; then
    34.   # delete kubefed types resources
    35.   for kubefed in `kubectl api-resources --namespaced=true --api-group=types.kubefed.io -o name`
    36.   do
    37.     kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null
    38.   done
    39.   for kubefed in `kubectl api-resources --namespaced=false --api-group=types.kubefed.io -o name`
    40.   do
    41.     kubectl delete $kubefed --all 2>/dev/null
    42.   done
    43.   # delete kubefed core resouces
    44.   for kubefed in `kubectl api-resources --namespaced=true --api-group=core.kubefed.io -o name`
    45.   do
    46.     kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null
    47.   done
    48.   for kubefed in `kubectl api-resources --namespaced=false --api-group=core.kubefed.io -o name`
    49.   do
    50.     kubectl delete $kubefed --all 2>/dev/null
    51.   done
    52.   # uninstall kubefed chart
    53.   helm uninstall -n kube-federation-system kubefed 2>/dev/null
    54. fi
    55.  
    56.  
    57. helm uninstall -n kube-system snapshot-controller 2>/dev/null
    58.  
    59. # delete kubesphere deployment & statefulset
    60. kubectl delete deployment -n kubesphere-system `kubectl get deployment -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
    61. kubectl delete statefulset -n kubesphere-system `kubectl get statefulset -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
    62.  
    63. # delete monitor resources
    64. kubectl delete prometheus -n kubesphere-monitoring-system k8s 2>/dev/null
    65. kubectl delete Alertmanager -n kubesphere-monitoring-system main 2>/dev/null
    66. kubectl delete DaemonSet -n kubesphere-monitoring-system node-exporter 2>/dev/null
    67. kubectl delete statefulset -n kubesphere-monitoring-system `kubectl get statefulset -n kubesphere-monitoring-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
    68.  
    69. # delete grafana
    70. kubectl delete deployment -n kubesphere-monitoring-system grafana 2>/dev/null
    71. kubectl --no-headers=true get pvc -n kubesphere-monitoring-system -o custom-columns=:metadata.namespace,:metadata.name | grep -E kubesphere-monitoring-system | xargs -n2 kubectl delete pvc -n 2>/dev/null
    72.  
    73. # delete pvc
    74. pvcs="kubesphere-system|openpitrix-system|kubesphere-devops-system|kubesphere-logging-system"
    75. kubectl --no-headers=true get pvc --all-namespaces -o custom-columns=:metadata.namespace,:metadata.name | grep -E $pvcs | xargs -n2 kubectl delete pvc -n 2>/dev/null
    76.  
    77.  
    78. # delete rolebindings
    79. delete_role_bindings() {
    80.   for rolebinding in `kubectl -n $1 get rolebindings -l iam.kubesphere.io/user-ref -o jsonpath="{.items[*].metadata.name}"`
    81.   do
    82.     kubectl -n $1 delete rolebinding $rolebinding 2>/dev/null
    83.   done
    84. }
    85.  
    86. # delete roles
    87. delete_roles() {
    88.   kubectl -n $1 delete role admin 2>/dev/null
    89.   kubectl -n $1 delete role operator 2>/dev/null
    90.   kubectl -n $1 delete role viewer 2>/dev/null
    91.   for role in `kubectl -n $1 get roles -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
    92.   do
    93.     kubectl -n $1 delete role $role 2>/dev/null
    94.   done
    95. }
    96.  
    97. # remove useless labels and finalizers
    98. for ns in `kubectl get ns -o jsonpath="{.items[*].metadata.name}"`
    99. do
    100.   kubectl label ns $ns kubesphere.io/workspace-
    101.   kubectl label ns $ns kubesphere.io/namespace-
    102.   kubectl patch ns $ns -p '{"metadata":{"finalizers":null,"ownerReferences":null}}'
    103.   delete_role_bindings $ns
    104.   delete_roles $ns
    105. done
    106.  
    107. # delete clusterroles
    108. delete_cluster_roles() {
    109.   for role in `kubectl get clusterrole -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
    110.   do
    111.     kubectl delete clusterrole $role 2>/dev/null
    112.   done
    113.  
    114.   for role in `kubectl get clusterroles | grep "kubesphere" | awk '{print $1}'| paste -sd " "`
    115.   do
    116.     kubectl delete clusterrole $role 2>/dev/null
    117.   done
    118. }
    119. delete_cluster_roles
    120.  
    121. # delete clusterrolebindings
    122. delete_cluster_role_bindings() {
    123.   for rolebinding in `kubectl get clusterrolebindings -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
    124.   do
    125.     kubectl delete clusterrolebindings $rolebinding 2>/dev/null
    126.   done
    127.  
    128.   for rolebinding in `kubectl get clusterrolebindings | grep "kubesphere" | awk '{print $1}'| paste -sd " "`
    129.   do
    130.     kubectl delete clusterrolebindings $rolebinding 2>/dev/null
    131.   done
    132. }
    133. delete_cluster_role_bindings
    134.  
    135. # delete clusters
    136. for cluster in `kubectl get clusters -o jsonpath="{.items[*].metadata.name}"`
    137. do
    138.   kubectl patch cluster $cluster -p '{"metadata":{"finalizers":null}}' --type=merge
    139. done
    140. kubectl delete clusters --all 2>/dev/null
    141.  
    142. # delete workspaces
    143. for ws in `kubectl get workspaces -o jsonpath="{.items[*].metadata.name}"`
    144. do
    145.   kubectl patch workspace $ws -p '{"metadata":{"finalizers":null}}' --type=merge
    146. done
    147. kubectl delete workspaces --all 2>/dev/null
    148.  
    149. # make DevOps CRs deletable
    150. for devops_crd in $(kubectl get crd -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep "devops.kubesphere.io"); do
    151.     for ns in $(kubectl get ns -ojsonpath='{.items..metadata.name}'); do
    152.         for devops_res in $(kubectl get $devops_crd -n $ns -oname); do
    153.             kubectl patch $devops_res -n $ns -p '{"metadata":{"finalizers":[]}}' --type=merge
    154.         done
    155.     done
    156. done
    157.  
    158. # delete validatingwebhookconfigurations
    159. for webhook in ks-events-admission-validate users.iam.kubesphere.io network.kubesphere.io validating-webhook-configuration resourcesquotas.quota.kubesphere.io
    160. do
    161.   kubectl delete validatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null
    162. done
    163.  
    164. # delete mutatingwebhookconfigurations
    165. for webhook in ks-events-admission-mutate logsidecar-injector-admission-mutate mutating-webhook-configuration
    166. do
    167.   kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null
    168. done
    169.  
    170. # delete users
    171. for user in `kubectl get users -o jsonpath="{.items[*].metadata.name}"`
    172. do
    173.   kubectl patch user $user -p '{"metadata":{"finalizers":null}}' --type=merge
    174. done
    175. kubectl delete users --all 2>/dev/null
    176.  
    177.  
    178. # delete helm resources
    179. for resource_type in `echo helmcategories helmapplications helmapplicationversions helmrepos helmreleases`; do
    180.   for resource_name in `kubectl get ${resource_type}.application.kubesphere.io -o jsonpath="{.items[*].metadata.name}"`; do
    181.     kubectl patch ${resource_type}.application.kubesphere.io ${resource_name} -p '{"metadata":{"finalizers":null}}' --type=merge
    182.   done
    183.   kubectl delete ${resource_type}.application.kubesphere.io --all 2>/dev/null
    184. done
    185.  
    186. # delete workspacetemplates
    187. for workspacetemplate in `kubectl get workspacetemplates.tenant.kubesphere.io -o jsonpath="{.items[*].metadata.name}"`
    188. do
    189.   kubectl patch workspacetemplates.tenant.kubesphere.io $workspacetemplate -p '{"metadata":{"finalizers":null}}' --type=merge
    190. done
    191. kubectl delete workspacetemplates.tenant.kubesphere.io --all 2>/dev/null
    192.  
    193. # delete federatednamespaces in namespace kubesphere-monitoring-federated
    194. for resource in $(kubectl get federatednamespaces.types.kubefed.io -n kubesphere-monitoring-federated -oname); do
    195.   kubectl patch "${resource}" -p '{"metadata":{"finalizers":null}}' --type=merge -n kubesphere-monitoring-federated
    196. done
    197.  
    198. # delete crds
    199. for crd in `kubectl get crds -o jsonpath="{.items[*].metadata.name}"`
    200. do
    201.   if [[ $crd == *kubesphere.io ]] || [[ $crd == *kubefed.io ]] ; then kubectl delete crd $crd 2>/dev/null; fi
    202. done
    203.  
    204. # delete relevance ns
    205. for ns in kube-federation-system kubesphere-alerting-system kubesphere-controls-system kubesphere-devops-system kubesphere-devops-worker kubesphere-logging-system kubesphere-monitoring-system kubesphere-monitoring-federated openpitrix-system kubesphere-system
    206. do
    207.   kubectl delete ns $ns 2>/dev/null
    208. done

    6.修改镜像拉去规则
     

    kubectl -n kubesphere-system edit deployments.apps ks-apiserver

    kubectl -n kubesphere-system edit deployments.apps ks-console

    kubectl -n kubesphere-system edit deployments.apps ks-controller-manager

    IfNotPresent

    只有当镜像在本地不存在时才会拉取

    参考博客:https://www.toutiao.com/c/user/token/MS4wLjABAAAA0YFomuMNm87NNysXeUsQdI0Tt3gOgz8WG_0B3MzxsmI/?is_new_connect=0&is_new_user=0&tab=article&wid=1649086635305

  • 相关阅读:
    绿色便携方式安装apache+mysql+tomcat+php集成环境并提供控制面板
    记一次服务器Cuda驱动崩溃修复过程
    Redis（11）Hyperloglog
    墨香戏韵，重塑经典
    SpringCloud微服务之基于zuul搭建服务网关
    【C++】vector 的常用接口
    Centos 安装/操作 Docker
    Hive mapjoin使用
    非 root 用户安装和配置 NodeJS
    PyQt5可视化编程-菜单和工具栏
  • 原文地址：https://blog.csdn.net/qq_35583325/article/details/126950754