一、准备工作
1、创建一个k8s集群，搭建步骤参见我的另外一篇博文 k8s搭建文档
[root@kubernetes-master ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
kubernetes-master Ready control-plane,master 152d v1.23.4
kubernetes-node1 Ready 152d v1.23.4
kubernetes-node2 Ready 152d v1.23.4
1.2、阿里云oss账号，并创建bucket。这个没什么好说的，参考阿里云官方文档
1.3、在每台机器上安装阿里云ossfs软件。这个软件必须要安装，因为pv/pvc如果想要用阿里云的oss的话，这是必须的软件。安装步骤参见阿里云oss安装文档，我的机器是三节点centos,下边我就写一下centos 该怎么安装ossfs。
# 下载安装包
wget https://gosspublic.alicdn.com/ossfs/ossfs_1.80.6_centos7.0_x86_64.rpm
yum install ossfs_1.80.6_centos7.0_x86_64.rpm
二、yaml文件准备
2.1、rbac.yaml
# This YAML file contains all RBAC objects that are necessary to run external
# CSI provisioner.
#
# In production, each CSI driver deployment has to be customized:
# - to avoid conflicts, use non-default namespace and different names
# for non-namespaced entities like the ClusterRole
# - decide whether the deployment replicates the external CSI
# provisioner, in which case leadership election must be enabled;
# this influences the RBAC setup, see below
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
# replace with the same namespace name with plugin
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: alicloud-csi-plugin
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create", "list", "watch", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: alicloud-csi-plugin
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
roleRef:
kind: ClusterRole
name: alicloud-csi-plugin
apiGroup: rbac.authorization.k8s.io
2.2、oss-plugin.yaml
apiVersion: storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
name: ossplugin.csi.alibabacloud.com
spec:
attachRequired: false
---
# This YAML defines all API objects to create RBAC roles for csi node plugin.
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-ossplugin
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-ossplugin
template:
metadata:
labels:
app: csi-ossplugin
spec:
tolerations:
- operator: Exists
priorityClassName: system-node-critical
serviceAccount: admin
hostNetwork: true
hostPID: true
containers:
- name: driver-registrar
image: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.1.0
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/ossplugin.csi.alibabacloud.com /registration/ossplugin.csi.alibabacloud.com-reg.sock"]
args:
- "--v=5"
- "--csi-address=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
volumeMounts:
- name: kubelet-dir
mountPath: /var/lib/kubelet/
- name: registration-dir
mountPath: /registration
- name: csi-ossplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: registry.cn-hangzhou.aliyuncs.com/acs/csi-plugin:v1.14.8.32-c77e277b-aliyun
imagePullPolicy: "Always"
args:
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--driver=ossplugin.csi.alibabacloud.com"
- "--nodeid=$(KUBE_NODE_NAME)"
env:
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: kubelet-dir
mountPath: /var/lib/kubelet/
mountPropagation: "Bidirectional"
- name: etc
mountPath: /host/etc
- mountPath: /var/log/
name: host-log
- mountPath: /host/usr/
name: flexvolumedir
volumes:
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet/
type: Directory
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate
- name: etc
hostPath:
path: /etc
- name: flexvolumedir
hostPath:
path: /usr/
- name: host-log
hostPath:
path: /var/log/
updateStrategy:
type: RollingUpdate
2.3、pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: oss-csi-pv
labels:
alicloud-pvname: oss-csi-pv
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
csi:
driver: ossplugin.csi.alibabacloud.com
# set volumeHandle same value pv name
volumeHandle: oss-csi-pv
volumeAttributes:
bucket: "*****" #重要
url: "******" #重要
otherOpts: "-o max_stat_cache_size=0 -o allow_other"
akId: "****" #重要
akSecret: "*******" #重要
path: "/"
说明：
2.4、pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: oss-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
selector:
matchLabels:
alicloud-pvname: oss-csi-pv
2.5、deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deployment-oss
labels:
app: nginx
spec:
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
volumeMounts:
- name: oss-pvc
mountPath: "/data"
volumes:
- name: oss-pvc
persistentVolumeClaim:
claimName: oss-pvc
三、部署服务
#创建rbac权限
$ kubectl create -f ./rbac.yaml
serviceaccount/admin created
clusterrole.rbac.authorization.k8s.io/alicloud-csi-plugin created
clusterrolebinding.rbac.authorization.k8s.io/alicloud-csi-plugin created
#创建oss-plugin
$ kubectl create -f ./oss-plugin.yaml
#检查创建情况
$ kubectl get pod -n kube-system | grep csi-oss
kube-system csi-ossplugin-9jdhw 2/2 Running 0 55m
kube-system csi-ossplugin-f7n5f 2/2 Running 0 55m
kube-system csi-ossplugin-vgkcp 2/2 Running 0 55m
#查验CSIDriver安装情况
$ kubectl get CSIDriver
NAME CREATED AT
ossplugin.csi.alibabacloud.com 2020-06-23T14:48:18Z
#创建pv
$ kubectl create -f ./pv.yaml
#创建pvc
$ kubectl create -f ./pvc.yaml
#检验一下阿里云oss是否可以成功挂载到k8s集群中做pv使用
$ kubectl create -f ./deploy.yaml
四、验证
$ kubectl get pod
NAME READY STATUS RESTARTS AGE
deployment-oss-795894886d-lhpsx 1/1 Running 0 11h
#pod成功后通过kubectl exec 进入到pod中，你能看到你账号下bucket里边的所有文件。样例如下：
$ kubectl exec -it deployment-oss-795894886d-lhpsx -- sh
$ ls
bin boot data dev etc home lib lib64 media mnt opt proc root run sbin selinux srv sys tmp usr var
$ cd data
$ ls
osstest
五、参考文档
5.1、 K8S有状态服务-OSS存储使用最佳实践
5.2、 阿里云oss CSI安装步骤
5.3、 阿里云oss官方文档