Helm部署ES及Kibana（默认开启SSL）

由于之前使用helm部署EFK，感觉在过程上有些麻。因此我在helm-charts的7.16分支上写了一个job，使得用户名密码及ssl证书可以自动生成并在k8s里创建secret。所以部署时ssl是默认开启的。
helm-charts:7.16分支的改动纯属个人兴趣，仅作参考。
官方 elastic/helm-charts在最新的改动中也实现了自动生成证书的功能。

下载charts

$ git clone https://github.com/cloudenmin/helm-charts.git 
$ git checkout 7.16
1
2

elasticsearch

values.yaml
默认用户名：elastic
默认密码：P@ssw0rD

security:
  username: "elastic"
  password: "P@ssw0rD"
1
2
3

部署es

$ cd elasticsearch
$ helm install elasticsearch . -n efk --create-namespace
1
2

部署结果：

$ kubectl get pod -n efk
NAME                     READY   STATUS    RESTARTS   AGE
elasticsearch-master-0   1/1     Running   0          2m
elasticsearch-master-1   1/1     Running   0          2m
elasticsearch-master-2   1/1     Running   0          2m
1
2
3
4
5

部署Kibana

修改values.yaml

elasticsearchHosts: "https://elasticsearch-master-headless.efk.svc.cluster.local:9200"
1

部署Kibana

$ cd kibana
$ helm install kibana . -n efk
1
2

部署结果：

NAME                      READY   STATUS    RESTARTS   AGE
elasticsearch-master-0    1/1     Running   0          13m
elasticsearch-master-1    1/1     Running   0          13m
elasticsearch-master-2    1/1     Running   0          13m
kibana-79465dfb9f-chxft   1/1     Running   0          72s
1
2
3
4
5

访问https://${host_ip}:30601

文件改动

创建了一个job.yaml

{{- if .Values.security.enable }}
{{- $serviceAccountName := .Values.security.rbac.serviceAccountName }}
---
# 因为涉及到secret的操作，所以创建了一个新的seviceaccount，并赋予相关权限
apiVersion: v1
kind: ServiceAccount
metadata:
  name: {{ $serviceAccountName }}
  namespace: {{ .Release.Namespace }}
  labels:
    heritage: {{ .Release.Service | quote }}
    release: {{ .Release.Name | quote }}
    chart: "{{ .Chart.Name }}"
    app: "{{ template "elasticsearch.uname" . }}"
  annotations:
    "helm.sh/hook": pre-install,post-delete
    "helm.sh/hook-weight": "-7"
    "helm.sh/hook-delete-policy": before-hook-creation
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: {{ $serviceAccountName }}
  namespace: {{ .Release.Namespace }}
  labels:
    heritage: {{ .Release.Service | quote }}
    release: {{ .Release.Name | quote }}
    chart: "{{ .Chart.Name }}"
    app: "{{ template "elasticsearch.uname" . }}"
  annotations:
    "helm.sh/hook": pre-install,post-delete
    "helm.sh/hook-weight": "-6"
    "helm.sh/hook-delete-policy": before-hook-creation
rules:
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - get
      - list
      - create
      - update
      - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: {{ $serviceAccountName }}
  namespace: {{ .Release.Namespace }}
  labels:
    heritage: {{ .Release.Service | quote }}
    release: {{ .Release.Name | quote }}
    chart: "{{ .Chart.Name }}"
    app: "{{ template "elasticsearch.uname" . }}"
  annotations:
    "helm.sh/hook": pre-install,post-delete
    "helm.sh/hook-weight": "-5"
    "helm.sh/hook-delete-policy": before-hook-creation
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: {{ $serviceAccountName }}
subjects:
  - kind: ServiceAccount
    name: {{ $serviceAccountName }}
    namespace: {{ .Release.Namespace }}
---
# 执行一个job，创建用户及证书相关secret
# job在执行完成后自动删除。
apiVersion: batch/v1
kind: Job
metadata:
  name: elastic-security-config
  namespace: {{ .Release.Namespace | quote }}
  annotations:
    "helm.sh/hook": pre-install,post-delete
    "helm.sh/hook-weight": "-4"
    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
  labels:
    heritage: {{ .Release.Service | quote }}
    release: {{ .Release.Name | quote }}
    chart: "{{ .Chart.Name }}"
    app: "{{ template "elasticsearch.uname" . }}"
spec:
  ttlSecondsAfterFinished: 100
  template:
    spec:
      serviceAccountName: {{ $serviceAccountName }}
      restartPolicy: OnFailure
      containers:
        - name: create-security-config
          image: "{{ .Values.image }}:{{ .Values.imageTag }}"
          imagePullPolicy: "{{ .Values.imagePullPolicy }}"
          env:
            - name: USERNAME
              value: {{ .Values.security.username | b64enc}}
            - name: PASSWORD
              value: {{ .Values.security.password | b64enc}}
            - name: NAMESPACE
              value: {{ .Release.Namespace }}
          command: 
{{ toYaml .Values.security.command | indent 12 -}}
  {{- end }}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

job执行的脚本：
定义在values.yaml里

security:
  command:
    - bash
    - -c
    - |
      #!/bin/bash
      KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
      KUBE_CERT='/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
      SECRET_URL=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/${NAMESPACE}/secrets

      delete_secret(){
        if [ $(curl -sw '%{http_code}' --cacert ${KUBE_CERT} -X GET $SECRET_URL/$1 -H 'Content-Type: application/json' -H 'Authorization: Bearer '${KUBE_TOKEN} -o /dev/null) -eq 200 ]; then
           if [ $(curl -sw '%{http_code}' --cacert ${KUBE_CERT} -X DELETE $SECRET_URL/$1 -H 'Content-Type: application/json' -H 'Authorization: Bearer '${KUBE_TOKEN} -o /dev/null) -eq 200 ]; then
              echo "deleting "$1" successfully!"
           fi
        else
           echo $1" does not exist"
        fi
      }
      
      # 删除旧的secret
      delete_secret elastic-credentials
      delete_secret elastic-certificates
      delete_secret elastic-certificate-pem
      delete_secret elastic-certificate-crt

      elasticsearch-certutil ca --out elastic-stack-ca.p12 --pass ''
      elasticsearch-certutil cert --name security-master --dns security-master --ca elastic-stack-ca.p12 --pass '' --ca-pass '' --out elastic-certificates.p12
      openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem
      openssl x509 -outform der -in elastic-certificate.pem -out elastic-certificate.crt

      create_user_secret(){
        DATA='{"apiVersion":"v1","kind":"Secret","type":"Opaque","metadata":{"name":"elastic-credentials","namespace":"'${NAMESPACE}'"},"data":{"password":"'${PASSWORD}'","username":"'${USERNAME}'"}}'
        HTTP_CODE=$(curl -sw '%{http_code}' --cacert ${KUBE_CERT} -X POST $SECRET_URL -H 'Content-Type: application/json' -H 'Authorization: Bearer '${KUBE_TOKEN} -d $DATA -o /dev/null)
        if [ $HTTP_CODE -eq 201 ]; then
           echo $HTTP_CODE": creating elastic-credentials successfully!"
        else
           echo $HTTP_CODE": failed to create elastic-credentials!"
        fi
      }

      # 创建用户名密码
      create_user_secret

      create_certifcate_secret(){
        DATA='{"apiVersion":"v1","kind":"Secret","type":"Opaque","metadata":{"name":"'$1'","namespace":"'${NAMESPACE}'"},"data":{"'$2'":"'$(cat $2 | base64 -w0)'"}}'
        HTTP_CODE=$(curl -sw '%{http_code}' --cacert ${KUBE_CERT} -X POST $SECRET_URL -H 'Content-Type: application/json' -H 'Authorization: Bearer '${KUBE_TOKEN} -d $DATA -o /dev/null)
        if [ $HTTP_CODE -eq 201 ]; then
          echo $HTTP_CODE": creating "$1" successfully!"
        else
          echo $HTTP_CODE": failed to create a "$1"!"
        fi
      }

      # 创建证书
      create_certifcate_secret elastic-certificates elastic-certificates.p12
      create_certifcate_secret elastic-certificate-pem elastic-certificate.pem
      create_certifcate_secret elastic-certificate-crt elastic-certificate.crt

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59