-
docker compose 部署ELK 8.X及监控
1. 前置环境
安装docker-compose
相关版本：
ES：8.5.0
kibana：8.5.0
logstash：8.5.0
2. docker-compose yaml文件
# 在指定路径创建配置文件 vim docker-compose-es-kibana-logstash.yaml- 1
- 2
version: "3" services: elasticsearch: restart: always image: docker.elastic.co/elasticsearch/elasticsearch:8.5.0 container_name: elasticsearch hostname: elasticsearch network_mode: host privileged: true ulimits: memlock: soft: -1 hard: -1 environment: - "ES_JAVA_OPTS=-Xms8192m -Xmx8192m" - "http.host=0.0.0.0" - "node.name=es_node01" - "cluster.name=es_cluster" - "discovery.type=single-node" ports: - "9200:9200" - "9300:9300" volumes: - "/opt/config/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - "/opt/config/es/plugins:/usr/share/elasticsearch/plugins:rw" - "/opt/data/es/data:/usr/share/elasticsearch/data:rw" kibana: restart: always container_name: kibana image: docker.elastic.co/kibana/kibana:8.5.0 network_mode: host environment: - XPACK_MONITORING_COLLECTION_ENABLED="true" ports: - "5601:5601" volumes: - "/opt/config/kibana/config:/usr/share/kibana/config" logstash: image: docker.elastic.co/logstash/logstash:8.5.0 container_name: logstash network_mode: host ports: - "9600:9600" restart: always volumes: - "/opt/config/logstash/config:/usr/share/logstash/config" - "/opt/config/logstash/pipeline:/usr/share/logstash/pipeline" - "/opt/data/logstash:/usr/share/logstash/data" - "/opt/logs/logstash:/usr/share/logstash/logs"- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
3. elasticsearch 配置
-
数据存储映射路径配置777权限
chmod -R 777 /opt/data/es/data- 1
-
es配置文件
# 根据配置文件映射的路径，创建es配置文件 vim /opt/config/es/config/elasticsearch.yml- 1
- 2
# 集群节点名称 node.name: "es_node01" # 设置集群名称为elasticsearch cluster.name: "es_cluster" # 网络访问限制 network.host: 0.0.0.0 # 以单一节点模式启动 discovery.type: single-node # 注意：ES 8.X的版本，需要使用 node.roles配置角色，否则启动报错 #当前该节点是不是有资格竞选主节点 #node.master: true #当前该节点是否存储数据 #node.data: true node.roles: [master,data,remote_cluster_client] # 是否支持跨域 http.cors.enabled: true # 表示支持所有域名 http.cors.allow-origin: "*" # 设置映射端口 http.port: 9200 # 内部节点之间沟通端口 transport.port: 9300 # 内存交换的选项，官网建议为true bootstrap.memory_lock: true # 修改安全配置、关闭证书校验、启动xpack监控 xpack.security.http.ssl: enabled: false xpack.security.transport.ssl: enabled: false xpack.security.enabled: false xpack.monitoring.collection.enabled: true # http传输内容的最大容量 http.max_content_length: 200mb- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
4. logstash 配置
-
数据存储映射路径配置777权限
chmod -R 777 /opt/data/logstash- 1
-
logstash配置文件
vim /opt/config/logstash/config/logstash.yml- 1
# 开启xpack监控 xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: ["http://localhost:9200"] xpack.monitoring.collection.interval: 10s- 1
- 2
- 3
- 4
-
日志文件配置
vim /opt/config/logstash/config/log4j2.properties- 1
status = error name = LogstashPropertiesConfig appender.console.type = Console appender.console.name = plain_console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n appender.json_console.type = Console appender.json_console.name = json_console appender.json_console.layout.type = JSONLayout appender.json_console.layout.compact = true appender.json_console.layout.eventEol = true rootLogger.level = ${sys:ls.log.level} rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
-
pipelines配置文件
vim /opt/config/logstash/config/pipelines.yml- 1
# This file is where you define your pipelines. You can define multiple. # For more information on multiple pipelines, see the documentation: # https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html - pipeline.id: main path.config: "/usr/share/logstash/pipeline"- 1
- 2
- 3
- 4
- 5
- 6
-
配置logstash数据处理流程，依据实际项目需求配置，此处以input为kafka，output为es举例
vim /opt/config/logstash/pipeline/logstash-kafka-es.conf- 1
input { kafka { bootstrap_servers => "localhost:9092" topics => ["topic1"] codec => "json" } } output { if [fields][type] == "topic1" { elasticsearch { hosts => ["http://localhost:9200"] index => "topic1-%{+YYYY.MM.dd}" } } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
5. kibana 配置
vim /opt/config/kibana/config/kibana.yml- 1
# Default Kibana configuration for docker target i18n.locale: zh-CN server.host: "0.0.0.0" server.shutdownTimeout: "5s" elasticsearch.hosts: [ "http://localhost:9200" ] monitoring.ui.container.elasticsearch.enabled: true #此处为新生成的kibana账号和密码 elasticsearch.username: "kibana" elasticsearch.password: "123456" xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: ["http://localhost:9200"] xpack.monitoring.kibana.collection.enabled: true xpack.monitoring.kibana.collection.interval: 10000- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
6. docker compose 启动/停止
在docker-compose yaml所在的路径运行启动或停车命令
# 启动 docker-compose -f docker-compose-es-kibana-logstash.yaml up -d # 停止 docker-compose -f docker-compose-es-kibana-logstash.yaml down- 1
- 2
- 3
- 4
- 5
7. 访问测试
es：http://ip:9200
kibana：http://ip:5601
kibana 监控集群状态：kibana --> Management --> 堆栈监测
-
相关阅读:
六大设计原则
Windows中将tomcat以服务的形式安装,然后在服务进行启动管理
利用抽象工厂模式提升游戏开发的精度与灵活性
简易备忘录
位图+布隆过滤器+海量数据问题(它们都是哈希的应用)
「Verilog学习笔记」使用3-8译码器①实现逻辑函数
nacos不同局域网如何相互调用？nacos微服务云开发，远程联调部署，内网穿透，frp部署
[附源码]java毕业设计校园飞毛腿系统
使用环形缓冲区ringbuffer实现串口数据接收
第1章 入门
- 原文地址：https://blog.csdn.net/u013071014/article/details/133796628
