• DockerCompose部署系列：构建ELK并同步MySQL数据

    1. # 拉取镜像
    2. docker pull mysql:5.7
    3. # 启动镜像
    4. docker run -p 3306:3306 --name mysql -v /data/mysql/log:/var/log/mysql -v /data/mysql/data:/var/lib/mysql -v /data/mysql/conf:/etc/mysql -e MYSQL_ROOT_PASSWORD=123456 -d mysql:5.7
    5.  
    6. # 配置MySQL
    7. vim /data/mysql/conf/my.cnf
    8.  
    9. [client]
    10. default-character-set=utf8
    11. [mysql]
    12. default-character-set=utf8
    13. [mysqld]
    14. init_connect='SET collation_connection = utf8_unicode_ci'
    15. init_connect='SET NAMES utf8'
    16. character-set-server=utf8
    17. collation-server=utf8_unicode_ci
    18. skip-character-set-client-handshake
    19. skip-name-resolve
    20. # 重启MySQL
    21. docker restart mysql
    22.  
    23. # 建表
    24. DROP TABLE IF EXISTS `sys_log`;
    25. CREATE TABLE `sys_log`  (
    26.   `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '日志主键',
    27.   `title` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '模块标题',
    28.   `business_type` int(2) NULL DEFAULT 0 COMMENT '业务类型（0其它 1新增 2修改 3删除）',
    29.   `method` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '方法名称',
    30.   `request_method` varchar(10) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '请求方式',
    31.   `oper_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '操作人员',
    32.   `oper_url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '请求URL',
    33.   `oper_ip` varchar(128) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT '' COMMENT '主机地址',
    34.   `oper_time` datetime(0) NULL DEFAULT NULL COMMENT '操作时间',
    35.   PRIMARY KEY (`id`) USING BTREE
    36. ) ENGINE = InnoDB AUTO_INCREMENT = 1585197503834284034 CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '操作日志记录' ROW_FORMAT = Dynamic;
    37.  
    38. SET FOREIGN_KEY_CHECKS = 1;
    39.  
    40. # ELK搭建准备
    41. # elasticsearch挂载
    42. mkdir -p /data/elk/elasticsearch/{config,plugins,data,logs}
    43. # 赋权
    44. chmod 777 /data/elk/elasticsearch/{config,plugins,data,logs}
    45.  
    46. # kibana挂载
    47. mkdir -p /data/elk/kibana/config
    48.  
    49. # logstash挂载
    50. mkdir -p /data/elk/logstash/config
    51. # 赋权
    52. chmod 777 /data/elk/logstash/config
    53.  
    54. # 配置elasticsearch
    55. vim /data/elk/elasticsearch/config/elasticsearch.yml
    56.  
    57. http.host: 0.0.0.0
    58. xpack.security.enabled: false
    59.  
    60. # 配置kibana
    61. vim /data/elk/kibana/config/kibana.yml
    62.  
    63. server.host: 0.0.0.0
    64. elasticsearch.hosts: [ "http://192.168.5.11:9200" ]
    65.  
    66. # 配置logstash
    67. vim /data/elk/logstash/config/logstash.yml
    68.  
    69. http.host: 0.0.0.0
    70. xpack.monitoring.elasticsearch.hosts: [ "http://192.168.5.11:9200" ]
    71.  
    72. # 创建日志文件
    73. touch /data/elk/logstash/config/log
    74. chmod 777 /data/elk/logstash/config/log
    75.  
    76. # 自行下载mysql-connector-java-8.0.28.jar 放到/data/elk/logstash/config/
    77. # 配置logstash文件
    78. vim /data/elk/logstash/config/logstash.conf
    79. input {
    80.     stdin {
    81.     }
    82.     jdbc {
    83.       jdbc_connection_string => "jdbc:mysql://192.168.5.11:3306/test?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC"
    84.       jdbc_user => "root"
    85.       jdbc_password => "123456"
    86.       jdbc_driver_library => "/usr/share/logstash/config/mysql-connector-java-8.0.28.jar"
    87.       jdbc_driver_class => "com.mysql.jdbc.Driver"
    88.       jdbc_paging_enabled => "true"
    89.       jdbc_page_size => "300000"
    90.       statement => "SELECT id, title, business_type, method, request_method, oper_name, oper_url, oper_ip, oper_time FROM sys_log"
    91.       schedule => "*/1 * * * *"
    92.       use_column_value => false
    93.       tracking_column_type => "timestamp"
    94.       tracking_column => "oper_time"
    95.       record_last_run => true
    96.       jdbc_default_timezone => "Asia/Shanghai"
    97.       last_run_metadata_path => "/usr/share/logstash/config/log"
    98.     }
    99. }
    100.  
    101. output {
    102.     elasticsearch {
    103.         hosts => ["192.168.5.11:9200"]
    104.         index => "sys_log"
    105.         document_id => "%{id}"
    106.     }
    107.     stdout {
    108.         codec => json_lines
    109.     }
    110. }
    111.  
    112. # 配置流水线
    113. vim /data/elk/logstash/config/pipelines.yml
    114.  
    115. - pipeline.id: sys_log
    116.   path.config: "/usr/share/logstash/config/logstash.conf"
    117.   
    118. # 编辑dockerCompose一键搭建
    119. vim docker-compose.yml
    120.  
    121. version: '3'
    122. services:
    123.   elasticsearch:
    124.     image: elasticsearch:7.17.7
    125.     container_name: elasticsearch
    126.     ports:
    127.       - "9200:9200"
    128.       - "9300:9300"
    129.     environment:
    130.       - cluster.name=elasticsearch
    131.       - discovery.type=single-node
    132.       - "ES_JAVA_OPTS=-Xms64m -Xmx512m"
    133.     volumes:
    134.       - /data/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
    135.       - /data/elk/elasticsearch/data:/usr/share/elasticsearch/data
    136.       - /data/elk/elasticsearch/logs:/usr/share/elasticsearch/logs
    137.  
    138.   kibana:
    139.     image: kibana:7.17.7
    140.     container_name: kibana
    141.     ports:
    142.       - "5601:5601"
    143.     depends_on:
    144.       - elasticsearch
    145.     environment:
    146.       I18N_LOCALE: zh-CN
    147.     volumes:
    148.       - /data/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    149.  
    150.   logstash:
    151.     image: logstash:7.17.7
    152.     container_name: logstash
    153.     ports:
    154.       - "5044:5044"
    155.     volumes:
    156.       - /data/elk/logstash/config:/usr/share/logstash/config
    157.     depends_on:
    158.       - elasticsearch
    159.  
    160. # docker-compose.yml文件下执行	  
    161. docker-compose up -d  
    162.  
    163. # 测试访问Kibana
    164. http://192.168.5.11:5601/app/home#/
    165.  
    166.  
    167.  
    168. # 安装完之后，单个启动使用
    169. # 启动elasticsearch
    170. docker run --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms64m -Xmx512m" -v /data/elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /data/elk/elasticsearch/data:/usr/share/elasticsearch/data -v  /data/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins -d elasticsearch:7.17.7
    171. # 启动kibana
    172. docker run --name kibana -e ELASTICSEARCH_HOSTS=http://192.168.5.11:9200 -p 5601:5601 -d kibana:7.17.7
    173. # 启动logstash
    174. docker run --name logstash -p 5044:5044 -v /data/elk/logstash/config:/usr/share/logstash/config -d logstash:7.17.7

    在Kibana上创建索引，然后再数据库的sys_log表中添加测试数据。将会自动同步到ES中去

    1. # 创建与数据库表一样的索引结构
    2. PUT /sys_log
    3. {
    4.   "settings": {
    5.     "number_of_shards": 1,
    6.     "number_of_replicas": 0,
    7.     "index": {
    8.       "max_result_window": 100000000
    9.     }
    10.   },
    11.   "mappings": {
    12.     "dynamic": "strict",
    13.     "properties": {
    14.       "@timestamp": {
    15.         "type": "date"
    16.       },
    17.       "@version": {
    18.         "type": "text",
    19.         "fields": {
    20.           "keyword": {
    21.             "type": "keyword",
    22.             "ignore_above": 256
    23.           }
    24.         }
    25.       },
    26.  
    27.       "business_type": {
    28.         "type": "integer"
    29.       },
    30.       "title": {
    31.         "type": "text"
    32.       },
    33.       "method": {
    34.         "type": "text"
    35.       },
    36.       "request_method": {
    37.         "type": "text"
    38.       },
    39.       "oper_name": {
    40.         "type": "text"
    41.       },
    42.       "oper_url": {
    43.         "type": "text"
    44.       },
    45.       "oper_ip": {
    46.         "type": "text"
    47.       },
    48.       "oper_time": {
    49.         "type": "date"
    50.       },
    51.       "id": {
    52.         "type": "long"
    53.       }
    54.     }
    55.   }
    56. }
    57.  
    58.  
    59. # 查询索引
    60. GET /sys_log/_search
    61. {
    62.   "query": {
    63.     "match_all": {}
    64.   }
    65. }

  • 相关阅读:
    本、硕、博区别真的辣么大吗？
    JPA如何查询部分字段
    ansible的介绍安装与模块
    LC926. 将字符串翻转到单调递增(JAVA - 动态规划)
    django-rest-framework 基础四 过滤、排序、分页、异常处理
    数据化运营15 活跃（上）：如何通过运营手法提升⽤户活跃度？
    Docker镜像仓库：存储与分发Docker镜像的中央仓库
    「数据结构详解·二」二叉树的初步
    红黑树(Red Black Tree)
    MyBatis多条件查询、动态SQL、多表操作、注解开发详细教程
  • 原文地址：https://blog.csdn.net/CancerKing/article/details/128168156