• prometheus学习4Grafana监控mysql&blackbox了解

    Grafana部署及基本使用

    MySQL主从复制

    一、MySQL的主从安装配置

    [root@master ~]# yum -y install mariadb-server 
[root@master ~]# systemctl enable mariadb  --now
[root@master ~]# mysql_secure_installation   # 设置密码
enter
Set root password? [Y/n] y
[root@master ~]# mysql -uroot -p123456

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 修改主库配置
    [root@master ~]# cat /etc/my.cnf|grep -v '#'|grep -v '^$'
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
symbolic-links=0
binlog_format          = row
log_bin                = /data/logbin/mysql-bin
server-id              = 11
log-basename           = master
[mysqld_safe]
skip-name-resolve
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid
!includedir /etc/my.cnf.d
[root@master ~]# systemctl restart mariadb.service

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 从库配置
    [root@slave ~]# cat /etc/my.cnf|grep -v '#'|grep -v '^$'
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
symbolic-links=0
server_id          = 12
read_only          = ON
relay_log          = relay-log
relay_log_index    = relay-log.index
datadir            = /var/lib/mysql
socket             = /var/lib/mysql/mysql.sock
skip-name-resolve
[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid
!includedir /etc/my.cnf.d
[root@master ~]# systemctl restart mariadb.service

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 主库新建复制用户
    -- 登录mysql
[root@master ~]# mysql -uroot -p123456
create user 'mysqld_exporter'@'%' identified by 'ezdevops';
grant all privileges on *.* to 'mysqld_exporter'@'%';
#grant process, replication client, select on *.* to 'mysqld_exporter'@'%';
flush privileges;
select Host,User from mysql.user;
+-----------+-----------------+
| %         | mysqld_exporter |
show master status;
+-------------------+----------+--------------+------------------+
| File              | Position | Binlog_Do_DB | Binlog_Ignore_DB |
+-------------------+----------+--------------+------------------+
| master-bin.000001 |      712 |              |                  |
+-------------------+----------+--------------+------------------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15

    #slave exporter 账户创建

    create user 'mysqld_exporter'@'%' identified by 'ezdevops';
grant all privileges on *.* to 'mysqld_exporter'@'%';
flush privileges;
select Host,User from mysql.user;

    • 1
    • 2
    • 3
    • 4
    • 从库复制主库
    -- 指定主库复制
CHANGE MASTER TO
MASTER_HOST='192.168.1.11',
MASTER_USER='mysqld_exporter',
MASTER_PASSWORD='ezdevops',
MASTER_PORT=3306,
MASTER_LOG_FILE='master-bin.000001',
MASTER_LOG_POS=712,
MASTER_CONNECT_RETRY=10;

-- 启动slave
START SLAVE;
-- 查看slave信息
SHOW SLAVE STATUS\G
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
-- 查看进程信息
SHOW PROCESSLIST;
+----+-------------+-----------+------+---------+------+-----------------------------------------------------------------------------+------------------+----------+
| Id | User        | Host      | db   | Command | Time | State                                                                       | Info             | Progress |
+----+-------------+-----------+------+---------+------+-----------------------------------------------------------------------------+------------------+----------+
| 11 | root        | localhost | NULL | Query   |    0 | NULL                                                                        | SHOW PROCESSLIST |    0.000 |
| 12 | system user |           | NULL | Connect |   85 | Waiting for master to send event                                            | NULL             |    0.000 |
| 13 | system user |           | NULL | Connect |   85 | Slave has read all relay log; waiting for the slave I/O thread to update it | NULL             |    0.000 |
+----+-------------+-----------+------+---------+------+-----------------------------------------------------------------------------+------------------+----------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25

    验证

    [root@slave ~]# mysql -uroot -p123456
MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
+--------------------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    [root@master ~]# mysql -uroot -p123456
MariaDB [(none)]> create database testdb;
MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| testdb             |
+--------------------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    [root@slave ~]# mysql -uroot -p123456
MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| testdb             |
+--------------------+

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10

    二、配置mysql_exporter

    exporter官网

    监控master

    tar -xvf mysqld_exporter-0.13.0.linux-amd64.tar.gz
rm -rf /usr/local/mysqld_exporter*
mv ./mysqld_exporter-0.13.0.linux-amd64/ /usr/local/
ln -sv /usr/local/mysqld_exporter-0.13.0.linux-amd64 /usr/local/mysqld_exporter
mkdir /etc/mysqld_exporter &>/dev/null

    • 1
    • 2
    • 3
    • 4
    • 5
    [root@node1 ~]# cat /etc/my_master.cnf
[client]
host=192.168.1.11 
port=3306 
user=mysqld_exporter 
password=ezdevops

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    cat </usr/lib/systemd/system/mysqld_exporter_master.service
[Unit]
Description=Mysqld Exporter Master
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/mysqld_exporter/mysqld_exporter \
--config.my-cnf=/etc/my_master.cnf \
--collect.info_schema.processlist \
--collect.info_schema.innodb_tablespaces \
--collect.info_schema.innodb_metrics  \
--collect.perf_schema.tableiowaits \
--collect.perf_schema.indexiowaits \
--collect.perf_schema.tablelocks \
--collect.engine_innodb_status \
--collect.perf_schema.file_events \
--collect.binlog_size \
--collect.info_schema.clientstats \
--collect.perf_schema.eventswaits \
--web.listen-address=0.0.0.0:9105
Restart=always

[Install]
WantedBy=multi-user.target
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    systemctl daemon-reload
systemctl stop mysqld_exporter_master.service
systemctl start mysqld_exporter_master.service
systemctl restart mysqld_exporter_master.service

http://192.168.1.12:9105/metrics   #看不到上千条数据的话，确认下my_master.cnf账户密码是否正确

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6

    监控slave

    tar -xvf mysqld_exporter-0.13.0.linux-amd64.tar.gz
rm -rf /usr/local/mysqld_exporter*
mv ./mysqld_exporter-0.13.0.linux-amd64/ /usr/local/
ln -sv /usr/local/mysqld_exporter-0.13.0.linux-amd64 /usr/local/mysqld_exporter
mkdir /etc/mysqld_exporter &>/dev/null

    • 1
    • 2
    • 3
    • 4
    • 5
    [root@node1 ~]# cat /etc/my_slave.cnf
[client]
host=192.168.1.12 
port=3306
user=mysqld_exporter
password=ezdevops

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    cat </usr/lib/systemd/system/mysqld_exporter_slave.service
[Unit]
Description=Mysqld Exporter Master
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/mysqld_exporter/mysqld_exporter \
--config.my-cnf=/etc/my_slave.cnf \
--collect.info_schema.processlist \
--collect.info_schema.innodb_tablespaces \
--collect.info_schema.innodb_metrics  \
--collect.perf_schema.tableiowaits \
--collect.perf_schema.indexiowaits \
--collect.perf_schema.tablelocks \
--collect.engine_innodb_status \
--collect.perf_schema.file_events \
--collect.binlog_size \
--collect.info_schema.clientstats \
--collect.perf_schema.eventswaits \
--web.listen-address=0.0.0.0:9104
Restart=always

[Install]
WantedBy=multi-user.target
EOF

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    systemctl daemon-reload
systemctl start mysqld_exporter_slave.service

http://192.168.1.11:9104/metrics

    • 1
    • 2
    • 3
    • 4
    [root@prome nodes]# cat /usr/local/prometheus/prometheus.yml
...
  - job_name:  'mysql replication'
    file_sd_configs:
      - files:
        - /etc/prometheus/mysql/*.yaml
        refresh_interval: 5s
...
[root@prome nodes]# cat /etc/prometheus/mysql/nodes.yaml
- targets:
  - 192.168.1.11:9105
  labels:
    role: master
- targets:
  - 192.168.1.12:9104
  labels:
    role: slave
[root@prome ~]# systemctl restart prometheus.service
访问http://192.168.1.10:9090/targets

http://192.168.1.11:9105/metrics  UP instance="192.168.1.11:9105"job="scan yaml"role="master" 10.556s ago	8.546ms	
http://192.168.1.12:9104/metrics  UP instance="192.168.1.12:9104"job="scan yaml"role="slave"  13.91s ago	9.004ms	

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22

    在这里插入图片描述
    配置主从监控面板

    在这里插入图片描述
    在这里插入图片描述
    如果数据没有刷新出来涉及到相关标签变量，有些数据未显示。io sql状态查询需要调整

    mysql_slave_status_slave_io_running{channel_name="",connection_name="",master_host="192.168.1.11",master_uuid=""} 1
mysql_slave_status_slave_sql_running{channel_name="",connection_name="",master_host="192.168.1.11",master_uuid=""} 1

    • 1
    • 2

    三、MySQL自定义查询

    下载安装sql_exporter 官网

    1、配置启动sql_exporter

    root@master ~]# tar xf sql_exporter-0.5.linux-amd64.tar.gz
[root@master ~]# ls sql_exporter-0.5.linux-amd64
LICENSE  mssql_standard.collector.yml  README.md  sql_exporter  sql_exporter.yml

    • 1
    • 2
    • 3
    • sql_exporter.yml配置文件
    global:
  scrape_timeout_offset: 500ms
  min_interval: 0s
  max_connections: 3
  max_idle_connections: 3
target:
  data_source_name: 'mysql://mysqld_exporter:ezdevops@tcp(192.168.1.11:3306)/ezdevops'
  # mysql://user:passwd@protocol(host:port)/dbname。支持多种数据库，查询配置
  collectors: [collector_user]
collector_files:
  - "*.collector.yml"

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 定义查询语句,mysql.collector.yml,查询表的记录数量
    collector_name: collector_user
metrics:
  - metric_name: mysql_table_count
    type: counter
    help: 'Count Table Authors.'
    values: [count(*)]
    query: |
      select count(*)  from authors;

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 启动sql_exporter
    ./sql_exporter --config.file=./sql_exporter.yml  #提示默认启动的9399端口

    • 1

    2、测试数据

    CREATE TABLE authors (id INT, name VARCHAR(20), email VARCHAR(20));

    • 1
    INSERT INTO authors (id,name,email) VALUES(1,"user1","book1");
INSERT INTO authors (id,name,email) VALUES(2,"user2","book2");
INSERT INTO authors (id,name,email) VALUES(3,"user3","book3");

    • 1
    • 2
    • 3

    四、黑盒监控(black_exporter)

    balckexporter提供基于HTTP，ICMP，TCP端口等监控接口，可接入Prometheus后使用。

    • 监控面板预览
      在这里插入图片描述

    下载安装blackexporter

    [root@master ~]# tar xf blackbox_exporter-0.19.0.linux-amd64.tar.gz
[root@master ~]# ls blackbox_exporter-0.19.0.linux-amd64
blackbox_exporter  blackbox.yml  LICENSE  NOTICE
[root@master ~]# rm -rf /usr/local/blackbox_exporter*
[root@master ~]# mv blackbox_exporter-0.19.0.linux-amd64/ /usr/local/
[root@master ~]# ln -sv /usr/local/blackbox_exporter-0.19.0.linux-amd64/ /usr/local/blackbox_exporter
[root@master ~]# mkdir /etc/blackbox &>/dev/null

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • blackbox.yml配置文件
    modules:
  http_2xx:
    prober: http
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 启动blackexporter
    ./blackbox_exporter  --config.file=./blackbox.yml --web.listen-address=:9115

    • 1
    [root@master ~]# cat </usr/lib/systemd/system/blackbox_exporter.service
[Unit]
Description=Blackbox Exporter Service
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/blackbox_exporter/blackbox_exporter \
  --config.file=/etc/blackbox/blackbox.yml \
  --web.listen-address=:9115

Restart=always

[Install]
WantedBy=multi-user.target
EOF

[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl start blackbox_exporter

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22

    在这里插入图片描述
    在这里插入图片描述

    Prometheus接入blackexporter

    • prometheus.yml配置文件,检测HTTP
    [root@prome ~]# cat /usr/local/prometheus/prometheus.yml
...
  - job_name: "http_check"
    metrics_path: /probe
    params:
      module: [http_2xx]  #使用http模块
    file_sd_configs:
    - refresh_interval: 1m
      files:
      - "/etc/prometheus/blackbox/black-http.yml"
    relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: 192.168.1.12:9115

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    [root@prome ~]# mkdir -p  /etc/prometheus/blackbox/
[root@prome ~]# vim /etc/prometheus/blackbox/black-http.yml
- targets:
  - https://www.baidu.com
[root@prome ~]# systemctl restart prometheus.service

    • 1
    • 2
    • 3
    • 4
    • 5

    在这里插入图片描述
    在这里插入图片描述

    • prometheus.yml配置文件,检测ICMP
      - job_name: "icmp_ping"
    metrics_path: /probe
    params:
      module: [icmp]  # 使用icmp模块
    file_sd_configs:
    - refresh_interval: 10s
      files:
      - "/etc/prometheus/node/black-icmp.yml"
    relabel_configs:
    - source_labels: [__address__]
      regex: (.*)(:80)?
      target_label: __param_target
      replacement: ${1}
    - source_labels: [__param_target]
      target_label: instance
    - source_labels: [__param_target]
      regex: (.*)
      target_label: ping
      replacement: ${1}
    - source_labels: []
      regex: .*
      target_label: __address__
      replacement: 192.168.0.100:9115

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • prometheus.yml配置文件,检测TCP端口
    
  - job_name: 'prometheus_port_status'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets: ['192.168.1.11:3306']
        labels:
          instance: 'port_status'
          group: 'tcp'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 192.168.0.100:9115

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17

    检测目标配置

    • black-http.yml
    - targets:
  - https://www.sina.cn
  - https://www.baidu.com

    • 1
    • 2
    • 3
    • black-icmp.yml
    - targets: ['192.168.0.1','192.168.0.100']
  labels:
    group: '内网节点'
- targets: ['114.114.114.114','8.8.8.8']
  labels:
    group: '域名服务商'

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6

    配置blackexporter面板

    ID-7587 import导入这个id即可

    使用consul方式注册

    • 注册脚本,添加一个自定义的标签，最后将他转为param_target
    curl -X PUT -d '{
  "id": "http_2xx",
  "name": "http_200_check",
  "Meta": {
		"target": "https://www.sina.cn"
  }
}' http://192.168.1.11:8500/v1/agent/service/register

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • prometheus配置
      - job_name: "http_200_check"
    metrics_path: /probe
    params:
      module: [http_2xx]  #使用http模块
    consul_sd_configs:
       - server: 192.168.1.11:8500
    relabel_configs:
    - source_labels: [__meta_consul_service_metadata_target]
      regex: ""
      action: drop
    - source_labels: [__meta_consul_service_metadata_target]
      target_label: __param_target
    - source_labels: [__meta_consul_service_metadata_target]
      target_label: instance
    - target_label: __address__
      replacement: 192.168.1.12:9115

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16

    http://192.168.1.11:8500/v1/agent/service/register

    
- prometheus配置

```yml
  - job_name: "http_200_check"
    metrics_path: /probe
    params:
      module: [http_2xx]  #使用http模块
    consul_sd_configs:
       - server: 192.168.1.11:8500
    relabel_configs:
    - source_labels: [__meta_consul_service_metadata_target]
      regex: ""
      action: drop
    - source_labels: [__meta_consul_service_metadata_target]
      target_label: __param_target
    - source_labels: [__meta_consul_service_metadata_target]
      target_label: instance
    - target_label: __address__
      replacement: 192.168.1.12:9115

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
  • 相关阅读:
    洗地机哪个品牌最好最实用?口碑最好的洗地机
    快速幂的模板
    【冲刺 NOIP2022 模拟赛 B 组 Day9】开采矿物【DP】
    GCC - 基于win10平台搭建Cmake + MinGW + gcc-arm-none 开源开发环境
    STM32H5开发(7)----LCD显示TOF检测数据
    组播，单播和广播相关
    传输线理论基础01——相关定义、信号速率、分布参数与电报方程
    ubuntu搭建nginx rtmp服务
    无锡矽杰微电子----XC8P9520MCU
    Unity 将是驱动 C# 增长的引擎吗 ？
  • 原文地址：https://blog.csdn.net/weixin_60092693/article/details/126235366