CentOS7搭建keepalived+DRBD+NFS高可用共享存储

CentOS7搭建keepalived+DRBD+NFS高可用共享存储

一、服务器信息

二、两台主机关闭防火墙，禁用SELinux

systemctl stop firewalld
iptables -F
sed -i '/^SELINUX=/ cSELINUX=disabled' /etc/selinux/config
setenforce 0
1
2
3
4

三、两台主机设置hosts文件，打通通道

vim /etc/hosts
172.25.10.100 node1
172.25.10.110 node2
1
2
3

重启虚拟机

配置root用户免密钥互信
#主服务器

ssh-keygen -f ~/.ssh/id_rsa -P '' -q
ssh-copy-id node2
1
2

#备服务器

ssh-keygen -f ~/.ssh/id_rsa -P '' -q
ssh-copy-id node1
1
2

四、两台主机配置drbd的yum的epel源，方便软件安装

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum localinstall -y https://mirrors.tuna.tsinghua.edu.cn/elrepo/elrepo/el7/x86_64/RPMS/elrepo-release-7.0-6.el7.elrepo.noarch.rpm
wget -O /etc/yum.repos.d/elrepo.repo https://mirrors.aliyun.com/repo/epel-7.repo
1
2
3

安装DRBD软件包
yum install -y drbd90-utils kmod-drbd90
启动drbd的内核

modprobe drbd
echo drbd > /etc/modules-load.d/drbd.conf
lsmod | grep drbd
1
2
3

移除默认配置
cp /etc/drbd.d/global_common.conf /etc/drbd.d/global_common.conf.orig

六、两台主机安装配置DRBD

vim /etc/drbd.d/global_common.conf
global {
	usage-count no;
}
common {	
        protocol C;
        
	handlers {
	        pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
		pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
		local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";

	}

	startup {
	}

	options {
	}

	disk {
	        on-io-error detach;
	}
	
    net {

        }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

配置资源

vim /etc/drbd.d/nfs.res

resource nfs {
  disk /dev/sdb;
  device /dev/drbd0;
  meta-disk internal;
  

  on node1 {
    address 172.25.10.100:7789;
  }

  on node2 {
    address 172.25.10.110:7789;
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

六、启动DRBD

1. 首次启用资源
   创建设备的元数据
   drbdadm create-md nfs
   
   启用资源
   drbdadm up nfs
   初始设备同步,只在一个主节点上执行
   drbdadm primary --force nfs

监视进度
drbdadm status

等待一段时间，检查同步情况

连接状态
drbdadm cstate nfs来观察资源的连接状态

2. 常用命令

drbdadm cstate nfs      连接状态
drbdadm dstate nfs      磁盘状态
drbdadm role nfs         资源角色
drbdadm primary nfs   提升资源
drbdadm secondary nfs   降级资源
1
2
3
4
5

七、验证drbd主从同步

两台节点都创建挂载点：mkdir /data

主节点上操作：

mkfs.xfs /dev/drdb0 /data
mount /dev/drbd0 /data
1
2

创建测试文件

touch /data/file{1..3}
ls -l /data
1
2

卸载文件系统并切换备节点

umount /data
drbdadm secondary nfs
1
2

在从节点上执行以下命令确认文件

drbdadm primary nfs
mount /dev/drbd0 /data
ls -l /data
1
2
3

验证完成

还原，将node1设置为主，node2为备

启用服务

systemctl start drbd
systemctl enable drbd
1
2

八、安装nfs

两台主机安装nfs

yum -y install rpcbind nfs-utils
vim letc/exports

/data 172.25.0.0/16(rw,sync,no_root_squash,no_all_squash)

systemctl restart rpcbind
systemctl restart nfs
systemctl enable nfs-server rpcbind
showmount -e localhost
1
2
3
4
5
6
7
8
9

九、安装配置keepalived

yum install -y keepalived

2. 配置keepalived配置文件：

##MASTER配置文件
[root@node1 ~]# vim /etc/keepalived/keepalived.conf
 
! Configuration File for keepalived
 
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL_NFS1
}
 
vrrp_script chk_nfs {
    script "killall -0 nfsd"
    interval 2
    weight -40
}
 
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
 
    track_script {
        chk_nfs
    }
 
    notify_stop /etc/keepalived/notify_stop.sh  #当服务停止时执行的脚本
    notify_master /etc/keepalived/notify_master.sh  #当切换成主时执行的脚本
    notify_backup /etc/keepalived/notify_backup.sh  #当切换成备时执行的脚本
 
    virtual_ipaddress {
        172.25.10.200/24 dev ens33 label ens33:1  #表示将VIP绑定在指定网卡的指定子接口上
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

##BACKUP配置文件：
[root@node2 ~]# vim /etc/keepalived/keepalived.conf
 
! Configuration File for keepalived
 
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL_NFS2
}
 
vrrp_script chk_nfs {
    script "killall -0 nfsd"
    interval 2
    weight -40
}
 
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
 
    track_script {
        chk_nfs
    }
 
    notify_stop /etc/keepalived/notify_stop.sh  #当服务停止时执行的脚本
    notify_master /etc/keepalived/notify_master.sh  #当切换成主时执行的脚本
    notify_backup /etc/keepalived/notify_backup.sh  #当切换成备时执行的脚本
 
    virtual_ipaddress {
        172.25.10.200/24 dev ens33 label ens33:1
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

3. 配置从BACKUP切换到MASTER触发的脚本，主备脚本一致：

[root@node1 keepalived]# vim notify_master.sh
 
#!/bin/bash
 
time=$(date +"%F %T")
logname=notify_master
logdir=/etc/keepalived/logs
mountdir=/dev/drbd0
resname=nfs
 
[ -d ${logdir} ] || mkdir -p $logdir
echo -e "$time ----${logname}----\n" >> $logdir/${logname}.log
drbdadm primary $resname &>> $logdir/${logname}.log  ##升级资源角色
mount $mountdir /data &>> $logdir/${logname}.log  ##将设备挂载到挂载点上
systemctl start nfs-server &>> $logdir/${logname}.log  ##启动nfs服务
echo -e "\n" >> $logdir/${logname}.log
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

4. 配置从MASTER切换到BACKUP触发的脚本，主备脚本一致：

[root@node1 keepalived]# vim notify_backup.sh
 
#!/bin/bash
 
time=$(date +"%F %T")
logname=notify_backup
logdir=/etc/keepalived/logs
mountdir=/dev/drbd0
resname=nfs
 
[ -d ${logdir} ] || mkdir -p $logdir
echo -e "$time ----${logname}----\n" >> $logdir/${logname}.log
systemctl stop nfs-server &>> $logdir/${logname}.log  ##停止nfs服务
umount /data &>> $logdir/${logname}.log  ##卸载
drbdadm secondary $resname &>> $logdir/${logname}.log  ##降级资源角色
echo -e "\n" >> $logdir/${logname}.log
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

5. 配置停止服务时触发的脚本，主备脚本一致：

[root@node1 keepalived]# vim notify_stop.sh
 
#!/bin/bash
 
time=$(date +"%F %T")
logname=notify_stop
logdir=/etc/keepalived/logs
mountdir=/dev/drbd0
resname=nfs
 
[ -d ${logdir} ] || mkdir -p $logdir
echo -e "$time ----${logname}----\n" >> $logdir/${logname}.log
systemctl stop nfs-server &>> $logdir/${logname}.log
umount /data &>> $logdir/${logname}.log
drbdadm secondary $resname &>> $logdir/${logname}.log
echo -e "\n" >> $logdir/${logname}.log
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

6. 为脚本添加执行权限，主备节点操作一致：

[root@node1 keepalived]# chmod +x notify_stop.sh 
 
[root@node1 keepalived]# chmod +x notify_backup.sh 
 
[root@node1 keepalived]# chmod +x notify_master.sh 
1
2
3
4
5

7. 重启keepalived：

[root@node1 ~]# systemctl restart keepalived.service
1

五. 测试高可用，共享存储是否正常：

1. 查看VIP的位置是否在node1上：

[root@node1 ~]# ip a | grep 172.25.10.200
    inet 172.25.10.200/24 scope global secondary ens33:1
1
2

2. 查看node1上的nfs是否启动，node2上的nfs是否是关闭：

[root@node1 ~]# systemctl is-active nfs
active
 
[root@node2 ~]# systemctl is-active nfs
inactive
1
2
3
4
5

3. 查看node1:/data/下是否有数据，node2:/data/下是否没有数据：

[root@node1 ~]# ls /data
centos-release  centos-release-upstream  cron.deny  crontab  crypttab  
csh.cshrc  csh.login
 
[root@node2 ~]# ls /data/
[root@node2 ~]# 
1
2
3
4
5
6

    4.  node1停止nfs服务，node2开启nfs服务；并查看VIP是否飘移：
1

[root@node1 ~]# systemctl stop nfs
 
[root@node2 ~]# systemctl start nfs
 
[root@node1 ~]# ip a | grep 172.25.10.200
[root@node1 ~]# 
 
[root@node2 ~]# ip a | grep 172.25.10.200
    inet 172.25.10.200/24 scope global secondary ens33:1
1
2
3
4
5
6
7
8
9

5. 查看node2:/data/下是否有数据：

[root@node2 ~]# ls /data/
centos-release  centos-release-upstream  cron.deny  crontab  crypttab  
csh.cshrc  csh.login

1
2
3
4