Logstash 部署及基础使用

一、部署Logstash环境及基础使用

1.部署logstash环境

yum -y localinstall logstash-7.17.3-x86_64.rpm 

ln -sv /usr/share/logstash/bin/logstash /usr/local/bin/

下载地址:
    https://www.elastic.co/downloads/past-releases#logstash
1
2
3
4
5
6

2.修改logstash的配置文件

(1)编写配置⽂件
cat > conf.d/01-stdin-to-stdout.conf <<'EOF'
input {
  stdin {}
}

output {
  stdout {}
}
EOF

(2)检查配置⽂件语法
logstash -tf conf.d/01-stdin-to-stdout.conf

(3)启动logstash实例
logstash -f conf.d/01-stdin-to-stdout.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

3.input插件基于file案例

input {
  file {
      # 指定收集的路径
      path => ["/tmp/test/*.txt"]
      # 指定⽂件的读取位置，仅在".sincedb*"⽂件中没有记录的情况下⽣效!
      start_position => "beginning"     
      # start_position => "end"     
  }
}

output {
  stdout {}
}
1
2
3
4
5
6
7
8
9
10
11
12
13

4.input插件基于tcp案例

input {
  tcp {
     port => 8888
  }

  tcp {
     port => 9999
  }

}

output {
  stdout {}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14

5.input插件基于http案例

input {
  http {
     port => 8888
  }

  http {
     port => 9999
  }

}

output {
  stdout {}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14

6.input插件基于redis案例

    filebeat的配置:(仅供参考)
filebeat.inputs:
- type: tcp
  host: "0.0.0.0:9000"

output.redis:
  # 写⼊redis的主机地址
  hosts: ["10.0.0.101:6379"]
  # 指定redis的认证⼝令
  password: "oldboyedu"
  # 指定连接数据库的编号
  db: 5
  # 指定的key值
  key: "oldboyedu-linux80-filebeat"
  # 规定超时时间.
  timeout: 3

    logstash的配置:
input {
  redis {
     # 指定的是REDIS的键(key)的类型
     data_type => "list"
     # 指定数据库的编号,默认值是0号数据库
     db => 5
     # 指定数据库的ip地址,默认值是localhost
     host => "10.0.0.101"
     # 指定数据库的端⼝号，默认值为6379
     port => 6379
     # 指定redis的认证密码
     password => "oldboyedu"
     # 指定从redis的哪个key取数据
     key => "oldboyedu-linux80-filebeat" 
  }

}

output {
  stdout {}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

7.input插件基于beats案例

    filbeat配置:
filebeat.inputs:
- type: tcp
  host: "0.0.0.0:9000"

output.logstash:
  hosts: ["10.0.0.101:5044"]

    logstsh配置:
input {
  beats {
     port => 5044
  }

}

output {
  stdout {}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

8.output插件基于redis案例

input {
  tcp {
     port => 9999
  }

}

output {
  stdout {}

  redis {
      # 指定redis的主机地址
      host => "10.0.0.101"
      # 指定redis的端⼝号
      port => "6379"
      # 指定redis数据库编号
      db => 10
      # 指定redis的密码
      password => "oldboyedu"
      # 指定写⼊数据的key类型
      data_type => "list"
      # 指定的写⼊的key名称
      key => "oldboyedu-linux80-logstash"
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

9.output插件基于file案例

input {
  tcp {
     port => 9999
  }

}

output {
  stdout {}

  file {
      # 指定磁盘的落地位置
      path => "/tmp/oldboyedu-linux80-logstash.log"
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

10.logstash综合案例

    (1)filebeat-to-redis参考笔记
filebeat.inputs:
- type: tcp
  host: "0.0.0.0:8888"

output.redis:
  # 写⼊redis的主机地址
  hosts: ["10.0.0.101:6379"]
  # 指定redis的认证⼝令
  password: "oldboyedu"
  # 指定连接数据库的编号
  db: 5
  # 指定的key值
  key: "oldboyedu-linux80-filebeat"
  # 规定超时时间.
  timeout: 3  

    (2)filebeat-to-logstash参考笔记
filebeat.inputs:
- type: tcp
  host: "0.0.0.0:9999"

output.logstash:
  hosts: ["10.0.0.101:7777"]
  
      (3)logstash配置⽂件
input {
  tcp {
     type => "oldboyedu-tcp"
     port => 6666
  }

  beats {
     type => "oldboyedu-beat"
     port => 7777
  }

  redis {
     type => "oldboyedu-redis"
     data_type => "list"
     db => 5
     host => "10.0.0.101"
     port => 6379
     password => "oldboyedu"
     key => "oldboyedu-linux80-filebeat" 
  }

}

output {
  stdout {}
  
  if [type] == "oldboyedu-tcp" {
     elasticsearch {
        hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
        index => "oldboyedu-linux80-tcp-%{+YYYY.MM.dd}"
     }
  } else if [type] == "oldboyedu-beat" {
    elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-beat-%{+YYYY.MM.dd}"
    }
  } else if [type] == "oldboyedu-redis" {
    elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-redis-%{+YYYY.MM.dd}"
    }
  } else {
    elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-other-%{+YYYY.MM.dd}"
    }
  }

}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

11.今日作业

(1)完成课堂的所有练习，要求能够⼿绘架构图;
(2)如上图所示，按照上述要求完成作业;
1
2

11.1.运行一个logstash版本

[root@elk101.oldboyedu.com ~]# cat config-logstash/11-many-to-es.conf 
input {
  beats {
     port => 8888
  }

  redis {
     data_type => "list"
     db => 8
     host => "10.0.0.101"
     port => 6379
     password => "oldboyedu"
     key => "oldboyedu-linux80-filebeat" 
  }
}

output {
  stdout {}

  elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-logstash-%{+YYYY.MM.dd}"
  }
}
[root@elk101.oldboyedu.com ~]# 
[root@elk101.oldboyedu.com ~]# logstash -f config-logstash/11-many-to-es.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

11.2.运行二个logstash版本

    logstash接受redis示例：
[root@elk101.oldboyedu.com ~]# cat config-logstash/13-redis-to-es.conf   
         
input {
  redis {
     data_type => "list"
     db => 8
     host => "10.0.0.101"
     port => 6379
     password => "oldboyedu"
     key => "oldboyedu-linux80-filebeat" 
  }
}

output {
  stdout {}

  elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-logstash-%{+YYYY.MM.dd}"
  }
}
[root@elk101.oldboyedu.com ~]# logstash -f config-logstash/13-redis-to-
es.conf 

    logstash接受beats示例：
[root@elk101.oldboyedu.com ~]# cat config-logstash/12-beat-to-es.conf   
         
input {
  beats {
     port => 8888
  }
}

output {
  stdout {}

  elasticsearch {
      hosts => ["10.0.0.101:9200","10.0.0.102:9200","10.0.0.103:9200"]
      index => "oldboyedu-linux80-logstash-%{+YYYY.MM.dd}"
  }
}
[root@elk101.oldboyedu.com ~]# logstash -f config-logstash/12-beat-to-es.conf --path.data /tmp/logstash
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43