elasticsearch + java实现复杂查询

elasticsearch + java实现复杂查询

首先项目中需要集成elasticsearch，如不会集成请参考《集成地址》，如不会安装elasticsearch请参考《安装地址》

---

---

前言

按照现阶段项目的要求，简单的查询已不满足现阶段。这方面的资料在全网都非常少， 接下来下面会罗列出比如汇总查询，must、should filter组合查询等。

---

一、分组统计查询

注意：分组如果在字符串字段上，需要建立字段对应的.keyword字段，该字段支持聚合处理，直接用字符串字段会报错。

1.根据src_ip单个字段做分组

@Autowired
private RestHighLevelClient esClient;

@Test
void groupBy() {
    try {
        SearchRequest searchRequest = new SearchRequest("sh.security-*");
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("src_ip").field("src_ip" + ".keyword").size(100000);
        sourceBuilder.aggregation(aggregationBuilder);
        searchRequest.source(sourceBuilder);
        SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);
        if (RestStatus.OK.equals(searchResponse.status())) {
            // 获取聚合结果
            Aggregations aggregations = searchResponse.getAggregations();
            Terms byAgeAggregation = aggregations.get("src_ip");
            for (Terms.Bucket buck : byAgeAggregation.getBuckets()) {
                System.out.println("key: " + buck.getKeyAsString());
                System.out.println("docCount: " + buck.getDocCount());
                System.out.println("docCountError: " + buck.getDocCountError());

            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
    
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

2.根据src_ip和src_port多个字段进行分组

@Test
void MultipleGroupBy(){
     try {

         // 1、创建search请求
         SearchRequest searchRequest = new SearchRequest("sh.security-*");
         // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         sourceBuilder.size(0);

         //加入聚合
         //字段值项分组聚合
         TermsAggregationBuilder aggregation = AggregationBuilders.terms("src_ip")
                 .script(new Script("doc['src_ip.keyword'] +'#'+doc['src_port.keyword']"))
                 //.field("src_ip.keyword")
                 .size(Integer.MAX_VALUE);
         sourceBuilder.aggregation(aggregation);
         searchRequest.source(sourceBuilder);
         //3、发送请求
         SearchResponse searchResponse = esClient.search(searchRequest,RequestOptions.DEFAULT);
         //4、处理响应
         //搜索结果状态信息
         if(RestStatus.OK.equals(searchResponse.status())) {
             // 获取聚合结果
             Aggregations aggregations = searchResponse.getAggregations();
             Terms byAgeAggregation = aggregations.get("src_ip");
             for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
                 System.out.println("key: " + buck.getKeyAsString());
                 System.out.println("docCount: " + buck.getDocCount());
                 System.out.println("docCountError: " + buck.getDocCountError());
             }
         }
     }catch (Exception e){
         e.printStackTrace();
     }
 }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

二、echart统计数据

根据时间统计

 @Test 
    void chart(){
        try {
            String startTimeStr = "2022-08-01 00:00:00";
            String endTimeStr = "2022-08-31 23:59:59";
            // 判断索引是否为空
            SearchRequest request = new SearchRequest("sh.security-*");
            // 开启真实总数
            request.source().trackTotalHits(true);
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
            // 时间范围
            long startTime = DateUtils.parseTimezone(startTimeStr)*1000L;
            long endTime = DateUtils.parseTimezone(endTimeStr)*1000L;
            boolQuery.must(QueryBuilders.rangeQuery("occur_time").gt(startTime).lt(endTime));
            // 间隔时间
            DateHistogramInterval dateHistogramInterval  = DateHistogramInterval.days(1);
            AggregationBuilder agg = AggregationBuilders.dateHistogram("agg_by_time")
                    .dateHistogramInterval(dateHistogramInterval)
                    .field("occur_time")
                    .minDocCount(0);
            // 请求
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
                    .query(boolQuery)
                    .aggregation(agg)
                    .size(0);
            request.source(searchSourceBuilder);
            SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);

            Aggregations aggregations = response.getAggregations();
            ParsedDateHistogram terms = aggregations.get("agg_by_time");
            for (Histogram.Bucket bucket : terms.getBuckets()) {
                System.out.println(DateUtils.getFormatDate(Long.parseLong(bucket.getKeyAsString())));
                System.out.println(bucket.getDocCount());
            }
        }catch (IOException e){
            throw new RuntimeException(e);
        }
    }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

下次在更