写的一款简易的热点词汇记录工具

项目需要对用户提交的附件、文章、搜索框内容等做热词分析。如下图：

公司有大数据团队。本着不麻烦别人就不麻烦别人的原则，写了一款简易的记录工具，原理也简单，手工在业务插入锚点，用分词器分好词，排掉字母、数字、符号、敏感词。将词汇按年度累加记录到数据库中即可。代码如下:

@Component
public class HotWordHelper {

    private static HotWordMapper hotWordMapper;

    static List<Character> FILTER_CHARS = new ArrayList<>();

    static {
        String number = "123456789abcdefghijklnmopqrstuvwxyzABCDEFGHIJKLNMOPQRSTUVWXYZ";
        char[] chars = number.toCharArray();
        for (char aChar : chars) {
            FILTER_CHARS.add(aChar);
        }
    }

    @Autowired
    public ZYHotWordHelper(HotWordMapper hotWordMapper) {
        ZYHotWordHelper.hotWordMapper = hotWordMapper;
    }

    public static List<HotWord> loaderHotWordTen(String moduleCode) {
        LocalDate now = LocalDate.now();
        int year = now.getYear();
        return loaderHotWord(year, 10, moduleCode);
    }

    public static List<HotWord> loaderHotWord(int top, String moduleCode) {
        LocalDate now = LocalDate.now();
        int year = now.getYear();
        return loaderHotWord(year, top, moduleCode);
    }

    public static List<HotWord> loaderHotWord(int year, int top, String moduleCode) {
        LambdaQueryWrapper<HotWord> wrapper = Wrappers.lambdaQuery();
        wrapper.eq(HotWord::getRecordYear, year);
        wrapper.eq(HotWord::getModuleCode, moduleCode);
        return hotWordMapper.selectTop(wrapper, HotWord::getAppearTimes, top);
    }

	// 直接词汇，如字典之类的。
    public static void putDirectHotWord(String text, String moduleCode) {
    	// 为不影响主业务速度，改成异常
        Runnable runnable = () -> putHotWord(true, text, moduleCode);
        AsyncExecutor.execute(runnable);
    }
	
	// 分析词汇
    public static void putAnalyzeHotWord(String text, String moduleCode) {
    	// 为不影响主业务速度，改成异常
        Runnable runnable = () -> putHotWord(false, text, moduleCode);
        AsyncExecutor.execute(runnable);
    }

	// 附件
    public static void putAttachmentAsync(StringsField attachmentIds, String moduleCode) {
        if (ZYListUtils.isEmptyList(attachmentIds)) {
            return;
        }
        Runnable runnable = () -> doPutAttachmentAsync(attachmentIds, moduleCode);
        AsyncExecutor.execute(runnable);
    }

	// 解析附件
    private static void doPutAttachmentAsync(StringsField attachmentIds, String moduleCode) {
        FileInfoMapper fileInfoMapper = SpringUtils.getBean(FileInfoMapper.class);
        List<FileInfo> fileInfos = fileInfoMapper.selectBatchIds(attachmentIds);
        if (ZYListUtils.isEmptyList(fileInfos)) {
            return;
        }

        FileStoreService storeService= ZYSpringUtils.getBean(FileStoreService.class);
        List<FileWrapper> fileWrappers = ZYListUtils.list2list(fileInfos, FileInfo::toFileWrapper);
        for (FileWrapper fileWrapper : fileWrappers) {
            try (InputStream objectStream = storeService.getObjectStream(fileWrapper)) {
                String text = IOUtils.toString(objectStream, StandardCharsets.UTF_8);
                putAnalyzeHotWord(text, moduleCode);
            } catch (Exception e) {
                return;
            }

        }
    }

    private static void putHotWord(boolean isDirect, String text, String moduleCode) {
        if (ZYStrUtils.isAnyNull(text, moduleCode)) {
            return;
        }

        List<String> words = analyzerWords(isDirect, text);
        if (ZYListUtils.isEmptyList(words)) {
            return;
        }
        //  List smallWordCompare = new ArrayList<>(words);
        words.removeIf(w -> {
            if (!matchLength(w)) {
                return true;
            }
            char[] chars = w.toCharArray();
            for (char aChar : chars) {
                // 不要数字字母
                if (FILTER_CHARS.contains(aChar)) {
                    return true;
                }
            }

            // 存在误判，还是不用这段代码
      /*      for (String compareWord : smallWordCompare) {
                if (!w.equals(compareWord) && compareWord.contains(w)) {
                    return true;
                }
            }
*/
            return false;
        });
        Map<String, Integer> wordCount = ZYMapUtils.countField(words, w -> w);

        LocalDate now = LocalDate.now();
        int year = now.getYear();
        LambdaQueryWrapper<HotWord> wrapper = Wrappers.lambdaQuery();
        wrapper.in(HotWord::getHotWord, words);
        wrapper.eq(HotWord::getRecordYear, year);
        wrapper.eq(HotWord::getModuleCode, moduleCode);
        List<HotWord> existsWords = hotWordMapper.selectList(wrapper);
        Map<String, HotWord> wordIdContainer = ZYListUtils.groupModel(existsWords, HotWord::getHotWord);

        List<HotWord> addHotWords = new ArrayList<>();
        List<HotWord> editHotWords = new ArrayList<>();
        wordCount.forEach((w, times) -> {
            HotWord hotWord = wordIdContainer.get(w);
            if (null != hotWord) {
                Integer appearTimes = hotWord.getAppearTimes();
                appearTimes += times;
                hotWord.setAppearTimes(appearTimes);
                editHotWords.add(hotWord);
            } else {
                HotWord newHotWord = new HotWord();
                newHotWord.setRecordYear(year);
                newHotWord.setAppearTimes(times);
                newHotWord.setHotWord(w);
                newHotWord.setModuleCode(moduleCode);
                addHotWords.add(newHotWord);
            }
        });

        if (ZYListUtils.isNotEmptyList(addHotWords)) {
            hotWordMapper.insertBatch(addHotWords);
        }
        if (ZYListUtils.isNotEmptyList(editHotWords)) {
            for (HotWord editHotWord : editHotWords) {
                hotWordMapper.updateById(editHotWord);
            }
        }
    }

    private static List<String> analyzerWords(boolean isDirect, String text) {
        List<String> words;
        if (isDirect) {
            words = Collections.singletonList(text);
        } else {
            words = ZYDirtyWordHelper.analyze(text);
            if (matchLength(text) && !words.contains(text)) {
                words.add(text);
            }
        }
        return words;
    }
    
    private static boolean matchLength(String text) {
        int length = text.length();
        return length > 1 && length < 6;
    }

}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173

记录表中效果图,实际效果还阔以，实时性和记录速度都非常快。