• ElasticSearch异常情况监控处理

    1.异常类型

    在公司使用Canal-Server作为数据同步监听,在同步过程中,会出现各种问题异常,

    比如常见一下几种:

    1.表结构发生变更,导出BinLog字段对比异常

    2.数据库连接失败,账号密码错误, 权限不足等

    3.Binlog文件过期或者丢失

    2.传统监控手段-ELK

    1.应用端按固定格式,把日志写入到log文件中

    2.logstash读取日志文件, 写入到ES中

    3.通过logstash分析日志,按特定规则配置告警规则,发送钉钉报警

    3.轻量级监控手段logback

    1.自定义LogDingDingAppende

    1. package com.logtest.appender;
    2.  
    3.  
    4. import ch.qos.logback.classic.Level;
    5. import ch.qos.logback.classic.PatternLayout;
    6. import ch.qos.logback.classic.spi.LoggingEvent;
    7. import ch.qos.logback.core.UnsynchronizedAppenderBase;
    8. import com.alibaba.fastjson.JSON;
    9. import com.logtest.appender.endity.DingtalkMessage;
    10. import com.logtest.util.HttpsUtils;
    11. import lombok.extern.slf4j.Slf4j;
    12. import org.springframework.util.StringUtils;
    13.  
    14. import java.util.HashMap;
    15. import java.util.Map;
    16.  
    17.  
    18.  
    19. @Slf4j
    20. public class LogDingDingAppender extends UnsynchronizedAppenderBase<LoggingEvent> {
    21.  
    22.  
    23.     private String pattern = "%level\n%m\n%logger\n%ex";
    24.     private PatternLayout layout;
    25.     private String profilesActive = "unknown";
    26.     private String dingdingUrl;
    27.     private Level nowLevel = Level.ERROR;
    28.     private String secret;
    29.     private String serverName;
    30.  
    31.     /**
    32.      * 排除的类集合用逗号分隔
    33.      */
    34.     private String excludeThrowableClasses;
    35.  
    36.     public String getProfilesActive() {
    37.         return profilesActive;
    38.     }
    39.  
    40.     public void setProfilesActive(String profilesActive) {
    41.         this.profilesActive = profilesActive;
    42.     }
    43.  
    44.     public void setPattern(String pattern) {
    45.         this.pattern = pattern;
    46.     }
    47.  
    48.     public String getServerName() {
    49.         return serverName;
    50.     }
    51.  
    52.     public void setServerName(String serverName) {
    53.         this.serverName = serverName;
    54.     }
    55.  
    56.     public String getPattern() {
    57.         return pattern;
    58.     }
    59.  
    60.     public PatternLayout getLayout() {
    61.         return layout;
    62.     }
    63.  
    64.     public void setLayout(PatternLayout layout) {
    65.         this.layout = layout;
    66.     }
    67.  
    68.     public String getDingdingUrl() {
    69.         return dingdingUrl;
    70.     }
    71.  
    72.     public void setDingdingUrl(String dingdingUrl) {
    73.         this.dingdingUrl = dingdingUrl;
    74.     }
    75.  
    76.     public void setLevel(String level) {
    77.         this.nowLevel = Level.toLevel(level, Level.ERROR);
    78.     }
    79.  
    80.     public void setExcludeThrowableClasses(String excludeThrowableClasses) {
    81.         this.excludeThrowableClasses = excludeThrowableClasses;
    82.     }
    83.  
    84.     public String getSecret() {
    85.         return secret;
    86.     }
    87.  
    88.     public void setSecret(String secret) {
    89.         this.secret = secret;
    90.     }
    91.  
    92.  
    93.     @Override
    94.     protected void append(LoggingEvent eventObject) {
    95.         try {
    96.  
    97.             if (eventObject.getLevel().isGreaterOrEqual(nowLevel)) {
    98.                 if (excludeThrowableClasses != null && eventObject.getThrowableProxy() != null) {
    99.                     if (excludeThrowableClasses.contains(eventObject.getThrowableProxy().getClassName())) {
    100.                         return;
    101.                     }
    102.                 }
    103.                 String toMsg = layout.doLayout(eventObject);
    104.                 sendMarkDownMsg("Canal-Server监听异常", toMsg);
    105.             }
    106.         } catch (Exception e) {
    107.             e.printStackTrace();
    108.         }
    109.     }
    110.  
    111.     @Override
    112.     public void start() {
    113.         //sendStartMsg();
    114.         System.out.println("Canal-Server开始监控,应用名:" + getServerName());
    115.         PatternLayout patternLayout = new PatternLayout();
    116.         patternLayout.setContext(context);
    117.         patternLayout.setPattern(getPattern());
    118.         patternLayout.start();
    119.         this.layout = patternLayout;
    120.         super.start();
    121.  
    122.  
    123.     }
    124.  
    125.  
    126.     /**
    127.      * @Author
    128.      * @Description 开始监听
    129.      * @Date 2021-09-29 18:05
    130.      * @Param
    131.      * @return
    132.      **/
    133.     public void sendStartMsg() {
    134.  
    135.         String title = "Canal-Server开始监控";
    136.         String prefixEnd = "</font> \n";
    137.         String prefixStart = "####";
    138.         String msg = "### " + title + prefixEnd +
    139.                 prefixStart + " 应用名：<font color=#1A73E8 size=2>" + getServerName() + prefixEnd;
    140.  
    141.         DingtalkMessage message = new DingtalkMessage();
    142.         DingtalkMessage.Markdown markdown = new DingtalkMessage.Markdown();
    143.         markdown.setTitle(title);
    144.         markdown.setText(msg);
    145.         message.setMarkdown(markdown);
    146.         Map<String, String> header = new HashMap<>();
    147.         header.put("Content-Type", "application/json");
    148.         String sendMsgRet = HttpsUtils.post(dingdingUrl, JSON.toJSONString(message), header);
    149.         log.info("钉钉机器人返回:{}" + sendMsgRet);
    150.  
    151.     }
    152.  
    153.     /**
    154.      * @Author
    155.      * @Description 钉钉发送信息
    156.      * @Date 2021-09-29 18:04
    157.      * @Param
    158.      * @return
    159.      **/
    160.     public void sendMarkDownMsg(String title, String msg) {
    161.  
    162.         DingtalkMessage message = new DingtalkMessage();
    163.         DingtalkMessage.Markdown markdown = new DingtalkMessage.Markdown();
    164.         markdown.setTitle(title);
    165.         markdown.setText(getMarkdown(title, msg));
    166.         message.setMarkdown(markdown);
    167.         Map<String, String> header = new HashMap<>();
    168.         header.put("Content-Type", "application/json");
    169.         String sendMsgRet = HttpsUtils.post(dingdingUrl, JSON.toJSONString(message), header);
    170.         log.info("钉钉机器人返回:{}" + sendMsgRet);
    171.  
    172.     }
    173.  
    174.     /**
    175.      * @Author
    176.      * @Description 格式化钉钉文本内容
    177.      * @Date 2021-09-29 18:04
    178.      * @Param
    179.      * @return
    180.      **/
    181.     private String getMarkdown(String title, String msg) {
    182.  
    183.         if (StringUtils.isEmpty(msg)) {
    184.             return null;
    185.         }
    186.         String resultStr = "";
    187.  
    188.         //日志格式logback配置<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS}|-|${appName}|-|%thread|-|%level|-|%logger|-|%msg%n</pattern>
    189.         String[] msgs = msg.split("\\|-\\|");
    190.         if (msgs.length > 0) {
    191.  
    192.             String prefixEnd = "</font> \n";
    193.             String prefixStart = "####";
    194.             resultStr = "### " + title + prefixEnd +
    195.                     prefixStart + " 应用名：<font color=#1A73E8 size=2>" + getServerName() + prefixEnd +
    196.                     prefixStart + " 类路径：<font color=#666666 size=2>" + msgs[4] + prefixEnd +
    197.                     prefixStart + " 异常时间：<font color=#666666 size=2>" + msgs[0] + prefixEnd +
    198.                     prefixStart + " 异常信息：<font color=#FF0000 size=2>" + msgs[5] + prefixEnd;
    199.         }
    200.  
    201.  
    202.         return resultStr;
    203.     }
    204. }

    2.配置logback.xml

    1.     <appender name="LogDingDing" class="com.logtest.appender.LogDingDingAppender">
    2.         <serverName>log-test</serverName>
    3.         <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS}|-|${appName}|-|%thread|-|%level|-|%logger|-|%msg%n</pattern>
    4.         <!--钉钉机器人Webhook url-->
    5.         <dingdingUrl>*****</dingdingUrl>
    6.         <!--机器人签名密钥 新版需要签名推送-->
    7.         <secret></secret>
    8.         <!--排除的异常类,用逗号分隔-->
    9.         <excludeThrowableClasses></excludeThrowableClasses>
    10.         <!--机器人的推送级别-->
    11.         <level>ERROR</level>
    12.     </appender>
    13.  
    14.     <root level="INFO">
    15.         <!--<root level="DEBUG">-->
    16.         <appender-ref ref="STDOUT"/>
    17.         <appender-ref ref="infoAppender"/>
    18.         <appender-ref ref="debugAppender"/>
    19.         <appender-ref ref="warnAppender"/>
    20.         <appender-ref ref="errorAppender"/>
    21.         <!--机器人的推送-->
    22.         <appender-ref ref="LogDingDing"/>
    23.  
    24.     </root>

    3.项目所需POM文件

    1.     <dependencies>
    2.  
    3.         <dependency>
    4.             <groupId>org.springframework.boot</groupId>
    5.             <artifactId>spring-boot-starter-logging</artifactId>
    6.         </dependency>
    7.  
    8.         <dependency>
    9.             <groupId>org.apache.httpcomponents</groupId>
    10.             <artifactId>httpclient</artifactId>
    11.             <version>4.5.5</version>
    12.         </dependency>
    13.  
    14.         <dependency>
    15.             <groupId>com.alibaba</groupId>
    16.             <artifactId>fastjson</artifactId>
    17.             <version>1.2.58</version>
    18.         </dependency>
    19.  
    20.         <dependency>
    21.             <groupId>org.springframework.boot</groupId>
    22.             <artifactId>spring-boot-starter-web</artifactId>
    23.         </dependency>
    24.  
    25.         <dependency>
    26.             <groupId>org.springframework.boot</groupId>
    27.             <artifactId>spring-boot-starter-test</artifactId>
    28.             <scope>test</scope>
    29.         </dependency>
    30.         <dependency>
    31.             <groupId>org.springframework.boot</groupId>
    32.             <artifactId>spring-boot-autoconfigure</artifactId>
    33.             <version>2.0.1.RELEASE</version>
    34.         </dependency>
    35.         <dependency>
    36.             <groupId>org.projectlombok</groupId>
    37.             <artifactId>lombok</artifactId>
    38.             <version>1.18.16</version>
    39.         </dependency>
    40.     </dependencies>

    4.新建SpringBoot项目&测试

    1. package com.logtest;
    2.  
    3. import org.springframework.boot.SpringApplication;
    4. import org.springframework.boot.autoconfigure.SpringBootApplication;
    5. import lombok.extern.slf4j.Slf4j;
    6.  
    7. @SpringBootApplication
    8. @Slf4j
    9. public class LogTestApplication {
    10.  
    11.     public static void main(String[] args) {
    12.         SpringApplication.run(LogTestApplication.class, args);
    13.  
    14.         log.error("测试钉钉日志告警");
    15.  
    16.     }
    17.  
    18. }

    测试结果如下:

    5.项目源码

    链接：百度网盘 请输入提取码

    提取码：b293

     

  • 相关阅读:
    Mybatis的生命周期和作用域
    MySQL 过滤重复数据
    Scheduling in Kubernetes
    学习Vue3 第一章
    机器学习-最小二乘法
    win10 IIS部署.NET Core api
    【校招VIP】产品深入分析之电商运营
    iOS获取当前网络连接状态WiFi、5G、4G、3G、2G
    “量化交易”、“算法交易”、“电子交易”等新型交易模式是如何发展起来的呢？
    打破焦虑！AI 时代的程序员为什么需要云端 IDE？
  • 原文地址：https://blog.csdn.net/qq_19734597/article/details/125563071