pom:
- <?xml version="1.0" encoding="UTF-8"?>
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>com.zy</groupId>
- <artifactId>data-sync</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <name>data-sync</name>
- <properties>
- <java.version>1.8</java.version>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <spring-boot.version>2.3.7.RELEASE</spring-boot.version>
- <commons-pool2.version>2.6.2</commons-pool2.version>
- <fastjson.version>1.2.79</fastjson.version>
- <hutool.version>5.8.16</hutool.version>
- <mybatis-plus.version>3.2.0</mybatis-plus.version>
- <druid.version>1.2.6</druid.version>
- <mysql.version>8.0.30</mysql.version>
- <orika-core.version>1.5.1</orika-core.version>
- <aspectjrt.version>1.9.7</aspectjrt.version>
- <commons-lang3.version>3.12.0</commons-lang3.version>
- <debezium.version>1.9.5.Final</debezium.version>
- </properties>
-
- <dependencies>
-
- <!-- spring-boot-starter-web -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-web</artifactId>
- </dependency>
-
- <!-- 阿里JSON解析器 -->
- <dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>fastjson</artifactId>
- <version>${fastjson.version}</version>
- </dependency>
-
- <!-- hutool -->
- <dependency>
- <groupId>cn.hutool</groupId>
- <artifactId>hutool-all</artifactId>
- <version>${hutool.version}</version>
- </dependency>
-
- <!-- mybatis-plus -->
- <dependency>
- <groupId>com.baomidou</groupId>
- <artifactId>mybatis-plus-boot-starter</artifactId>
- <version>${mybatis-plus.version}</version>
- </dependency>
-
- <!-- 阿里数据库连接池 -->
- <dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>druid-spring-boot-starter</artifactId>
- <version>${druid.version}</version>
- </dependency>
-
- <!-- Mysql驱动包 -->
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.version}</version>
- </dependency>
-
- <!-- 对象转换工具类 -->
- <dependency>
- <groupId>ma.glasnost.orika</groupId>
- <artifactId>orika-core</artifactId>
- <version>${orika-core.version}</version>
- </dependency>
-
- <!-- aop切面 -->
- <dependency>
- <groupId>org.aspectj</groupId>
- <artifactId>aspectjrt</artifactId>
- <version>${aspectjrt.version}</version>
- </dependency>
-
- <!-- commons-lang3-->
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
- </dependency>
-
- <!-- aop-->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-aop</artifactId>
- </dependency>
-
- <!-- dynamic-datasource-spring-boot-starter -->
- <dependency>
- <groupId>com.baomidou</groupId>
- <artifactId>dynamic-datasource-spring-boot-starter</artifactId>
- <version>3.5.0</version>
- </dependency>
-
- <!--瀚高驱动-->
- <dependency>
- <groupId>com.highgo</groupId>
- <artifactId>HgdbJdbc</artifactId>
- <version>6.2.2</version>
- </dependency>
-
- <!-- clickhouse -->
- <dependency>
- <groupId>com.clickhouse</groupId>
- <artifactId>clickhouse-jdbc</artifactId>
- <version>0.3.2</version>
- </dependency>
-
- <dependency>
- <groupId>io.debezium</groupId>
- <artifactId>debezium-api</artifactId>
- <version>${debezium.version}</version>
- </dependency>
- <dependency>
- <groupId>io.debezium</groupId>
- <artifactId>debezium-embedded</artifactId>
- <version>${debezium.version}</version>
- </dependency>
- <dependency>
- <groupId>io.debezium</groupId>
- <artifactId>debezium-connector-mysql</artifactId>
- <version>${debezium.version}</version>
- <exclusions>
- <exclusion>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.debezium</groupId>
- <artifactId>debezium-connector-postgres</artifactId>
- <version>${debezium.version}</version>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </dependency>
-
- </dependencies>
-
- <repositories>
- <repository>
- <id>nexus-aliyun</id>
- <name>nexus-aliyun</name>
- <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>${spring-boot.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.8.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <version>${spring-boot.version}</version>
- <configuration>
- <mainClass>com.zy.data.sync.DataSyncApplication</mainClass>
- <skip>true</skip>
- </configuration>
- <executions>
- <execution>
- <id>repackage</id>
- <goals>
- <goal>repackage</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- </project>
-
- package com.zy.data.sync.moudles.init.service;
-
- import com.zy.data.sync.utils.DebeziumDataHande;
- import com.zy.data.sync.utils.DebeziumSqlHande;
- import io.debezium.engine.ChangeEvent;
- import io.debezium.engine.DebeziumEngine;
- import io.debezium.engine.format.Json;
- import lombok.extern.slf4j.Slf4j;
- import org.codehaus.plexus.util.StringUtils;
- import org.springframework.stereotype.Service;
-
- import javax.annotation.PostConstruct;
- import java.util.HashMap;
- import java.util.Map;
- import java.util.Properties;
- import java.util.concurrent.ExecutorService;
- import java.util.concurrent.Executors;
-
- @Service
- @Slf4j
- public class CdcInitService
- {
-
- //参数参考: https://blog.51cto.com/maxiaobian/3014474
- //日期转换参考: https://blog.csdn.net/qq_30529079/article/details/127809317
- @PostConstruct
- public void init() {
- final Properties props = new Properties();
-
- props.setProperty("name", "instala-core");
- props.setProperty("connector.class", "io.debezium.connector.mysql.MySqlConnector");
- //偏移量持久化,用来容错
- props.setProperty("offset.storage","org.apache.kafka.connect.storage.FileOffsetBackingStore");
- //偏移量持久化文件路径,默认/tmp/offsets.dat,如果路径配置不正确可能导致无法存储偏移量 可能会导致重复消费变更
- props.setProperty("offset.storage.file.filename", "D:/tmp/offsets.dat");
- //如果连接器重新启动,它将使用最后记录的偏移量来知道它应该恢复读取源信息中的哪个位置。
- props.setProperty("offset.flush.interval.ms", "60000");
-
- //需要监听的数据库名称
- props.setProperty("database.whitelist", "report_sharing_center");
- //initial(默认) 连接器执行数据库的初始一致性快照,快照完成后,连接器开始为后续数据库更改流式传输事件记录。
- //initial_only 连接器只执行数据库的初始一致性快照,不允许捕获任何后续更改的事件。
- //schema_only 连接器只捕获所有相关表的表结构,不捕获初始数据,但是会同步后续数据库的更改记录
- //schema_only_recovery 设置此选项可恢复丢失或损坏的数据库历史主题(database.history.kafka.topic)。
- props.setProperty("snapshot.mode", "schema_only");
- //数据库地址
- props.setProperty("database.hostname", "xxx");
- //数据库端口
- props.setProperty("database.port","3306");
- //数据库用户名
- props.setProperty("database.user", "xxx");
- //数据库密码
- props.setProperty("database.password", "xxx");
- //server.id起到唯一标识作用,随意起
- props.setProperty("database.server.id", "xxx");
- //server.name起到唯一标识作用,随意起
- props.setProperty("database.server.name", "xxx");
-
- //历史变更记录
- props.setProperty("database.history", "io.debezium.relational.history.FileDatabaseHistory");
- //历史变更记录存储位置
- props.setProperty("database.history.file.filename", "D:/tmp/dbhistory.dat");
-
- //格式化日期
- props.setProperty("converters", "dateConverters");
- props.setProperty("dateConverters.type", "com.zy.data.sync.common.conver.MySqlDateTimeConverter");
- props.setProperty("dateConverters.format.date", "yyyy-MM-dd");
- props.setProperty("dateConverters.format.time", "HH:mm:ss");
- props.setProperty("dateConverters.format.datetime", "yyyy-MM-dd HH:mm:ss");
- props.setProperty("dateConverters.format.timestamp", "yyyy-MM-dd HH:mm:ss");
- props.setProperty("dateConverters.format.timestamp.zone", "UTC+8");
-
- //全局读写锁,会影响在线业务,所以跳过锁设置
- props.setProperty("debezium.snapshot.locking.mode","none");
- //是否包含数据库表结构层面的变更,建议使用默认值true
- props.setProperty("include.schema.changes", "true");
-
- //指定 BIGINT UNSIGNED 列应如何在更改事件中表示。可能的设置有
- //long使用 Java 的 表示值long,这可能无法提供精确度,但在消费者中易于使用。long通常是首选设置。
- props.setProperty("bigint.unsigned.handling.mode","long");
- //decimal类型转换为double
- props.setProperty("decimal.handling.mode","double");
-
- DebeziumEngine<ChangeEvent<String, String>> engine = DebeziumEngine.create(Json.class)
- .using(props)
- .notifying(record -> {
- String recordStr = record.value();
- if(StringUtils.isNotEmpty(recordStr)){
- System.out.println("recordStr-->"+recordStr);
- Map<String, Object> payload = DebeziumDataHande.getPayload(recordStr);
-
- //to do something
- if(payload.containsKey("op")&&payload.get("op").equals("c")){
-
- }
-
- }
-
- }).build();
-
- Executors.newSingleThreadExecutor().execute(engine);
-
- // Run the engine asynchronously ...
- ExecutorService executor = Executors.newSingleThreadExecutor();
- executor.execute(engine);
- }
- }
- package com.zy.data.sync.common.conver;
-
- import io.debezium.spi.converter.CustomConverter;
- import io.debezium.spi.converter.RelationalColumn;
- import org.apache.kafka.connect.data.SchemaBuilder;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
-
- import java.time.*;
- import java.time.format.DateTimeFormatter;
- import java.util.Properties;
- import java.util.function.Consumer;
-
- public class MySqlDateTimeConverter implements CustomConverter<SchemaBuilder, RelationalColumn> {
- private final static Logger logger = LoggerFactory.getLogger(MySqlDateTimeConverter.class);
-
- private DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_DATE;
- private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_TIME;
- private DateTimeFormatter datetimeFormatter = DateTimeFormatter.ISO_DATE_TIME;
- private DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_DATE_TIME;
-
- private ZoneId timestampZoneId = ZoneId.systemDefault();
-
- @Override
- public void configure(Properties props) {
- readProps(props, "format.date", p -> dateFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.time", p -> timeFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.datetime", p -> datetimeFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.timestamp", p -> timestampFormatter = DateTimeFormatter.ofPattern(p));
- readProps(props, "format.timestamp.zone", z -> timestampZoneId = ZoneId.of(z));
- }
-
- private void readProps(Properties properties, String settingKey, Consumer<String> callback) {
- String settingValue = (String) properties.get(settingKey);
- if (settingValue == null || settingValue.length() == 0) {
- return;
- }
- try {
- callback.accept(settingValue.trim());
- } catch (IllegalArgumentException | DateTimeException e) {
- logger.error("The {} setting is illegal: {}",settingKey,settingValue);
- throw e;
- }
- }
-
-
-
- @Override
- public void converterFor(RelationalColumn column, ConverterRegistration<SchemaBuilder> registration) {
- String sqlType = column.typeName().toUpperCase();
- SchemaBuilder schemaBuilder = null;
- Converter converter = null;
- if ("DATE".equals(sqlType)) {
- schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.date.string");
- converter = this::convertDate;
- }
- if ("TIME".equals(sqlType)) {
- schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.time.string");
- converter = this::convertTime;
- }
- if ("DATETIME".equals(sqlType)) {
- schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.datetime.string");
- converter = this::convertDateTime;
- }
- if ("TIMESTAMP".equals(sqlType)) {
- schemaBuilder = SchemaBuilder.string().optional().name("com.darcytech.debezium.timestamp.string");
- converter = this::convertTimestamp;
- }
- if (schemaBuilder != null) {
- registration.register(schemaBuilder, converter);
- }
- }
-
- private String convertDate(Object input) {
- if (input instanceof LocalDate) {
- return dateFormatter.format((LocalDate) input);
- }
- if (input instanceof Integer) {
- LocalDate date = LocalDate.ofEpochDay((Integer) input);
- return dateFormatter.format(date);
- }
- return null;
- }
-
- private String convertTime(Object input) {
- if (input instanceof Duration) {
- Duration duration = (Duration) input;
- long seconds = duration.getSeconds();
- int nano = duration.getNano();
- LocalTime time = LocalTime.ofSecondOfDay(seconds).withNano(nano);
- return timeFormatter.format(time);
- }
- return null;
- }
-
- private String convertDateTime(Object input) {
- if (input instanceof LocalDateTime) {
- return datetimeFormatter.format((LocalDateTime) input);
- }
- return null;
- }
-
- private String convertTimestamp(Object input) {
- if (input instanceof ZonedDateTime) {
- // mysql的timestamp会转成UTC存储,这里的zonedDatetime都是UTC时间
- ZonedDateTime zonedDateTime = (ZonedDateTime) input;
- LocalDateTime localDateTime = zonedDateTime.withZoneSameInstant(timestampZoneId).toLocalDateTime();
- return timestampFormatter.format(localDateTime);
- }
- return null;
- }
- }
DebeziumDataHande:
- package com.zy.data.sync.utils;
-
- import com.alibaba.fastjson.JSON;
-
- import java.util.Map;
-
- public class DebeziumDataHande {
-
- public static Map<String, Object> getPayload(String value) {
- Map<String, Object> map = JSON.parseObject(value, Map.class);
- if(map.containsKey("payload")){
- return JSON.parseObject(JSON.toJSONString(map.get("payload")), Map.class);
- }
- return null;
- }
-
- public static Map<String, Object> getBefore(Map<String, Object> payload) {
- Map<String, Object> map = JSON.parseObject(payload.get("before").toString(), Map.class);
- return map;
- }
-
- public static Map<String, Object> getAfter(Map<String, Object> payload) {
- Map<String, Object> map = JSON.parseObject(payload.get("after").toString(), Map.class);
- return map;
- }
-
- public static Map<String, Object> getSource(Map<String, Object> payload) {
- Map<String, Object> map = JSON.parseObject(payload.get("source").toString(), Map.class);
- return map;
- }
-
- public static String getTableName(Map<String, Object> payload) {
- Map<String, Object> sourceMap = getSource(payload);
- String tableName = sourceMap.get("table").toString();
- return tableName;
- }
-
- }
DebeziumSqlHande:
- package com.zy.data.sync.utils;
-
- import java.util.Map;
-
- public class DebeziumSqlHande {
-
- public static String getInsertSql(Map<String, Object> insertParam,String tableName) {
- String insertSql = SQLUtils.genSqlInsert(insertParam, tableName);
- return insertSql;
- }
-
- public static String getDeleteSql(Map<String, Object> deleteParam, String tableName) {
- String deleteSql = SQLUtils.genSqlDelete(deleteParam, tableName);
- return deleteSql;
- }
-
- }
SQLUtils:
- package com.zy.data.sync.utils;
-
- import java.util.Map;
-
-
- public class SQLUtils {
-
- public static String genSqlInsert(Map<String, Object> dataMap,String tableName) {
- if(dataMap.size()==0) {
- return null;
- }
-
- //生成INSERT INTO table(field1,field2) 部分
- StringBuffer sbField = new StringBuffer();
- //生成VALUES('value1','value2') 部分
- StringBuffer sbValue = new StringBuffer();
-
- sbField.append("INSERT INTO " + tableName.toLowerCase() + "(");
-
- for(Map.Entry<String, Object> entry : dataMap.entrySet()){
- String mapKey = entry.getKey();
- Object mapValue = entry.getValue();
- if(mapValue instanceof String){
- sbValue.append("'" + mapValue + "',");
- }else{
- sbValue.append(mapValue + ",");
- }
-
- sbField.append("`" + mapKey + "`,");
- }
-
- String sbFieldStr = sbField.toString();
- String sbValueStr = sbValue.toString();
- if(sbFieldStr.endsWith(",")){
- sbFieldStr = sbFieldStr.substring(0,sbFieldStr.length()-1);
- }
-
- if(sbValueStr.endsWith(",")){
- sbValueStr = sbValueStr.substring(0,sbValueStr.length()-1);
- }
-
- return sbFieldStr + ") VALUES(" + sbValueStr + ")";
- }
-
- public static String genSqlDelete(Map<String, Object> dataMap,String tableName) {
- if(dataMap.size()==0) {
- return null;
- }
-
- //生成DELETE FROM xxx where 部分
- StringBuffer sbField = new StringBuffer();
- //生成id = zhangsan and sex = 1
- StringBuffer sbValue = new StringBuffer();
-
- sbField.append("DELETE FROM " + tableName.toLowerCase() + " where ");
-
- for(Map.Entry<String, Object> entry : dataMap.entrySet()){
- String mapKey = entry.getKey();
- Object mapValue = entry.getValue();
- if(mapValue instanceof String){
- sbValue.append("and "+mapKey+"='" + mapValue + "' ");
- }else{
- sbValue.append("and "+mapKey+"=" + mapValue + " ");
- }
- }
-
- String sbFieldStr = sbField.toString();
- String sbValueStr = sbValue.toString();
-
- if(sbValueStr.startsWith("and")){
- sbValueStr = sbValueStr.substring(3);
- }
-
- if(sbValueStr.endsWith(",")){
- sbValueStr = sbValueStr.substring(0,sbValueStr.length()-1);
- }
-
- return sbFieldStr + sbValueStr ;
- }
-
- }