-
redis高级案列case
案列一 双写一致性
案例二 双锁策略
package com.redis.redis01.service; import com.redis.redis01.bean.RedisBs; import com.redis.redis01.mapper.RedisBsMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; import java.beans.Transient; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @Slf4j @Service public class RedisBsService { //定义key前缀/命名空间 public static final String CACHE_KEY_USER = "user:"; @Autowired private RedisBsMapper mapper; @Resource private RedisTemplate<String, Object> redisTemplate; private static ReentrantLock lock = new ReentrantLock(); /** * 业务逻辑没有写错，对于中小长(qps<=1000)可以使用，但是大厂不行：大长需要采用双检加锁策略 * * @param id * @return */ @Transactional public RedisBs findUserById(Integer id,int type,int qps) { //qps<=1000 if(qps<=1000){ return qpsSmall1000(id); } //qps>1000 return qpsBig1000(id, type); } /** * 加强补充，避免突然key失效了，或者不存在的key穿透redis打爆mysql，做一下预防，尽量不出现缓存击穿的情况，进行排队等候 * @param id * @param type 0使用synchronized重锁，1ReentrantLock轻量锁 * @return */ private RedisBs qpsBig1000(Integer id, int type) { RedisBs redisBs = null; String key = CACHE_KEY_USER + id; //1先从redis里面查询，如果有直接返回，没有再去查mysql redisBs = (RedisBs) redisTemplate.opsForValue().get(key); if (null == redisBs) { switch (type) { case 0: //加锁，假设请求量很大，缓存过期，大厂用，对于高qps的优化，进行加锁保证一个请求操作，让外面的redis等待一下，避免击穿mysql synchronized (RedisBsService.class) { //第二次查询缓存目的防止加锁之前刚好被其他线程缓存了 redisBs = (RedisBs) redisTemplate.opsForValue().get(key); if (null != redisBs) { //查询到数据直接返回 return redisBs; } else { //数据缓存 //查询mysql，回写到redis中 redisBs = mapper.findUserById(id); if (null == redisBs) { // 3 redis+mysql都没有数据，防止多次穿透(redis为防弹衣，mysql为人，穿透直接伤人，就是直接访问mysql)，优化：记录这个null值的key，列入黑名单或者记录或者异常 return new RedisBs(-1, "当前值已经列入黑名单"); } //4 mysql有，回写保证数据一致性 //setifabsent redisTemplate.opsForValue().setIfAbsent(key, redisBs,7l, TimeUnit.DAYS); } } break; case 1: //加锁，大厂用，对于高qps的优化，进行加锁保证一个请求操作，让外面的redis等待一下，避免击穿mysql lock.lock(); try { //第二次查询缓存目的防止加锁之前刚好被其他线程缓存了 redisBs = (RedisBs) redisTemplate.opsForValue().get(key); if (null != redisBs) { //查询到数据直接返回 return redisBs; } else { //数据缓存 //查询mysql，回写到redis中 redisBs = mapper.findUserById(id); if (null == redisBs) { // 3 redis+mysql都没有数据，防止多次穿透(redis为防弹衣，mysql为人，穿透直接伤人，就是直接访问mysql)，优化：记录这个null值的key，列入黑名单或者记录或者异常 return new RedisBs(-1, "当前值已经列入黑名单"); } //4 mysql有，回写保证数据一致性 redisTemplate.opsForValue().set(key, redisBs); } } catch (Exception e) { e.printStackTrace(); } finally { //解锁 lock.unlock(); } } } return redisBs; } private RedisBs qpsSmall1000(Integer id) { RedisBs redisBs = null; String key = CACHE_KEY_USER + id; //1先从redis里面查询，如果有直接返回，没有再去查mysql redisBs = (RedisBs) redisTemplate.opsForValue().get(key); if (null == redisBs) { //2查询mysql，回写到redis中 redisBs = mapper.findUserById(id); if (null == redisBs) { // 3 redis+mysql都没有数据，防止多次穿透(redis为防弹衣，mysql为人，穿透直接伤人，就是直接访问mysql)，优化：记录这个null值的key，列入黑名单或者记录或者异常 return new RedisBs(-1, "当前值已经列入黑名单"); } //4 mysql有，回写保证数据一致性 redisTemplate.opsForValue().set(key, redisBs); } return redisBs; } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
案列三 mysql+redis实时同步
下载canal监控端admin和服务端deployer
登录mysql授权canal连接mysql账户
DROP USER IF EXISTS 'canal'@'%'; CREATE USER 'canal'@'%' IDENTIFIED BY 'canal'; GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' IDENTIFIED BY 'canal'; FLUSH PRIVILEGES;- 1
- 2
- 3
- 4
配置canal
修改mysql ip
启动./startup.bat
Canal客户端（Java编写）
非springboot项目
<dependency> <groupId>com.alibaba.otter</groupId> <artifactId>canal.client</artifactId> <version>1.1.0</version> </dependency>- 1
- 2
- 3
- 4
- 5
server.port=8002 #连接数据源 spring.datasource.druid.username=root spring.datasource.druid.password=xgm@2023.. spring.datasource.druid.url=jdbc:mysql://172.16.204.51:3306/redis?serverTimezone=GMT%2B8 spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver spring.datasource.druid.initial-size=5 ##指定缓存类型redis #spring.cache.type=redis ##一个小时，以毫秒为单位 #spring.cache.redis.time-to-live=3600000 ##给缓存的建都起一个前缀。 如果指定了前缀就用我们指定的，如果没有就默认使用缓存的名字作为前缀,一般不指定 #spring.cache.redis.key-prefix=CACHE_ ##指定是否使用前缀 #spring.cache.redis.use-key-prefix=true ##是否缓存空值，防止缓存穿透 #spring.cache.redis.cache-null-values=true #redis spring.redis.host=172.16.204.51 spring.redis.port=6379 spring.redis.password=123456 spring.redis.database=1 # mybatis配置 mybatis: check-config-location: true # mybatis框架配置文件，对mybatis的生命周期起作用 config-location: "classpath:mybatis/mybatis-config.xml" # 配置xml路径 mapper-locations: "classpath:mybatis/mapper/*Mapper.xml" # 配置model包路径 type-aliases-package: "com.redis.redis01.bean.*" #日志 logging.level.root=info #logging.level.io.lettuce.core=debug #logging.level.org.springframework.data.redis=debug #canal安装地址 canal.server=172.16.204.51:11111 canal.destination=example #控制台刷新时间，每隔5秒检查一下数据库数据是否更新 根据需求设置其他时间 canal.timeout=5 spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver spring.datasource.username=canal spring.datasource.password=canal spring.datasource.url=jdbc:mysql://172.16.204.51:3306/redis?autoReconnect=true&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=false&serverTimezone=CTT&allowMultiQueries=true- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
package com.redis.redis01; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.client.CanalConnectors; import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.client.CanalConnectors; import com.alibaba.otter.canal.client.CanalConnector; import com.alibaba.otter.canal.common.utils.AddressUtils; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.CanalEntry.Column; import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; import com.alibaba.otter.canal.protocol.CanalEntry.EventType; import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; import com.alibaba.otter.canal.protocol.CanalEntry.RowData; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; import org.json.JSONException; import org.json.JSONObject; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; import java.net.InetSocketAddress; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; @Slf4j public class CanalTest { public static final Integer _60SECONDS = 60; public static final String REDIS_IP_ADDR = "172.16.204.51"; private void redisInsert(List<Column> columns) throws JSONException { JSONObject jsonObject = new JSONObject(); for (Column column : columns) { System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated()); jsonObject.put(column.getName(), column.getValue()); } if (columns.size() > 0) { try (Jedis jedis = new RedisUtils().getJedis()){ jedis.set(columns.get(0).getValue(), jsonObject.toString()); } catch (Exception e) { e.printStackTrace(); } } } public class RedisUtils { public final String REDIS_IP_ADDR = "172.16.204.51"; public final String REDIS_pwd = "123456"; public JedisPool jedisPool; public Jedis getJedis() throws Exception { JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); jedisPoolConfig.setMaxTotal(20); jedisPoolConfig.setMaxIdle(10); jedisPool=new JedisPool(jedisPoolConfig, REDIS_IP_ADDR,6379,10000,REDIS_pwd); if (null != jedisPool) { return jedisPool.getResource(); } throw new Exception("Jedispool is not ok"); } } private void redisDelete(List<Column> columns) throws JSONException { JSONObject jsonObject = new JSONObject(); for (Column column : columns) { jsonObject.put(column.getName(), column.getValue()); } if (columns.size() > 0) { try (Jedis jedis = new RedisUtils().getJedis()) { jedis.del(columns.get(0).getValue()); } catch (Exception e) { e.printStackTrace(); } } } private void redisUpdate(List<Column> columns) throws JSONException { JSONObject jsonObject = new JSONObject(); for (Column column : columns) { System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated()); jsonObject.put(column.getName(), column.getValue()); } if (columns.size() > 0) { try (Jedis jedis =new RedisUtils().getJedis()){ jedis.set(columns.get(0).getValue(), jsonObject.toString()); System.out.println("---------update after: " + jedis.get(columns.get(0).getValue())); } catch (Exception e) { e.printStackTrace(); } } } public void printEntry(List<Entry> entrys) throws JSONException { for (Entry entry : entrys) { if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) { continue; } RowChange rowChage = null; try { //获取变更的row数据 rowChage = RowChange.parseFrom(entry.getStoreValue()); } catch (Exception e) { throw new RuntimeException("ERROR ## parser of eromanga-event has an error,data:" + entry.toString(), e); } //获取变动类型 EventType eventType = rowChage.getEventType(); System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType)); for (RowData rowData : rowChage.getRowDatasList()) { if (eventType == EventType.INSERT) { redisInsert(rowData.getAfterColumnsList()); } else if (eventType == EventType.DELETE) { redisDelete(rowData.getBeforeColumnsList()); } else {//EventType.UPDATE redisUpdate(rowData.getAfterColumnsList()); } } } } public static void main(String[] args) { System.out.println("---------O(∩_∩)O哈哈~ initCanal() main方法-----------"); //================================= // 创建链接canal服务端 CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(REDIS_IP_ADDR, 11111), "example", "", ""); // 这里用户名和密码如果在这写了，会覆盖canal配置文件的账号密码，如果不填从配置文件中读 int batchSize = 1000; //空闲空转计数器 int emptyCount = 0; System.out.println("---------------------canal init OK，开始监听mysql变化------"); try { connector.connect(); //connector.subscribe(".*\\..*"); connector.subscribe("redis.redis_syc"); // 设置监听哪个表 connector.rollback(); int totalEmptyCount = 10 * _60SECONDS; while (emptyCount < totalEmptyCount) { System.out.println("我是canal，每秒一次正在监听:" + UUID.randomUUID().toString()); Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 long batchId = message.getId(); int size = message.getEntries().size(); if (batchId == -1 || size == 0) { emptyCount++; try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { e.printStackTrace(); } } else { //计数器重新置零 emptyCount = 0; new CanalTest().printEntry(message.getEntries()); } connector.ack(batchId); // 提交确认 // connector.rollback(batchId); // 处理失败, 回滚数据 } System.out.println("已经监听了" + totalEmptyCount + "秒，无任何消息，请重启重试......"); } catch (JSONException e) { throw new RuntimeException(e); } finally { connector.disconnect(); } } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
截图
spingboot项目
<dependency> <groupId>top.javatool</groupId> <artifactId>canal-spring-boot-starter</artifactId> <version>1.2.1-RELEASE</version> </dependency- 1
- 2
- 3
- 4
- 5
- 6
# canal starter配置信息 canal.server=127.0.0.1:11111 canal.destination=example logging.level.root=info logging.level.top.javatool.canal.client.client.AbstractCanalClient=error- 1
- 2
- 3
- 4
- 5
- 6
- 7
package com.redis.redis01.canal; import com.redis.redis01.bean.RedisSyc; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import top.javatool.canal.client.annotation.CanalTable; import top.javatool.canal.client.handler.EntryHandler; @Component @CanalTable(value = "redis_syc") @Slf4j public class RedisCanalClientExample implements EntryHandler<RedisSyc> { @Override public void insert(RedisSyc redisSyc) { EntryHandler.super.insert(redisSyc); log.info("新增 ---> {}",redisSyc); } @Override public void update(RedisSyc before, RedisSyc after) { EntryHandler.super.update(before, after); log.info("更新前 --->{} , 更新后 --->{} ", before, after); } @Override public void delete(RedisSyc redisSyc) { EntryHandler.super.delete(redisSyc); log.info("删除 --->{} " , redisSyc); } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
私服监听
注意：canal依赖stater在中央仓库是不存在的，需要手动放进本地仓库或者你公司里面的nexus
<!--canal依赖--> <dependency> <groupId>com.xpand</groupId> <artifactId>starter-canal</artifactId> <version>0.0.1-SNAPSHOT</version> </dependency>- 1
- 2
- 3
- 4
- 5
- 6
@SpringBootApplication @EnableCanalClient public class CanalApplication { public static void main(String[] args) { SpringApplication.run(CanalApplication.class,args); } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
@CanalEventListener public class CanalDataEventListener { /*** * 增加数据监听 * @param eventType * @param rowData */ @InsertListenPoint public void onEventInsert(CanalEntry.EventType eventType, CanalEntry.RowData rowData) { rowData.getAfterColumnsList().forEach((c) -> System.out.println("By--Annotation: " + c.getName() + " :: " + c.getValue())); } /*** * 修改数据监听 * @param rowData */ @UpdateListenPoint public void onEventUpdate(CanalEntry.RowData rowData) { System.out.println("UpdateListenPoint"); rowData.getAfterColumnsList().forEach((c) -> System.out.println("By--Annotation: " + c.getName() + " :: " + c.getValue())); } /*** * 删除数据监听 * @param eventType */ @DeleteListenPoint public void onEventDelete(CanalEntry.EventType eventType) { System.out.println("DeleteListenPoint"); } /*** * 自定义数据修改监听 * @param eventType * @param rowData */ @ListenPoint(destination = "example", schema = "torlesse_test", table = {"tb_user", "tb_order"}, eventType = CanalEntry.EventType.UPDATE) public void onEventCustomUpdate(CanalEntry.EventType eventType, CanalEntry.RowData rowData) { System.err.println("DeleteListenPoint"); rowData.getAfterColumnsList().forEach((c) -> System.out.println("By--Annotation: " + c.getName() + " :: " + c.getValue())); } @ListenPoint(destination = "example", schema = "test_canal", //所要监听的数据库名 table = {"tb_user"}, //所要监听的数据库表名 eventType = {CanalEntry.EventType.UPDATE, CanalEntry.EventType.INSERT, CanalEntry.EventType.DELETE}) public void onEventCustomUpdateForTbUser(CanalEntry.EventType eventType, CanalEntry.RowData rowData){ getChangeValue(eventType,rowData); } public static void getChangeValue(CanalEntry.EventType eventType, CanalEntry.RowData rowData){ if(eventType == CanalEntry.EventType.DELETE){ rowData.getBeforeColumnsList().forEach(column -> { //获取删除前的数据 System.out.println(column.getName() + " == " + column.getValue()); }); }else { rowData.getBeforeColumnsList().forEach(column -> { //打印改变前的字段名和值 System.out.println(column.getName() + " == " + column.getValue()); }); rowData.getAfterColumnsList().forEach(column -> { //打印改变后的字段名和值 System.out.println(column.getName() + " == " + column.getValue()); }); } } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
案列四 统计千亿级别PV
UV: Unique Visitor ,独立访客数，是指在一个统计周期内，访问网站的人数之和。一般理解客户ip，需要去重
PV : Page View，浏览量，是指在一个统计周期内，浏览页面的数之和。不需要去重
DAU: Daily Active User 日活跃用户数量；去重
DNU：Daily New User，日新增用户数
MAU：Monthly New User，月活跃用户；去重
需要使用redis hyperloglog基数统计数据结构来实现
基数统计：数据集中不重复的元素的个数模拟后台1万用户点击首页
package com.redis.redis01.service; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; import javax.annotation.Resource; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @Slf4j @Service public class HyperLogService { @Resource private RedisTemplate<String, Object> redisTemplate; /** * 模拟后台1万用户点击首页，每个用户来自不同的ip地址 */ public void hyperloglogUvTest() { StopWatch stopWatch=new StopWatch(); stopWatch.start(); CountDownLatch countDownLatch=new CountDownLatch(10000); //主子线程传递共享连接资源redisTemplate ExecutorService executorService = Executors.newFixedThreadPool(200); executorService.execute(new Runnable() { @Override public void run() { //模拟1万用户 for (int i = 0; i < 10000; i++) { countDownLatch.countDown(); Random random = new Random(); String ipAddress = random.nextInt(256) + "." + random.nextInt(256) + "." + random.nextInt(256) + "." + random.nextInt(256); redisTemplate.opsForHyperLogLog().add("uv_click", ipAddress); System.out.println("countDownLatch=" + countDownLatch.getCount()); } } }); try { countDownLatch.await(); stopWatch.stop(); Long uvClick1 = redisTemplate.opsForHyperLogLog().size("uv_click"); //用户访问首页次数uv=10059 System.out.println("用户访问首页次数uv=" + uvClick1); //共耗时=3:秒 System.out.println("共耗时=" + stopWatch.getTotalTimeMillis()/1000+":秒"); } catch (InterruptedException e) { throw new RuntimeException(e); } } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
案列五 布隆过滤器方案实现
利用bitmap实现，一个bitmap=2^32bit最大能存512M，一个用户一天签到用1个bit，一年365个bit就可以实现，1千万个用户一年只需要435MB还不到一个bitmap最大存储能力
优点- 高效地插入和查询，内存占用 bit 空间少
缺点
- 不能删除元素
- 因为删除元素会导致误判率增加，因为hash冲突同一个位置可能存的东西是多个共有的，你删除一个元素的同时可能也把其他的删除了
- 存在误判，不能精准过滤
- 有，可能有
- 无，绝对无
package com.redis.redis01.service; import com.google.common.collect.Lists; import com.redis.redis01.bean.RedisBs; import com.redis.redis01.mapper.RedisBsMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.Arrays; import java.util.List; import java.util.concurrent.locks.ReentrantLock; @Slf4j @Service public class BitmapService { @Resource private RedisTemplate<String, Object> redisTemplate; private static ReentrantLock lock = new ReentrantLock(); @Autowired private RedisBsMapper redisBsMapper; /** * 场景一：布隆过滤器解决缓存穿透问题(null/黑客攻击)；利用redis+bitmap实现 * 有可能有，没有一定没有 * 无-------------》mysql查询 * 有--------》redis查询----------》有-----------》返回 * 请求-----》布隆过滤器-----------》 * 无-------终止 * * @param type:0初始化，1常规查询 */ public void booleanFilterBitmap(int type, Integer id) { switch (type) { case 0://初始化数据 for (int i = 0; i < 10; i++) { RedisBs initBs = RedisBs.builder().id(i).name("赵三" + i).phone("1580080569" + i).build(); //1 插入数据库 redisBsMapper.insert(initBs); //2 插入redis redisTemplate.opsForValue().set("customer:info" + i, initBs); } //3 将用户id插入布隆过滤器中,作为白名单 for (int i = 0; i < 10; i++) { String booleanKey = "customer:booleanFilter:" + i; //3.1 计算hashvalue int abs = Math.abs(booleanKey.hashCode()); //3.2 通过abs和2的32次方取余，获得布隆过滤器/bitmap对应的下标坑位/index long index = (long) (abs % Math.pow(2, 32)); log.info("坑位:{}", index); //3.3 设置redis里面的bitmap对应类型的白名单 redisTemplate.opsForValue().setBit("whiteListCustomer", index, true); } break; case 1://常规查询 //1 获取当前传过来的id对应的哈希值 String inputBooleanKey = "customer:booleanFilter:" + id; int abs = Math.abs(inputBooleanKey.hashCode()); long index = (long) (abs % Math.pow(2, 32)); Boolean whiteListCustomer = redisTemplate.opsForValue().getBit("whiteListCustomer", index); //加入双检锁 //加锁，大厂用，对于高qps的优化，进行加锁保证一个请求操作，让外面的redis等待一下，避免击穿mysql lock.lock(); try { if (null == whiteListCustomer) { whiteListCustomer = redisTemplate.opsForValue().getBit("whiteListCustomer", index); if (null != whiteListCustomer && whiteListCustomer) {//布隆过滤器中存在，则可能存在 //2 查找redis Object queryCustomer = redisTemplate.opsForValue().get("customer:info" + id); if (null != queryCustomer) { log.info("返回客户信息:{}", queryCustomer); break; } else { //3 redis没有查找mysql RedisBs userById = redisBsMapper.findUserById(id); if (null != userById) { log.info("返回客户信息:{}", queryCustomer); redisTemplate.opsForValue().set("customer:info" + id, userById); break; } else { log.info("当前客户信息不存在:{}", id); break; } } } else {//redis没有，去mysql中查询 //3 redis没有查找mysql RedisBs userById = redisBsMapper.findUserById(id); if (null != userById) { log.info("返回客户信息:{}", userById); redisTemplate.opsForValue().set("customer:info" + id, userById); break; } else { log.info("当前客户信息不存在:{}", id); break; } } } } finally { lock.unlock(); } log.info("当前客户信息不存在:{}", id); break; default: break; } } }- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
-
相关阅读:
Perl 和 StrawberryPerl 与 ActivePerl 的区别详解
如何通过cpolar内网穿透工具实现远程访问本地postgreSQL
ubuntn20.4安装git
【MySQL】MySQL的安装，登录，配置和相关命令
【HarmonyOS】遇见的问题汇总
Window系统安装JDK8与Maven
算法-分数
netty系列之:channel,ServerChannel和netty中的实现
算法编程技巧
Java项目：SSM律师事务所律师管理系统
- 原文地址：https://blog.csdn.net/weixin_38501485/article/details/134445138