ClickHouse副本节点数据损坏恢复

参考链接：https://blog.csdn.net/qq_42082701/article/details/127771766

参考链接：https://kb.altinity.com/altinity-kb-setup-and-maintenance/suspiciously-many-broken-parts/

# 背景
	CK配置为1分片2副本

# 配置参数,这里我们将max_suspicious_broken_parts设置为0
<?xml version="1.0"?>
<yandex>
     <merge_tree>
         <max_suspicious_broken_parts>0</max_suspicious_broken_parts>
     </merge_tree>
</yandex>
	

# 创建表/clickhouse/tables/{shard}/test,{shard}在配置各个节点的配置文件中可能不同，这里不使用占位符
	create database test on cluster default;
	create table test.test on cluster default (A UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/shard0/test', '{replica}') order by A settings max_suspicious_broken_parts=0;
	insert into test.test select number from numbers(100000);
	

# 分离表并使数据损坏
	detach table test.test;

# 损坏数据，/var/lib/clickhouse/data/default/test/all_*** 使数据文件损坏：对 2 个或更多数据文件重复此操作。
I have no name!@clickhouse-0-0:/$ ls /var/lib/clickhouse/data/test/test/all_0_0_0
checksums.txt  columns.txt  count.txt  data.bin  data.mrk3  default_compression_codec.txt  primary.idx

I have no name!@clickhouse-0-0:/$ echo > /var/lib/clickhouse/data/test/test/all_0_0_0/data.mrk3
I have no name!@clickhouse-0-0:/$ echo > /var/lib/clickhouse/data/test/test/all_0_0_0/data.bin

# 附上表格
clickhouse-0-0.clickhouse-0.maas.svc.cluster.local :) attach table test.test;
ATTACH TABLE test.test
Query id: 6cc3e7d7-8439-4f04-bc6a-5231ffce3ecb
0 rows in set. Elapsed: 0.013 sec. 
Received exception from server (version 22.9.5):
Code: 231. DB::Exception: Received from localhost:9000. DB::Exception: Suspiciously many (1 parts, 308.00 B in total) broken parts to remove while maximum allowed broken parts count is 0. You can change the maximum value with merge tree setting 'max_suspicious_broken_parts' in <merge_tree> configuration section or in table settings in .sql file (don't forget to return setting back to default value). (TOO_MANY_UNEXPECTED_DATA_PARTS)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

• 方法一

# 设置force_restore_data标志,然后重启Clickhouse,
# 在 ClickHouse Keeper 中使用任意内容创建节点/path_to_table/replica_name/flags/force_restore_data，或运行命令来恢复所有复制表
# 启动时，服务器删除这些标志并开始恢复
I have no name!@clickhouse-0-0:/$ touch /var/lib/clickhouse/flags/force_restore_data

# 重启后，正常启动，不报错，查询数据是否从另外一个副本集中copy过来

# 异常数据被移到了detached目录
I have no name!@clickhouse-0-0:/$ ls /var/lib/clickhouse/data/test/test/detached/
broken-on-start_all_0_0_0
1
2
3
4
5
6
7
8
9
10

• 方法二

# 如果可以接受数据丢失，可以将参数max_suspicious_broken_parts设置得大一些
<?xml version="1.0"?>
<yandex>
     <merge_tree>
         <max_suspicious_broken_parts>1000</max_suspicious_broken_parts>
     </merge_tree>
</yandex>
1
2
3
4
5
6
7