Ceph RGW误删index对象恢复

版本

[root@control1 ~]# ceph -v
ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351) nautilus (stable)

问题

RGW将对象的索引信息存储在 .rgw.buckets.index pool中，在不分片的情况下，一个对象就对应一个bucket，分片的情况下，多个对象对应同一个bucket，在该对象的omap中存储这bucket的索引信息，如果将这个对象误删除，那么就无法看到该bucket中都有哪些对象，但是对象的真实数据是存储在 .rgw.buckets.data pool中的，只要data中的数据没有被删除，那么仍然可以正常get到该对象，此时只要object rewrite或者重新向该bucket中上传这些对象就可以自动修复index。

示例

1. 准备测试数据

[root@control1 ~]# s3cmd mb s3://nice
Bucket 's3://nice/' created
[root@control1 ~]#
[root@control1 ~]#
[root@control1 ~]# s3cmd  ls s3://nice
[root@control1 ~]#
[root@control1 ~]# s3cmd put pushgateway-1.4.2.linux-amd64.tar.gz  s3://nice
[root@control1 ~]# s3cmd put node_exporter-1.1.2.linux-amd64.tar.gz s3://nice
[root@control1 ~]#
[root@control1 ~]#
[root@control1 ~]# s3cmd ls s3://nice
2023-02-13 06:12      9246179  s3://nice/node_exporter-1.1.2.linux-amd64.tar.gz
2023-02-13 06:09      9189846  s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz

2. 查看底层数据结构

2.1 找到bucket对应的id

[root@control1 ~]# radosgw-admin bucket stats --bucket nice
{
    "bucket": "nice",
    "num_shards": 0,
    "tenant": "",
    "zonegroup": "cc0580a9-21d5-4377-b64b-fabf1f55fb4e",
    "placement_rule": "default-placement",
    "explicit_placement": {
        "data_pool": "",
        "data_extra_pool": "",
        "index_pool": ""
    },
    "id": "8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2",
    "marker": "8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2",
    "index_type": "Normal",
    "owner": "1034c29d7fec46e897aa531f832b6e5c",
    "ver": "0#3",
    "master_ver": "0#0",
    "mtime": "2023-02-13 06:07:37.785853Z",
    "max_marker": "0#",
    "usage": {
        "rgw.main": {
            "size": 18436025,
            "size_actual": 18440192,
            "size_utilized": 18436025,
            "size_kb": 18004,
            "size_kb_actual": 18008,
            "size_kb_utilized": 18004,
            "num_objects": 2
        }
    },
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    }
}

2.2 在index pool中查看该对象的信息

[root@control1 ~]# rados ls -p cn-1.rgw.buckets.index  | grep 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
.dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
 
[root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapkeys .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
node_exporter-1.1.2.linux-amd64.tar.gz
pushgateway-1.4.2.linux-amd64.tar.gz
[root@control1 ~]#
[root@control1 ~]#
[root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapvals .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
node_exporter-1.1.2.linux-amd64.tar.gz
value (293 bytes) :
00000000  08 03 1f 01 00 00 26 00  00 00 6e 6f 64 65 5f 65  |......&...node_e|
00000010  78 70 6f 72 74 65 72 2d  31 2e 31 2e 32 2e 6c 69  |xporter-1.1.2.li|
00000020  6e 75 78 2d 61 6d 64 36  34 2e 74 61 72 2e 67 7a  |nux-amd64.tar.gz|
00000030  03 00 00 00 00 00 00 00  01 07 03 91 00 00 00 01  |................|
00000040  e3 15 8d 00 00 00 00 00  ea d4 e9 63 7b eb 01 07  |...........c{...|
00000050  20 00 00 00 36 31 65 32  62 39 36 33 66 36 36 66  | ...61e2b963f66f|
00000060  31 65 30 30 36 34 39 63  38 65 34 64 31 66 34 37  |1e00649c8e4d1f47|
00000070  32 39 65 35 20 00 00 00  31 30 33 34 63 32 39 64  |29e5 ...1034c29d|
00000080  37 66 65 63 34 36 65 38  39 37 61 61 35 33 31 66  |7fec46e897aa531f|
00000090  38 33 32 62 36 65 35 63  05 00 00 00 61 64 6d 69  |832b6e5c....admi|
000000a0  6e 12 00 00 00 61 70 70  6c 69 63 61 74 69 6f 6e  |n....application|
000000b0  2f 78 2d 67 7a 69 70 e3  15 8d 00 00 00 00 00 00  |/x-gzip.........|
000000c0  00 00 00 08 00 00 00 53  54 41 4e 44 41 52 44 00  |.......STANDARD.|
000000d0  00 00 00 00 00 00 00 00  01 01 02 00 00 00 0c 03  |................|
000000e0  02 32 00 00 00 38 63 30  36 35 64 31 33 2d 66 35  |.2...8c065d13-f5|
000000f0  66 30 2d 34 39 32 33 2d  39 65 61 30 2d 37 35 31  |f0-4923-9ea0-751|
00000100  38 62 37 37 36 30 34 39  35 2e 34 35 31 39 38 33  |8b7760495.451983|
00000110  37 37 2e 33 35 36 37 00  00 00 00 00 00 00 00 00  |77.3567.........|
00000120  00 00 00 00 00                                    |.....|
00000125

pushgateway-1.4.2.linux-amd64.tar.gz
value (291 bytes) :
00000000  08 03 1d 01 00 00 24 00  00 00 70 75 73 68 67 61  |......$...pushga|
00000010  74 65 77 61 79 2d 31 2e  34 2e 32 2e 6c 69 6e 75  |teway-1.4.2.linu|
00000020  78 2d 61 6d 64 36 34 2e  74 61 72 2e 67 7a 0b 00  |x-amd64.tar.gz..|
00000030  00 00 00 00 00 00 01 07  03 91 00 00 00 01 d6 39  |...............9|
00000040  8c 00 00 00 00 00 0a d4  e9 63 7f a9 85 1b 20 00  |.........c.... .|
00000050  00 00 38 35 63 33 62 39  61 30 30 36 36 37 66 34  |..85c3b9a00667f4|
00000060  38 36 32 64 37 62 38 36  31 36 35 66 63 64 30 34  |862d7b86165fcd04|
00000070  39 33 20 00 00 00 31 30  33 34 63 32 39 64 37 66  |93 ...1034c29d7f|
00000080  65 63 34 36 65 38 39 37  61 61 35 33 31 66 38 33  |ec46e897aa531f83|
00000090  32 62 36 65 35 63 05 00  00 00 61 64 6d 69 6e 12  |2b6e5c....admin.|
000000a0  00 00 00 61 70 70 6c 69  63 61 74 69 6f 6e 2f 78  |...application/x|
000000b0  2d 67 7a 69 70 d6 39 8c  00 00 00 00 00 00 00 00  |-gzip.9.........|
000000c0  00 08 00 00 00 53 54 41  4e 44 41 52 44 00 00 00  |.....STANDARD...|
000000d0  00 00 00 00 00 00 01 01  02 00 00 00 0c 0b 01 32  |...............2|
000000e0  00 00 00 38 63 30 36 35  64 31 33 2d 66 35 66 30  |...8c065d13-f5f0|
000000f0  2d 34 39 32 33 2d 39 65  61 30 2d 37 35 31 38 62  |-4923-9ea0-7518b|
00000100  37 37 36 30 34 39 35 2e  34 35 31 39 38 33 37 37  |7760495.45198377|
00000110  2e 33 35 35 33 00 00 00  00 00 00 00 00 00 00 00  |.3553...........|
00000120  00 00 00                                          |...|
00000123

可以看到index对象通过omap的方式存储了该bucket跟该bucket中对象的索引关系，以及对象的一些元数据信息，比如修改时间，文件类型等，可以通过下面命令来查看：

[root@control1 ~]# radosgw-admin bucket list --bucket nice
[
    {
        "name": "node_exporter-1.1.2.linux-amd64.tar.gz",
        "instance": "",
        "ver": {
            "pool": 12,
            "epoch": 3
        },
        "locator": "",
        "exists": "true",
        "meta": {
            "category": 1,
            "size": 9246179,
            "mtime": "2023-02-13 06:12:58.117566Z",
            "etag": "61e2b963f66f1e00649c8e4d1f4729e5",
            "storage_class": "STANDARD",
            "owner": "1034c29d7fec46e897aa531f832b6e5c",
            "owner_display_name": "admin",
            "content_type": "application/x-gzip",
            "accounted_size": 9246179,
            "user_data": "",
            "appendable": "false"
        },
        "tag": "8c065d13-f5f0-4923-9ea0-7518b7760495.45198377.3567",
        "flags": 0,
        "pending_map": [],
        "versioned_epoch": 0
    },
    {
        "name": "pushgateway-1.4.2.linux-amd64.tar.gz",
        "instance": "",
        "ver": {
            "pool": 12,
            "epoch": 11
        },
        "locator": "",
        "exists": "true",
        "meta": {
            "category": 1,
            "size": 9189846,
            "mtime": "2023-02-13 06:09:14.461744Z",
            "etag": "85c3b9a00667f4862d7b86165fcd0493",
            "storage_class": "STANDARD",
            "owner": "1034c29d7fec46e897aa531f832b6e5c",
            "owner_display_name": "admin",
            "content_type": "application/x-gzip",
            "accounted_size": 9189846,
            "user_data": "",
            "appendable": "false"
        },
        "tag": "8c065d13-f5f0-4923-9ea0-7518b7760495.45198377.3553",
        "flags": 0,
        "pending_map": [],
        "versioned_epoch": 0
    }
]

2.3 查看data pool中该bucket的数据对象

[root@control1 ~]# rados -p cn-1.rgw.buckets.data ls | grep 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2_pushgateway-1.4.2.linux-amd64.tar.gz
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2_node_exporter-1.1.2.linux-amd64.tar.gz
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QjFsBiV-vhyjfGzwpTcBBjO7G6rm5Ld_2
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QjFsBiV-vhyjfGzwpTcBBjO7G6rm5Ld_1
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QU-DyeqL4u3CQLe3KkKlMsDVqde1wK3_2
8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QU-DyeqL4u3CQLe3KkKlMsDVqde1wK3_1

可以看到从data中对象的命名上，可以找到index和其中对象的关联关系，即8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2这个索引中有2个tar.gz对象，其他中间带 shadow 关键字的对象是上传的文件太大的情况下，做的数据分片。

3. 误删index对象

[root@control1 ~]# rados -p cn-1.rgw.buckets.index rm .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
[root@control1 ~]#
[root@control1 ~]# s3cmd ls s3://nice
ERROR: S3 error: 404 (NoSuchKey)
[root@control1 ~]#
[root@control1 ~]# radosgw-admin bucket list --bucket nice
ERROR: store->list_objects(): (2) No such file or directory
[root@control1 ~]#
[root@control1 ~]# s3cmd get s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz
download: 's3://nice/pushgateway-1.4.2.linux-amd64.tar.g

可以看到要列出该bucket中有哪些对象的操作，都是失败了的，但是如果知道文件名的话，仍然可以单独下载某个对象，即可以做get操作。

4. 恢复index对象

方法一：rewrite object

[root@control1 ~]# radosgw-admin object rewrite --bucket nice --object node_exporter-1.1.2.linux-amd64.tar.gz
[root@control1 ~]# s3cmd ls s3://nice
2023-02-13 06:12      9246179  s3://nice/node_exporter-1.1.2.linux-amd64.tar.gz
[root@control1 ~]#
[root@control1 ~]# rados -p cn-1.rgw.buckets.index ls | grep dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
.dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2

方法二：重新上传

[root@control1 ~]# s3cmd put pushgateway-1.4.2.linux-amd64.tar.gz s3://nice
upload: 'pushgateway-1.4.2.linux-amd64.tar.gz' -> 's3://nice/pushgateway-1.4.2.linux-amd64.tar.gz'  [1 of 1]
 
[root@control1 ~]# rados -p cn-1.rgw.buckets.index ls | grep dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
.dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
 
[root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapkeys .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
pushgateway-1.4.2.linux-amd64.tar.gz
 
[root@control1 ~]# s3cmd ls s3://nice
2023-02-13 06:42      9189846  s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz

方法一比较简单，不需要将该文件下载下来再上传，只需要指定object的名字，重新rewrite一遍就可以了。