/* This is the PCI capability header: */
struct virtio_pci_cap {
__u8 cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */
__u8 cap_next; /* Generic PCI field: next ptr. */
__u8 cap_len; /* Generic PCI field: capability length */
__u8 cfg_type; /* Identifies the structure. */
__u8 bar; /* Where to find it. */
__u8 id; /* Multiple capabilities of the same type */
__u8 padding[2]; /* Pad to full dword. */
__le32 offset; /* Offset within bar. */
__le32 length; /* Length of the structure, in bytes. */
};
从上可知,迁移的virtio设备,由于源端上BAR0地址的bit[4,7]内容为0xc,目的端BAR0地址的bit[4,7]内容为0,分析目的端日志: 2023-09-15T04:52:39.221154Z qemu-kvm: error while loading state for instance 0x0 of device ‘0000:00:0b.0/virtio-blk’ 迁移的设备是1个磁盘设备,其bdf号为00:0b.0,对比源端的virtio-pci设备:
virsh qemu-monitor-command {vm_uuid} --hmp info pci
...
Bus 0, device 9, function 0:
SCSI controller: PCI device 1af4:1001
IRQ 0.
BAR0: I/O at 0xd080 [0xd0bf].
BAR1: 32 bit memory at 0xfea59000 [0xfea59fff].
BAR4: 64 bit prefetchable memory at 0xfe208000 [0xfe20bfff].
id "virtio-disk0"
Bus 0, device 10, function 0:
Class 0255: PCI device 1af4:1002
IRQ 10.
BAR0: I/O at 0xd100 [0xd11f].
BAR4: 64 bit prefetchable memory at 0xfe20c000 [0xfe20ffff].
id "balloon0"
Bus 0, device 11, function 0:
SCSI controller: PCI device 1af4:1001
IRQ 0.
BAR0: I/O at 0xffc0 [0xffff].
BAR1: 32 bit memory at 0xfebff000 [0xfebfffff].
BAR4: 64 bit prefetchable memory at 0x4287fffc000 [0x4287fffffff].
id "virtio-disk1"
...
Bus 0, device 11, function 0:
SCSI controller: PCI device 1af4:1001
IRQ 0.
BAR0: I/O at 0xffc0 [0xffff].
BAR1: 32 bit memory at 0xfebff000 [0xfebfffff].
BAR4: 64 bit prefetchable memory at 0x4287fffc000 [0x4287fffffff].
id "virtio-disk1"
Bus 0, device 11, function 0:
SCSI controller: PCI device 1af4:1001
IRQ 0.
BAR0: I/O at 0xd080 [0xd0ff].
BAR1: 32 bit memory at 0xfea9a000 [0xfea9afff].
BAR4: 64 bit prefetchable memory at 0xfe210000 [0xfe213fff].
id "virtio-disk1"
37b06f8d46 virtio-blk: add DISCARD and WRITE_ZEROES features
20764be042 virtio-blk: set config size depending on the features enabled
ba550851f5 virtio-net: make VirtIOFeature usable for other virtio devices
5c81161f80 virtio-blk: add "discard" and "write-zeroes" properties
bbe8bd4d85 virtio-blk: add host_features field in VirtIOBlock
00f639fb8f virtio-blk: add acct_failed param to virtio_blk_handle_rw_error
caa1ee4313 vhost-user-blk: add discard/write zeroes features support
1
2
3
4
5
6
7
其中commit(caa1ee4313 vhost-user-blk: add discard/write zeroes features support)是vhost-user-blk实现TRIM/UNMAP的核心修改,为什么这个commit会导致virito-blk的BAR0 IO空间变长?因为这个commit需要定义virtio spec要求的discard/write_zeroes相关参数,因此扩展了virtio-blk的配置空间,配置空间原来提供的内容以下数据结构:
struct virtio_blk_config {
/* The capacity (in 512-byte sectors). */
uint64_t capacity;
/* The maximum segment size (if VIRTIO_BLK_F_SIZE_MAX) */
uint32_t size_max;
/* The maximum number of segments (if VIRTIO_BLK_F_SEG_MAX) */
uint32_t seg_max;
/* geometry of the device (if VIRTIO_BLK_F_GEOMETRY) */
struct virtio_blk_geometry {
uint16_t cylinders;
uint8_t heads;
uint8_t sectors;
} geometry;
/* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
uint32_t blk_size;
/* the next 4 entries are guarded by VIRTIO_BLK_F_TOPOLOGY */
/* exponent for physical block per logical block. */
uint8_t physical_block_exp;
/* alignment offset in logical blocks. */
uint8_t alignment_offset;
/* minimum I/O size without performance penalty in logical blocks. */
uint16_t min_io_size;
/* optimal sustained I/O size in logical blocks. */
uint32_t opt_io_size;
/* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
uint8_t wce;
uint8_t unused;
/* number of vqs, only available when VIRTIO_BLK_F_MQ is set */
uint16_t num_queues;
}
+
+ /* the next 3 entries are guarded by VIRTIO_BLK_F_DISCARD */
+ /*
+ * The maximum discard sectors (in 512-byte sectors) for
+ * one segment.
+ */
+ uint32_t max_discard_sectors;
+ /*
+ * The maximum number of discard segments in a
+ * discard command.
+ */
+ uint32_t max_discard_seg;
+ /* Discard commands must be aligned to this number of sectors. */
+ uint32_t discard_sector_alignment;
+
+ /* the next 3 entries are guarded by VIRTIO_BLK_F_WRITE_ZEROES */
+ /*
+ * The maximum number of write zeroes sectors (in 512-byte sectors) in
+ * one segment.
+ */
+ uint32_t max_write_zeroes_sectors;
+ /*
+ * The maximum number of segments in a write zeroes
+ * command.
+ */
+ uint32_t max_write_zeroes_seg;
+ /*
+ * Set if a VIRTIO_BLK_T_WRITE_ZEROES request may result in the
+ * deallocation of one or more of the sectors.
+ */
+ uint8_t write_zeroes_may_unmap;
+
+ uint8_t unused1[3];