aboutsummaryrefslogtreecommitdiff
path: root/hw/virtio/virtio.c
diff options
context:
space:
mode:
Diffstat (limited to 'hw/virtio/virtio.c')
-rw-r--r--hw/virtio/virtio.c982
1 files changed, 637 insertions, 345 deletions
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 3a1f6c520c..fb6b4ccd83 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -13,13 +13,17 @@
#include "qemu/osdep.h"
#include "qapi/error.h"
-#include "cpu.h"
+#include "qapi/qapi-commands-virtio.h"
#include "trace.h"
+#include "qemu/defer-call.h"
#include "qemu/error-report.h"
#include "qemu/log.h"
#include "qemu/main-loop.h"
#include "qemu/module.h"
+#include "qom/object_interfaces.h"
+#include "hw/core/cpu.h"
#include "hw/virtio/virtio.h"
+#include "hw/virtio/vhost.h"
#include "migration/qemu-file-types.h"
#include "qemu/atomic.h"
#include "hw/virtio/virtio-bus.h"
@@ -27,7 +31,25 @@
#include "hw/virtio/virtio-access.h"
#include "sysemu/dma.h"
#include "sysemu/runstate.h"
+#include "virtio-qmp.h"
+
#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/vhost_types.h"
+#include "standard-headers/linux/virtio_blk.h"
+#include "standard-headers/linux/virtio_console.h"
+#include "standard-headers/linux/virtio_gpu.h"
+#include "standard-headers/linux/virtio_net.h"
+#include "standard-headers/linux/virtio_scsi.h"
+#include "standard-headers/linux/virtio_i2c.h"
+#include "standard-headers/linux/virtio_balloon.h"
+#include "standard-headers/linux/virtio_iommu.h"
+#include "standard-headers/linux/virtio_mem.h"
+#include "standard-headers/linux/virtio_vsock.h"
+
+/*
+ * Maximum size of virtio device config space
+ */
+#define VHOST_USER_MAX_CONFIG_SIZE 256
/*
* The alignment to use between consumer and producer parts of vring.
@@ -125,7 +147,6 @@ struct VirtQueue
uint16_t vector;
VirtIOHandleOutput handle_output;
- VirtIOHandleAIOOutput handle_aio_output;
VirtIODevice *vdev;
EventNotifier guest_notifier;
EventNotifier host_notifier;
@@ -133,6 +154,56 @@ struct VirtQueue
QLIST_ENTRY(VirtQueue) node;
};
+const char *virtio_device_names[] = {
+ [VIRTIO_ID_NET] = "virtio-net",
+ [VIRTIO_ID_BLOCK] = "virtio-blk",
+ [VIRTIO_ID_CONSOLE] = "virtio-serial",
+ [VIRTIO_ID_RNG] = "virtio-rng",
+ [VIRTIO_ID_BALLOON] = "virtio-balloon",
+ [VIRTIO_ID_IOMEM] = "virtio-iomem",
+ [VIRTIO_ID_RPMSG] = "virtio-rpmsg",
+ [VIRTIO_ID_SCSI] = "virtio-scsi",
+ [VIRTIO_ID_9P] = "virtio-9p",
+ [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan",
+ [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial",
+ [VIRTIO_ID_CAIF] = "virtio-caif",
+ [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon",
+ [VIRTIO_ID_GPU] = "virtio-gpu",
+ [VIRTIO_ID_CLOCK] = "virtio-clk",
+ [VIRTIO_ID_INPUT] = "virtio-input",
+ [VIRTIO_ID_VSOCK] = "vhost-vsock",
+ [VIRTIO_ID_CRYPTO] = "virtio-crypto",
+ [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal",
+ [VIRTIO_ID_PSTORE] = "virtio-pstore",
+ [VIRTIO_ID_IOMMU] = "virtio-iommu",
+ [VIRTIO_ID_MEM] = "virtio-mem",
+ [VIRTIO_ID_SOUND] = "virtio-sound",
+ [VIRTIO_ID_FS] = "virtio-user-fs",
+ [VIRTIO_ID_PMEM] = "virtio-pmem",
+ [VIRTIO_ID_RPMB] = "virtio-rpmb",
+ [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim",
+ [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder",
+ [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder",
+ [VIRTIO_ID_SCMI] = "virtio-scmi",
+ [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod",
+ [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c",
+ [VIRTIO_ID_WATCHDOG] = "virtio-watchdog",
+ [VIRTIO_ID_CAN] = "virtio-can",
+ [VIRTIO_ID_DMABUF] = "virtio-dmabuf",
+ [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv",
+ [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol",
+ [VIRTIO_ID_BT] = "virtio-bluetooth",
+ [VIRTIO_ID_GPIO] = "virtio-gpio"
+};
+
+static const char *virtio_id_to_name(uint16_t device_id)
+{
+ assert(device_id < G_N_ELEMENTS(virtio_device_names));
+ const char *name = virtio_device_names[device_id];
+ assert(name != NULL);
+ return name;
+}
+
/* Called within call_rcu(). */
static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
{
@@ -154,7 +225,7 @@ static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
}
}
-static void virtio_init_region_cache(VirtIODevice *vdev, int n)
+void virtio_init_region_cache(VirtIODevice *vdev, int n)
{
VirtQueue *vq = &vdev->vq[n];
VRingMemoryRegionCaches *old = vq->vring.caches;
@@ -247,13 +318,10 @@ static void vring_packed_event_read(VirtIODevice *vdev,
hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
- address_space_read_cached(cache, off_flags, &e->flags,
- sizeof(e->flags));
+ e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags);
/* Make sure flags is seen before off_wrap */
smp_rmb();
- address_space_read_cached(cache, off_off, &e->off_wrap,
- sizeof(e->off_wrap));
- virtio_tswap16s(vdev, &e->off_wrap);
+ e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off);
virtio_tswap16s(vdev, &e->flags);
}
@@ -263,8 +331,7 @@ static void vring_packed_off_wrap_write(VirtIODevice *vdev,
{
hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
- virtio_tswap16s(vdev, &off_wrap);
- address_space_write_cached(cache, off, &off_wrap, sizeof(off_wrap));
+ virtio_stw_phys_cached(vdev, cache, off, off_wrap);
address_space_cache_invalidate(cache, off, sizeof(off_wrap));
}
@@ -273,8 +340,7 @@ static void vring_packed_flags_write(VirtIODevice *vdev,
{
hwaddr off = offsetof(VRingPackedDescEvent, flags);
- virtio_tswap16s(vdev, &flags);
- address_space_write_cached(cache, off, &flags, sizeof(flags));
+ virtio_stw_phys_cached(vdev, cache, off, flags);
address_space_cache_invalidate(cache, off, sizeof(flags));
}
@@ -347,6 +413,19 @@ static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
}
+/* Called within rcu_read_lock(). */
+static inline uint16_t vring_used_flags(VirtQueue *vq)
+{
+ VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
+ hwaddr pa = offsetof(VRingUsed, flags);
+
+ if (!caches) {
+ return 0;
+ }
+
+ return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
+}
+
/* Called within rcu_read_lock(). */
static uint16_t vring_used_idx(VirtQueue *vq)
{
@@ -507,11 +586,9 @@ static void vring_packed_desc_read_flags(VirtIODevice *vdev,
MemoryRegionCache *cache,
int i)
{
- address_space_read_cached(cache,
- i * sizeof(VRingPackedDesc) +
- offsetof(VRingPackedDesc, flags),
- flags, sizeof(*flags));
- virtio_tswap16s(vdev, flags);
+ hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
+
+ *flags = virtio_lduw_phys_cached(vdev, cache, off);
}
static void vring_packed_desc_read(VirtIODevice *vdev,
@@ -564,8 +641,7 @@ static void vring_packed_desc_write_flags(VirtIODevice *vdev,
{
hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
- virtio_tswap16s(vdev, &desc->flags);
- address_space_write_cached(cache, off, &desc->flags, sizeof(desc->flags));
+ virtio_stw_phys_cached(vdev, cache, off, desc->flags);
address_space_cache_invalidate(cache, off, sizeof(desc->flags));
}
@@ -893,6 +969,7 @@ static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
if (vq->used_idx >= vq->vring.num) {
vq->used_idx -= vq->vring.num;
vq->used_wrap_counter ^= 1;
+ vq->signalled_used_valid = false;
}
}
@@ -921,7 +998,12 @@ void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
/* Called within rcu_read_lock(). */
static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
{
- uint16_t num_heads = vring_avail_idx(vq) - idx;
+ uint16_t avail_idx, num_heads;
+
+ /* Use shadow index whenever possible. */
+ avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx
+ : vring_avail_idx(vq);
+ num_heads = avail_idx - idx;
/* Check it isn't doing very strange things with descriptor numbers. */
if (num_heads > vq->vring.num) {
@@ -929,8 +1011,15 @@ static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
idx, vq->shadow_avail_idx);
return -EINVAL;
}
- /* On success, callers read a descriptor at vq->last_avail_idx.
- * Make sure descriptor read does not bypass avail index read. */
+ /*
+ * On success, callers read a descriptor at vq->last_avail_idx.
+ * Make sure descriptor read does not bypass avail index read.
+ *
+ * This is necessary even if we are using a shadow index, since
+ * the shadow index could have been initialized by calling
+ * vring_avail_idx() outside of this function, i.e., by a guest
+ * memory read not accompanied by a barrier.
+ */
if (num_heads) {
smp_rmb();
}
@@ -961,9 +1050,10 @@ enum {
VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
};
+/* Reads the 'desc->next' descriptor into '*desc'. */
static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
MemoryRegionCache *desc_cache,
- unsigned int max, unsigned int *next)
+ unsigned int max)
{
/* If this descriptor says it doesn't chain, we're done. */
if (!(desc->flags & VRING_DESC_F_NEXT)) {
@@ -971,47 +1061,39 @@ static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
}
/* Check they're not leading us off end of descriptors. */
- *next = desc->next;
- /* Make sure compiler knows to grab that: we don't want it changing! */
- smp_wmb();
-
- if (*next >= max) {
- virtio_error(vdev, "Desc next is %u", *next);
+ if (desc->next >= max) {
+ virtio_error(vdev, "Desc next is %u", desc->next);
return VIRTQUEUE_READ_DESC_ERROR;
}
- vring_split_desc_read(vdev, desc, desc_cache, *next);
+ vring_split_desc_read(vdev, desc, desc_cache, desc->next);
return VIRTQUEUE_READ_DESC_MORE;
}
+/* Called within rcu_read_lock(). */
static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
unsigned int *in_bytes, unsigned int *out_bytes,
- unsigned max_in_bytes, unsigned max_out_bytes)
+ unsigned max_in_bytes, unsigned max_out_bytes,
+ VRingMemoryRegionCaches *caches)
{
VirtIODevice *vdev = vq->vdev;
- unsigned int max, idx;
+ unsigned int idx;
unsigned int total_bufs, in_total, out_total;
- VRingMemoryRegionCaches *caches;
- MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
+ MemoryRegionCache indirect_desc_cache;
int64_t len = 0;
int rc;
- RCU_READ_LOCK_GUARD();
+ address_space_cache_init_empty(&indirect_desc_cache);
idx = vq->last_avail_idx;
total_bufs = in_total = out_total = 0;
- max = vq->vring.num;
- caches = vring_get_region_caches(vq);
- if (!caches) {
- goto err;
- }
-
while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
MemoryRegionCache *desc_cache = &caches->desc;
unsigned int num_bufs;
VRingDesc desc;
unsigned int i;
+ unsigned int max = vq->vring.num;
num_bufs = total_bufs;
@@ -1064,7 +1146,7 @@ static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
goto done;
}
- rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
+ rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
} while (rc == VIRTQUEUE_READ_DESC_MORE);
if (rc == VIRTQUEUE_READ_DESC_ERROR) {
@@ -1124,39 +1206,37 @@ static int virtqueue_packed_read_next_desc(VirtQueue *vq,
return VIRTQUEUE_READ_DESC_MORE;
}
+/* Called within rcu_read_lock(). */
static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
unsigned int *in_bytes,
unsigned int *out_bytes,
unsigned max_in_bytes,
- unsigned max_out_bytes)
+ unsigned max_out_bytes,
+ VRingMemoryRegionCaches *caches)
{
VirtIODevice *vdev = vq->vdev;
- unsigned int max, idx;
+ unsigned int idx;
unsigned int total_bufs, in_total, out_total;
+ MemoryRegionCache indirect_desc_cache;
MemoryRegionCache *desc_cache;
- VRingMemoryRegionCaches *caches;
- MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
int64_t len = 0;
VRingPackedDesc desc;
bool wrap_counter;
- RCU_READ_LOCK_GUARD();
+ address_space_cache_init_empty(&indirect_desc_cache);
+
idx = vq->last_avail_idx;
wrap_counter = vq->last_avail_wrap_counter;
total_bufs = in_total = out_total = 0;
- max = vq->vring.num;
- caches = vring_get_region_caches(vq);
- if (!caches) {
- goto err;
- }
-
for (;;) {
unsigned int num_bufs = total_bufs;
unsigned int i = idx;
int rc;
+ unsigned int max = vq->vring.num;
desc_cache = &caches->desc;
+
vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
if (!is_desc_avail(desc.flags, wrap_counter)) {
break;
@@ -1250,6 +1330,8 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
uint16_t desc_size;
VRingMemoryRegionCaches *caches;
+ RCU_READ_LOCK_GUARD();
+
if (unlikely(!vq->vring.desc)) {
goto err;
}
@@ -1268,10 +1350,12 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
- max_in_bytes, max_out_bytes);
+ max_in_bytes, max_out_bytes,
+ caches);
} else {
virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
- max_in_bytes, max_out_bytes);
+ max_in_bytes, max_out_bytes,
+ caches);
}
return;
@@ -1319,7 +1403,8 @@ static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
is_write ?
DMA_DIRECTION_FROM_DEVICE :
- DMA_DIRECTION_TO_DEVICE);
+ DMA_DIRECTION_TO_DEVICE,
+ MEMTXATTRS_UNSPECIFIED);
if (!iov[num_sg].iov_base) {
virtio_error(vdev, "virtio: bogus descriptor or out of resources");
goto out;
@@ -1368,7 +1453,8 @@ static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
sg[i].iov_base = dma_memory_map(vdev->dma_as,
addr[i], &len, is_write ?
DMA_DIRECTION_FROM_DEVICE :
- DMA_DIRECTION_TO_DEVICE);
+ DMA_DIRECTION_TO_DEVICE,
+ MEMTXATTRS_UNSPECIFIED);
if (!sg[i].iov_base) {
error_report("virtio: error trying to map MMIO memory");
exit(1);
@@ -1413,7 +1499,7 @@ static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
{
unsigned int i, head, max;
VRingMemoryRegionCaches *caches;
- MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
+ MemoryRegionCache indirect_desc_cache;
MemoryRegionCache *desc_cache;
int64_t len;
VirtIODevice *vdev = vq->vdev;
@@ -1424,6 +1510,8 @@ static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
VRingDesc desc;
int rc;
+ address_space_cache_init_empty(&indirect_desc_cache);
+
RCU_READ_LOCK_GUARD();
if (virtio_queue_empty_rcu(vq)) {
goto done;
@@ -1513,7 +1601,7 @@ static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
goto err_undo_map;
}
- rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
+ rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
} while (rc == VIRTQUEUE_READ_DESC_MORE);
if (rc == VIRTQUEUE_READ_DESC_ERROR) {
@@ -1550,7 +1638,7 @@ static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
{
unsigned int i, max;
VRingMemoryRegionCaches *caches;
- MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
+ MemoryRegionCache indirect_desc_cache;
MemoryRegionCache *desc_cache;
int64_t len;
VirtIODevice *vdev = vq->vdev;
@@ -1562,6 +1650,8 @@ static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
uint16_t id;
int rc;
+ address_space_cache_init_empty(&indirect_desc_cache);
+
RCU_READ_LOCK_GUARD();
if (virtio_queue_packed_empty_rcu(vq)) {
goto done;
@@ -1703,6 +1793,8 @@ static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
VirtIODevice *vdev = vq->vdev;
VRingPackedDesc desc;
+ RCU_READ_LOCK_GUARD();
+
caches = vring_get_region_caches(vq);
if (!caches) {
return 0;
@@ -1978,6 +2070,58 @@ static enum virtio_device_endian virtio_current_cpu_endian(void)
}
}
+static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i)
+{
+ vdev->vq[i].vring.desc = 0;
+ vdev->vq[i].vring.avail = 0;
+ vdev->vq[i].vring.used = 0;
+ vdev->vq[i].last_avail_idx = 0;
+ vdev->vq[i].shadow_avail_idx = 0;
+ vdev->vq[i].used_idx = 0;
+ vdev->vq[i].last_avail_wrap_counter = true;
+ vdev->vq[i].shadow_avail_wrap_counter = true;
+ vdev->vq[i].used_wrap_counter = true;
+ virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
+ vdev->vq[i].signalled_used = 0;
+ vdev->vq[i].signalled_used_valid = false;
+ vdev->vq[i].notification = true;
+ vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
+ vdev->vq[i].inuse = 0;
+ virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
+}
+
+void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index)
+{
+ VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+
+ if (k->queue_reset) {
+ k->queue_reset(vdev, queue_index);
+ }
+
+ __virtio_queue_reset(vdev, queue_index);
+}
+
+void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index)
+{
+ VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+
+ /*
+ * TODO: Seabios is currently out of spec and triggering this error.
+ * So this needs to be fixed in Seabios, then this can
+ * be re-enabled for new machine types only, and also after
+ * being converted to LOG_GUEST_ERROR.
+ *
+ if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+ error_report("queue_enable is only supported in devices of virtio "
+ "1.0 or later.");
+ }
+ */
+
+ if (k->queue_enable) {
+ k->queue_enable(vdev, queue_index);
+ }
+}
+
void virtio_reset(void *opaque)
{
VirtIODevice *vdev = opaque;
@@ -1993,6 +2137,10 @@ void virtio_reset(void *opaque)
vdev->device_endian = virtio_default_endian();
}
+ if (vdev->vhost_started && k->get_vhost) {
+ vhost_reset_device(k->get_vhost(vdev));
+ }
+
if (k->reset) {
k->reset(vdev);
}
@@ -2009,211 +2157,7 @@ void virtio_reset(void *opaque)
virtio_notify_vector(vdev, vdev->config_vector);
for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
- vdev->vq[i].vring.desc = 0;
- vdev->vq[i].vring.avail = 0;
- vdev->vq[i].vring.used = 0;
- vdev->vq[i].last_avail_idx = 0;
- vdev->vq[i].shadow_avail_idx = 0;
- vdev->vq[i].used_idx = 0;
- vdev->vq[i].last_avail_wrap_counter = true;
- vdev->vq[i].shadow_avail_wrap_counter = true;
- vdev->vq[i].used_wrap_counter = true;
- virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
- vdev->vq[i].signalled_used = 0;
- vdev->vq[i].signalled_used_valid = false;
- vdev->vq[i].notification = true;
- vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
- vdev->vq[i].inuse = 0;
- virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
- }
-}
-
-uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint8_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = ldub_p(vdev->config + addr);
- return val;
-}
-
-uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint16_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = lduw_p(vdev->config + addr);
- return val;
-}
-
-uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint32_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = ldl_p(vdev->config + addr);
- return val;
-}
-
-void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint8_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stb_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
- }
-}
-
-void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint16_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stw_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
- }
-}
-
-void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint32_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stl_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
- }
-}
-
-uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint8_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = ldub_p(vdev->config + addr);
- return val;
-}
-
-uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint16_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = lduw_le_p(vdev->config + addr);
- return val;
-}
-
-uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint32_t val;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return (uint32_t)-1;
- }
-
- k->get_config(vdev, vdev->config);
-
- val = ldl_le_p(vdev->config + addr);
- return val;
-}
-
-void virtio_config_modern_writeb(VirtIODevice *vdev,
- uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint8_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stb_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
- }
-}
-
-void virtio_config_modern_writew(VirtIODevice *vdev,
- uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint16_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stw_le_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
- }
-}
-
-void virtio_config_modern_writel(VirtIODevice *vdev,
- uint32_t addr, uint32_t data)
-{
- VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
- uint32_t val = data;
-
- if (addr + sizeof(val) > vdev->config_len) {
- return;
- }
-
- stl_le_p(vdev->config + addr, val);
-
- if (k->set_config) {
- k->set_config(vdev, vdev->config);
+ __virtio_queue_reset(vdev, i);
}
}
@@ -2311,24 +2255,6 @@ void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
}
}
-static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
-{
- bool ret = false;
-
- if (vq->vring.desc && vq->handle_aio_output) {
- VirtIODevice *vdev = vq->vdev;
-
- trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
- ret = vq->handle_aio_output(vdev, vq);
-
- if (unlikely(vdev->start_on_kick)) {
- virtio_set_started(vdev, true);
- }
- }
-
- return ret;
-}
-
static void virtio_queue_notify_vq(VirtQueue *vq)
{
if (vq->vring.desc && vq->handle_output) {
@@ -2407,9 +2333,7 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
vdev->vq[i].vring.num_default = queue_size;
vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
vdev->vq[i].handle_output = handle_output;
- vdev->vq[i].handle_aio_output = NULL;
- vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) *
- queue_size);
+ vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size);
return &vdev->vq[i];
}
@@ -2419,7 +2343,6 @@ void virtio_delete_queue(VirtQueue *vq)
vq->vring.num = 0;
vq->vring.num_default = 0;
vq->handle_output = NULL;
- vq->handle_aio_output = NULL;
g_free(vq->used_elems);
vq->used_elems = NULL;
virtio_virtqueue_reset_region_cache(vq);
@@ -2523,6 +2446,16 @@ static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
}
}
+/* Batch irqs while inside a defer_call_begin()/defer_call_end() section */
+static void virtio_notify_irqfd_deferred_fn(void *opaque)
+{
+ EventNotifier *notifier = opaque;
+ VirtQueue *vq = container_of(notifier, VirtQueue, guest_notifier);
+
+ trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq);
+ event_notifier_set(notifier);
+}
+
void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
{
WITH_RCU_READ_LOCK_GUARD() {
@@ -2549,7 +2482,7 @@ void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
* to an atomic operation.
*/
virtio_set_isr(vq->vdev, 0x1);
- event_notifier_set(&vq->guest_notifier);
+ defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier);
}
static void virtio_irq(VirtQueue *vq)
@@ -2661,7 +2594,7 @@ static const VMStateDescription vmstate_virtqueue = {
.name = "virtqueue_state",
.version_id = 1,
.minimum_version_id = 1,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_UINT64(vring.avail, struct VirtQueue),
VMSTATE_UINT64(vring.used, struct VirtQueue),
VMSTATE_END_OF_LIST()
@@ -2672,7 +2605,7 @@ static const VMStateDescription vmstate_packed_virtqueue = {
.name = "packed_virtqueue_state",
.version_id = 1,
.minimum_version_id = 1,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
VMSTATE_UINT16(used_idx, struct VirtQueue),
@@ -2687,7 +2620,7 @@ static const VMStateDescription vmstate_virtio_virtqueues = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_virtqueue_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
VMSTATE_END_OF_LIST()
@@ -2699,7 +2632,7 @@ static const VMStateDescription vmstate_virtio_packed_virtqueues = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_packed_virtqueue_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
VMSTATE_END_OF_LIST()
@@ -2710,7 +2643,7 @@ static const VMStateDescription vmstate_ringsize = {
.name = "ringsize_state",
.version_id = 1,
.minimum_version_id = 1,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_UINT32(vring.num_default, struct VirtQueue),
VMSTATE_END_OF_LIST()
}
@@ -2721,7 +2654,7 @@ static const VMStateDescription vmstate_virtio_ringsize = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_ringsize_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
VMSTATE_END_OF_LIST()
@@ -2764,7 +2697,7 @@ static const VMStateDescription vmstate_virtio_extra_state = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_extra_state_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
{
.name = "extra_state",
.version_id = 0,
@@ -2783,7 +2716,7 @@ static const VMStateDescription vmstate_virtio_device_endian = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_device_endian_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_UINT8(device_endian, VirtIODevice),
VMSTATE_END_OF_LIST()
}
@@ -2794,7 +2727,7 @@ static const VMStateDescription vmstate_virtio_64bit_features = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_64bit_features_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_UINT64(guest_features, VirtIODevice),
VMSTATE_END_OF_LIST()
}
@@ -2805,7 +2738,7 @@ static const VMStateDescription vmstate_virtio_broken = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_broken_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_BOOL(broken, VirtIODevice),
VMSTATE_END_OF_LIST()
}
@@ -2816,7 +2749,7 @@ static const VMStateDescription vmstate_virtio_started = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_started_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_BOOL(started, VirtIODevice),
VMSTATE_END_OF_LIST()
}
@@ -2827,7 +2760,7 @@ static const VMStateDescription vmstate_virtio_disabled = {
.version_id = 1,
.minimum_version_id = 1,
.needed = &virtio_disabled_needed,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_BOOL(disabled, VirtIODevice),
VMSTATE_END_OF_LIST()
}
@@ -2837,11 +2770,10 @@ static const VMStateDescription vmstate_virtio = {
.name = "virtio",
.version_id = 1,
.minimum_version_id = 1,
- .minimum_version_id_old = 1,
- .fields = (VMStateField[]) {
+ .fields = (const VMStateField[]) {
VMSTATE_END_OF_LIST()
},
- .subsections = (const VMStateDescription*[]) {
+ .subsections = (const VMStateDescription * const []) {
&vmstate_virtio_device_endian,
&vmstate_virtio_64bit_features,
&vmstate_virtio_virtqueues,
@@ -2923,8 +2855,9 @@ static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
}
/* A wrapper for use as a VMState .get function */
-static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
- const VMStateField *field)
+static int coroutine_mixed_fn
+virtio_device_get(QEMUFile *f, void *opaque, size_t size,
+ const VMStateField *field)
{
VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
@@ -2951,6 +2884,39 @@ static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
return bad ? -1 : 0;
}
+typedef struct VirtioSetFeaturesNocheckData {
+ Coroutine *co;
+ VirtIODevice *vdev;
+ uint64_t val;
+ int ret;
+} VirtioSetFeaturesNocheckData;
+
+static void virtio_set_features_nocheck_bh(void *opaque)
+{
+ VirtioSetFeaturesNocheckData *data = opaque;
+
+ data->ret = virtio_set_features_nocheck(data->vdev, data->val);
+ aio_co_wake(data->co);
+}
+
+static int coroutine_mixed_fn
+virtio_set_features_nocheck_maybe_co(VirtIODevice *vdev, uint64_t val)
+{
+ if (qemu_in_coroutine()) {
+ VirtioSetFeaturesNocheckData data = {
+ .co = qemu_coroutine_self(),
+ .vdev = vdev,
+ .val = val,
+ };
+ aio_bh_schedule_oneshot(qemu_get_current_aio_context(),
+ virtio_set_features_nocheck_bh, &data);
+ qemu_coroutine_yield();
+ return data.ret;
+ } else {
+ return virtio_set_features_nocheck(vdev, val);
+ }
+}
+
int virtio_set_features(VirtIODevice *vdev, uint64_t val)
{
int ret;
@@ -2961,6 +2927,13 @@ int virtio_set_features(VirtIODevice *vdev, uint64_t val)
if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
return -EINVAL;
}
+
+ if (val & (1ull << VIRTIO_F_BAD_FEATURE)) {
+ qemu_log_mask(LOG_GUEST_ERROR,
+ "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n",
+ __func__, vdev->name);
+ }
+
ret = virtio_set_features_nocheck(vdev, val);
if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
/* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
@@ -2980,11 +2953,12 @@ int virtio_set_features(VirtIODevice *vdev, uint64_t val)
return ret;
}
-size_t virtio_feature_get_config_size(const VirtIOFeature *feature_sizes,
- uint64_t host_features)
+size_t virtio_get_config_size(const VirtIOConfigSizeParams *params,
+ uint64_t host_features)
{
- size_t config_size = 0;
- int i;
+ size_t config_size = params->min_size;
+ const VirtIOFeature *feature_sizes = params->feature_sizes;
+ size_t i;
for (i = 0; feature_sizes[i].flags != 0; i++) {
if (host_features & feature_sizes[i].flags) {
@@ -2992,10 +2966,12 @@ size_t virtio_feature_get_config_size(const VirtIOFeature *feature_sizes,
}
}
+ assert(config_size <= params->max_size);
return config_size;
}
-int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
+int coroutine_mixed_fn
+virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
{
int i, ret;
int32_t config_len;
@@ -3112,14 +3088,14 @@ int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
* host_features.
*/
uint64_t features64 = vdev->guest_features;
- if (virtio_set_features_nocheck(vdev, features64) < 0) {
+ if (virtio_set_features_nocheck_maybe_co(vdev, features64) < 0) {
error_report("Features 0x%" PRIx64 " unsupported. "
"Allowed features: 0x%" PRIx64,
features64, vdev->host_features);
return -1;
}
} else {
- if (virtio_set_features_nocheck(vdev, features) < 0) {
+ if (virtio_set_features_nocheck_maybe_co(vdev, features) < 0) {
error_report("Features 0x%x unsupported. "
"Allowed features: 0x%" PRIx64,
features, vdev->host_features);
@@ -3238,8 +3214,7 @@ void virtio_instance_init_common(Object *proxy_obj, void *data,
qdev_alias_all_properties(vdev, proxy_obj);
}
-void virtio_init(VirtIODevice *vdev, const char *name,
- uint16_t device_id, size_t config_size)
+void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size)
{
BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
@@ -3253,12 +3228,13 @@ void virtio_init(VirtIODevice *vdev, const char *name,
vdev->start_on_kick = false;
vdev->started = false;
+ vdev->vhost_started = false;
vdev->device_id = device_id;
vdev->status = 0;
qatomic_set(&vdev->isr, 0);
vdev->queue_sel = 0;
vdev->config_vector = VIRTIO_NO_VECTOR;
- vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
+ vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX);
vdev->vm_running = runstate_is_running();
vdev->broken = false;
for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
@@ -3268,7 +3244,7 @@ void virtio_init(VirtIODevice *vdev, const char *name,
vdev->vq[i].host_notifier_enabled = false;
}
- vdev->name = name;
+ vdev->name = virtio_id_to_name(device_id);
vdev->config_len = config_size;
if (vdev->config_len) {
vdev->config = g_malloc0(config_size);
@@ -3392,6 +3368,18 @@ static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
return vdev->vq[n].last_avail_idx;
}
+static uint32_t virtio_queue_split_get_vring_states(VirtIODevice *vdev,
+ int n)
+{
+ struct VirtQueue *vq = &vdev->vq[n];
+ uint16_t avail, used;
+
+ avail = vq->last_avail_idx;
+ used = vq->used_idx;
+
+ return avail | (uint32_t)used << 16;
+}
+
unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
{
if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
@@ -3401,6 +3389,33 @@ unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
}
}
+unsigned int virtio_queue_get_vring_states(VirtIODevice *vdev, int n)
+{
+ if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
+ return -1;
+ } else {
+ return virtio_queue_split_get_vring_states(vdev, n);
+ }
+}
+
+static void virtio_queue_split_set_vring_states(VirtIODevice *vdev,
+ int n, uint32_t idx)
+{
+ struct VirtQueue *vq = &vdev->vq[n];
+ vq->last_avail_idx = (uint16_t)(idx & 0xffff);
+ vq->shadow_avail_idx = (uint16_t)(idx & 0xffff);
+ vq->used_idx = (uint16_t)(idx >> 16);
+}
+
+void virtio_queue_set_vring_states(VirtIODevice *vdev, int n, uint32_t idx)
+{
+ if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
+ return;
+ } else {
+ virtio_queue_split_set_vring_states(vdev, n, idx);
+ }
+}
+
static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
int n, unsigned int idx)
{
@@ -3410,7 +3425,7 @@ static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
vq->last_avail_wrap_counter =
vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
idx >>= 16;
- vq->used_idx = idx & 0x7ffff;
+ vq->used_idx = idx & 0x7fff;
vq->used_wrap_counter = !!(idx & 0x8000);
}
@@ -3502,7 +3517,14 @@ static void virtio_queue_guest_notifier_read(EventNotifier *n)
virtio_irq(vq);
}
}
+static void virtio_config_guest_notifier_read(EventNotifier *n)
+{
+ VirtIODevice *vdev = container_of(n, VirtIODevice, config_notifier);
+ if (event_notifier_test_and_clear(n)) {
+ virtio_notify_config(vdev);
+ }
+}
void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
bool with_irqfd)
{
@@ -3519,17 +3541,26 @@ void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
}
}
-EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
+void virtio_config_set_guest_notifier_fd_handler(VirtIODevice *vdev,
+ bool assign, bool with_irqfd)
{
- return &vq->guest_notifier;
+ EventNotifier *n;
+ n = &vdev->config_notifier;
+ if (assign && !with_irqfd) {
+ event_notifier_set_handler(n, virtio_config_guest_notifier_read);
+ } else {
+ event_notifier_set_handler(n, NULL);
+ }
+ if (!assign) {
+ /* Test and clear notifier before closing it,*/
+ /* in case poll callback didn't have time to run. */
+ virtio_config_guest_notifier_read(n);
+ }
}
-static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
+EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
{
- VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
- if (event_notifier_test_and_clear(n)) {
- virtio_queue_notify_aio_vq(vq);
- }
+ return &vq->guest_notifier;
}
static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
@@ -3544,11 +3575,14 @@ static bool virtio_queue_host_notifier_aio_poll(void *opaque)
EventNotifier *n = opaque;
VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
- if (!vq->vring.desc || virtio_queue_empty(vq)) {
- return false;
- }
+ return vq->vring.desc && !virtio_queue_empty(vq);
+}
+
+static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier *n)
+{
+ VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
- return virtio_queue_notify_aio_vq(vq);
+ virtio_queue_notify_vq(vq);
}
static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
@@ -3559,24 +3593,75 @@ static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
virtio_queue_set_notification(vq, 1);
}
-void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
- VirtIOHandleAIOOutput handle_output)
+void virtio_queue_aio_attach_host_notifier(VirtQueue *vq, AioContext *ctx)
{
- if (handle_output) {
- vq->handle_aio_output = handle_output;
- aio_set_event_notifier(ctx, &vq->host_notifier, true,
- virtio_queue_host_notifier_aio_read,
- virtio_queue_host_notifier_aio_poll);
- aio_set_event_notifier_poll(ctx, &vq->host_notifier,
- virtio_queue_host_notifier_aio_poll_begin,
- virtio_queue_host_notifier_aio_poll_end);
- } else {
- aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
- /* Test and clear notifier before after disabling event,
- * in case poll callback didn't have time to run. */
- virtio_queue_host_notifier_aio_read(&vq->host_notifier);
- vq->handle_aio_output = NULL;
+ /*
+ * virtio_queue_aio_detach_host_notifier() can leave notifications disabled.
+ * Re-enable them. (And if detach has not been used before, notifications
+ * being enabled is still the default state while a notifier is attached;
+ * see virtio_queue_host_notifier_aio_poll_end(), which will always leave
+ * notifications enabled once the polling section is left.)
+ */
+ if (!virtio_queue_get_notification(vq)) {
+ virtio_queue_set_notification(vq, 1);
}
+
+ aio_set_event_notifier(ctx, &vq->host_notifier,
+ virtio_queue_host_notifier_read,
+ virtio_queue_host_notifier_aio_poll,
+ virtio_queue_host_notifier_aio_poll_ready);
+ aio_set_event_notifier_poll(ctx, &vq->host_notifier,
+ virtio_queue_host_notifier_aio_poll_begin,
+ virtio_queue_host_notifier_aio_poll_end);
+
+ /*
+ * We will have ignored notifications about new requests from the guest
+ * while no notifiers were attached, so "kick" the virt queue to process
+ * those requests now.
+ */
+ event_notifier_set(&vq->host_notifier);
+}
+
+/*
+ * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use
+ * this for rx virtqueues and similar cases where the virtqueue handler
+ * function does not pop all elements. When the virtqueue is left non-empty
+ * polling consumes CPU cycles and should not be used.
+ */
+void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue *vq, AioContext *ctx)
+{
+ /* See virtio_queue_aio_attach_host_notifier() */
+ if (!virtio_queue_get_notification(vq)) {
+ virtio_queue_set_notification(vq, 1);
+ }
+
+ aio_set_event_notifier(ctx, &vq->host_notifier,
+ virtio_queue_host_notifier_read,
+ NULL, NULL);
+
+ /*
+ * See virtio_queue_aio_attach_host_notifier().
+ * Note that this may be unnecessary for the type of virtqueues this
+ * function is used for. Still, it will not hurt to have a quick look into
+ * whether we can/should process any of the virtqueue elements.
+ */
+ event_notifier_set(&vq->host_notifier);
+}
+
+void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx)
+{
+ aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL);
+
+ /*
+ * aio_set_event_notifier_poll() does not guarantee whether io_poll_end()
+ * will run after io_poll_begin(), so by removing the notifier, we do not
+ * know whether virtio_queue_host_notifier_aio_poll_end() has run after a
+ * previous virtio_queue_host_notifier_aio_poll_begin(), i.e. whether
+ * notifications are enabled or disabled. It does not really matter anyway;
+ * we just removed the notifier, so we do not care about notifications until
+ * we potentially re-attach it. The attach_host_notifier functions will
+ * ensure that notifications are enabled again when they are needed.
+ */
}
void virtio_queue_host_notifier_read(EventNotifier *n)
@@ -3592,6 +3677,11 @@ EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
return &vq->host_notifier;
}
+EventNotifier *virtio_config_get_guest_notifier(VirtIODevice *vdev)
+{
+ return &vdev->config_notifier;
+}
+
void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
{
vq->host_notifier_enabled = enabled;
@@ -3616,7 +3706,7 @@ void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
vdev->bus_name = g_strdup(bus_name);
}
-void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
+void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
{
va_list ap;
@@ -3670,6 +3760,7 @@ static void virtio_device_realize(DeviceState *dev, Error **errp)
}
vdev->listener.commit = virtio_memory_listener_commit;
+ vdev->listener.name = "virtio";
memory_listener_register(&vdev->listener, vdev->dma_as);
}
@@ -3868,6 +3959,207 @@ bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
return virtio_bus_ioeventfd_enabled(vbus);
}
+VirtQueueStatus *qmp_x_query_virtio_queue_status(const char *path,
+ uint16_t queue,
+ Error **errp)
+{
+ VirtIODevice *vdev;
+ VirtQueueStatus *status;
+
+ vdev = qmp_find_virtio_device(path);
+ if (vdev == NULL) {
+ error_setg(errp, "Path %s is not a VirtIODevice", path);
+ return NULL;
+ }
+
+ if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) {
+ error_setg(errp, "Invalid virtqueue number %d", queue);
+ return NULL;
+ }
+
+ status = g_new0(VirtQueueStatus, 1);
+ status->name = g_strdup(vdev->name);
+ status->queue_index = vdev->vq[queue].queue_index;
+ status->inuse = vdev->vq[queue].inuse;
+ status->vring_num = vdev->vq[queue].vring.num;
+ status->vring_num_default = vdev->vq[queue].vring.num_default;
+ status->vring_align = vdev->vq[queue].vring.align;
+ status->vring_desc = vdev->vq[queue].vring.desc;
+ status->vring_avail = vdev->vq[queue].vring.avail;
+ status->vring_used = vdev->vq[queue].vring.used;
+ status->used_idx = vdev->vq[queue].used_idx;
+ status->signalled_used = vdev->vq[queue].signalled_used;
+ status->signalled_used_valid = vdev->vq[queue].signalled_used_valid;
+
+ if (vdev->vhost_started) {
+ VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
+ struct vhost_dev *hdev = vdc->get_vhost(vdev);
+
+ /* check if vq index exists for vhost as well */
+ if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) {
+ status->has_last_avail_idx = true;
+
+ int vhost_vq_index =
+ hdev->vhost_ops->vhost_get_vq_index(hdev, queue);
+ struct vhost_vring_state state = {
+ .index = vhost_vq_index,
+ };
+
+ status->last_avail_idx =
+ hdev->vhost_ops->vhost_get_vring_base(hdev, &state);
+ }
+ } else {
+ status->has_shadow_avail_idx = true;
+ status->has_last_avail_idx = true;
+ status->last_avail_idx = vdev->vq[queue].last_avail_idx;
+ status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx;
+ }
+
+ return status;
+}
+
+static strList *qmp_decode_vring_desc_flags(uint16_t flags)
+{
+ strList *list = NULL;
+ strList *node;
+ int i;
+
+ struct {
+ uint16_t flag;
+ const char *value;
+ } map[] = {
+ { VRING_DESC_F_NEXT, "next" },
+ { VRING_DESC_F_WRITE, "write" },
+ { VRING_DESC_F_INDIRECT, "indirect" },
+ { 1 << VRING_PACKED_DESC_F_AVAIL, "avail" },
+ { 1 << VRING_PACKED_DESC_F_USED, "used" },
+ { 0, "" }
+ };
+
+ for (i = 0; map[i].flag; i++) {
+ if ((map[i].flag & flags) == 0) {
+ continue;
+ }
+ node = g_malloc0(sizeof(strList));
+ node->value = g_strdup(map[i].value);
+ node->next = list;
+ list = node;
+ }
+
+ return list;
+}
+
+VirtioQueueElement *qmp_x_query_virtio_queue_element(const char *path,
+ uint16_t queue,
+ bool has_index,
+ uint16_t index,
+ Error **errp)
+{
+ VirtIODevice *vdev;
+ VirtQueue *vq;
+ VirtioQueueElement *element = NULL;
+
+ vdev = qmp_find_virtio_device(path);
+ if (vdev == NULL) {
+ error_setg(errp, "Path %s is not a VirtIO device", path);
+ return NULL;
+ }
+
+ if (queue >= VIRTIO_QUEUE_MAX || !virtio_queue_get_num(vdev, queue)) {
+ error_setg(errp, "Invalid virtqueue number %d", queue);
+ return NULL;
+ }
+ vq = &vdev->vq[queue];
+
+ if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
+ error_setg(errp, "Packed ring not supported");
+ return NULL;
+ } else {
+ unsigned int head, i, max;
+ VRingMemoryRegionCaches *caches;
+ MemoryRegionCache indirect_desc_cache;
+ MemoryRegionCache *desc_cache;
+ VRingDesc desc;
+ VirtioRingDescList *list = NULL;
+ VirtioRingDescList *node;
+ int rc; int ndescs;
+
+ address_space_cache_init_empty(&indirect_desc_cache);
+
+ RCU_READ_LOCK_GUARD();
+
+ max = vq->vring.num;
+
+ if (!has_index) {
+ head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num);
+ } else {
+ head = vring_avail_ring(vq, index % vq->vring.num);
+ }
+ i = head;
+
+ caches = vring_get_region_caches(vq);
+ if (!caches) {
+ error_setg(errp, "Region caches not initialized");
+ return NULL;
+ }
+ if (caches->desc.len < max * sizeof(VRingDesc)) {
+ error_setg(errp, "Cannot map descriptor ring");
+ return NULL;
+ }
+
+ desc_cache = &caches->desc;
+ vring_split_desc_read(vdev, &desc, desc_cache, i);
+ if (desc.flags & VRING_DESC_F_INDIRECT) {
+ int64_t len;
+ len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
+ desc.addr, desc.len, false);
+ desc_cache = &indirect_desc_cache;
+ if (len < desc.len) {
+ error_setg(errp, "Cannot map indirect buffer");
+ goto done;
+ }
+
+ max = desc.len / sizeof(VRingDesc);
+ i = 0;
+ vring_split_desc_read(vdev, &desc, desc_cache, i);
+ }
+
+ element = g_new0(VirtioQueueElement, 1);
+ element->avail = g_new0(VirtioRingAvail, 1);
+ element->used = g_new0(VirtioRingUsed, 1);
+ element->name = g_strdup(vdev->name);
+ element->index = head;
+ element->avail->flags = vring_avail_flags(vq);
+ element->avail->idx = vring_avail_idx(vq);
+ element->avail->ring = head;
+ element->used->flags = vring_used_flags(vq);
+ element->used->idx = vring_used_idx(vq);
+ ndescs = 0;
+
+ do {
+ /* A buggy driver may produce an infinite loop */
+ if (ndescs >= max) {
+ break;
+ }
+ node = g_new0(VirtioRingDescList, 1);
+ node->value = g_new0(VirtioRingDesc, 1);
+ node->value->addr = desc.addr;
+ node->value->len = desc.len;
+ node->value->flags = qmp_decode_vring_desc_flags(desc.flags);
+ node->next = list;
+ list = node;
+
+ ndescs++;
+ rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max);
+ } while (rc == VIRTQUEUE_READ_DESC_MORE);
+ element->descs = list;
+done:
+ address_space_cache_destroy(&indirect_desc_cache);
+ }
+
+ return element;
+}
+
static const TypeInfo virtio_device_info = {
.name = TYPE_VIRTIO_DEVICE,
.parent = TYPE_DEVICE,
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-24 18:35:52 +0000