blob: 3bca9b0adb626324c4a09afe97effb8badfa6f6b [file] [log] [blame]
Kevin Wolf4f999d02009-10-22 17:54:37 +02001/*
2 * QEMU System Emulator
3 *
4 * Copyright (c) 2003-2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
Peter Maydelld38ea872016-01-29 17:50:05 +000025#include "qemu/osdep.h"
Markus Armbrusterda34e652016-03-14 09:01:28 +010026#include "qapi/error.h"
Kevin Wolf4f999d02009-10-22 17:54:37 +020027#include "qemu-common.h"
Paolo Bonzini737e1502012-12-17 18:19:44 +010028#include "block/aio.h"
Stefan Hajnoczi9b342772013-03-07 13:41:47 +010029#include "block/thread-pool.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010030#include "qemu/main-loop.h"
Paolo Bonzini0ceb8492014-07-07 15:18:04 +020031#include "qemu/atomic.h"
Paolo Bonzini0187f5c2016-07-04 18:33:20 +020032#include "block/raw-aio.h"
Kevin Wolf9a1e9482009-10-22 17:54:38 +020033
Kevin Wolf4f999d02009-10-22 17:54:37 +020034/***********************************************************/
35/* bottom halves (can be seen as timers which expire ASAP) */
36
37struct QEMUBH {
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +020038 AioContext *ctx;
Kevin Wolf4f999d02009-10-22 17:54:37 +020039 QEMUBHFunc *cb;
40 void *opaque;
Kevin Wolf4f999d02009-10-22 17:54:37 +020041 QEMUBH *next;
Stefan Weil9b47b172012-04-29 19:08:45 +020042 bool scheduled;
43 bool idle;
44 bool deleted;
Kevin Wolf4f999d02009-10-22 17:54:37 +020045};
46
Paolo Bonzinif627aab2012-10-29 23:45:23 +010047QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
Kevin Wolf4f999d02009-10-22 17:54:37 +020048{
49 QEMUBH *bh;
Paolo Bonziniee823102014-12-17 16:10:00 +010050 bh = g_new(QEMUBH, 1);
51 *bh = (QEMUBH){
52 .ctx = ctx,
53 .cb = cb,
54 .opaque = opaque,
55 };
Liu Ping Fandcc772e2013-07-16 12:28:58 +080056 qemu_mutex_lock(&ctx->bh_lock);
Paolo Bonzinif627aab2012-10-29 23:45:23 +010057 bh->next = ctx->first_bh;
Liu Ping Fandcc772e2013-07-16 12:28:58 +080058 /* Make sure that the members are ready before putting bh into list */
59 smp_wmb();
Paolo Bonzinif627aab2012-10-29 23:45:23 +010060 ctx->first_bh = bh;
Liu Ping Fandcc772e2013-07-16 12:28:58 +080061 qemu_mutex_unlock(&ctx->bh_lock);
Kevin Wolf4f999d02009-10-22 17:54:37 +020062 return bh;
63}
64
Pavel Dovgalyukdf281b82015-09-17 19:24:50 +030065void aio_bh_call(QEMUBH *bh)
66{
67 bh->cb(bh->opaque);
68}
69
Liu Ping Fandcc772e2013-07-16 12:28:58 +080070/* Multiple occurrences of aio_bh_poll cannot be called concurrently */
Paolo Bonzinif627aab2012-10-29 23:45:23 +010071int aio_bh_poll(AioContext *ctx)
Kevin Wolf4f999d02009-10-22 17:54:37 +020072{
Kevin Wolf7887f622011-06-07 17:51:21 +020073 QEMUBH *bh, **bhp, *next;
Kevin Wolf4f999d02009-10-22 17:54:37 +020074 int ret;
Kevin Wolf648fb0e2011-09-01 16:16:10 +020075
Paolo Bonzinif627aab2012-10-29 23:45:23 +010076 ctx->walking_bh++;
Kevin Wolf4f999d02009-10-22 17:54:37 +020077
78 ret = 0;
Paolo Bonzinif627aab2012-10-29 23:45:23 +010079 for (bh = ctx->first_bh; bh; bh = next) {
Liu Ping Fandcc772e2013-07-16 12:28:58 +080080 /* Make sure that fetching bh happens before accessing its members */
81 smp_read_barrier_depends();
Kevin Wolf7887f622011-06-07 17:51:21 +020082 next = bh->next;
Paolo Bonzinie8d3b1a2015-04-07 17:16:19 +020083 /* The atomic_xchg is paired with the one in qemu_bh_schedule. The
84 * implicit memory barrier ensures that the callback sees all writes
85 * done by the scheduling thread. It also ensures that the scheduling
86 * thread sees the zero before bh->cb has run, and thus will call
87 * aio_notify again if necessary.
88 */
89 if (!bh->deleted && atomic_xchg(&bh->scheduled, 0)) {
Stefan Hajnoczica96ac42015-07-28 18:34:09 +020090 /* Idle BHs and the notify BH don't count as progress */
91 if (!bh->idle && bh != ctx->notify_dummy_bh) {
Kevin Wolf4f999d02009-10-22 17:54:37 +020092 ret = 1;
Stefan Hajnoczica96ac42015-07-28 18:34:09 +020093 }
Kevin Wolf4f999d02009-10-22 17:54:37 +020094 bh->idle = 0;
Pavel Dovgalyukdf281b82015-09-17 19:24:50 +030095 aio_bh_call(bh);
Kevin Wolf4f999d02009-10-22 17:54:37 +020096 }
97 }
98
Paolo Bonzinif627aab2012-10-29 23:45:23 +010099 ctx->walking_bh--;
Kevin Wolf648fb0e2011-09-01 16:16:10 +0200100
Kevin Wolf4f999d02009-10-22 17:54:37 +0200101 /* remove deleted bhs */
Paolo Bonzinif627aab2012-10-29 23:45:23 +0100102 if (!ctx->walking_bh) {
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800103 qemu_mutex_lock(&ctx->bh_lock);
Paolo Bonzinif627aab2012-10-29 23:45:23 +0100104 bhp = &ctx->first_bh;
Kevin Wolf648fb0e2011-09-01 16:16:10 +0200105 while (*bhp) {
106 bh = *bhp;
107 if (bh->deleted) {
108 *bhp = bh->next;
109 g_free(bh);
110 } else {
111 bhp = &bh->next;
112 }
113 }
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800114 qemu_mutex_unlock(&ctx->bh_lock);
Kevin Wolf4f999d02009-10-22 17:54:37 +0200115 }
116
117 return ret;
118}
119
120void qemu_bh_schedule_idle(QEMUBH *bh)
121{
Kevin Wolf4f999d02009-10-22 17:54:37 +0200122 bh->idle = 1;
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800123 /* Make sure that idle & any writes needed by the callback are done
124 * before the locations are read in the aio_bh_poll.
125 */
Paolo Bonzinie8d3b1a2015-04-07 17:16:19 +0200126 atomic_mb_set(&bh->scheduled, 1);
Kevin Wolf4f999d02009-10-22 17:54:37 +0200127}
128
129void qemu_bh_schedule(QEMUBH *bh)
130{
Stefan Hajnoczi924fe122014-06-03 11:21:01 +0200131 AioContext *ctx;
132
Stefan Hajnoczi924fe122014-06-03 11:21:01 +0200133 ctx = bh->ctx;
Kevin Wolf4f999d02009-10-22 17:54:37 +0200134 bh->idle = 0;
Paolo Bonzinie8d3b1a2015-04-07 17:16:19 +0200135 /* The memory barrier implicit in atomic_xchg makes sure that:
Stefan Hajnoczi924fe122014-06-03 11:21:01 +0200136 * 1. idle & any writes needed by the callback are done before the
137 * locations are read in the aio_bh_poll.
138 * 2. ctx is loaded before scheduled is set and the callback has a chance
139 * to execute.
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800140 */
Paolo Bonzinie8d3b1a2015-04-07 17:16:19 +0200141 if (atomic_xchg(&bh->scheduled, 1) == 0) {
142 aio_notify(ctx);
143 }
Kevin Wolf4f999d02009-10-22 17:54:37 +0200144}
145
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800146
147/* This func is async.
148 */
Kevin Wolf4f999d02009-10-22 17:54:37 +0200149void qemu_bh_cancel(QEMUBH *bh)
150{
151 bh->scheduled = 0;
152}
153
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800154/* This func is async.The bottom half will do the delete action at the finial
155 * end.
156 */
Kevin Wolf4f999d02009-10-22 17:54:37 +0200157void qemu_bh_delete(QEMUBH *bh)
158{
159 bh->scheduled = 0;
160 bh->deleted = 1;
161}
162
Paolo Bonzini845ca102014-07-09 11:53:01 +0200163int64_t
164aio_compute_timeout(AioContext *ctx)
Kevin Wolf4f999d02009-10-22 17:54:37 +0200165{
Paolo Bonzini845ca102014-07-09 11:53:01 +0200166 int64_t deadline;
167 int timeout = -1;
Kevin Wolf4f999d02009-10-22 17:54:37 +0200168 QEMUBH *bh;
169
Paolo Bonzinif627aab2012-10-29 23:45:23 +0100170 for (bh = ctx->first_bh; bh; bh = bh->next) {
Kevin Wolf4f999d02009-10-22 17:54:37 +0200171 if (!bh->deleted && bh->scheduled) {
172 if (bh->idle) {
173 /* idle bottom halves will be polled at least
174 * every 10ms */
Paolo Bonzini845ca102014-07-09 11:53:01 +0200175 timeout = 10000000;
Kevin Wolf4f999d02009-10-22 17:54:37 +0200176 } else {
177 /* non-idle bottom halves will be executed
178 * immediately */
Paolo Bonzini845ca102014-07-09 11:53:01 +0200179 return 0;
Kevin Wolf4f999d02009-10-22 17:54:37 +0200180 }
181 }
182 }
Kevin Wolf4f999d02009-10-22 17:54:37 +0200183
Paolo Bonzini845ca102014-07-09 11:53:01 +0200184 deadline = timerlistgroup_deadline_ns(&ctx->tlg);
Alex Bligh533a8cf2013-08-21 16:02:51 +0100185 if (deadline == 0) {
Paolo Bonzini845ca102014-07-09 11:53:01 +0200186 return 0;
Alex Bligh533a8cf2013-08-21 16:02:51 +0100187 } else {
Paolo Bonzini845ca102014-07-09 11:53:01 +0200188 return qemu_soonest_timeout(timeout, deadline);
Alex Bligh533a8cf2013-08-21 16:02:51 +0100189 }
Paolo Bonzini845ca102014-07-09 11:53:01 +0200190}
Alex Bligh533a8cf2013-08-21 16:02:51 +0100191
Paolo Bonzini845ca102014-07-09 11:53:01 +0200192static gboolean
193aio_ctx_prepare(GSource *source, gint *timeout)
194{
195 AioContext *ctx = (AioContext *) source;
196
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200197 atomic_or(&ctx->notify_me, 1);
198
Paolo Bonzini845ca102014-07-09 11:53:01 +0200199 /* We assume there is no timeout already supplied */
200 *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx));
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200201
202 if (aio_prepare(ctx)) {
203 *timeout = 0;
204 }
205
Paolo Bonzini845ca102014-07-09 11:53:01 +0200206 return *timeout == 0;
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200207}
208
209static gboolean
210aio_ctx_check(GSource *source)
211{
212 AioContext *ctx = (AioContext *) source;
213 QEMUBH *bh;
214
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200215 atomic_and(&ctx->notify_me, ~1);
Paolo Bonzini05e514b2015-07-21 16:07:53 +0200216 aio_notify_accept(ctx);
Paolo Bonzini21a03d12015-07-21 16:07:52 +0200217
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200218 for (bh = ctx->first_bh; bh; bh = bh->next) {
219 if (!bh->deleted && bh->scheduled) {
220 return true;
Cao jin6977d902016-07-14 21:10:43 +0800221 }
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200222 }
Alex Bligh533a8cf2013-08-21 16:02:51 +0100223 return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0);
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200224}
225
226static gboolean
227aio_ctx_dispatch(GSource *source,
228 GSourceFunc callback,
229 gpointer user_data)
230{
231 AioContext *ctx = (AioContext *) source;
232
233 assert(callback == NULL);
Paolo Bonzinie4c7e2d2014-07-09 11:53:05 +0200234 aio_dispatch(ctx);
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200235 return true;
236}
237
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200238static void
239aio_ctx_finalize(GSource *source)
240{
241 AioContext *ctx = (AioContext *) source;
242
Stefan Hajnoczica96ac42015-07-28 18:34:09 +0200243 qemu_bh_delete(ctx->notify_dummy_bh);
Stefan Hajnoczi9b342772013-03-07 13:41:47 +0100244 thread_pool_free(ctx->thread_pool);
Stefan Hajnoczia0769722015-07-28 18:34:08 +0200245
Paolo Bonzini0187f5c2016-07-04 18:33:20 +0200246#ifdef CONFIG_LINUX_AIO
247 if (ctx->linux_aio) {
248 laio_detach_aio_context(ctx->linux_aio, ctx);
249 laio_cleanup(ctx->linux_aio);
250 ctx->linux_aio = NULL;
251 }
252#endif
253
Stefan Hajnoczia0769722015-07-28 18:34:08 +0200254 qemu_mutex_lock(&ctx->bh_lock);
255 while (ctx->first_bh) {
256 QEMUBH *next = ctx->first_bh->next;
257
258 /* qemu_bh_delete() must have been called on BHs in this AioContext */
259 assert(ctx->first_bh->deleted);
260
261 g_free(ctx->first_bh);
262 ctx->first_bh = next;
263 }
264 qemu_mutex_unlock(&ctx->bh_lock);
265
Fam Zhengdca21ef2015-10-23 11:08:05 +0800266 aio_set_event_notifier(ctx, &ctx->notifier, false, NULL);
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200267 event_notifier_cleanup(&ctx->notifier);
Stefan Hajnoczi98563fc2014-03-03 11:30:04 +0100268 rfifolock_destroy(&ctx->lock);
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800269 qemu_mutex_destroy(&ctx->bh_lock);
Alex Blighdae21b92013-08-21 16:02:49 +0100270 timerlistgroup_deinit(&ctx->tlg);
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200271}
272
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200273static GSourceFuncs aio_source_funcs = {
274 aio_ctx_prepare,
275 aio_ctx_check,
276 aio_ctx_dispatch,
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200277 aio_ctx_finalize
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200278};
279
280GSource *aio_get_g_source(AioContext *ctx)
281{
282 g_source_ref(&ctx->source);
283 return &ctx->source;
284}
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200285
Stefan Hajnoczi9b342772013-03-07 13:41:47 +0100286ThreadPool *aio_get_thread_pool(AioContext *ctx)
287{
288 if (!ctx->thread_pool) {
289 ctx->thread_pool = thread_pool_new(ctx);
290 }
291 return ctx->thread_pool;
292}
293
Paolo Bonzini0187f5c2016-07-04 18:33:20 +0200294#ifdef CONFIG_LINUX_AIO
295LinuxAioState *aio_get_linux_aio(AioContext *ctx)
296{
297 if (!ctx->linux_aio) {
298 ctx->linux_aio = laio_init();
299 laio_attach_aio_context(ctx->linux_aio, ctx);
300 }
301 return ctx->linux_aio;
302}
303#endif
304
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200305void aio_notify(AioContext *ctx)
306{
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200307 /* Write e.g. bh->scheduled before reading ctx->notify_me. Pairs
308 * with atomic_or in aio_ctx_prepare or atomic_add in aio_poll.
309 */
Paolo Bonzini0ceb8492014-07-07 15:18:04 +0200310 smp_mb();
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200311 if (ctx->notify_me) {
Paolo Bonzini0ceb8492014-07-07 15:18:04 +0200312 event_notifier_set(&ctx->notifier);
Paolo Bonzini05e514b2015-07-21 16:07:53 +0200313 atomic_mb_set(&ctx->notified, true);
314 }
315}
316
317void aio_notify_accept(AioContext *ctx)
318{
319 if (atomic_xchg(&ctx->notified, false)) {
320 event_notifier_test_and_clear(&ctx->notifier);
Paolo Bonzini0ceb8492014-07-07 15:18:04 +0200321 }
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200322}
323
Alex Blighd5541d82013-08-21 16:02:50 +0100324static void aio_timerlist_notify(void *opaque)
325{
326 aio_notify(opaque);
327}
328
Stefan Hajnoczida5e1de2015-06-03 10:15:33 +0100329static void aio_rfifolock_cb(void *opaque)
330{
Stefan Hajnoczica96ac42015-07-28 18:34:09 +0200331 AioContext *ctx = opaque;
332
Stefan Hajnoczida5e1de2015-06-03 10:15:33 +0100333 /* Kick owner thread in case they are blocked in aio_poll() */
Stefan Hajnoczica96ac42015-07-28 18:34:09 +0200334 qemu_bh_schedule(ctx->notify_dummy_bh);
335}
336
337static void notify_dummy_bh(void *opaque)
338{
339 /* Do nothing, we were invoked just to force the event loop to iterate */
Stefan Hajnoczida5e1de2015-06-03 10:15:33 +0100340}
341
Paolo Bonzini21a03d12015-07-21 16:07:52 +0200342static void event_notifier_dummy_cb(EventNotifier *e)
343{
344}
345
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300346AioContext *aio_context_new(Error **errp)
Paolo Bonzinif627aab2012-10-29 23:45:23 +0100347{
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300348 int ret;
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200349 AioContext *ctx;
Fam Zheng37fcee52015-10-30 12:06:28 +0800350
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200351 ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
Cao jin7e003462016-07-15 18:28:44 +0800352 aio_context_setup(ctx);
353
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300354 ret = event_notifier_init(&ctx->notifier, false);
355 if (ret < 0) {
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300356 error_setg_errno(errp, -ret, "Failed to initialize event notifier");
Fam Zheng37fcee52015-10-30 12:06:28 +0800357 goto fail;
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300358 }
Paolo Bonzinifcf5def2014-12-17 16:09:58 +0100359 g_source_set_can_recurse(&ctx->source, true);
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300360 aio_set_event_notifier(ctx, &ctx->notifier,
Fam Zhengdca21ef2015-10-23 11:08:05 +0800361 false,
Chrysostomos Nanakos2f78e492014-09-18 14:30:49 +0300362 (EventNotifierHandler *)
Paolo Bonzini21a03d12015-07-21 16:07:52 +0200363 event_notifier_dummy_cb);
Paolo Bonzini0187f5c2016-07-04 18:33:20 +0200364#ifdef CONFIG_LINUX_AIO
365 ctx->linux_aio = NULL;
366#endif
Stefan Hajnoczi9b342772013-03-07 13:41:47 +0100367 ctx->thread_pool = NULL;
Liu Ping Fandcc772e2013-07-16 12:28:58 +0800368 qemu_mutex_init(&ctx->bh_lock);
Stefan Hajnoczida5e1de2015-06-03 10:15:33 +0100369 rfifolock_init(&ctx->lock, aio_rfifolock_cb, ctx);
Alex Blighd5541d82013-08-21 16:02:50 +0100370 timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx);
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200371
Stefan Hajnoczica96ac42015-07-28 18:34:09 +0200372 ctx->notify_dummy_bh = aio_bh_new(ctx, notify_dummy_bh, NULL);
373
Paolo Bonzini2f4dc3c2012-09-24 18:44:14 +0200374 return ctx;
Fam Zheng37fcee52015-10-30 12:06:28 +0800375fail:
376 g_source_destroy(&ctx->source);
377 return NULL;
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200378}
379
380void aio_context_ref(AioContext *ctx)
381{
382 g_source_ref(&ctx->source);
383}
384
385void aio_context_unref(AioContext *ctx)
386{
387 g_source_unref(&ctx->source);
Paolo Bonzinif627aab2012-10-29 23:45:23 +0100388}
Stefan Hajnoczi98563fc2014-03-03 11:30:04 +0100389
390void aio_context_acquire(AioContext *ctx)
391{
392 rfifolock_lock(&ctx->lock);
393}
394
395void aio_context_release(AioContext *ctx)
396{
397 rfifolock_unlock(&ctx->lock);
398}