From b2df431407a099ad2fe87a76188d12d99ab12841 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 12 Jul 2012 18:52:31 +0200 Subject: ide scsi: Mess with geometry only for hard disk devices Legacy -drive cyls=... are now ignored completely when the drive doesn't back a hard disk device. Before, they were first checked against a hard disk's limits, then ignored. Signed-off-by: Markus Armbruster Acked-by: Paolo Bonzini Signed-off-by: Kevin Wolf --- hw/ide/qdev.c | 3 ++- hw/scsi-disk.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c index 22e58dfc8a..5ea9b8f4b2 100644 --- a/hw/ide/qdev.c +++ b/hw/ide/qdev.c @@ -149,7 +149,8 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind) } blkconf_serial(&dev->conf, &dev->serial); - if (blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255) < 0) { + if (kind != IDE_CD + && blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255) < 0) { return -1; } diff --git a/hw/scsi-disk.c b/hw/scsi-disk.c index a9c727905a..c8d5edd86e 100644 --- a/hw/scsi-disk.c +++ b/hw/scsi-disk.c @@ -1958,7 +1958,8 @@ static int scsi_initfn(SCSIDevice *dev) } blkconf_serial(&s->qdev.conf, &s->serial); - if (blkconf_geometry(&dev->conf, NULL, 65535, 255, 255) < 0) { + if (dev->type == TYPE_DISK + && blkconf_geometry(&dev->conf, NULL, 65535, 255, 255) < 0) { return -1; } -- cgit v1.2.3 From 05290d80c88de1e4d02f2f00605e09206430cca9 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 24 Jul 2012 13:03:39 +0200 Subject: qapi: generalize documentation of streaming commands Talk about background operations in general, rather than specifically about streaming. Signed-off-by: Paolo Bonzini Signed-off-by: Kevin Wolf --- hmp-commands.hx | 2 +- qapi-schema.json | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hmp-commands.hx b/hmp-commands.hx index eea8b32894..9bbc7f7555 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -101,7 +101,7 @@ ETEXI .name = "block_job_cancel", .args_type = "device:B", .params = "device", - .help = "stop an active block streaming operation", + .help = "stop an active background block operation", .mhandler.cmd = hmp_block_job_cancel, }, diff --git a/qapi-schema.json b/qapi-schema.json index cddf63a878..bd9c450029 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -1660,7 +1660,7 @@ # Returns: Nothing on success # If the job type does not support throttling, NotSupported # If the speed value is invalid, InvalidParameter -# If streaming is not active on this device, DeviceNotActive +# If no background operation is active on this device, DeviceNotActive # # Since: 1.1 ## @@ -1670,9 +1670,9 @@ ## # @block-job-cancel: # -# Stop an active block streaming operation. +# Stop an active background block operation. # -# This command returns immediately after marking the active block streaming +# This command returns immediately after marking the active background block # operation for cancellation. It is an error to call this command if no # operation is in progress. # @@ -1680,16 +1680,15 @@ # BLOCK_JOB_CANCELLED event. Before that happens the job is still visible when # enumerated using query-block-jobs. # -# The image file retains its backing file unless the streaming operation happens -# to complete just as it is being cancelled. -# -# A new block streaming operation can be started at a later time to finish -# copying all data from the backing file. +# For streaming, the image file retains its backing file unless the streaming +# operation happens to complete just as it is being cancelled. A new streaming +# operation can be started at a later time to finish copying all data from the +# backing file. # # @device: the device name # # Returns: Nothing on success -# If streaming is not active on this device, DeviceNotActive +# If no background operation is active on this device, DeviceNotActive # If cancellation already in progress, DeviceInUse # # Since: 1.1 -- cgit v1.2.3 From e77964f79b905760ce7f0406fc6b988daeabbb65 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:01:27 +0100 Subject: qemu-iotests: add qed.py image manipulation utility The qed.py utility can inspect and manipulate QED image files. It can be used for testing to see the state of image metadata and also to inject corruptions into the image file. It also has a scrubbing feature to copy just the metadata out of an image file, allowing users to share broken image files without revealing data in bug reports. This has lived in my local repo for a long time but could be useful to others. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/qed.py | 235 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100755 tests/qemu-iotests/qed.py diff --git a/tests/qemu-iotests/qed.py b/tests/qemu-iotests/qed.py new file mode 100755 index 0000000000..52ff845590 --- /dev/null +++ b/tests/qemu-iotests/qed.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# +# Tool to manipulate QED image files +# +# Copyright (C) 2010 IBM, Corp. +# +# Authors: +# Stefan Hajnoczi +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +import sys +import struct +import random +import optparse + +# This can be used as a module +__all__ = ['QED_F_NEED_CHECK', 'QED'] + +QED_F_NEED_CHECK = 0x02 + +header_fmt = ']- Show header or l1/l2 tables''' + if not args or args[0] == 'header': + print qed.header + elif args[0] == 'l1': + print qed.l1_table + elif len(args) == 2 and args[0] == 'l2': + offset = int(args[1]) + print qed.read_table(offset) + else: + err('unrecognized sub-command') + +def cmd_duplicate(qed, table_level): + '''duplicate l1|l2 - Duplicate a random table element''' + if table_level == 'l1': + offset = qed.header['l1_table_offset'] + table = qed.l1_table + elif table_level == 'l2': + _, offset = random_table_item(qed.l1_table) + table = qed.read_table(offset) + else: + err('unrecognized sub-command') + corrupt_table_duplicate(table) + qed.write_table(offset, table) + +def cmd_invalidate(qed, table_level): + '''invalidate l1|l2 - Plant an invalid table element at random''' + if table_level == 'l1': + offset = qed.header['l1_table_offset'] + table = qed.l1_table + elif table_level == 'l2': + _, offset = random_table_item(qed.l1_table) + table = qed.read_table(offset) + else: + err('unrecognized sub-command') + corrupt_table_invalidate(qed, table) + qed.write_table(offset, table) + +def cmd_need_check(qed, *args): + '''need-check [on|off] - Test, set, or clear the QED_F_NEED_CHECK header bit''' + if not args: + print bool(qed.header['features'] & QED_F_NEED_CHECK) + return + + if args[0] == 'on': + qed.header['features'] |= QED_F_NEED_CHECK + elif args[0] == 'off': + qed.header['features'] &= ~QED_F_NEED_CHECK + else: + err('unrecognized sub-command') + qed.store_header() + +def cmd_zero_cluster(qed, pos, *args): + '''zero-cluster [] - Zero data clusters''' + pos, n = int(pos), 1 + if args: + if len(args) != 1: + err('expected one argument') + n = int(args[0]) + + for i in xrange(n): + l1_index = pos / qed.header['cluster_size'] / len(qed.l1_table) + if qed.l1_table[l1_index] == 0: + err('no l2 table allocated') + + l2_offset = qed.l1_table[l1_index] + l2_table = qed.read_table(l2_offset) + + l2_index = (pos / qed.header['cluster_size']) % len(qed.l1_table) + l2_table[l2_index] = 1 # zero the data cluster + qed.write_table(l2_offset, l2_table) + pos += qed.header['cluster_size'] + +def cmd_copy_metadata(qed, outfile): + '''copy-metadata - Copy metadata only (for scrubbing corrupted images)''' + out = open(outfile, 'wb') + + # Match file size + out.seek(qed.filesize - 1) + out.write('\0') + + # Copy header clusters + out.seek(0) + header_size_bytes = qed.header['header_size'] * qed.header['cluster_size'] + out.write(qed.raw_pread(0, header_size_bytes)) + + # Copy L1 table + out.seek(qed.header['l1_table_offset']) + s = ''.join(pack_table_elem(x) for x in qed.l1_table) + out.write(s) + + # Copy L2 tables + for l2_offset in qed.l1_table: + if l2_offset == 0: + continue + l2_table = qed.read_table(l2_offset) + out.seek(l2_offset) + s = ''.join(pack_table_elem(x) for x in l2_table) + out.write(s) + + out.close() + +def usage(): + print 'Usage: %s [, ...]' % sys.argv[0] + print + print 'Supported commands:' + for cmd in sorted(x for x in globals() if x.startswith('cmd_')): + print globals()[cmd].__doc__ + sys.exit(1) + +def main(): + if len(sys.argv) < 3: + usage() + filename, cmd = sys.argv[1:3] + + cmd = 'cmd_' + cmd.replace('-', '_') + if cmd not in globals(): + usage() + + qed = QED(open(filename, 'r+b')) + try: + globals()[cmd](qed, *sys.argv[3:]) + except TypeError, e: + sys.stderr.write(globals()[cmd].__doc__ + '\n') + sys.exit(1) + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 0f6d767aa84676a2374d38797f42df4595415369 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:18 +0100 Subject: docs: add dirty bit to qcow2 specification The dirty bit will make it possible to perform lazy refcount updates, where the image file is not kept consistent all the time. Upon opening a dirty image file, it is necessary to perform a consistency check and repair any incorrect refcounts. Therefore the dirty bit must be an incompatible feature bit. We don't want old programs accessing a file with stale refcounts. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- docs/specs/qcow2.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt index 87bf785fe0..339cdc1b58 100644 --- a/docs/specs/qcow2.txt +++ b/docs/specs/qcow2.txt @@ -75,7 +75,12 @@ in the description of a field. Bitmask of incompatible features. An implementation must fail to open an image if an unknown bit is set. - Bits 0-63: Reserved (set to 0) + Bit 0: Dirty bit. If this bit is set then refcounts + may be inconsistent, make sure to scan L1/L2 + tables to repair refcounts before accessing the + image. + + Bits 1-63: Reserved (set to 0) 80 - 87: compatible_features Bitmask of compatible features. An implementation can -- cgit v1.2.3 From c61d0004bc37182cae0f92a865ad2f0625762610 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:19 +0100 Subject: qcow2: introduce dirty bit This patch adds an incompatible feature bit to mark images that have not been closed cleanly. When a dirty image file is opened a consistency check and repair is performed. Update qemu-iotests 031 and 036 since the extension header size changes when we add feature bit table entries. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- block/qcow2.c | 50 +++++++++++++++++++++++++++++++++++++++++++--- block/qcow2.h | 8 ++++++++ tests/qemu-iotests/031.out | 20 +++++++++---------- tests/qemu-iotests/036.out | 4 ++-- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index 870148ddf8..7fe156712a 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -214,6 +214,27 @@ static void report_unsupported_feature(BlockDriverState *bs, } } +/* + * Clears the dirty bit and flushes before if necessary. Only call this + * function when there are no pending requests, it does not guard against + * concurrent requests dirtying the image. + */ +static int qcow2_mark_clean(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + + if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) { + int ret = bdrv_flush(bs); + if (ret < 0) { + return ret; + } + + s->incompatible_features &= ~QCOW2_INCOMPAT_DIRTY; + return qcow2_update_header(bs); + } + return 0; +} + static int qcow2_open(BlockDriverState *bs, int flags) { BDRVQcowState *s = bs->opaque; @@ -287,12 +308,13 @@ static int qcow2_open(BlockDriverState *bs, int flags) s->compatible_features = header.compatible_features; s->autoclear_features = header.autoclear_features; - if (s->incompatible_features != 0) { + if (s->incompatible_features & ~QCOW2_INCOMPAT_MASK) { void *feature_table = NULL; qcow2_read_extensions(bs, header.header_length, ext_end, &feature_table); report_unsupported_feature(bs, feature_table, - s->incompatible_features); + s->incompatible_features & + ~QCOW2_INCOMPAT_MASK); ret = -ENOTSUP; goto fail; } @@ -412,6 +434,22 @@ static int qcow2_open(BlockDriverState *bs, int flags) /* Initialise locks */ qemu_co_mutex_init(&s->lock); + /* Repair image if dirty */ + if ((s->incompatible_features & QCOW2_INCOMPAT_DIRTY) && + !bs->read_only) { + BdrvCheckResult result = {0}; + + ret = qcow2_check_refcounts(bs, &result, BDRV_FIX_ERRORS); + if (ret < 0) { + goto fail; + } + + ret = qcow2_mark_clean(bs); + if (ret < 0) { + goto fail; + } + } + #ifdef DEBUG_ALLOC { BdrvCheckResult result = {0}; @@ -785,6 +823,8 @@ static void qcow2_close(BlockDriverState *bs) qcow2_cache_flush(bs, s->l2_table_cache); qcow2_cache_flush(bs, s->refcount_block_cache); + qcow2_mark_clean(bs); + qcow2_cache_destroy(bs, s->l2_table_cache); qcow2_cache_destroy(bs, s->refcount_block_cache); @@ -949,7 +989,11 @@ int qcow2_update_header(BlockDriverState *bs) /* Feature table */ Qcow2Feature features[] = { - /* no feature defined yet */ + { + .type = QCOW2_FEAT_TYPE_INCOMPATIBLE, + .bit = QCOW2_INCOMPAT_DIRTY_BITNR, + .name = "dirty bit", + }, }; ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE, diff --git a/block/qcow2.h b/block/qcow2.h index 455b6d7cfe..b5fefc08f7 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -110,6 +110,14 @@ enum { QCOW2_FEAT_TYPE_AUTOCLEAR = 2, }; +/* Incompatible feature bits */ +enum { + QCOW2_INCOMPAT_DIRTY_BITNR = 0, + QCOW2_INCOMPAT_DIRTY = 1 << QCOW2_INCOMPAT_DIRTY_BITNR, + + QCOW2_INCOMPAT_MASK = QCOW2_INCOMPAT_DIRTY, +}; + typedef struct Qcow2Feature { uint8_t type; uint8_t bit; diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out index d3cab301d4..297b4587e3 100644 --- a/tests/qemu-iotests/031.out +++ b/tests/qemu-iotests/031.out @@ -54,8 +54,8 @@ header_length 72 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -68,7 +68,7 @@ No errors were found on the image. magic 0x514649fb version 2 -backing_file_offset 0x98 +backing_file_offset 0xc8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -92,8 +92,8 @@ data 'host_device' Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -155,8 +155,8 @@ header_length 104 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -169,7 +169,7 @@ No errors were found on the image. magic 0x514649fb version 3 -backing_file_offset 0xb8 +backing_file_offset 0xe8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -193,8 +193,8 @@ data 'host_device' Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out index 6953e37ab6..ca0fda13d3 100644 --- a/tests/qemu-iotests/036.out +++ b/tests/qemu-iotests/036.out @@ -46,7 +46,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data *** done -- cgit v1.2.3 From dae8796d009ffc4d980db1f87f26eed0074be4ef Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:20 +0100 Subject: docs: add lazy refcounts bit to qcow2 specification The lazy refcounts bit indicates that this image can take advantage of the dirty bit and that refcount updates can be postponed. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- docs/specs/qcow2.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt index 339cdc1b58..36a559d886 100644 --- a/docs/specs/qcow2.txt +++ b/docs/specs/qcow2.txt @@ -86,7 +86,12 @@ in the description of a field. Bitmask of compatible features. An implementation can safely ignore any unknown bits that are set. - Bits 0-63: Reserved (set to 0) + Bit 0: Lazy refcounts bit. If this bit is set then + lazy refcount updates can be used. This means + marking the image file dirty and postponing + refcount metadata updates. + + Bits 1-63: Reserved (set to 0) 88 - 95: autoclear_features Bitmask of auto-clear features. An implementation may only -- cgit v1.2.3 From 91cf8a35e777cdbdc3bf0d39c319ac9f76b7a296 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:21 +0100 Subject: qemu-iotests: ignore qemu-img create lazy_refcounts output Hide the default lazy_refcounts=off output from qemu-img like we do with other image creation options. This ensures that existing golden outputs continue to pass despite the new option that has been added. Note that this patch applies before the one that actually introduces the lazy_refcounts=on|off option. This ensures git-bisect(1) continues to work. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/common.rc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index 5e3a524bc8..cc4e39bbbf 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -113,7 +113,8 @@ _make_test_img() sed -e "s# table_size=0##g" | \ sed -e "s# compat='[^']*'##g" | \ sed -e "s# compat6=off##g" | \ - sed -e "s# static=off##g" + sed -e "s# static=off##g" | \ + sed -e "s# lazy_refcounts=off##g" } _cleanup_test_img() -- cgit v1.2.3 From bfe8043e9214d2fc6572cc72b5f2218308747acd Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:22 +0100 Subject: qcow2: implement lazy refcounts Lazy refcounts is a performance optimization for qcow2 that postpones refcount metadata updates and instead marks the image dirty. In the case of crash or power failure the image will be left in a dirty state and repaired next time it is opened. Reducing metadata I/O is important for cache=writethrough and cache=directsync because these modes guarantee that data is on disk after each write (hence we cannot take advantage of caching updates in RAM). Refcount metadata is not needed for guest->file block address translation and therefore does not need to be on-disk at the time of write completion - this is the motivation behind the lazy refcount optimization. The lazy refcount optimization must be enabled at image creation time: qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=on a.qcow2 10G qemu-system-x86_64 -drive if=virtio,file=a.qcow2,cache=writethrough Update qemu-iotests 031 and 036 since the extension header size changes when we add feature bit table entries. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- block/qcow2-cluster.c | 5 +++- block/qcow2.c | 73 +++++++++++++++++++++++++++++++++++++++++++--- block/qcow2.h | 13 +++++++++ block_int.h | 26 +++++++++-------- tests/qemu-iotests/031.out | 12 ++++---- tests/qemu-iotests/036.out | 2 +- 6 files changed, 107 insertions(+), 24 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index d7e0e19d9c..e179211c57 100644 --- a/block/qcow2-cluster.c +++ b/block/qcow2-cluster.c @@ -662,7 +662,10 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m) qcow2_cache_depends_on_flush(s->l2_table_cache); } - qcow2_cache_set_dependency(bs, s->l2_table_cache, s->refcount_block_cache); + if (qcow2_need_accurate_refcounts(s)) { + qcow2_cache_set_dependency(bs, s->l2_table_cache, + s->refcount_block_cache); + } ret = get_cluster_table(bs, m->offset, &l2_table, &l2_index); if (ret < 0) { goto err; diff --git a/block/qcow2.c b/block/qcow2.c index 7fe156712a..fd5e214431 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -214,6 +214,41 @@ static void report_unsupported_feature(BlockDriverState *bs, } } +/* + * Sets the dirty bit and flushes afterwards if necessary. + * + * The incompatible_features bit is only set if the image file header was + * updated successfully. Therefore it is not required to check the return + * value of this function. + */ +static int qcow2_mark_dirty(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + uint64_t val; + int ret; + + assert(s->qcow_version >= 3); + + if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) { + return 0; /* already dirty */ + } + + val = cpu_to_be64(s->incompatible_features | QCOW2_INCOMPAT_DIRTY); + ret = bdrv_pwrite(bs->file, offsetof(QCowHeader, incompatible_features), + &val, sizeof(val)); + if (ret < 0) { + return ret; + } + ret = bdrv_flush(bs->file); + if (ret < 0) { + return ret; + } + + /* Only treat image as dirty if the header was updated successfully */ + s->incompatible_features |= QCOW2_INCOMPAT_DIRTY; + return 0; +} + /* * Clears the dirty bit and flushes before if necessary. Only call this * function when there are no pending requests, it does not guard against @@ -752,6 +787,11 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs, goto fail; } + if (l2meta.nb_clusters > 0 && + (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS)) { + qcow2_mark_dirty(bs); + } + cluster_offset = l2meta.cluster_offset; assert((cluster_offset & 511) == 0); @@ -994,6 +1034,11 @@ int qcow2_update_header(BlockDriverState *bs) .bit = QCOW2_INCOMPAT_DIRTY_BITNR, .name = "dirty bit", }, + { + .type = QCOW2_FEAT_TYPE_COMPATIBLE, + .bit = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR, + .name = "lazy refcounts", + }, }; ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE, @@ -1176,6 +1221,11 @@ static int qcow2_create2(const char *filename, int64_t total_size, header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE); } + if (flags & BLOCK_FLAG_LAZY_REFCOUNTS) { + header.compatible_features |= + cpu_to_be64(QCOW2_COMPAT_LAZY_REFCOUNTS); + } + ret = bdrv_pwrite(bs, 0, &header, sizeof(header)); if (ret < 0) { goto out; @@ -1289,6 +1339,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options) options->value.s); return -EINVAL; } + } else if (!strcmp(options->name, BLOCK_OPT_LAZY_REFCOUNTS)) { + flags |= options->value.n ? BLOCK_FLAG_LAZY_REFCOUNTS : 0; } options++; } @@ -1299,6 +1351,12 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options) return -EINVAL; } + if (version < 3 && (flags & BLOCK_FLAG_LAZY_REFCOUNTS)) { + fprintf(stderr, "Lazy refcounts only supported with compatibility " + "level 1.1 and above (use compat=1.1 or greater)\n"); + return -EINVAL; + } + return qcow2_create2(filename, sectors, backing_file, backing_fmt, flags, cluster_size, prealloc, options, version); } @@ -1485,10 +1543,12 @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs) return ret; } - ret = qcow2_cache_flush(bs, s->refcount_block_cache); - if (ret < 0) { - qemu_co_mutex_unlock(&s->lock); - return ret; + if (qcow2_need_accurate_refcounts(s)) { + ret = qcow2_cache_flush(bs, s->refcount_block_cache); + if (ret < 0) { + qemu_co_mutex_unlock(&s->lock); + return ret; + } } qemu_co_mutex_unlock(&s->lock); @@ -1603,6 +1663,11 @@ static QEMUOptionParameter qcow2_create_options[] = { .type = OPT_STRING, .help = "Preallocation mode (allowed values: off, metadata)" }, + { + .name = BLOCK_OPT_LAZY_REFCOUNTS, + .type = OPT_FLAG, + .help = "Postpone refcount updates", + }, { NULL } }; diff --git a/block/qcow2.h b/block/qcow2.h index b5fefc08f7..b4eb65470e 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -118,6 +118,14 @@ enum { QCOW2_INCOMPAT_MASK = QCOW2_INCOMPAT_DIRTY, }; +/* Compatible feature bits */ +enum { + QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR = 0, + QCOW2_COMPAT_LAZY_REFCOUNTS = 1 << QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR, + + QCOW2_COMPAT_FEAT_MASK = QCOW2_COMPAT_LAZY_REFCOUNTS, +}; + typedef struct Qcow2Feature { uint8_t type; uint8_t bit; @@ -245,6 +253,11 @@ static inline int qcow2_get_cluster_type(uint64_t l2_entry) } } +/* Check whether refcounts are eager or lazy */ +static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s) +{ + return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY); +} // FIXME Need qcow2_ prefix to global functions diff --git a/block_int.h b/block_int.h index d72317fbe3..6c1d9cafb1 100644 --- a/block_int.h +++ b/block_int.h @@ -31,8 +31,9 @@ #include "qemu-timer.h" #include "qapi-types.h" -#define BLOCK_FLAG_ENCRYPT 1 -#define BLOCK_FLAG_COMPAT6 4 +#define BLOCK_FLAG_ENCRYPT 1 +#define BLOCK_FLAG_COMPAT6 4 +#define BLOCK_FLAG_LAZY_REFCOUNTS 8 #define BLOCK_IO_LIMIT_READ 0 #define BLOCK_IO_LIMIT_WRITE 1 @@ -41,16 +42,17 @@ #define BLOCK_IO_SLICE_TIME 100000000 #define NANOSECONDS_PER_SECOND 1000000000.0 -#define BLOCK_OPT_SIZE "size" -#define BLOCK_OPT_ENCRYPT "encryption" -#define BLOCK_OPT_COMPAT6 "compat6" -#define BLOCK_OPT_BACKING_FILE "backing_file" -#define BLOCK_OPT_BACKING_FMT "backing_fmt" -#define BLOCK_OPT_CLUSTER_SIZE "cluster_size" -#define BLOCK_OPT_TABLE_SIZE "table_size" -#define BLOCK_OPT_PREALLOC "preallocation" -#define BLOCK_OPT_SUBFMT "subformat" -#define BLOCK_OPT_COMPAT_LEVEL "compat" +#define BLOCK_OPT_SIZE "size" +#define BLOCK_OPT_ENCRYPT "encryption" +#define BLOCK_OPT_COMPAT6 "compat6" +#define BLOCK_OPT_BACKING_FILE "backing_file" +#define BLOCK_OPT_BACKING_FMT "backing_fmt" +#define BLOCK_OPT_CLUSTER_SIZE "cluster_size" +#define BLOCK_OPT_TABLE_SIZE "table_size" +#define BLOCK_OPT_PREALLOC "preallocation" +#define BLOCK_OPT_SUBFMT "subformat" +#define BLOCK_OPT_COMPAT_LEVEL "compat" +#define BLOCK_OPT_LAZY_REFCOUNTS "lazy_refcounts" typedef struct BdrvTrackedRequest BdrvTrackedRequest; diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out index 297b4587e3..796c993df2 100644 --- a/tests/qemu-iotests/031.out +++ b/tests/qemu-iotests/031.out @@ -54,7 +54,7 @@ header_length 72 Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -68,7 +68,7 @@ No errors were found on the image. magic 0x514649fb version 2 -backing_file_offset 0xc8 +backing_file_offset 0xf8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -92,7 +92,7 @@ data 'host_device' Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -155,7 +155,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -169,7 +169,7 @@ No errors were found on the image. magic 0x514649fb version 3 -backing_file_offset 0xe8 +backing_file_offset 0x118 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -193,7 +193,7 @@ data 'host_device' Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out index ca0fda13d3..063ca22d66 100644 --- a/tests/qemu-iotests/036.out +++ b/tests/qemu-iotests/036.out @@ -46,7 +46,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 48 +length 96 data *** done -- cgit v1.2.3 From e01c30d3e20eb8cf068b08af46c532f99975527a Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:23 +0100 Subject: qemu-io: add "abort" command to simulate program crash Avoiding data loss and corruption is the top requirement for image file formats. The qemu-io "abort" command makes it possible to simulate program crashes and does not give the image format a chance to cleanly shut down. This command is useful for data integrity test cases. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- qemu-io.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qemu-io.c b/qemu-io.c index 8f3b94b838..d0f4fb70c7 100644 --- a/qemu-io.c +++ b/qemu-io.c @@ -1652,6 +1652,17 @@ static const cmdinfo_t map_cmd = { .oneline = "prints the allocated areas of a file", }; +static int abort_f(int argc, char **argv) +{ + abort(); +} + +static const cmdinfo_t abort_cmd = { + .name = "abort", + .cfunc = abort_f, + .flags = CMD_NOFILE_OK, + .oneline = "simulate a program crash using abort(3)", +}; static int close_f(int argc, char **argv) { @@ -1905,6 +1916,7 @@ int main(int argc, char **argv) add_command(&discard_cmd); add_command(&alloc_cmd); add_command(&map_cmd); + add_command(&abort_cmd); add_args_command(init_args_command); add_check_command(init_check_command); -- cgit v1.2.3 From dc68afe0f3e8be913a0eb639afc9196e0abab261 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:24 +0100 Subject: qemu-iotests: add 039 qcow2 lazy refcounts test This tests establishes the basic post-conditions of the qcow2 lazy refcounts features: 1. If the image was closed normally, it is marked clean. 2. If an allocating write was performed and the image was not closed normally, then it is marked dirty. a. Written data can be read back successfully. b. The image file can be repaired and will be marked clean again. c. The image file is automatically repaired when opened read/write. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/039 | 136 +++++++++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/039.out | 53 ++++++++++++++++++ tests/qemu-iotests/group | 1 + 3 files changed, 190 insertions(+) create mode 100755 tests/qemu-iotests/039 create mode 100644 tests/qemu-iotests/039.out diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039 new file mode 100755 index 0000000000..a749fcf23b --- /dev/null +++ b/tests/qemu-iotests/039 @@ -0,0 +1,136 @@ +#!/bin/bash +# +# Test qcow2 lazy refcounts +# +# Copyright (C) 2012 Red Hat, Inc. +# Copyright IBM, Corp. 2010 +# +# Based on test 038. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# creator +owner=stefanha@linux.vnet.ibm.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +_supported_fmt qcow2 +_supported_proto generic +_supported_os Linux + +size=128M + +echo +echo "== Checking that image is clean on shutdown ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +$QEMU_IO -c "write -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +echo +echo "== Creating a dirty image file ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +echo +echo "== Read-only access must still work ==" + +$QEMU_IO -r -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Repairing the image file must succeed ==" + +$QEMU_IMG check -r all $TEST_IMG + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Data should still be accessible after repair ==" + +$QEMU_IO -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +echo +echo "== Opening a dirty image read/write should repair it ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +$QEMU_IO -c "write 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Creating an image file with lazy_refcounts=off ==" + +IMGOPTS="compat=1.1,lazy_refcounts=off" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must not be set since lazy_refcounts=off +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 + diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out new file mode 100644 index 0000000000..8ad570d33f --- /dev/null +++ b/tests/qemu-iotests/039.out @@ -0,0 +1,53 @@ +QA output created by 039 + +== Checking that image is clean on shutdown == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 +No errors were found on the image. + +== Creating a dirty image file == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +ERROR cluster 5 refcount=0 reference=1 + +2 errors were found on the image. +Data may be corrupted, or further writes to the image may corrupt it. + +== Read-only access must still work == +read 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 + +== Repairing the image file must succeed == +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +Repairing cluster 5 refcount=0 reference=1 +No errors were found on the image. +incompatible_features 0x0 + +== Data should still be accessible after repair == +read 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +== Opening a dirty image read/write should repair it == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +Repairing cluster 5 refcount=0 reference=1 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 + +== Creating an image file with lazy_refcounts=off == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 +No errors were found on the image. +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 7a2c92b6e9..ebb5ca4b41 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -45,3 +45,4 @@ 036 rw auto quick 037 rw auto backing 038 rw auto backing +039 rw auto -- cgit v1.2.3 From b0869a46b2dc2e6e4d6d6cca8bb5c5bf611f33d0 Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 27 Jul 2012 15:14:28 +0200 Subject: qemu-iotests: Be more flexible with image creation options qemu-iotests already filters out image creation options that may be present or not in order to get the same output in both cases. However, often it only considers the default value of the option. Cover all valid values instead so that ./check -o name=value can be used successfull for all of them. Signed-off-by: Kevin Wolf --- tests/qemu-iotests/039.out | 6 +++--- tests/qemu-iotests/common.rc | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out index 8ad570d33f..155a05e109 100644 --- a/tests/qemu-iotests/039.out +++ b/tests/qemu-iotests/039.out @@ -1,14 +1,14 @@ QA output created by 039 == Checking that image is clean on shutdown == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x0 No errors were found on the image. == Creating a dirty image file == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x1 @@ -34,7 +34,7 @@ read 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) == Opening a dirty image read/write should repair it == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x1 diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index cc4e39bbbf..7782808a26 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -110,11 +110,11 @@ _make_test_img() sed -e "s#$IMGFMT#IMGFMT#g" | \ sed -e "s# encryption=off##g" | \ sed -e "s# cluster_size=[0-9]\\+##g" | \ - sed -e "s# table_size=0##g" | \ + sed -e "s# table_size=[0-9]\\+##g" | \ sed -e "s# compat='[^']*'##g" | \ - sed -e "s# compat6=off##g" | \ - sed -e "s# static=off##g" | \ - sed -e "s# lazy_refcounts=off##g" + sed -e "s# compat6=\\(on\\|off\\)##g" | \ + sed -e "s# static=\\(on\\|off\\)##g" | \ + sed -e "s# lazy_refcounts=\\(on\\|off\\)##g" } _cleanup_test_img() -- cgit v1.2.3 From 20caf0f766b48fbbf718eb20aaddb7ffa356a21f Mon Sep 17 00:00:00 2001 From: Dong Xu Wang Date: Mon, 6 Aug 2012 10:18:42 +0800 Subject: qemu-img: use QemuOpts instead of QEMUOptionParameter in resize function Signed-off-by: Dong Xu Wang Reviewed-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- qemu-img.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/qemu-img.c b/qemu-img.c index b866f8081e..94a31ad9f0 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -1567,14 +1567,19 @@ static int img_resize(int argc, char **argv) const char *filename, *fmt, *size; int64_t n, total_size; BlockDriverState *bs = NULL; - QEMUOptionParameter *param; - QEMUOptionParameter resize_options[] = { - { - .name = BLOCK_OPT_SIZE, - .type = OPT_SIZE, - .help = "Virtual disk size" + QemuOpts *param; + static QemuOptsList resize_options = { + .name = "resize_options", + .head = QTAILQ_HEAD_INITIALIZER(resize_options.head), + .desc = { + { + .name = BLOCK_OPT_SIZE, + .type = QEMU_OPT_SIZE, + .help = "Virtual disk size" + }, { + /* end of list */ + } }, - { NULL } }; /* Remove size from argv manually so that negative numbers are not treated @@ -1624,14 +1629,15 @@ static int img_resize(int argc, char **argv) } /* Parse size */ - param = parse_option_parameters("", resize_options, NULL); - if (set_option_parameter(param, BLOCK_OPT_SIZE, size)) { + param = qemu_opts_create(&resize_options, NULL, 0, NULL); + if (qemu_opt_set(param, BLOCK_OPT_SIZE, size)) { /* Error message already printed when size parsing fails */ ret = -1; + qemu_opts_del(param); goto out; } - n = get_option_parameter(param, BLOCK_OPT_SIZE)->value.n; - free_option_parameters(param); + n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0); + qemu_opts_del(param); bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR); if (!bs) { -- cgit v1.2.3