bitmaps patches for 2020-05-26

- fix non-blockdev migration of bitmaps when mirror job is in use
 - add bitmap sizing to 'qemu-img measure'
 - add 'qemu-img convert --bitmaps'
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEccLMIrHEYCkn0vOqp6FrSiUnQ2oFAl7QAA8ACgkQp6FrSiUn
 Q2pRxgf+N+X929X+o6kELEFBXMckOn8LC94A/6RjZKYywY3nV1ynESYflwznbYqx
 xQzXA2fTHK0O9n9av9pSMqs612HvxQyCJ51btit9QyOKW79//5QiNBkmVzXRwaN7
 hNnq5eHgDtH7O0jJPbJ1kxY6oufaudl/npEbBINlXknkIyvtChRfMhkAEOcV0N/y
 OxnmFm02r3TENoDmLdCmmi6iSDBvUl2th4p5ICZ0D9qUOnL83NNnMoYNF1goH25Z
 AlJWLe577mV5rw3w+HpoPutJZffPPiveCgW04WaEOhF4b0RzJFYazw9DM7PQiC+T
 hDzL3m15ZmgocgDLbbXEJiCVsDUwTA==
 =83Zm
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/ericb/tags/pull-bitmaps-2020-05-26-v3' into staging

bitmaps patches for 2020-05-26

- fix non-blockdev migration of bitmaps when mirror job is in use
- add bitmap sizing to 'qemu-img measure'
- add 'qemu-img convert --bitmaps'

# gpg: Signature made Thu 28 May 2020 19:16:47 BST
# gpg:                using RSA key 71C2CC22B1C4602927D2F3AAA7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>" [full]
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>" [full]
# gpg:                 aka "[jpeg image of size 6874]" [full]
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-bitmaps-2020-05-26-v3:
  iotests: Add test 291 to for qemu-img bitmap coverage
  qemu-img: Add convert --bitmaps option
  qemu-img: Factor out code for merging bitmaps
  qcow2: Expose bitmaps' size during measure
  iotests: Fix test 178
  migration: forbid bitmap migration by generated node-name
  migration: add_bitmaps_to_list: check disk name once
  iotests: 194: test also migration of dirty bitmap
  migration: fix bitmaps pre-blockdev migration with mirror job
  block/dirty-bitmap: add bdrv_has_named_bitmaps helper
  migration: refactor init_dirty_bitmap_migration

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2020-05-29 19:25:54 +01:00
commit ce20db593f
21 changed files with 582 additions and 77 deletions

View File

@ -552,7 +552,7 @@ static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts,
* Unallocated blocks are still encrypted so allocation status makes no
* difference to the file size.
*/
info = g_new(BlockMeasureInfo, 1);
info = g_new0(BlockMeasureInfo, 1);
info->fully_allocated = luks_payload_size + size;
info->required = luks_payload_size + size;
return info;

View File

@ -818,6 +818,19 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)
return false;
}
bool bdrv_has_named_bitmaps(BlockDriverState *bs)
{
BdrvDirtyBitmap *bm;
QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
if (bdrv_dirty_bitmap_name(bm)) {
return true;
}
}
return false;
}
/* Called with BQL taken. */
void bdrv_dirty_bitmap_set_persistence(BdrvDirtyBitmap *bitmap, bool persistent)
{

View File

@ -1755,3 +1755,39 @@ bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs)
return s->qcow_version >= 3;
}
/*
* Compute the space required for bitmaps in @bs.
*
* The computation is based as if copying to a new image with the
* given @cluster_size, which may differ from the cluster size in @bs.
*/
uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs,
uint32_t cluster_size)
{
uint64_t bitmaps_size = 0;
BdrvDirtyBitmap *bm;
size_t bitmap_dir_size = 0;
FOR_EACH_DIRTY_BITMAP(bs, bm) {
if (bdrv_dirty_bitmap_get_persistence(bm)) {
const char *name = bdrv_dirty_bitmap_name(bm);
uint32_t granularity = bdrv_dirty_bitmap_granularity(bm);
uint64_t bmbytes =
get_bitmap_bytes_needed(bdrv_dirty_bitmap_size(bm),
granularity);
uint64_t bmclusters = DIV_ROUND_UP(bmbytes, cluster_size);
/* Assume the entire bitmap is allocated */
bitmaps_size += bmclusters * cluster_size;
/* Also reserve space for the bitmap table entries */
bitmaps_size += ROUND_UP(bmclusters * sizeof(uint64_t),
cluster_size);
/* And space for contribution to bitmap directory size */
bitmap_dir_size += calc_dir_entry_size(strlen(name), 0);
}
}
bitmaps_size += ROUND_UP(bitmap_dir_size, cluster_size);
return bitmaps_size;
}

View File

@ -4953,16 +4953,24 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
required = virtual_size;
}
info = g_new(BlockMeasureInfo, 1);
info = g_new0(BlockMeasureInfo, 1);
info->fully_allocated =
qcow2_calc_prealloc_size(virtual_size, cluster_size,
ctz32(refcount_bits)) + luks_payload_size;
/* Remove data clusters that are not required. This overestimates the
/*
* Remove data clusters that are not required. This overestimates the
* required size because metadata needed for the fully allocated file is
* still counted.
* still counted. Show bitmaps only if both source and destination
* would support them.
*/
info->required = info->fully_allocated - virtual_size + required;
info->has_bitmaps = version >= 3 && in_bs &&
bdrv_supports_persistent_dirty_bitmap(in_bs);
if (info->has_bitmaps) {
info->bitmaps = qcow2_get_persistent_dirty_bitmap_size(in_bs,
cluster_size);
}
return info;
err:

View File

@ -783,6 +783,8 @@ int qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,
const char *name,
Error **errp);
bool qcow2_supports_persistent_dirty_bitmap(BlockDriverState *bs);
uint64_t qcow2_get_persistent_dirty_bitmap_size(BlockDriverState *bs,
uint32_t cluster_size);
ssize_t coroutine_fn
qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,

View File

@ -359,7 +359,7 @@ static BlockMeasureInfo *raw_measure(QemuOpts *opts, BlockDriverState *in_bs,
BDRV_SECTOR_SIZE);
}
info = g_new(BlockMeasureInfo, 1);
info = g_new0(BlockMeasureInfo, 1);
info->required = required;
/* Unallocated sectors count towards the file size in raw images */

View File

@ -162,6 +162,10 @@ Parameters to convert subcommand:
.. program:: qemu-img-convert
.. option:: --bitmaps
Additionally copy all persistent bitmaps from the top layer of the source
.. option:: -n
Skip the creation of the target volume
@ -397,7 +401,7 @@ Command description:
4
Error on reading data
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
Convert the disk image *FILENAME* or a snapshot *SNAPSHOT_PARAM*
to disk image *OUTPUT_FILENAME* using format *OUTPUT_FMT*. It can
@ -616,6 +620,7 @@ Command description:
required size: 524288
fully allocated size: 1074069504
bitmaps size: 0
The ``required size`` is the file size of the new image. It may be smaller
than the virtual disk size if the image format supports compact representation.
@ -625,6 +630,12 @@ Command description:
occupy with the exception of internal snapshots, dirty bitmaps, vmstate data,
and other advanced image format features.
The ``bitmaps size`` is the additional size required in order to
copy bitmaps from a source image in addition to the guest-visible
data; the line is omitted if either source or destination lacks
bitmap support, or 0 if bitmaps are supported but there is nothing
to copy.
.. option:: snapshot [--object OBJECTDEF] [--image-opts] [-U] [-q] [-l | -a SNAPSHOT | -c SNAPSHOT | -d SNAPSHOT] FILENAME
List, apply, create or delete snapshots in image *FILENAME*.

View File

@ -95,6 +95,7 @@ int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
void bdrv_dirty_bitmap_truncate(BlockDriverState *bs, int64_t bytes);
bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap);
bool bdrv_has_readonly_bitmaps(BlockDriverState *bs);
bool bdrv_has_named_bitmaps(BlockDriverState *bs);
bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap *bitmap);
bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);

View File

@ -268,57 +268,118 @@ static void dirty_bitmap_mig_cleanup(void)
}
/* Called with iothread lock taken. */
static int init_dirty_bitmap_migration(void)
static int add_bitmaps_to_list(BlockDriverState *bs, const char *bs_name)
{
BlockDriverState *bs;
BdrvDirtyBitmap *bitmap;
DirtyBitmapMigBitmapState *dbms;
Error *local_err = NULL;
bitmap = bdrv_dirty_bitmap_first(bs);
if (!bitmap) {
return 0;
}
if (!bs_name || strcmp(bs_name, "") == 0) {
error_report("Bitmap '%s' in unnamed node can't be migrated",
bdrv_dirty_bitmap_name(bitmap));
return -1;
}
if (bs_name[0] == '#') {
error_report("Bitmap '%s' in a node with auto-generated "
"name '%s' can't be migrated",
bdrv_dirty_bitmap_name(bitmap), bs_name);
return -1;
}
FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
if (!bdrv_dirty_bitmap_name(bitmap)) {
continue;
}
if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
error_report_err(local_err);
return -1;
}
bdrv_ref(bs);
bdrv_dirty_bitmap_set_busy(bitmap, true);
dbms = g_new0(DirtyBitmapMigBitmapState, 1);
dbms->bs = bs;
dbms->node_name = bs_name;
dbms->bitmap = bitmap;
dbms->total_sectors = bdrv_nb_sectors(bs);
dbms->sectors_per_chunk = CHUNK_SIZE * 8 *
bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS;
if (bdrv_dirty_bitmap_enabled(bitmap)) {
dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
}
if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
}
QSIMPLEQ_INSERT_TAIL(&dirty_bitmap_mig_state.dbms_list,
dbms, entry);
}
return 0;
}
/* Called with iothread lock taken. */
static int init_dirty_bitmap_migration(void)
{
BlockDriverState *bs;
DirtyBitmapMigBitmapState *dbms;
GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
BlockBackend *blk;
dirty_bitmap_mig_state.bulk_completed = false;
dirty_bitmap_mig_state.prev_bs = NULL;
dirty_bitmap_mig_state.prev_bitmap = NULL;
dirty_bitmap_mig_state.no_bitmaps = false;
/*
* Use blockdevice name for direct (or filtered) children of named block
* backends.
*/
for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
const char *name = blk_name(blk);
if (!name || strcmp(name, "") == 0) {
continue;
}
bs = blk_bs(blk);
/* Skip filters without bitmaps */
while (bs && bs->drv && bs->drv->is_filter &&
!bdrv_has_named_bitmaps(bs))
{
if (bs->backing) {
bs = bs->backing->bs;
} else if (bs->file) {
bs = bs->file->bs;
} else {
bs = NULL;
}
}
if (bs && bs->drv && !bs->drv->is_filter) {
if (add_bitmaps_to_list(bs, name)) {
goto fail;
}
g_hash_table_add(handled_by_blk, bs);
}
}
for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
const char *name = bdrv_get_device_or_node_name(bs);
if (g_hash_table_contains(handled_by_blk, bs)) {
continue;
}
FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
if (!bdrv_dirty_bitmap_name(bitmap)) {
continue;
}
if (!name || strcmp(name, "") == 0) {
error_report("Found bitmap '%s' in unnamed node %p. It can't "
"be migrated", bdrv_dirty_bitmap_name(bitmap), bs);
goto fail;
}
if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT,
&local_err)) {
error_report_err(local_err);
goto fail;
}
bdrv_ref(bs);
bdrv_dirty_bitmap_set_busy(bitmap, true);
dbms = g_new0(DirtyBitmapMigBitmapState, 1);
dbms->bs = bs;
dbms->node_name = name;
dbms->bitmap = bitmap;
dbms->total_sectors = bdrv_nb_sectors(bs);
dbms->sectors_per_chunk = CHUNK_SIZE * 8 *
bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS;
if (bdrv_dirty_bitmap_enabled(bitmap)) {
dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
}
if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
}
QSIMPLEQ_INSERT_TAIL(&dirty_bitmap_mig_state.dbms_list,
dbms, entry);
if (add_bitmaps_to_list(bs, bdrv_get_node_name(bs))) {
goto fail;
}
}
@ -331,9 +392,12 @@ static int init_dirty_bitmap_migration(void)
dirty_bitmap_mig_state.no_bitmaps = true;
}
g_hash_table_destroy(handled_by_blk);
return 0;
fail:
g_hash_table_destroy(handled_by_blk);
dirty_bitmap_mig_cleanup();
return -1;

View File

@ -636,18 +636,24 @@
# efficiently so file size may be smaller than virtual disk size.
#
# The values are upper bounds that are guaranteed to fit the new image file.
# Subsequent modification, such as internal snapshot or bitmap creation, may
# require additional space and is not covered here.
# Subsequent modification, such as internal snapshot or further bitmap
# creation, may require additional space and is not covered here.
#
# @required: Size required for a new image file, in bytes.
# @required: Size required for a new image file, in bytes, when copying just
# allocated guest-visible contents.
#
# @fully-allocated: Image file size, in bytes, once data has been written
# to all sectors.
# to all sectors, when copying just guest-visible contents.
#
# @bitmaps: Additional size required if all the top-level bitmap metadata
# in the source image were to be copied to the destination,
# present only when source and destination both support
# persistent bitmaps. (since 5.1)
#
# Since: 2.10
##
{ 'struct': 'BlockMeasureInfo',
'data': {'required': 'int', 'fully-allocated': 'int'} }
'data': {'required': 'int', 'fully-allocated': 'int', '*bitmaps': 'int'} }
##
# @query-block:

View File

@ -46,9 +46,9 @@ SRST
ERST
DEF("convert", img_convert,
"convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
"convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
SRST
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
ERST
DEF("create", img_create,

View File

@ -78,6 +78,7 @@ enum {
OPTION_ENABLE = 272,
OPTION_DISABLE = 273,
OPTION_MERGE = 274,
OPTION_BITMAPS = 275,
};
typedef enum OutputFormat {
@ -191,6 +192,7 @@ static void QEMU_NORETURN help(void)
" hiding corruption that has already occurred.\n"
"\n"
"Parameters to convert subcommand:\n"
" '--bitmaps' copies all top-level persistent bitmaps to destination\n"
" '-m' specifies how many coroutines work in parallel during the convert\n"
" process (defaults to 8)\n"
" '-W' allow to write to the target out of order rather than sequential\n"
@ -1638,6 +1640,24 @@ out4:
return ret;
}
/* Convenience wrapper around qmp_block_dirty_bitmap_merge */
static void do_dirty_bitmap_merge(const char *dst_node, const char *dst_name,
const char *src_node, const char *src_name,
Error **errp)
{
BlockDirtyBitmapMergeSource *merge_src;
BlockDirtyBitmapMergeSourceList *list;
merge_src = g_new0(BlockDirtyBitmapMergeSource, 1);
merge_src->type = QTYPE_QDICT;
merge_src->u.external.node = g_strdup(src_node);
merge_src->u.external.name = g_strdup(src_name);
list = g_new0(BlockDirtyBitmapMergeSourceList, 1);
list->value = merge_src;
qmp_block_dirty_bitmap_merge(dst_node, dst_name, list, errp);
qapi_free_BlockDirtyBitmapMergeSourceList(list);
}
enum ImgConvertBlockStatus {
BLK_DATA,
BLK_ZERO,
@ -2121,6 +2141,39 @@ static int convert_do_copy(ImgConvertState *s)
return s->ret;
}
static int convert_copy_bitmaps(BlockDriverState *src, BlockDriverState *dst)
{
BdrvDirtyBitmap *bm;
Error *err = NULL;
FOR_EACH_DIRTY_BITMAP(src, bm) {
const char *name;
if (!bdrv_dirty_bitmap_get_persistence(bm)) {
continue;
}
name = bdrv_dirty_bitmap_name(bm);
qmp_block_dirty_bitmap_add(dst->node_name, name,
true, bdrv_dirty_bitmap_granularity(bm),
true, true,
true, !bdrv_dirty_bitmap_enabled(bm),
&err);
if (err) {
error_reportf_err(err, "Failed to create bitmap %s: ", name);
return -1;
}
do_dirty_bitmap_merge(dst->node_name, name, src->node_name, name,
&err);
if (err) {
error_reportf_err(err, "Failed to populate bitmap %s: ", name);
return -1;
}
}
return 0;
}
#define MAX_BUF_SECTORS 32768
static int img_convert(int argc, char **argv)
@ -2142,6 +2195,7 @@ static int img_convert(int argc, char **argv)
int64_t ret = -EINVAL;
bool force_share = false;
bool explict_min_sparse = false;
bool bitmaps = false;
ImgConvertState s = (ImgConvertState) {
/* Need at least 4k of zeros for sparse detection */
@ -2161,6 +2215,7 @@ static int img_convert(int argc, char **argv)
{"target-image-opts", no_argument, 0, OPTION_TARGET_IMAGE_OPTS},
{"salvage", no_argument, 0, OPTION_SALVAGE},
{"target-is-zero", no_argument, 0, OPTION_TARGET_IS_ZERO},
{"bitmaps", no_argument, 0, OPTION_BITMAPS},
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WU",
@ -2286,6 +2341,9 @@ static int img_convert(int argc, char **argv)
*/
s.has_zero_init = true;
break;
case OPTION_BITMAPS:
bitmaps = true;
break;
}
}
@ -2347,7 +2405,6 @@ static int img_convert(int argc, char **argv)
goto fail_getopt;
}
/* ret is still -EINVAL until here */
ret = bdrv_parse_cache_mode(src_cache, &src_flags, &src_writethrough);
if (ret < 0) {
@ -2507,6 +2564,20 @@ static int img_convert(int argc, char **argv)
}
}
/* Determine if bitmaps need copying */
if (bitmaps) {
if (s.src_num > 1) {
error_report("Copying bitmaps only possible with single source");
ret = -1;
goto out;
}
if (!bdrv_supports_persistent_dirty_bitmap(blk_bs(s.src[0]))) {
error_report("Source lacks bitmap support");
ret = -1;
goto out;
}
}
/*
* The later open call will need any decryption secrets, and
* bdrv_create() will purge "opts", so extract them now before
@ -2515,9 +2586,7 @@ static int img_convert(int argc, char **argv)
if (!skip_create) {
open_opts = qdict_new();
qemu_opt_foreach(opts, img_add_key_secrets, open_opts, &error_abort);
}
if (!skip_create) {
/* Create the new image */
ret = bdrv_create(drv, out_filename, opts, &local_err);
if (ret < 0) {
@ -2555,6 +2624,13 @@ static int img_convert(int argc, char **argv)
}
out_bs = blk_bs(s.target);
if (bitmaps && !bdrv_supports_persistent_dirty_bitmap(out_bs)) {
error_report("Format driver '%s' does not support bitmaps",
out_bs->drv->format_name);
ret = -1;
goto out;
}
if (s.compressed && !block_driver_can_compress(out_bs->drv)) {
error_report("Compression not supported for this file format");
ret = -1;
@ -2614,6 +2690,12 @@ static int img_convert(int argc, char **argv)
}
ret = convert_do_copy(&s);
/* Now copy the bitmaps */
if (bitmaps && ret == 0) {
ret = convert_copy_bitmaps(blk_bs(s.src[0]), out_bs);
}
out:
if (!ret) {
qemu_progress_print(100, 0);
@ -4714,21 +4796,11 @@ static int img_bitmap(int argc, char **argv)
qmp_block_dirty_bitmap_disable(bs->node_name, bitmap, &err);
op = "disable";
break;
case BITMAP_MERGE: {
BlockDirtyBitmapMergeSource *merge_src;
BlockDirtyBitmapMergeSourceList *list;
merge_src = g_new0(BlockDirtyBitmapMergeSource, 1);
merge_src->type = QTYPE_QDICT;
merge_src->u.external.node = g_strdup(src_bs->node_name);
merge_src->u.external.name = g_strdup(act->src);
list = g_new0(BlockDirtyBitmapMergeSourceList, 1);
list->value = merge_src;
qmp_block_dirty_bitmap_merge(bs->node_name, bitmap, list, &err);
qapi_free_BlockDirtyBitmapMergeSourceList(list);
case BITMAP_MERGE:
do_dirty_bitmap_merge(bs->node_name, bitmap, src_bs->node_name,
act->src, &err);
op = "merge";
break;
}
default:
g_assert_not_reached();
}
@ -5302,6 +5374,9 @@ static int img_measure(int argc, char **argv)
if (output_format == OFORMAT_HUMAN) {
printf("required size: %" PRIu64 "\n", info->required);
printf("fully allocated size: %" PRIu64 "\n", info->fully_allocated);
if (info->has_bitmaps) {
printf("bitmaps size: %" PRIu64 "\n", info->bitmaps);
}
} else {
dump_json_block_measure_info(info);
}

View File

@ -13,7 +13,7 @@ qemu-img: Invalid option list: ,
qemu-img: Invalid parameter 'snapshot.foo'
qemu-img: Failed in parsing snapshot param 'snapshot.foo'
qemu-img: --output must be used with human or json as argument.
qemu-img: Image size must be less than 8 EiB!
qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807.
qemu-img: Unknown file format 'foo'
== Size calculation for a new file (human) ==
@ -37,6 +37,7 @@ qemu-img: The image size is too large (try using a larger cluster size)
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
required size: 196608
fully allocated size: 196608
bitmaps size: 0
converted image file size in bytes: 196608
@ -45,6 +46,7 @@ converted image file size in bytes: 196608
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
required size: 393216
fully allocated size: 1074135040
bitmaps size: 0
wrote 512/512 bytes at offset 512
512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 65536/65536 bytes at offset 65536
@ -53,6 +55,7 @@ wrote 64512/64512 bytes at offset 134217728
63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
required size: 589824
fully allocated size: 1074135040
bitmaps size: 0
converted image file size in bytes: 524288
@ -60,6 +63,7 @@ converted image file size in bytes: 524288
required size: 524288
fully allocated size: 1074135040
bitmaps size: 0
converted image file size in bytes: 458752
@ -67,16 +71,19 @@ converted image file size in bytes: 458752
required size: 1074135040
fully allocated size: 1074135040
bitmaps size: 0
== qcow2 input image and LUKS encryption ==
required size: 2686976
fully allocated size: 1076232192
bitmaps size: 0
== qcow2 input image and preallocation (human) ==
required size: 1074135040
fully allocated size: 1074135040
bitmaps size: 0
converted image file size in bytes: 1074135040
@ -87,6 +94,7 @@ wrote 8388608/8388608 bytes at offset 0
8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
required size: 8716288
fully allocated size: 8716288
bitmaps size: 0
converted image file size in bytes: 8716288
@ -173,6 +181,7 @@ qemu-img: The image size is too large (try using a larger cluster size)
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
{
"bitmaps": 0,
"required": 196608,
"fully-allocated": 196608
}
@ -183,6 +192,7 @@ converted image file size in bytes: 196608
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
{
"bitmaps": 0,
"required": 393216,
"fully-allocated": 1074135040
}
@ -193,6 +203,7 @@ wrote 65536/65536 bytes at offset 65536
wrote 64512/64512 bytes at offset 134217728
63 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
{
"bitmaps": 0,
"required": 589824,
"fully-allocated": 1074135040
}
@ -202,6 +213,7 @@ converted image file size in bytes: 524288
== qcow2 input image with internal snapshot (json) ==
{
"bitmaps": 0,
"required": 524288,
"fully-allocated": 1074135040
}
@ -211,6 +223,7 @@ converted image file size in bytes: 458752
== qcow2 input image and a backing file (json) ==
{
"bitmaps": 0,
"required": 1074135040,
"fully-allocated": 1074135040
}
@ -218,6 +231,7 @@ converted image file size in bytes: 458752
== qcow2 input image and LUKS encryption ==
{
"bitmaps": 0,
"required": 2686976,
"fully-allocated": 1076232192
}
@ -225,6 +239,7 @@ converted image file size in bytes: 458752
== qcow2 input image and preallocation (json) ==
{
"bitmaps": 0,
"required": 1074135040,
"fully-allocated": 1074135040
}
@ -237,6 +252,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8388608
wrote 8388608/8388608 bytes at offset 0
8 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
{
"bitmaps": 0,
"required": 8716288,
"fully-allocated": 8716288
}

View File

@ -13,7 +13,7 @@ qemu-img: Invalid option list: ,
qemu-img: Invalid parameter 'snapshot.foo'
qemu-img: Failed in parsing snapshot param 'snapshot.foo'
qemu-img: --output must be used with human or json as argument.
qemu-img: Image size must be less than 8 EiB!
qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807.
qemu-img: Unknown file format 'foo'
== Size calculation for a new file (human) ==

View File

@ -2,7 +2,7 @@
#
# qemu-img measure sub-command tests on huge qcow2 files
#
# Copyright (C) 2017 Red Hat, Inc.
# Copyright (C) 2017-2020 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -42,7 +42,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2
_supported_proto file
echo "== Huge file =="
echo "== Huge file without bitmaps =="
echo
_make_test_img -o 'cluster_size=2M' 2T
@ -51,6 +51,49 @@ $QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"
$QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"
$QEMU_IMG measure -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG"
echo
echo "== Huge file with bitmaps =="
echo
$QEMU_IMG bitmap --add --granularity 512 -f qcow2 "$TEST_IMG" b1
$QEMU_IMG bitmap --add -g 2M -f qcow2 "$TEST_IMG" b2
# No bitmap without a source
$QEMU_IMG measure -O qcow2 --size 10M
# No bitmap output, since raw does not support it
$QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"
# No bitmap output, since no bitmaps on raw source. Munge required size, as
# some filesystems store the qcow2 file with less sparseness than others
$QEMU_IMG measure -O qcow2 -f raw "$TEST_IMG" |
sed '/^required size:/ s/[0-9][0-9]*/SIZE/'
# No bitmap output, since v2 does not support it
$QEMU_IMG measure -O qcow2 -o compat=0.10 -f qcow2 "$TEST_IMG"
# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory
echo
val2T=$((2*1024*1024*1024*1024))
cluster=$((64*1024))
b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster ))
b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster ))
echo expected bitmap $((b1clusters * cluster +
(b1clusters * 8 + cluster - 1) / cluster * cluster +
b2clusters * cluster +
(b2clusters * 8 + cluster - 1) / cluster * cluster +
cluster))
$QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"
# Compute expected output: bitmap clusters + bitmap tables + bitmaps directory
echo
cluster=$((2*1024*1024))
b1clusters=$(( (val2T/512/8 + cluster - 1) / cluster ))
b2clusters=$(( (val2T/2/1024/1024/8 + cluster - 1) / cluster ))
echo expected bitmap $((b1clusters * cluster +
(b1clusters * 8 + cluster - 1) / cluster * cluster +
b2clusters * cluster +
(b2clusters * 8 + cluster - 1) / cluster * cluster +
cluster))
$QEMU_IMG measure --output=json -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG"
# success, all done
echo "*** done"
rm -f $seq.full

View File

@ -1,11 +1,36 @@
QA output created by 190
== Huge file ==
== Huge file without bitmaps ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552
required size: 2199023255552
fully allocated size: 2199023255552
required size: 335806464
fully allocated size: 2199359062016
bitmaps size: 0
required size: 18874368
fully allocated size: 2199042129920
bitmaps size: 0
== Huge file with bitmaps ==
required size: 327680
fully allocated size: 10813440
required size: 2199023255552
fully allocated size: 2199023255552
required size: SIZE
fully allocated size: 17170432
required size: 335806464
fully allocated size: 2199359062016
expected bitmap 537198592
required size: 335806464
fully allocated size: 2199359062016
bitmaps size: 537198592
expected bitmap 545259520
{
"bitmaps": 545259520,
"required": 18874368,
"fully-allocated": 2199042129920
}
*** done

View File

@ -42,6 +42,8 @@ with iotests.FilePath('source.img') as source_img_path, \
.add_incoming('unix:{0}'.format(migration_sock_path))
.launch())
source_vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0')
iotests.log('Launching NBD server on destination...')
iotests.log(dest_vm.qmp('nbd-server-start', addr={'type': 'unix', 'data': {'path': nbd_sock_path}}))
iotests.log(dest_vm.qmp('nbd-server-add', device='drive0', writable=True))
@ -61,12 +63,14 @@ with iotests.FilePath('source.img') as source_img_path, \
filters=[iotests.filter_qmp_event])
iotests.log('Starting migration...')
source_vm.qmp('migrate-set-capabilities',
capabilities=[{'capability': 'events', 'state': True}])
dest_vm.qmp('migrate-set-capabilities',
capabilities=[{'capability': 'events', 'state': True}])
capabilities = [{'capability': 'events', 'state': True},
{'capability': 'dirty-bitmaps', 'state': True}]
source_vm.qmp('migrate-set-capabilities', capabilities=capabilities)
dest_vm.qmp('migrate-set-capabilities', capabilities=capabilities)
iotests.log(source_vm.qmp('migrate', uri='unix:{0}'.format(migration_sock_path)))
source_vm.qmp_log('migrate-start-postcopy')
while True:
event1 = source_vm.event_wait('MIGRATION')
iotests.log(event1, filters=[iotests.filter_qmp_event])
@ -82,3 +86,5 @@ with iotests.FilePath('source.img') as source_img_path, \
iotests.log('Stopping the NBD server on destination...')
iotests.log(dest_vm.qmp('nbd-server-stop'))
break
iotests.log(source_vm.qmp('query-block')['return'][0]['dirty-bitmaps'])

View File

@ -1,4 +1,6 @@
Launching VMs...
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": "drive0"}}
{"return": {}}
Launching NBD server on destination...
{"return": {}}
{"return": {}}
@ -8,11 +10,15 @@ Waiting for `drive-mirror` to complete...
{"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_READY", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
Starting migration...
{"return": {}}
{"execute": "migrate-start-postcopy", "arguments": {}}
{"return": {}}
{"data": {"status": "setup"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"status": "active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"status": "postcopy-active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"status": "completed"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
Gracefully ending the `drive-mirror` job on source...
{"return": {}}
{"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
Stopping the NBD server on destination...
{"return": {}}
[{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", "persistent": false, "recording": true, "status": "active"}]

112
tests/qemu-iotests/291 Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env bash
#
# Test qemu-img bitmap handling
#
# Copyright (C) 2018-2020 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
seq="$(basename $0)"
echo "QA output created by $seq"
status=1 # failure is the default!
_cleanup()
{
_cleanup_test_img
nbd_server_stop
}
trap "_cleanup; exit \$status" 0 1 2 3 15
# get standard environment, filters and checks
. ./common.rc
. ./common.filter
. ./common.nbd
_supported_fmt qcow2
_supported_proto file
_supported_os Linux
_require_command QEMU_NBD
echo
echo "=== Initial image setup ==="
echo
# Create backing image with one bitmap
TEST_IMG="$TEST_IMG.base" _make_test_img 10M
$QEMU_IMG bitmap --add -f $IMGFMT "$TEST_IMG.base" b0
$QEMU_IO -c 'w 3M 1M' -f $IMGFMT "$TEST_IMG.base" | _filter_qemu_io
# Create initial image and populate two bitmaps: one active, one inactive.
ORIG_IMG=$TEST_IMG
TEST_IMG=$TEST_IMG.orig
_make_test_img -b "$ORIG_IMG.base" -F $IMGFMT 10M
$QEMU_IO -c 'w 0 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
$QEMU_IMG bitmap --add -g 512k -f $IMGFMT "$TEST_IMG" b1
$QEMU_IMG bitmap --add --disable -f $IMGFMT "$TEST_IMG" b2
$QEMU_IO -c 'w 3M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
$QEMU_IMG bitmap --clear -f $IMGFMT "$TEST_IMG" b1
$QEMU_IO -c 'w 1M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
$QEMU_IMG bitmap --disable -f $IMGFMT "$TEST_IMG" b1
$QEMU_IMG bitmap --enable -f $IMGFMT "$TEST_IMG" b2
$QEMU_IO -c 'w 2M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
echo
echo "=== Bitmap preservation not possible to non-qcow2 ==="
echo
TEST_IMG=$ORIG_IMG
$QEMU_IMG convert --bitmaps -O raw "$TEST_IMG.orig" "$TEST_IMG" &&
echo "unexpected success"
echo
echo "=== Convert with bitmap preservation ==="
echo
# Only bitmaps from the active layer are copied
$QEMU_IMG convert --bitmaps -O qcow2 "$TEST_IMG.orig" "$TEST_IMG"
$QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific
# But we can also merge in bitmaps from other layers. This test is a bit
# contrived to cover more code paths, in reality, you could merge directly
# into b0 without going through tmp
$QEMU_IMG bitmap --add --disable -f $IMGFMT "$TEST_IMG" b0
$QEMU_IMG bitmap --add --merge b0 -b "$TEST_IMG.base" -F $IMGFMT \
-f $IMGFMT "$TEST_IMG" tmp
$QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0
$QEMU_IMG bitmap --remove --image-opts \
driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp
$QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific
echo
echo "=== Check bitmap contents ==="
echo
# x-dirty-bitmap is a hack for reading bitmaps; it abuses block status to
# report "data":false for portions of the bitmap which are set
IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket"
nbd_server_start_unix_socket -r -f qcow2 -B b0 "$TEST_IMG"
$QEMU_IMG map --output=json --image-opts \
"$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b0" | _filter_qemu_img_map
nbd_server_start_unix_socket -r -f qcow2 -B b1 "$TEST_IMG"
$QEMU_IMG map --output=json --image-opts \
"$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b1" | _filter_qemu_img_map
nbd_server_start_unix_socket -r -f qcow2 -B b2 "$TEST_IMG"
$QEMU_IMG map --output=json --image-opts \
"$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b2" | _filter_qemu_img_map
# success, all done
echo '*** done'
rm -f $seq.full
status=0

View File

@ -0,0 +1,80 @@
QA output created by 291
=== Initial image setup ===
Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=10485760
wrote 1048576/1048576 bytes at offset 3145728
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=10485760 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT
wrote 1048576/1048576 bytes at offset 0
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 3145728
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 1048576
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 2097152
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== Bitmap preservation not possible to non-qcow2 ===
qemu-img: Format driver 'raw' does not support bitmaps
=== Convert with bitmap preservation ===
image: TEST_DIR/t.IMGFMT
file format: IMGFMT
virtual size: 10 MiB (10485760 bytes)
disk size: 4.39 MiB
Format specific information:
compat: 1.1
compression type: zlib
lazy refcounts: false
bitmaps:
[0]:
flags:
name: b1
granularity: 524288
[1]:
flags:
[0]: auto
name: b2
granularity: 65536
refcount bits: 16
corrupt: false
image: TEST_DIR/t.IMGFMT
file format: IMGFMT
virtual size: 10 MiB (10485760 bytes)
disk size: 4.48 MiB
Format specific information:
compat: 1.1
compression type: zlib
lazy refcounts: false
bitmaps:
[0]:
flags:
name: b1
granularity: 524288
[1]:
flags:
[0]: auto
name: b2
granularity: 65536
[2]:
flags:
name: b0
granularity: 65536
refcount bits: 16
corrupt: false
=== Check bitmap contents ===
[{ "start": 0, "length": 3145728, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
{ "start": 3145728, "length": 1048576, "depth": 0, "zero": false, "data": false},
{ "start": 4194304, "length": 6291456, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
[{ "start": 0, "length": 1048576, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
{ "start": 1048576, "length": 1048576, "depth": 0, "zero": false, "data": false},
{ "start": 2097152, "length": 8388608, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
[{ "start": 0, "length": 2097152, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
{ "start": 2097152, "length": 1048576, "depth": 0, "zero": false, "data": false},
{ "start": 3145728, "length": 7340032, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
*** done

View File

@ -299,5 +299,6 @@
288 quick
289 rw quick
290 rw auto quick
291 rw quick
292 rw auto quick
297 meta