OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

bitmaps patches for 2020-08-21

Browse Source 
- Andrey Shinkevich: Enhance qcow2.py for iotest inspection of qcow2 images
 - Max Reitz: Add block-bitmap-mapping migration parameter
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEccLMIrHEYCkn0vOqp6FrSiUnQ2oFAl8/1JMACgkQp6FrSiUn
 Q2rfQAf+IxDfjueqHm5+KdrA6FJ1qfoeLDndvOopm7ax2KPIsrhNRJQZ2i0Ts/RU
 oIQaY9BESYiQkTdw6q4THmCBlkMDYqDnJuWQRRFC5NAuTA4Q4EbO4j3WZhvh7Vfb
 OPcvdBYo16+ujD/h76mSpeuQvA8fnFUJ1pRhmJvBX78nj0uHE5UMxXjB9v2hoFOx
 tg5ApM3l4Fzm/eUz/5MY5+eX7XUMpmeN7G4qMfJxZGgIAeh6UiZaDVz7J2bVNo/c
 L3EqjPiWJcgTk3Tt8DhAZynVlqkdvgwyGwsntYybAyz+GSPCOPd5gdoAxIdv0wxx
 roVRJ3ARrQl5agYLc2A2nXcmxz9WZA==
 =cec/
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/ericb/tags/pull-bitmaps-2020-08-21' into staging

bitmaps patches for 2020-08-21

- Andrey Shinkevich: Enhance qcow2.py for iotest inspection of qcow2 images
- Max Reitz: Add block-bitmap-mapping migration parameter

# gpg: Signature made Fri 21 Aug 2020 15:05:07 BST
# gpg:                using RSA key 71C2CC22B1C4602927D2F3AAA7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>" [full]
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>" [full]
# gpg:                 aka "[jpeg image of size 6874]" [full]
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-bitmaps-2020-08-21:
  iotests: Test node/bitmap aliases during migration
  iotests.py: Let wait_migration() return on failure
  migration: Add block-bitmap-mapping parameter
  iotests: dump QCOW2 header in JSON in #303
  qcow2_format.py: support dumping metadata in JSON format
  qcow2_format.py: collect fields to dump in JSON format
  qcow2.py: Introduce '-j' key to dump in JSON format
  qcow2_format.py: Dump bitmap table serialized entries
  qcow2_format.py: pass cluster size to substructures
  qcow2_format.py: Dump bitmap directory information
  qcow2_format.py: dump bitmap flags in human readable way.
  qcow2_format.py: change Qcow2BitmapExt initialization method
  qcow2_format.py: make printable data an extension class member
  iotests: add test for QCOW2 header dump

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2020-08-22 19:57:23 +01:00
commit 66e01f1cdc
13 changed files with 1568 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

362
migration/block-dirty-bitmap.c
View File

@ -29,10 +29,10 @@
*
* # Header (shared for different chunk types)
* 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
* [ 1 byte: node name size ] \ flags & DEVICE_NAME
* [ n bytes: node name ] /
* [ 1 byte: bitmap name size ] \ flags & BITMAP_NAME
* [ n bytes: bitmap name ] /
* [ 1 byte: node alias size ] \ flags & DEVICE_NAME
* [ n bytes: node alias ] /
* [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME
* [ n bytes: bitmap alias ] /
*
* # Start of bitmap migration (flags & START)
* header
@ -72,7 +72,9 @@
#include "migration/register.h"
#include "qemu/hbitmap.h"
#include "qemu/cutils.h"
#include "qemu/id.h"
#include "qapi/error.h"
#include "qapi/qapi-commands-migration.h"
#include "trace.h"
#define CHUNK_SIZE (1 << 10)
@ -104,7 +106,8 @@
typedef struct SaveBitmapState {
/* Written during setup phase. */
BlockDriverState *bs;
const char *node_name;
char *node_alias;
char *bitmap_alias;
BdrvDirtyBitmap *bitmap;
uint64_t total_sectors;
uint64_t sectors_per_chunk;
@ -138,8 +141,9 @@ typedef struct LoadBitmapState {
/* State of the dirty bitmap migration (DBM) during load process */
typedef struct DBMLoadState {
uint32_t flags;
char node_name[256];
char bitmap_name[256];
char node_alias[256];
char bitmap_alias[256];
char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
BlockDriverState *bs;
BdrvDirtyBitmap *bitmap;
@ -165,6 +169,188 @@ typedef struct DBMState {
static DBMState dbm_state;
/* For hash tables that map node/bitmap names to aliases */
typedef struct AliasMapInnerNode {
char *string;
GHashTable *subtree;
} AliasMapInnerNode;
static void free_alias_map_inner_node(void *amin_ptr)
{
AliasMapInnerNode *amin = amin_ptr;
g_free(amin->string);
g_hash_table_unref(amin->subtree);
g_free(amin);
}
/**
* Construct an alias map based on the given QMP structure.
*
* (Note that we cannot store such maps in the MigrationParameters
* object, because that struct is defined by the QAPI schema, which
* makes it basically impossible to have dicts with arbitrary keys.
* Therefore, we instead have to construct these maps when migration
* starts.)
*
* @bbm is the block_bitmap_mapping from the migration parameters.
*
* If @name_to_alias is true, the returned hash table will map node
* and bitmap names to their respective aliases (for outgoing
* migration).
*
* If @name_to_alias is false, the returned hash table will map node
* and bitmap aliases to their respective names (for incoming
* migration).
*
* The hash table maps node names/aliases to AliasMapInnerNode
* objects, whose .string is the respective node alias/name, and whose
* .subtree table maps bitmap names/aliases to the respective bitmap
* alias/name.
*/
static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
bool name_to_alias,
Error **errp)
{
GHashTable *alias_map;
size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
g_free, free_alias_map_inner_node);
for (; bbm; bbm = bbm->next) {
const BitmapMigrationNodeAlias *bmna = bbm->value;
const BitmapMigrationBitmapAliasList *bmbal;
AliasMapInnerNode *amin;
GHashTable *bitmaps_map;
const char *node_map_from, *node_map_to;
if (!id_wellformed(bmna->alias)) {
error_setg(errp, "The node alias '%s' is not well-formed",
bmna->alias);
goto fail;
}
if (strlen(bmna->alias) > UINT8_MAX) {
error_setg(errp, "The node alias '%s' is longer than %u bytes",
bmna->alias, UINT8_MAX);
goto fail;
}
if (strlen(bmna->node_name) > max_node_name_len) {
error_setg(errp, "The node name '%s' is longer than %zu bytes",
bmna->node_name, max_node_name_len);
goto fail;
}
if (name_to_alias) {
if (g_hash_table_contains(alias_map, bmna->node_name)) {
error_setg(errp, "The node name '%s' is mapped twice",
bmna->node_name);
goto fail;
}
node_map_from = bmna->node_name;
node_map_to = bmna->alias;
} else {
if (g_hash_table_contains(alias_map, bmna->alias)) {
error_setg(errp, "The node alias '%s' is used twice",
bmna->alias);
goto fail;
}
node_map_from = bmna->alias;
node_map_to = bmna->node_name;
}
bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal,
g_free, g_free);
amin = g_new(AliasMapInnerNode, 1);
*amin = (AliasMapInnerNode){
.string = g_strdup(node_map_to),
.subtree = bitmaps_map,
};
g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
const BitmapMigrationBitmapAlias *bmba = bmbal->value;
const char *bmap_map_from, *bmap_map_to;
if (strlen(bmba->alias) > UINT8_MAX) {
error_setg(errp,
"The bitmap alias '%s' is longer than %u bytes",
bmba->alias, UINT8_MAX);
goto fail;
}
if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
goto fail;
}
if (name_to_alias) {
bmap_map_from = bmba->name;
bmap_map_to = bmba->alias;
if (g_hash_table_contains(bitmaps_map, bmba->name)) {
error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
bmna->node_name, bmba->name);
goto fail;
}
} else {
bmap_map_from = bmba->alias;
bmap_map_to = bmba->name;
if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
bmna->alias, bmba->alias);
goto fail;
}
}
g_hash_table_insert(bitmaps_map,
g_strdup(bmap_map_from), g_strdup(bmap_map_to));
}
}
return alias_map;
fail:
g_hash_table_destroy(alias_map);
return NULL;
}
/**
* Run construct_alias_map() in both directions to check whether @bbm
* is valid.
* (This function is to be used by migration/migration.c to validate
* the user-specified block-bitmap-mapping migration parameter.)
*
* Returns true if and only if the mapping is valid.
*/
bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
Error **errp)
{
GHashTable *alias_map;
alias_map = construct_alias_map(bbm, true, errp);
if (!alias_map) {
return false;
}
g_hash_table_destroy(alias_map);
alias_map = construct_alias_map(bbm, false, errp);
if (!alias_map) {
return false;
}
g_hash_table_destroy(alias_map);
return true;
}
static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
{
uint8_t flags = qemu_get_byte(f);
@ -207,11 +393,11 @@ static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
qemu_put_bitmap_flags(f, flags);
if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
qemu_put_counted_string(f, dbms->node_name);
qemu_put_counted_string(f, dbms->node_alias);
}
if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
qemu_put_counted_string(f, bdrv_dirty_bitmap_name(bitmap));
qemu_put_counted_string(f, dbms->bitmap_alias);
}
}
@ -282,18 +468,25 @@ static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
bdrv_unref(dbms->bs);
g_free(dbms->node_alias);
g_free(dbms->bitmap_alias);
g_free(dbms);
}
}
/* Called with iothread lock taken. */
static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
const char *bs_name)
const char *bs_name, GHashTable *alias_map)
{
BdrvDirtyBitmap *bitmap;
SaveBitmapState *dbms;
GHashTable *bitmap_aliases;
const char *node_alias, *bitmap_name, *bitmap_alias;
Error *local_err = NULL;
/* When an alias map is given, @bs_name must be @bs's node name */
assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
if (bdrv_dirty_bitmap_name(bitmap)) {
break;
@ -303,21 +496,39 @@ static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
return 0;
}
bitmap_name = bdrv_dirty_bitmap_name(bitmap);
if (!bs_name || strcmp(bs_name, "") == 0) {
error_report("Bitmap '%s' in unnamed node can't be migrated",
bdrv_dirty_bitmap_name(bitmap));
bitmap_name);
return -1;
}
if (bs_name[0] == '#') {
if (alias_map) {
const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
if (!amin) {
/* Skip bitmaps on nodes with no alias */
return 0;
}
node_alias = amin->string;
bitmap_aliases = amin->subtree;
} else {
node_alias = bs_name;
bitmap_aliases = NULL;
}
if (node_alias[0] == '#') {
error_report("Bitmap '%s' in a node with auto-generated "
"name '%s' can't be migrated",
bdrv_dirty_bitmap_name(bitmap), bs_name);
bitmap_name, node_alias);
return -1;
}
FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
if (!bdrv_dirty_bitmap_name(bitmap)) {
bitmap_name = bdrv_dirty_bitmap_name(bitmap);
if (!bitmap_name) {
continue;
}
@ -326,12 +537,29 @@ static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
return -1;
}
if (bitmap_aliases) {
bitmap_alias = g_hash_table_lookup(bitmap_aliases, bitmap_name);
if (!bitmap_alias) {
/* Skip bitmaps with no alias */
continue;
}
} else {
if (strlen(bitmap_name) > UINT8_MAX) {
error_report("Cannot migrate bitmap '%s' on node '%s': "
"Name is longer than %u bytes",
bitmap_name, bs_name, UINT8_MAX);
return -1;
}
bitmap_alias = bitmap_name;
}
bdrv_ref(bs);
bdrv_dirty_bitmap_set_busy(bitmap, true);
dbms = g_new0(SaveBitmapState, 1);
dbms->bs = bs;
dbms->node_name = bs_name;
dbms->node_alias = g_strdup(node_alias);
dbms->bitmap_alias = g_strdup(bitmap_alias);
dbms->bitmap = bitmap;
dbms->total_sectors = bdrv_nb_sectors(bs);
dbms->sectors_per_chunk = CHUNK_SIZE * 8 *
@ -356,12 +584,20 @@ static int init_dirty_bitmap_migration(DBMSaveState *s)
SaveBitmapState *dbms;
GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
BlockBackend *blk;
const MigrationParameters *mig_params = &migrate_get_current()->parameters;
GHashTable *alias_map = NULL;
if (mig_params->has_block_bitmap_mapping) {
alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
&error_abort);
}
s->bulk_completed = false;
s->prev_bs = NULL;
s->prev_bitmap = NULL;
s->no_bitmaps = false;
if (!alias_map) {
/*
* Use blockdevice name for direct (or filtered) children of named block
* backends.
@ -389,19 +625,20 @@ static int init_dirty_bitmap_migration(DBMSaveState *s)
}
if (bs && bs->drv && !bs->drv->is_filter) {
if (add_bitmaps_to_list(s, bs, name)) {
if (add_bitmaps_to_list(s, bs, name, NULL)) {
goto fail;
}
g_hash_table_add(handled_by_blk, bs);
}
}
}
for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
if (g_hash_table_contains(handled_by_blk, bs)) {
continue;
}
if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs))) {
if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
goto fail;
}
}
@ -416,11 +653,17 @@ static int init_dirty_bitmap_migration(DBMSaveState *s)
}
g_hash_table_destroy(handled_by_blk);
if (alias_map) {
g_hash_table_destroy(alias_map);
}
return 0;
fail:
g_hash_table_destroy(handled_by_blk);
if (alias_map) {
g_hash_table_destroy(alias_map);
}
dirty_bitmap_do_save_cleanup(s);
return -1;
@ -770,8 +1013,10 @@ static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
return 0;
}
static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s)
static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
GHashTable *alias_map)
{
GHashTable *bitmap_alias_map = NULL;
Error *local_err = NULL;
bool nothing;
s->flags = qemu_get_bitmap_flags(f);
@ -780,28 +1025,75 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s)
nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
if (!qemu_get_counted_string(f, s->node_name)) {
error_report("Unable to read node name string");
if (!qemu_get_counted_string(f, s->node_alias)) {
error_report("Unable to read node alias string");
return -EINVAL;
}
if (!s->cancelled) {
s->bs = bdrv_lookup_bs(s->node_name, s->node_name, &local_err);
if (alias_map) {
const AliasMapInnerNode *amin;
amin = g_hash_table_lookup(alias_map, s->node_alias);
if (!amin) {
error_setg(&local_err, "Error: Unknown node alias '%s'",
s->node_alias);
s->bs = NULL;
} else {
bitmap_alias_map = amin->subtree;
s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
}
} else {
s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
&local_err);
}
if (!s->bs) {
error_report_err(local_err);
cancel_incoming_locked(s);
}
}
} else if (!s->bs && !nothing && !s->cancelled) {
} else if (s->bs) {
if (alias_map) {
const AliasMapInnerNode *amin;
/* Must be present in the map, or s->bs would not be set */
amin = g_hash_table_lookup(alias_map, s->node_alias);
assert(amin != NULL);
bitmap_alias_map = amin->subtree;
}
} else if (!nothing && !s->cancelled) {
error_report("Error: block device name is not set");
cancel_incoming_locked(s);
}
assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
if (!qemu_get_counted_string(f, s->bitmap_name)) {
error_report("Unable to read bitmap name string");
const char *bitmap_name;
if (!qemu_get_counted_string(f, s->bitmap_alias)) {
error_report("Unable to read bitmap alias string");
return -EINVAL;
}
if (!s->cancelled) {
if (bitmap_alias_map) {
bitmap_name = g_hash_table_lookup(bitmap_alias_map,
s->bitmap_alias);
if (!bitmap_name) {
error_report("Error: Unknown bitmap alias '%s' on node "
"'%s' (alias '%s')", s->bitmap_alias,
s->bs->node_name, s->node_alias);
cancel_incoming_locked(s);
}
} else {
bitmap_name = s->bitmap_alias;
}
}
if (!s->cancelled) {
g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
/*
@ -811,7 +1103,7 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s)
if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
error_report("Error: unknown dirty bitmap "
"'%s' for block device '%s'",
s->bitmap_name, s->node_name);
s->bitmap_name, s->bs->node_name);
cancel_incoming_locked(s);
}
}
@ -835,6 +1127,8 @@ static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s)
*/
static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
{
GHashTable *alias_map = NULL;
const MigrationParameters *mig_params = &migrate_get_current()->parameters;
DBMLoadState *s = &((DBMState *)opaque)->load;
int ret = 0;
@ -846,13 +1140,18 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
return -EINVAL;
}
if (mig_params->has_block_bitmap_mapping) {
alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
false, &error_abort);
}
do {
QEMU_LOCK_GUARD(&s->lock);
ret = dirty_bitmap_load_header(f, s);
ret = dirty_bitmap_load_header(f, s, alias_map);
if (ret < 0) {
cancel_incoming_locked(s);
return ret;
goto fail;
}
if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
@ -869,12 +1168,17 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
if (ret) {
cancel_incoming_locked(s);
return ret;
goto fail;
}
} while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
trace_dirty_bitmap_load_success();
return 0;
ret = 0;
fail:
if (alias_map) {
g_hash_table_destroy(alias_map);
}
return ret;
}
static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)

30
migration/migration.c
View File

@ -36,6 +36,7 @@
#include "block/block.h"
#include "qapi/error.h"
#include "qapi/clone-visitor.h"
#include "qapi/qapi-visit-migration.h"
#include "qapi/qapi-visit-sockets.h"
#include "qapi/qapi-commands-migration.h"
#include "qapi/qapi-events-migration.h"
@ -843,6 +844,13 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
params->has_announce_step = true;
params->announce_step = s->parameters.announce_step;
if (s->parameters.has_block_bitmap_mapping) {
params->has_block_bitmap_mapping = true;
params->block_bitmap_mapping =
QAPI_CLONE(BitmapMigrationNodeAliasList,
s->parameters.block_bitmap_mapping);
}
return params;
}
@ -1308,6 +1316,13 @@ static bool migrate_params_check(MigrationParameters *params, Error **errp)
"is invalid, it must be in the range of 1 to 10000 ms");
return false;
}
if (params->has_block_bitmap_mapping &&
!check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
return false;
}
return true;
}
@ -1402,6 +1417,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
if (params->has_announce_step) {
dest->announce_step = params->announce_step;
}
if (params->has_block_bitmap_mapping) {
dest->has_block_bitmap_mapping = true;
dest->block_bitmap_mapping = params->block_bitmap_mapping;
}
}
static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
@ -1514,6 +1534,16 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
if (params->has_announce_step) {
s->parameters.announce_step = params->announce_step;
}
if (params->has_block_bitmap_mapping) {
qapi_free_BitmapMigrationNodeAliasList(
s->parameters.block_bitmap_mapping);
s->parameters.has_block_bitmap_mapping = true;
s->parameters.block_bitmap_mapping =
QAPI_CLONE(BitmapMigrationNodeAliasList,
params->block_bitmap_mapping);
}
}
void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)

3
migration/migration.h
View File

@ -337,6 +337,9 @@ void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
void dirty_bitmap_mig_before_vm_start(void);
void dirty_bitmap_mig_cancel_outgoing(void);
void dirty_bitmap_mig_cancel_incoming(void);
bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
Error **errp);
void migrate_add_address(SocketAddress *address);
int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);

30
monitor/hmp-cmds.c
View File

@ -469,6 +469,32 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
monitor_printf(mon, "%s: '%s'\n",
MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
params->tls_authz);
if (params->has_block_bitmap_mapping) {
const BitmapMigrationNodeAliasList *bmnal;
monitor_printf(mon, "%s:\n",
MigrationParameter_str(
MIGRATION_PARAMETER_BLOCK_BITMAP_MAPPING));
for (bmnal = params->block_bitmap_mapping;
bmnal;
bmnal = bmnal->next)
{
const BitmapMigrationNodeAlias *bmna = bmnal->value;
const BitmapMigrationBitmapAliasList *bmbal;
monitor_printf(mon, " '%s' -> '%s'\n",
bmna->node_name, bmna->alias);
for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
const BitmapMigrationBitmapAlias *bmba = bmbal->value;
monitor_printf(mon, " '%s' -> '%s'\n",
bmba->name, bmba->alias);
}
}
}
}
qapi_free_MigrationParameters(params);
@ -1384,6 +1410,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
p->has_announce_step = true;
visit_type_size(v, param, &p->announce_step, &err);
break;
case MIGRATION_PARAMETER_BLOCK_BITMAP_MAPPING:
error_setg(&err, "The block-bitmap-mapping parameter can only be set "
"through QMP");
break;
default:
assert(0);
}

104
qapi/migration.json
View File

@ -508,6 +508,44 @@
'data': [ 'none', 'zlib',
{ 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
##
# @BitmapMigrationBitmapAlias:
#
# @name: The name of the bitmap.
#
# @alias: An alias name for migration (for example the bitmap name on
# the opposite site).
#
# Since: 5.2
##
{ 'struct': 'BitmapMigrationBitmapAlias',
'data': {
'name': 'str',
'alias': 'str'
} }
##
# @BitmapMigrationNodeAlias:
#
# Maps a block node name and the bitmaps it has to aliases for dirty
# bitmap migration.
#
# @node-name: A block node name.
#
# @alias: An alias block node name for migration (for example the
# node name on the opposite site).
#
# @bitmaps: Mappings for the bitmaps on this node.
#
# Since: 5.2
##
{ 'struct': 'BitmapMigrationNodeAlias',
'data': {
'node-name': 'str',
'alias': 'str',
'bitmaps': [ 'BitmapMigrationBitmapAlias' ]
} }
##
# @MigrationParameter:
#
@ -642,6 +680,25 @@
# will consume more CPU.
# Defaults to 1. (Since 5.0)
#
# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
# aliases for the purpose of dirty bitmap migration. Such
# aliases may for example be the corresponding names on the
# opposite site.
# The mapping must be one-to-one, but not necessarily
# complete: On the source, unmapped bitmaps and all bitmaps
# on unmapped nodes will be ignored. On the destination,
# encountering an unmapped alias in the incoming migration
# stream will result in a report, and all further bitmap
# migration data will then be discarded.
# Note that the destination does not know about bitmaps it
# does not receive, so there is no limitation or requirement
# regarding the number of bitmaps received, or how they are
# named, or on which nodes they are placed.
# By default (when this parameter has never been set), bitmap
# names are mapped to themselves. Nodes are mapped to their
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
# Since: 2.4
##
{ 'enum': 'MigrationParameter',
@ -656,7 +713,8 @@
'multifd-channels',
'xbzrle-cache-size', 'max-postcopy-bandwidth',
'max-cpu-throttle', 'multifd-compression',
'multifd-zlib-level' ,'multifd-zstd-level' ] }
'multifd-zlib-level' ,'multifd-zstd-level',
'block-bitmap-mapping' ] }
##
# @MigrateSetParameters:
@ -782,6 +840,25 @@
# will consume more CPU.
# Defaults to 1. (Since 5.0)
#
# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
# aliases for the purpose of dirty bitmap migration. Such
# aliases may for example be the corresponding names on the
# opposite site.
# The mapping must be one-to-one, but not necessarily
# complete: On the source, unmapped bitmaps and all bitmaps
# on unmapped nodes will be ignored. On the destination,
# encountering an unmapped alias in the incoming migration
# stream will result in a report, and all further bitmap
# migration data will then be discarded.
# Note that the destination does not know about bitmaps it
# does not receive, so there is no limitation or requirement
# regarding the number of bitmaps received, or how they are
# named, or on which nodes they are placed.
# By default (when this parameter has never been set), bitmap
# names are mapped to themselves. Nodes are mapped to their
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
# Since: 2.4
##
# TODO either fuse back into MigrationParameters, or make
@ -812,7 +889,8 @@
'*max-cpu-throttle': 'int',
'*multifd-compression': 'MultiFDCompression',
'*multifd-zlib-level': 'int',
'*multifd-zstd-level': 'int' } }
'*multifd-zstd-level': 'int',
'*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } }
##
# @migrate-set-parameters:
@ -958,6 +1036,25 @@
# will consume more CPU.
# Defaults to 1. (Since 5.0)
#
# @block-bitmap-mapping: Maps block nodes and bitmaps on them to
# aliases for the purpose of dirty bitmap migration. Such
# aliases may for example be the corresponding names on the
# opposite site.
# The mapping must be one-to-one, but not necessarily
# complete: On the source, unmapped bitmaps and all bitmaps
# on unmapped nodes will be ignored. On the destination,
# encountering an unmapped alias in the incoming migration
# stream will result in a report, and all further bitmap
# migration data will then be discarded.
# Note that the destination does not know about bitmaps it
# does not receive, so there is no limitation or requirement
# regarding the number of bitmaps received, or how they are
# named, or on which nodes they are placed.
# By default (when this parameter has never been set), bitmap
# names are mapped to themselves. Nodes are mapped to their
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
# Since: 2.4
##
{ 'struct': 'MigrationParameters',
@ -986,7 +1083,8 @@
'*max-cpu-throttle': 'uint8',
'*multifd-compression': 'MultiFDCompression',
'*multifd-zlib-level': 'uint8',
'*multifd-zstd-level': 'uint8' } }
'*multifd-zstd-level': 'uint8',
'*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } }
##
# @query-migrate-parameters:

593
tests/qemu-iotests/300 Executable file
View File

@ -0,0 +1,593 @@
#!/usr/bin/env python3
#
# Copyright (C) 2020 Red Hat, Inc.
#
# Tests for dirty bitmaps migration with node aliases
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import random
import re
from typing import Dict, List, Optional, Union
import iotests
import qemu
BlockBitmapMapping = List[Dict[str, Union[str, List[Dict[str, str]]]]]
assert iotests.sock_dir is not None
mig_sock = os.path.join(iotests.sock_dir, 'mig_sock')
class TestDirtyBitmapMigration(iotests.QMPTestCase):
src_node_name: str = ''
dst_node_name: str = ''
src_bmap_name: str = ''
dst_bmap_name: str = ''
def setUp(self) -> None:
self.vm_a = iotests.VM(path_suffix='-a')
self.vm_a.add_blockdev(f'node-name={self.src_node_name},'
'driver=null-co')
self.vm_a.launch()
self.vm_b = iotests.VM(path_suffix='-b')
self.vm_b.add_blockdev(f'node-name={self.dst_node_name},'
'driver=null-co')
self.vm_b.add_incoming(f'unix:{mig_sock}')
self.vm_b.launch()
result = self.vm_a.qmp('block-dirty-bitmap-add',
node=self.src_node_name,
name=self.src_bmap_name)
self.assert_qmp(result, 'return', {})
# Dirty some random megabytes
for _ in range(9):
mb_ofs = random.randrange(1024)
self.vm_a.hmp_qemu_io(self.src_node_name, f'discard {mb_ofs}M 1M')
result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
node=self.src_node_name,
name=self.src_bmap_name)
self.bitmap_hash_reference = result['return']['sha256']
caps = [{'capability': name, 'state': True}
for name in ('dirty-bitmaps', 'events')]
for vm in (self.vm_a, self.vm_b):
result = vm.qmp('migrate-set-capabilities', capabilities=caps)
self.assert_qmp(result, 'return', {})
def tearDown(self) -> None:
self.vm_a.shutdown()
self.vm_b.shutdown()
try:
os.remove(mig_sock)
except OSError:
pass
def check_bitmap(self, bitmap_name_valid: bool) -> None:
result = self.vm_b.qmp('x-debug-block-dirty-bitmap-sha256',
node=self.dst_node_name,
name=self.dst_bmap_name)
if bitmap_name_valid:
self.assert_qmp(result, 'return/sha256',
self.bitmap_hash_reference)
else:
self.assert_qmp(result, 'error/desc',
f"Dirty bitmap '{self.dst_bmap_name}' not found")
def migrate(self, bitmap_name_valid: bool = True,
migration_success: bool = True) -> None:
result = self.vm_a.qmp('migrate', uri=f'unix:{mig_sock}')
self.assert_qmp(result, 'return', {})
with iotests.Timeout(5, 'Timeout waiting for migration to complete'):
self.assertEqual(self.vm_a.wait_migration('postmigrate'),
migration_success)
self.assertEqual(self.vm_b.wait_migration('running'),
migration_success)
if migration_success:
self.check_bitmap(bitmap_name_valid)
def verify_dest_error(self, msg: Optional[str]) -> None:
"""
Check whether the given error message is present in vm_b's log.
(vm_b is shut down to do so.)
If @msg is None, check that there has not been any error.
"""
self.vm_b.shutdown()
if msg is None:
self.assertNotIn('qemu-system-', self.vm_b.get_log())
else:
self.assertIn(msg, self.vm_b.get_log())
@staticmethod
def mapping(node_name: str, node_alias: str,
bitmap_name: str, bitmap_alias: str) -> BlockBitmapMapping:
return [{
'node-name': node_name,
'alias': node_alias,
'bitmaps': [{
'name': bitmap_name,
'alias': bitmap_alias
}]
}]
def set_mapping(self, vm: iotests.VM, mapping: BlockBitmapMapping,
error: Optional[str] = None) -> None:
"""
Invoke migrate-set-parameters on @vm to set the given @mapping.
Check for success if @error is None, or verify the error message
if it is not.
On success, verify that "info migrate_parameters" on HMP returns
our mapping. (Just to check its formatting code.)
"""
result = vm.qmp('migrate-set-parameters',
block_bitmap_mapping=mapping)
if error is None:
self.assert_qmp(result, 'return', {})
result = vm.qmp('human-monitor-command',
command_line='info migrate_parameters')
m = re.search(r'^block-bitmap-mapping:\r?(\n .*)*\n',
result['return'], flags=re.MULTILINE)
hmp_mapping = m.group(0).replace('\r', '') if m else None
self.assertEqual(hmp_mapping, self.to_hmp_mapping(mapping))
else:
self.assert_qmp(result, 'error/desc', error)
@staticmethod
def to_hmp_mapping(mapping: BlockBitmapMapping) -> str:
result = 'block-bitmap-mapping:\n'
for node in mapping:
result += f" '{node['node-name']}' -> '{node['alias']}'\n"
assert isinstance(node['bitmaps'], list)
for bitmap in node['bitmaps']:
result += f" '{bitmap['name']}' -> '{bitmap['alias']}'\n"
return result
class TestAliasMigration(TestDirtyBitmapMigration):
src_node_name = 'node0'
dst_node_name = 'node0'
src_bmap_name = 'bmap0'
dst_bmap_name = 'bmap0'
def test_migration_without_alias(self) -> None:
self.migrate(self.src_node_name == self.dst_node_name and
self.src_bmap_name == self.dst_bmap_name)
# Check for error message on the destination
if self.src_node_name != self.dst_node_name:
self.verify_dest_error(f"Cannot find "
f"device={self.src_node_name} nor "
f"node_name={self.src_node_name}")
else:
self.verify_dest_error(None)
def test_alias_on_src_migration(self) -> None:
mapping = self.mapping(self.src_node_name, self.dst_node_name,
self.src_bmap_name, self.dst_bmap_name)
self.set_mapping(self.vm_a, mapping)
self.migrate()
self.verify_dest_error(None)
def test_alias_on_dst_migration(self) -> None:
mapping = self.mapping(self.dst_node_name, self.src_node_name,
self.dst_bmap_name, self.src_bmap_name)
self.set_mapping(self.vm_b, mapping)
self.migrate()
self.verify_dest_error(None)
def test_alias_on_both_migration(self) -> None:
src_map = self.mapping(self.src_node_name, 'node-alias',
self.src_bmap_name, 'bmap-alias')
dst_map = self.mapping(self.dst_node_name, 'node-alias',
self.dst_bmap_name, 'bmap-alias')
self.set_mapping(self.vm_a, src_map)
self.set_mapping(self.vm_b, dst_map)
self.migrate()
self.verify_dest_error(None)
class TestNodeAliasMigration(TestAliasMigration):
src_node_name = 'node-src'
dst_node_name = 'node-dst'
class TestBitmapAliasMigration(TestAliasMigration):
src_bmap_name = 'bmap-src'
dst_bmap_name = 'bmap-dst'
class TestFullAliasMigration(TestAliasMigration):
src_node_name = 'node-src'
dst_node_name = 'node-dst'
src_bmap_name = 'bmap-src'
dst_bmap_name = 'bmap-dst'
class TestLongBitmapNames(TestAliasMigration):
# Giving long bitmap names is OK, as long as there is a short alias for
# migration
src_bmap_name = 'a' * 512
dst_bmap_name = 'b' * 512
# Skip all tests that do not use the intermediate alias
def test_migration_without_alias(self) -> None:
pass
def test_alias_on_src_migration(self) -> None:
pass
def test_alias_on_dst_migration(self) -> None:
pass
class TestBlockBitmapMappingErrors(TestDirtyBitmapMigration):
src_node_name = 'node0'
dst_node_name = 'node0'
src_bmap_name = 'bmap0'
dst_bmap_name = 'bmap0'
"""
Note that mapping nodes or bitmaps that do not exist is not an error.
"""
def test_non_injective_node_mapping(self) -> None:
mapping: BlockBitmapMapping = [
{
'node-name': 'node0',
'alias': 'common-alias',
'bitmaps': [{
'name': 'bmap0',
'alias': 'bmap-alias0'
}]
},
{
'node-name': 'node1',
'alias': 'common-alias',
'bitmaps': [{
'name': 'bmap1',
'alias': 'bmap-alias1'
}]
}
]
self.set_mapping(self.vm_a, mapping,
"Invalid mapping given for block-bitmap-mapping: "
"The node alias 'common-alias' is used twice")
def test_non_injective_bitmap_mapping(self) -> None:
mapping: BlockBitmapMapping = [{
'node-name': 'node0',
'alias': 'node-alias0',
'bitmaps': [
{
'name': 'bmap0',
'alias': 'common-alias'
},
{
'name': 'bmap1',
'alias': 'common-alias'
}
]
}]
self.set_mapping(self.vm_a, mapping,
"Invalid mapping given for block-bitmap-mapping: "
"The bitmap alias 'node-alias0'/'common-alias' is "
"used twice")
def test_ambiguous_node_mapping(self) -> None:
mapping: BlockBitmapMapping = [
{
'node-name': 'node0',
'alias': 'node-alias0',
'bitmaps': [{
'name': 'bmap0',
'alias': 'bmap-alias0'
}]
},
{
'node-name': 'node0',
'alias': 'node-alias1',
'bitmaps': [{
'name': 'bmap0',
'alias': 'bmap-alias0'
}]
}
]
self.set_mapping(self.vm_a, mapping,
"Invalid mapping given for block-bitmap-mapping: "
"The node name 'node0' is mapped twice")
def test_ambiguous_bitmap_mapping(self) -> None:
mapping: BlockBitmapMapping = [{
'node-name': 'node0',
'alias': 'node-alias0',
'bitmaps': [
{
'name': 'bmap0',
'alias': 'bmap-alias0'
},
{
'name': 'bmap0',
'alias': 'bmap-alias1'
}
]
}]
self.set_mapping(self.vm_a, mapping,
"Invalid mapping given for block-bitmap-mapping: "
"The bitmap 'node0'/'bmap0' is mapped twice")
def test_migratee_node_is_not_mapped_on_src(self) -> None:
self.set_mapping(self.vm_a, [])
# Should just ignore all bitmaps on unmapped nodes
self.migrate(False)
self.verify_dest_error(None)
def test_migratee_node_is_not_mapped_on_dst(self) -> None:
self.set_mapping(self.vm_b, [])
self.migrate(False)
self.verify_dest_error(f"Unknown node alias '{self.src_node_name}'")
def test_migratee_bitmap_is_not_mapped_on_src(self) -> None:
mapping: BlockBitmapMapping = [{
'node-name': self.src_node_name,
'alias': self.dst_node_name,
'bitmaps': []
}]
self.set_mapping(self.vm_a, mapping)
# Should just ignore all unmapped bitmaps
self.migrate(False)
self.verify_dest_error(None)
def test_migratee_bitmap_is_not_mapped_on_dst(self) -> None:
mapping: BlockBitmapMapping = [{
'node-name': self.dst_node_name,
'alias': self.src_node_name,
'bitmaps': []
}]
self.set_mapping(self.vm_b, mapping)
self.migrate(False)
self.verify_dest_error(f"Unknown bitmap alias "
f"'{self.src_bmap_name}' "
f"on node '{self.dst_node_name}' "
f"(alias '{self.src_node_name}')")
def test_unused_mapping_on_dst(self) -> None:
# Let the source not send any bitmaps
self.set_mapping(self.vm_a, [])
# Establish some mapping on the destination
self.set_mapping(self.vm_b, [])
# The fact that there is a mapping on B without any bitmaps
# being received should be fine, not fatal
self.migrate(False)
self.verify_dest_error(None)
def test_non_wellformed_node_alias(self) -> None:
alias = '123-foo'
mapping: BlockBitmapMapping = [{
'node-name': self.src_node_name,
'alias': alias,
'bitmaps': []
}]
self.set_mapping(self.vm_a, mapping,
f"Invalid mapping given for block-bitmap-mapping: "
f"The node alias '{alias}' is not well-formed")
def test_node_alias_too_long(self) -> None:
alias = 'a' * 256
mapping: BlockBitmapMapping = [{
'node-name': self.src_node_name,
'alias': alias,
'bitmaps': []
}]
self.set_mapping(self.vm_a, mapping,
f"Invalid mapping given for block-bitmap-mapping: "
f"The node alias '{alias}' is longer than 255 bytes")
def test_bitmap_alias_too_long(self) -> None:
alias = 'a' * 256
mapping = self.mapping(self.src_node_name, self.dst_node_name,
self.src_bmap_name, alias)
self.set_mapping(self.vm_a, mapping,
f"Invalid mapping given for block-bitmap-mapping: "
f"The bitmap alias '{alias}' is longer than 255 "
f"bytes")
def test_bitmap_name_too_long(self) -> None:
name = 'a' * 256
result = self.vm_a.qmp('block-dirty-bitmap-add',
node=self.src_node_name,
name=name)
self.assert_qmp(result, 'return', {})
self.migrate(False, False)
# Check for the error in the source's log
self.vm_a.shutdown()
self.assertIn(f"Cannot migrate bitmap '{name}' on node "
f"'{self.src_node_name}': Name is longer than 255 bytes",
self.vm_a.get_log())
# Expect abnormal shutdown of the destination VM because of
# the failed migration
try:
self.vm_b.shutdown()
except qemu.machine.AbnormalShutdown:
pass
def test_aliased_bitmap_name_too_long(self) -> None:
# Longer than the maximum for bitmap names
self.dst_bmap_name = 'a' * 1024
mapping = self.mapping(self.dst_node_name, self.src_node_name,
self.dst_bmap_name, self.src_bmap_name)
# We would have to create this bitmap during migration, and
# that would fail, because the name is too long. Better to
# catch it early.
self.set_mapping(self.vm_b, mapping,
f"Invalid mapping given for block-bitmap-mapping: "
f"The bitmap name '{self.dst_bmap_name}' is longer "
f"than 1023 bytes")
def test_node_name_too_long(self) -> None:
# Longer than the maximum for node names
self.dst_node_name = 'a' * 32
mapping = self.mapping(self.dst_node_name, self.src_node_name,
self.dst_bmap_name, self.src_bmap_name)
# During migration, this would appear simply as a node that
# cannot be found. Still better to catch impossible node
# names early (similar to test_non_wellformed_node_alias).
self.set_mapping(self.vm_b, mapping,
f"Invalid mapping given for block-bitmap-mapping: "
f"The node name '{self.dst_node_name}' is longer "
f"than 31 bytes")
class TestCrossAliasMigration(TestDirtyBitmapMigration):
"""
Swap aliases, both to see that qemu does not get confused, and
that we can migrate multiple things at once.
So we migrate this:
node-a.bmap-a -> node-b.bmap-b
node-a.bmap-b -> node-b.bmap-a
node-b.bmap-a -> node-a.bmap-b
node-b.bmap-b -> node-a.bmap-a
"""
src_node_name = 'node-a'
dst_node_name = 'node-b'
src_bmap_name = 'bmap-a'
dst_bmap_name = 'bmap-b'
def setUp(self) -> None:
TestDirtyBitmapMigration.setUp(self)
# Now create another block device and let both have two bitmaps each
result = self.vm_a.qmp('blockdev-add',
node_name='node-b', driver='null-co')
self.assert_qmp(result, 'return', {})
result = self.vm_b.qmp('blockdev-add',
node_name='node-a', driver='null-co')
self.assert_qmp(result, 'return', {})
bmaps_to_add = (('node-a', 'bmap-b'),
('node-b', 'bmap-a'),
('node-b', 'bmap-b'))
for (node, bmap) in bmaps_to_add:
result = self.vm_a.qmp('block-dirty-bitmap-add',
node=node, name=bmap)
self.assert_qmp(result, 'return', {})
@staticmethod
def cross_mapping() -> BlockBitmapMapping:
return [
{
'node-name': 'node-a',
'alias': 'node-b',
'bitmaps': [
{
'name': 'bmap-a',
'alias': 'bmap-b'
},
{
'name': 'bmap-b',
'alias': 'bmap-a'
}
]
},
{
'node-name': 'node-b',
'alias': 'node-a',
'bitmaps': [
{
'name': 'bmap-b',
'alias': 'bmap-a'
},
{
'name': 'bmap-a',
'alias': 'bmap-b'
}
]
}
]
def verify_dest_has_all_bitmaps(self) -> None:
bitmaps = self.vm_b.query_bitmaps()
# Extract and sort bitmap names
for node in bitmaps:
bitmaps[node] = sorted((bmap['name'] for bmap in bitmaps[node]))
self.assertEqual(bitmaps,
{'node-a': ['bmap-a', 'bmap-b'],
'node-b': ['bmap-a', 'bmap-b']})
def test_alias_on_src(self) -> None:
self.set_mapping(self.vm_a, self.cross_mapping())
# Checks that node-a.bmap-a was migrated to node-b.bmap-b, and
# that is enough
self.migrate()
self.verify_dest_has_all_bitmaps()
self.verify_dest_error(None)
def test_alias_on_dst(self) -> None:
self.set_mapping(self.vm_b, self.cross_mapping())
# Checks that node-a.bmap-a was migrated to node-b.bmap-b, and
# that is enough
self.migrate()
self.verify_dest_has_all_bitmaps()
self.verify_dest_error(None)
if __name__ == '__main__':
iotests.main(supported_protocols=['file'])

5
tests/qemu-iotests/300.out Normal file
View File

@ -0,0 +1,5 @@
.....................................
----------------------------------------------------------------------
Ran 37 tests
OK

63
tests/qemu-iotests/303 Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python3
#
# Test for dumping of qcow2 image metadata
#
# Copyright (c) 2020 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import iotests
import subprocess
from iotests import qemu_img_create, qemu_io, file_path, log, filter_qemu_io
iotests.script_initialize(supported_fmts=['qcow2'])
disk = file_path('disk')
chunk = 1024 * 1024
def create_bitmap(bitmap_number, disabled):
granularity = 1 << (14 + bitmap_number)
bitmap_name = 'bitmap-' + str(bitmap_number)
args = ['bitmap', '--add', '-g', f'{granularity}', '-f', iotests.imgfmt,
disk, bitmap_name]
if disabled:
args.append('--disable')
iotests.qemu_img_pipe(*args)
def write_to_disk(offset, size):
write = f'write {offset} {size}'
log(qemu_io('-c', write, disk), filters=[filter_qemu_io])
def add_bitmap(num, begin, end, disabled):
log(f'Add bitmap {num}')
create_bitmap(num, disabled)
for i in range(begin, end):
write_to_disk((i) * chunk, chunk)
log('')
qemu_img_create('-f', iotests.imgfmt, disk, '10M')
add_bitmap(1, 0, 6, False)
add_bitmap(2, 6, 8, True)
dump = ['qcow2.py', disk, 'dump-header']
subprocess.run(dump)
# Dump the metadata in JSON format
dump.append('-j')
subprocess.run(dump)

158
tests/qemu-iotests/303.out Normal file
View File

@ -0,0 +1,158 @@
Add bitmap 1
wrote 1048576/1048576 bytes at offset 0
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 1048576
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 2097152
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 3145728
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 4194304
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 5242880
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
Add bitmap 2
wrote 1048576/1048576 bytes at offset 6291456
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 7340032
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
magic 0x514649fb
version 3
backing_file_offset 0x0
backing_file_size 0x0
cluster_bits 16
size 10485760
crypt_method 0
l1_size 1
l1_table_offset 0x30000
refcount_table_offset 0x10000
refcount_table_clusters 1
nb_snapshots 0
snapshot_offset 0x0
incompatible_features []
compatible_features []
autoclear_features [0]
refcount_order 4
header_length 112
Header extension:
magic 0x6803f857 (Feature table)
length 336
data <binary>
Header extension:
magic 0x23852875 (Bitmaps)
length 24
nb_bitmaps 2
reserved32 0
bitmap_directory_size 0x40
bitmap_directory_offset 0x9d0000
Bitmap name bitmap-1
bitmap_table_offset 0x9b0000
bitmap_table_size 1
flags 0x2 (['auto'])
type 1
granularity_bits 15
name_size 8
extra_data_size 0
Bitmap table type size offset
0 serialized 65536 10092544
Bitmap name bitmap-2
bitmap_table_offset 0x9c0000
bitmap_table_size 1
flags 0x0 ([])
type 1
granularity_bits 16
name_size 8
extra_data_size 0
Bitmap table type size offset
0 all-zeroes 0 0
{
"magic": 1363560955,
"version": 3,
"backing_file_offset": 0,
"backing_file_size": 0,
"cluster_bits": 16,
"size": 10485760,
"crypt_method": 0,
"l1_size": 1,
"l1_table_offset": 196608,
"refcount_table_offset": 65536,
"refcount_table_clusters": 1,
"nb_snapshots": 0,
"snapshot_offset": 0,
"incompatible_features": 0,
"compatible_features": 0,
"autoclear_features": 1,
"refcount_order": 4,
"header_length": 112
}
[
{
"name": "Feature table",
"magic": 1745090647,
"length": 336,
"data_str": "<binary>"
},
{
"name": "Bitmaps",
"magic": 595929205,
"length": 24,
"data": {
"nb_bitmaps": 2,
"reserved32": 0,
"bitmap_directory_size": 64,
"bitmap_directory_offset": 10289152,
"bitmap_directory": [
{
"name": "bitmap-1",
"bitmap_table_offset": 10158080,
"bitmap_table_size": 1,
"flags": 2,
"type": 1,
"granularity_bits": 15,
"name_size": 8,
"extra_data_size": 0,
"bitmap_table": [
{
"type": "serialized",
"offset": 10092544,
"reserved": 0
}
]
},
{
"name": "bitmap-2",
"bitmap_table_offset": 10223616,
"bitmap_table_size": 1,
"flags": 0,
"type": 1,
"granularity_bits": 16,
"name_size": 8,
"extra_data_size": 0,
"bitmap_table": [
{
"type": "all-zeroes",
"offset": 0,
"reserved": 0
}
]
}
]
}
}
]

2
tests/qemu-iotests/group
View File

@ -307,6 +307,8 @@
296 rw
297 meta
299 auto quick
300 migration
301 backing quick
302 quick
303 rw quick
304 rw quick

14
tests/qemu-iotests/iotests.py
View File

@ -729,16 +729,22 @@ class VM(qtest.QEMUQtestMachine):
}
]))
def wait_migration(self, expect_runstate):
def wait_migration(self, expect_runstate: Optional[str]) -> bool:
while True:
event = self.event_wait('MIGRATION')
log(event, filters=[filter_qmp_event])
if event['data']['status'] == 'completed':
if event['data']['status'] in ('completed', 'failed'):
break
if event['data']['status'] == 'completed':
# The event may occur in finish-migrate, so wait for the expected
# post-migration runstate
while self.qmp('query-status')['return']['status'] != expect_runstate:
pass
runstate = None
while runstate != expect_runstate:
runstate = self.qmp('query-status')['return']['status']
return True
else:
return False
def node_info(self, node_name):
nodes = self.qmp('query-named-block-nodes')

18
tests/qemu-iotests/qcow2.py
View File

@ -26,16 +26,19 @@ from qcow2_format import (
)
is_json = False
def cmd_dump_header(fd):
h = QcowHeader(fd)
h.dump()
h.dump(is_json)
print()
h.dump_extensions()
h.dump_extensions(is_json)
def cmd_dump_header_exts(fd):
h = QcowHeader(fd)
h.dump_extensions()
h.dump_extensions(is_json)
def cmd_set_header(fd, name, value):
@ -151,11 +154,14 @@ def main(filename, cmd, args):
def usage():
print("Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0])
print("Usage: %s <file> <cmd> [<arg>, ...] [<key>, ...]" % sys.argv[0])
print("")
print("Supported commands:")
for name, handler, num_args, desc in cmds:
print(" %-20s - %s" % (name, desc))
print("")
print("Supported keys:")
print(" %-20s - %s" % ('-j', 'Dump in JSON format'))
if __name__ == '__main__':
@ -163,4 +169,8 @@ if __name__ == '__main__':
usage()
sys.exit(1)
is_json = '-j' in sys.argv
if is_json:
sys.argv.remove('-j')
main(sys.argv[1], sys.argv[2], sys.argv[3:])

209
tests/qemu-iotests/qcow2_format.py
View File

@ -19,6 +19,15 @@
import struct
import string
import json
class ComplexEncoder(json.JSONEncoder):
def default(self, obj):
if hasattr(obj, 'to_json'):
return obj.to_json()
else:
return json.JSONEncoder.default(self, obj)
class Qcow2Field:
@ -40,6 +49,22 @@ class Flags64(Qcow2Field):
return str(bits)
class BitmapFlags(Qcow2Field):
flags = {
0x1: 'in-use',
0x2: 'auto'
}
def __str__(self):
bits = []
for bit in range(64):
flag = self.value & (1 << bit)
if flag:
bits.append(self.flags.get(flag, f'bit-{bit}'))
return f'{self.value:#x} ({bits})'
class Enum(Qcow2Field):
def __str__(self):
@ -93,7 +118,11 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
self.__dict__ = dict((field[2], values[i])
for i, field in enumerate(self.fields))
def dump(self):
def dump(self, is_json=False):
if is_json:
print(json.dumps(self.to_json(), indent=4, cls=ComplexEncoder))
return
for f in self.fields:
value = self.__dict__[f[2]]
if isinstance(f[1], str):
@ -103,6 +132,9 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
print('{:<25} {}'.format(f[2], value_str))
def to_json(self):
return dict((f[2], self.__dict__[f[2]]) for f in self.fields)
class Qcow2BitmapExt(Qcow2Struct):
@ -113,6 +145,131 @@ class Qcow2BitmapExt(Qcow2Struct):
('u64', '{:#x}', 'bitmap_directory_offset')
)
def __init__(self, fd, cluster_size):
super().__init__(fd=fd)
tail = struct.calcsize(self.fmt) % 8
if tail:
fd.seek(8 - tail, 1)
position = fd.tell()
self.cluster_size = cluster_size
self.read_bitmap_directory(fd)
fd.seek(position)
def read_bitmap_directory(self, fd):
fd.seek(self.bitmap_directory_offset)
self.bitmap_directory = \
[Qcow2BitmapDirEntry(fd, cluster_size=self.cluster_size)
for _ in range(self.nb_bitmaps)]
def dump(self):
super().dump()
for entry in self.bitmap_directory:
print()
entry.dump()
def to_json(self):
fields_dict = super().to_json()
fields_dict['bitmap_directory'] = self.bitmap_directory
return fields_dict
class Qcow2BitmapDirEntry(Qcow2Struct):
fields = (
('u64', '{:#x}', 'bitmap_table_offset'),
('u32', '{}', 'bitmap_table_size'),
('u32', BitmapFlags, 'flags'),
('u8', '{}', 'type'),
('u8', '{}', 'granularity_bits'),
('u16', '{}', 'name_size'),
('u32', '{}', 'extra_data_size')
)
def __init__(self, fd, cluster_size):
super().__init__(fd=fd)
self.cluster_size = cluster_size
# Seek relative to the current position in the file
fd.seek(self.extra_data_size, 1)
bitmap_name = fd.read(self.name_size)
self.name = bitmap_name.decode('ascii')
# Move position to the end of the entry in the directory
entry_raw_size = self.bitmap_dir_entry_raw_size()
padding = ((entry_raw_size + 7) & ~7) - entry_raw_size
fd.seek(padding, 1)
self.bitmap_table = Qcow2BitmapTable(fd=fd,
offset=self.bitmap_table_offset,
nb_entries=self.bitmap_table_size,
cluster_size=self.cluster_size)
def bitmap_dir_entry_raw_size(self):
return struct.calcsize(self.fmt) + self.name_size + \
self.extra_data_size
def dump(self):
print(f'{"Bitmap name":<25} {self.name}')
super(Qcow2BitmapDirEntry, self).dump()
self.bitmap_table.dump()
def to_json(self):
# Put the name ahead of the dict
return {
'name': self.name,
**super().to_json(),
'bitmap_table': self.bitmap_table
}
class Qcow2BitmapTableEntry(Qcow2Struct):
fields = (
('u64', '{}', 'entry'),
)
BME_TABLE_ENTRY_RESERVED_MASK = 0xff000000000001fe
BME_TABLE_ENTRY_OFFSET_MASK = 0x00fffffffffffe00
BME_TABLE_ENTRY_FLAG_ALL_ONES = 1
def __init__(self, fd):
super().__init__(fd=fd)
self.reserved = self.entry & self.BME_TABLE_ENTRY_RESERVED_MASK
self.offset = self.entry & self.BME_TABLE_ENTRY_OFFSET_MASK
if self.offset:
if self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES:
self.type = 'invalid'
else:
self.type = 'serialized'
elif self.entry & self.BME_TABLE_ENTRY_FLAG_ALL_ONES:
self.type = 'all-ones'
else:
self.type = 'all-zeroes'
def to_json(self):
return {'type': self.type, 'offset': self.offset,
'reserved': self.reserved}
class Qcow2BitmapTable:
def __init__(self, fd, offset, nb_entries, cluster_size):
self.cluster_size = cluster_size
position = fd.tell()
fd.seek(offset)
self.entries = [Qcow2BitmapTableEntry(fd) for _ in range(nb_entries)]
fd.seek(position)
def dump(self):
bitmap_table = enumerate(self.entries)
print(f'{"Bitmap table":<14} {"type":<15} {"size":<12} {"offset"}')
for i, entry in bitmap_table:
if entry.type == 'serialized':
size = self.cluster_size
else:
size = 0
print(f'{i:<14} {entry.type:<15} {size:<12} {entry.offset}')
def to_json(self):
return self.entries
QCOW2_EXT_MAGIC_BITMAPS = 0x23852875
@ -128,6 +285,9 @@ class QcowHeaderExtension(Qcow2Struct):
0x44415441: 'Data file'
}
def to_json(self):
return self.mapping.get(self.value, "<unknown>")
fields = (
('u32', Magic, 'magic'),
('u32', '{}', 'length')
@ -135,11 +295,13 @@ class QcowHeaderExtension(Qcow2Struct):
# then padding to next multiply of 8
)
def __init__(self, magic=None, length=None, data=None, fd=None):
def __init__(self, magic=None, length=None, data=None, fd=None,
cluster_size=None):
"""
Support both loading from fd and creation from user data.
For fd-based creation current position in a file will be used to read
the data.
The cluster_size value may be obtained by dependent structures.
This should be somehow refactored and functionality should be moved to
superclass (to allow creation of any qcow2 struct), but then, fields
@ -161,28 +323,43 @@ class QcowHeaderExtension(Qcow2Struct):
else:
assert all(v is None for v in (magic, length, data))
super().__init__(fd=fd)
if self.magic == QCOW2_EXT_MAGIC_BITMAPS:
self.obj = Qcow2BitmapExt(fd=fd, cluster_size=cluster_size)
self.data = None
else:
padded = (self.length + 7) & ~7
self.data = fd.read(padded)
assert self.data is not None
if self.magic == QCOW2_EXT_MAGIC_BITMAPS:
self.obj = Qcow2BitmapExt(data=self.data)
else:
self.obj = None
if self.data is not None:
data_str = self.data[:self.length]
if all(c in string.printable.encode(
'ascii') for c in data_str):
data_str = f"'{ data_str.decode('ascii') }'"
else:
data_str = '<binary>'
self.data_str = data_str
def dump(self):
super().dump()
if self.obj is None:
data = self.data[:self.length]
if all(c in string.printable.encode('ascii') for c in data):
data = f"'{ data.decode('ascii') }'"
else:
data = '<binary>'
print(f'{"data":<25} {data}')
print(f'{"data":<25} {self.data_str}')
else:
self.obj.dump()
def to_json(self):
# Put the name ahead of the dict
res = {'name': self.Magic(self.magic), **super().to_json()}
if self.obj is not None:
res['data'] = self.obj
else:
res['data_str'] = self.data_str
return res
@classmethod
def create(cls, magic, data):
return QcowHeaderExtension(magic, len(data), data)
@ -246,7 +423,7 @@ class QcowHeader(Qcow2Struct):
end = self.cluster_size
while fd.tell() < end:
ext = QcowHeaderExtension(fd=fd)
ext = QcowHeaderExtension(fd=fd, cluster_size=self.cluster_size)
if ext.magic == 0:
break
else:
@ -280,7 +457,11 @@ class QcowHeader(Qcow2Struct):
buf = buf[0:header_bytes-1]
fd.write(buf)
def dump_extensions(self):
def dump_extensions(self, is_json=False):
if is_json:
print(json.dumps(self.extensions, indent=4, cls=ComplexEncoder))
return
for ex in self.extensions:
print('Header extension:')
ex.dump()