OpenE2K
/
qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

Block patches: 

- Block-status cache for data regions
 - qcow2 optimization (when using subclusters)
 - iotests delinting, and let 297 (lint checker) cover named iotests
 - qcow2 check improvements
 - Added -F (target backing file format) option to qemu-img convert
 - Mirror job fix
 - Fix for when a migration is initiated while a backup job runs
 - Fix for uncached qemu-img convert to a volume with 4k sectors (for an
   unaligned image)
 - Minor gluster driver fix
 -----BEGIN PGP SIGNATURE-----
 
 iQJGBAABCAAwFiEEy2LXoO44KeRfAE00ofpA0JgBnN8FAmFCL78SHGhyZWl0ekBy
 ZWRoYXQuY29tAAoJEKH6QNCYAZzfrBgQALQ1eY3tScoNgutgVJs+oX9OvuziJGuB
 oX6SpvPfFZZyvpzykYtdBFmWW9f2GyAHftEMJgJ1qA+MvpYnBLMdsM/PwRUGNsd4
 Y6GrYrQLSTQPHsYvMgYpqUBIe4Sk0ECyXrtTfcx81XFKov93vLDQGIAH37T85sdQ
 QxoF1klslOT+RfrMB0lzmOvRY/QD6I2jIOrPUynWinyg16Bl12xCsht0iWTlPdHs
 ph2WhdUknIUUOM+cvFvTTNVqfybXJBqFk/vD/mNcr3PUVG/RhDqY6SAH2jh2Pcy4
 IZ1xt7c8O6Co4wHiB5gM+YixxjhKA50bk8c/+VksVN5KuQ/5C3tFHC8t9LWjxIKj
 DxbrNSUqoj+X0kiWY6MzvwPOL4FxdNpl94NKneZg0PU2nbGkHncEsJGUlsyBugoz
 iWbAWb7FeHUSOXRrjLSgX/+tFIZWhDYYol5ac4o7ONWf7Vr9/5D50I72D292P6lo
 d04B/RhoFxkL34h+ECpz00FLB86McMB2zECG8u/8KnQpZSqKOywUxw8gmc9uJYv2
 fC/zwzBktIDq8XCZUZDrU4NdS6FjiVtqglKN5yVPbNXSLOVMqbaLjUkAx/zklNma
 HzyzJPbEcVY3CvR7jic1yxHXntkUD3GNcD4hW7iqHGNPkLTBFQjctGpldmquYBlh
 jQypaaKUgRNa
 =/jVK
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/hreitz/tags/pull-block-2021-09-15' into staging

Block patches:
- Block-status cache for data regions
- qcow2 optimization (when using subclusters)
- iotests delinting, and let 297 (lint checker) cover named iotests
- qcow2 check improvements
- Added -F (target backing file format) option to qemu-img convert
- Mirror job fix
- Fix for when a migration is initiated while a backup job runs
- Fix for uncached qemu-img convert to a volume with 4k sectors (for an
  unaligned image)
- Minor gluster driver fix

# gpg: Signature made Wed 15 Sep 2021 18:39:11 BST
# gpg:                using RSA key CB62D7A0EE3829E45F004D34A1FA40D098019CDF
# gpg:                issuer "hreitz@redhat.com"
# gpg: Good signature from "Hanna Reitz <hreitz@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: CB62 D7A0 EE38 29E4 5F00  4D34 A1FA 40D0 9801 9CDF

* remotes/hreitz/tags/pull-block-2021-09-15: (32 commits)
  qemu-img: Add -F shorthand to convert
  qcow2-refcount: check_refblocks(): add separate message for reserved
  qcow2-refcount: check_refcounts_l1(): check reserved bits
  qcow2-refcount: improve style of check_refcounts_l1()
  qcow2-refcount: check_refcounts_l2(): check reserved bits
  qcow2-refcount: check_refcounts_l2(): check l2_bitmap
  qcow2-refcount: fix_l2_entry_by_zero(): also zero L2 entry bitmap
  qcow2-refcount: introduce fix_l2_entry_by_zero()
  qcow2: introduce qcow2_parse_compressed_l2_entry() helper
  qcow2: compressed read: simplify cluster descriptor passing
  qcow2-refcount: improve style of check_refcounts_l2()
  qemu-img: Allow target be aligned to sector size
  qcow2: handle_dependencies(): relax conflict detection
  qcow2: refactor handle_dependencies() loop body
  simplebench: add img_bench_templater.py
  block: bdrv_inactivate_recurse(): check for permissions and fix crash
  tests: add migrate-during-backup
  block/mirror: fix NULL pointer dereference in mirror_wait_on_conflicts()
  iotests/297: Cover tests/
  mirror-top-perms: Fix AbnormalShutdown path
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2021-09-15 18:55:59 +01:00
parent 7b7ab2d6c9 1899bf4737
commit 57b6f58c1d
26 changed files with 856 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
block.c
View File

@ -49,6 +49,8 @@
#include "qemu/timer.h"
#include "qemu/cutils.h"
#include "qemu/id.h"
#include "qemu/range.h"
#include "qemu/rcu.h"
#include "block/coroutines.h"
#ifdef CONFIG_BSD
@ -401,6 +403,9 @@ BlockDriverState *bdrv_new(void)
qemu_co_queue_init(&bs->flush_queue);
qemu_co_mutex_init(&bs->bsc_modify_lock);
bs->block_status_cache = g_new0(BdrvBlockStatusCache, 1);
for (i = 0; i < bdrv_drain_all_count; i++) {
bdrv_drained_begin(bs);
}
@ -4694,6 +4699,8 @@ static void bdrv_close(BlockDriverState *bs)
bs->explicit_options = NULL;
qobject_unref(bs->full_open_options);
bs->full_open_options = NULL;
g_free(bs->block_status_cache);
bs->block_status_cache = NULL;
bdrv_release_named_dirty_bitmaps(bs);
assert(QLIST_EMPTY(&bs->dirty_bitmaps));
@ -6319,6 +6326,7 @@ static int bdrv_inactivate_recurse(BlockDriverState *bs)
{
BdrvChild *child, *parent;
int ret;
uint64_t cumulative_perms, cumulative_shared_perms;
if (!bs->drv) {
return -ENOMEDIUM;
@ -6349,6 +6357,13 @@ static int bdrv_inactivate_recurse(BlockDriverState *bs)
}
}
bdrv_get_cumulative_perm(bs, &cumulative_perms,
&cumulative_shared_perms);
if (cumulative_perms & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
/* Our inactive parents still need write access. Inactivation failed. */
return -EPERM;
}
bs->open_flags |= BDRV_O_INACTIVE;
/*
@ -7684,3 +7699,76 @@ BlockDriverState *bdrv_backing_chain_next(BlockDriverState *bs)
{
return bdrv_skip_filters(bdrv_cow_bs(bdrv_skip_filters(bs)));
}
/**
* Check whether [offset, offset + bytes) overlaps with the cached
* block-status data region.
*
* If so, and @pnum is not NULL, set *pnum to `bsc.data_end - offset`,
* which is what bdrv_bsc_is_data()'s interface needs.
* Otherwise, *pnum is not touched.
*/
static bool bdrv_bsc_range_overlaps_locked(BlockDriverState *bs,
int64_t offset, int64_t bytes,
int64_t *pnum)
{
BdrvBlockStatusCache *bsc = qatomic_rcu_read(&bs->block_status_cache);
bool overlaps;
overlaps =
qatomic_read(&bsc->valid) &&
ranges_overlap(offset, bytes, bsc->data_start,
bsc->data_end - bsc->data_start);
if (overlaps && pnum) {
*pnum = bsc->data_end - offset;
}
return overlaps;
}
/**
* See block_int.h for this function's documentation.
*/
bool bdrv_bsc_is_data(BlockDriverState *bs, int64_t offset, int64_t *pnum)
{
RCU_READ_LOCK_GUARD();
return bdrv_bsc_range_overlaps_locked(bs, offset, 1, pnum);
}
/**
* See block_int.h for this function's documentation.
*/
void bdrv_bsc_invalidate_range(BlockDriverState *bs,
int64_t offset, int64_t bytes)
{
RCU_READ_LOCK_GUARD();
if (bdrv_bsc_range_overlaps_locked(bs, offset, bytes, NULL)) {
qatomic_set(&bs->block_status_cache->valid, false);
}
}
/**
* See block_int.h for this function's documentation.
*/
void bdrv_bsc_fill(BlockDriverState *bs, int64_t offset, int64_t bytes)
{
BdrvBlockStatusCache *new_bsc = g_new(BdrvBlockStatusCache, 1);
BdrvBlockStatusCache *old_bsc;
*new_bsc = (BdrvBlockStatusCache) {
.valid = true,
.data_start = offset,
.data_end = offset + bytes,
};
QEMU_LOCK_GUARD(&bs->bsc_modify_lock);
old_bsc = qatomic_rcu_read(&bs->block_status_cache);
qatomic_rcu_set(&bs->block_status_cache, new_bsc);
if (old_bsc) {
g_free_rcu(old_bsc, rcu);
}
}

7
block/file-posix.c
View File

@ -2744,7 +2744,8 @@ static int find_allocation(BlockDriverState *bs, off_t start,
* the specified offset) that are known to be in the same
* allocated/unallocated state.
*
* 'bytes' is the max value 'pnum' should be set to.
* 'bytes' is a soft cap for 'pnum'. If the information is free, 'pnum' may
* well exceed it.
*/
static int coroutine_fn raw_co_block_status(BlockDriverState *bs,
bool want_zero,
@ -2782,7 +2783,7 @@ static int coroutine_fn raw_co_block_status(BlockDriverState *bs,
} else if (data == offset) {
/* On a data extent, compute bytes to the end of the extent,
* possibly including a partial sector at EOF. */
*pnum = MIN(bytes, hole - offset);
*pnum = hole - offset;
/*
* We are not allowed to return partial sectors, though, so
@ -2801,7 +2802,7 @@ static int coroutine_fn raw_co_block_status(BlockDriverState *bs,
} else {
/* On a hole, compute bytes to the beginning of the next extent. */
assert(hole == offset);
*pnum = MIN(bytes, data - offset);
*pnum = data - offset;
ret = BDRV_BLOCK_ZERO;
}
*map = offset;

23
block/gluster.c
View File

@ -1461,7 +1461,8 @@ exit:
* the specified offset) that are known to be in the same
* allocated/unallocated state.
*
* 'bytes' is the max value 'pnum' should be set to.
* 'bytes' is a soft cap for 'pnum'. If the information is free, 'pnum' may
* well exceed it.
*
* (Based on raw_co_block_status() from file-posix.c.)
*/
@ -1477,6 +1478,8 @@ static int coroutine_fn qemu_gluster_co_block_status(BlockDriverState *bs,
off_t data = 0, hole = 0;
int ret = -EINVAL;
assert(QEMU_IS_ALIGNED(offset | bytes, bs->bl.request_alignment));
if (!s->fd) {
return ret;
}
@ -1500,12 +1503,26 @@ static int coroutine_fn qemu_gluster_co_block_status(BlockDriverState *bs,
} else if (data == offset) {
/* On a data extent, compute bytes to the end of the extent,
* possibly including a partial sector at EOF. */
*pnum = MIN(bytes, hole - offset);
*pnum = hole - offset;
/*
* We are not allowed to return partial sectors, though, so
* round up if necessary.
*/
if (!QEMU_IS_ALIGNED(*pnum, bs->bl.request_alignment)) {
int64_t file_length = qemu_gluster_getlength(bs);
if (file_length > 0) {
/* Ignore errors, this is just a safeguard */
assert(hole == file_length);
}
*pnum = ROUND_UP(*pnum, bs->bl.request_alignment);
}
ret = BDRV_BLOCK_DATA;
} else {
/* On a hole, compute bytes to the beginning of the next extent. */
assert(hole == offset);
*pnum = MIN(bytes, data - offset);
*pnum = data - offset;
ret = BDRV_BLOCK_ZERO;
}

68
block/io.c
View File

@ -1883,6 +1883,9 @@ static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
return -ENOTSUP;
}
/* Invalidate the cached block-status data range if this write overlaps */
bdrv_bsc_invalidate_range(bs, offset, bytes);
assert(alignment % bs->bl.request_alignment == 0);
head = offset % alignment;
tail = (offset + bytes) % alignment;
@ -2447,9 +2450,65 @@ static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
if (bs->drv->bdrv_co_block_status) {
ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
aligned_bytes, pnum, &local_map,
&local_file);
/*
* Use the block-status cache only for protocol nodes: Format
* drivers are generally quick to inquire the status, but protocol
* drivers often need to get information from outside of qemu, so
* we do not have control over the actual implementation. There
* have been cases where inquiring the status took an unreasonably
* long time, and we can do nothing in qemu to fix it.
* This is especially problematic for images with large data areas,
* because finding the few holes in them and giving them special
* treatment does not gain much performance. Therefore, we try to
* cache the last-identified data region.
*
* Second, limiting ourselves to protocol nodes allows us to assume
* the block status for data regions to be DATA | OFFSET_VALID, and
* that the host offset is the same as the guest offset.
*
* Note that it is possible that external writers zero parts of
* the cached regions without the cache being invalidated, and so
* we may report zeroes as data. This is not catastrophic,
* however, because reporting zeroes as data is fine.
*/
if (QLIST_EMPTY(&bs->children) &&
bdrv_bsc_is_data(bs, aligned_offset, pnum))
{
ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
local_file = bs;
local_map = aligned_offset;
} else {
ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
aligned_bytes, pnum, &local_map,
&local_file);
/*
* Note that checking QLIST_EMPTY(&bs->children) is also done when
* the cache is queried above. Technically, we do not need to check
* it here; the worst that can happen is that we fill the cache for
* non-protocol nodes, and then it is never used. However, filling
* the cache requires an RCU update, so double check here to avoid
* such an update if possible.
*/
if (ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
QLIST_EMPTY(&bs->children))
{
/*
* When a protocol driver reports BLOCK_OFFSET_VALID, the
* returned local_map value must be the same as the offset we
* have passed (aligned_offset), and local_bs must be the node
* itself.
* Assert this, because we follow this rule when reading from
* the cache (see the `local_file = bs` and
* `local_map = aligned_offset` assignments above), and the
* result the cache delivers must be the same as the driver
* would deliver.
*/
assert(local_file == bs);
assert(local_map == aligned_offset);
bdrv_bsc_fill(bs, aligned_offset, *pnum);
}
}
} else {
/* Default code for filters */
@ -3002,6 +3061,9 @@ int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
return 0;
}
/* Invalidate the cached block-status data range if this discard overlaps */
bdrv_bsc_invalidate_range(bs, offset, bytes);
/* Discard is advisory, but some devices track and coalesce
* unaligned requests, so we must pass everything down rather than
* round here. Still, most devices will just silently ignore

3
block/iscsi.c
View File

@ -781,9 +781,6 @@ retry:
iscsi_allocmap_set_allocated(iscsilun, offset, *pnum);
}
if (*pnum > bytes) {
*pnum = bytes;
}
out_unlock:
qemu_mutex_unlock(&iscsilun->mutex);
g_free(iTask.err_str);

25
block/mirror.c
View File

@ -160,18 +160,25 @@ static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
if (ranges_overlap(self_start_chunk, self_nb_chunks,
op_start_chunk, op_nb_chunks))
{
/*
* If the operation is already (indirectly) waiting for us, or
* will wait for us as soon as it wakes up, then just go on
* (instead of producing a deadlock in the former case).
*/
if (op->waiting_for_op) {
continue;
if (self) {
/*
* If the operation is already (indirectly) waiting for us,
* or will wait for us as soon as it wakes up, then just go
* on (instead of producing a deadlock in the former case).
*/
if (op->waiting_for_op) {
continue;
}
self->waiting_for_op = op;
}
self->waiting_for_op = op;
qemu_co_queue_wait(&op->waiting_requests, NULL);
self->waiting_for_op = NULL;
if (self) {
self->waiting_for_op = NULL;
}
break;
}
}

78
block/qcow2-cluster.c
View File

@ -556,8 +556,7 @@ static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,
* offset needs to be aligned to a cluster boundary.
*
* If the cluster is unallocated then *host_offset will be 0.
* If the cluster is compressed then *host_offset will contain the
* complete compressed cluster descriptor.
* If the cluster is compressed then *host_offset will contain the l2 entry.
*
* On entry, *bytes is the maximum number of contiguous bytes starting at
* offset that we are interested in.
@ -660,7 +659,7 @@ int qcow2_get_host_offset(BlockDriverState *bs, uint64_t offset,
ret = -EIO;
goto fail;
}
*host_offset = l2_entry & L2E_COMPRESSED_OFFSET_SIZE_MASK;
*host_offset = l2_entry;
break;
case QCOW2_SUBCLUSTER_ZERO_PLAIN:
case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
@ -1400,29 +1399,47 @@ static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
if (end <= old_start || start >= old_end) {
/* No intersection */
continue;
}
if (old_alloc->keep_old_clusters &&
(end <= l2meta_cow_start(old_alloc) ||
start >= l2meta_cow_end(old_alloc)))
{
/*
* Clusters intersect but COW areas don't. And cluster itself is
* already allocated. So, there is no actual conflict.
*/
continue;
}
/* Conflict */
if (start < old_start) {
/* Stop at the start of a running allocation */
bytes = old_start - start;
} else {
if (start < old_start) {
/* Stop at the start of a running allocation */
bytes = old_start - start;
} else {
bytes = 0;
}
bytes = 0;
}
/* Stop if already an l2meta exists. After yielding, it wouldn't
* be valid any more, so we'd have to clean up the old L2Metas
* and deal with requests depending on them before starting to
* gather new ones. Not worth the trouble. */
if (bytes == 0 && *m) {
*cur_bytes = 0;
return 0;
}
/*
* Stop if an l2meta already exists. After yielding, it wouldn't
* be valid any more, so we'd have to clean up the old L2Metas
* and deal with requests depending on them before starting to
* gather new ones. Not worth the trouble.
*/
if (bytes == 0 && *m) {
*cur_bytes = 0;
return 0;
}
if (bytes == 0) {
/* Wait for the dependency to complete. We need to recheck
* the free/allocated clusters when we continue. */
qemu_co_queue_wait(&old_alloc->dependent_requests, &s->lock);
return -EAGAIN;
}
if (bytes == 0) {
/*
* Wait for the dependency to complete. We need to recheck
* the free/allocated clusters when we continue.
*/
qemu_co_queue_wait(&old_alloc->dependent_requests, &s->lock);
return -EAGAIN;
}
}
@ -2463,3 +2480,18 @@ fail:
g_free(l1_table);
return ret;
}
void qcow2_parse_compressed_l2_entry(BlockDriverState *bs, uint64_t l2_entry,
uint64_t *coffset, int *csize)
{
BDRVQcow2State *s = bs->opaque;
int nb_csectors;
assert(qcow2_get_cluster_type(bs, l2_entry) == QCOW2_CLUSTER_COMPRESSED);
*coffset = l2_entry & s->cluster_offset_mask;
nb_csectors = ((l2_entry >> s->csize_shift) & s->csize_mask) + 1;
*csize = nb_csectors * QCOW2_COMPRESSED_SECTOR_SIZE -
(*coffset & (QCOW2_COMPRESSED_SECTOR_SIZE - 1));
}

328
block/qcow2-refcount.c
View File

@ -1177,11 +1177,11 @@ void qcow2_free_any_cluster(BlockDriverState *bs, uint64_t l2_entry,
switch (ctype) {
case QCOW2_CLUSTER_COMPRESSED:
{
int64_t offset = (l2_entry & s->cluster_offset_mask)
& QCOW2_COMPRESSED_SECTOR_MASK;
int size = QCOW2_COMPRESSED_SECTOR_SIZE *
(((l2_entry >> s->csize_shift) & s->csize_mask) + 1);
qcow2_free_clusters(bs, offset, size, type);
uint64_t coffset;
int csize;
qcow2_parse_compressed_l2_entry(bs, l2_entry, &coffset, &csize);
qcow2_free_clusters(bs, coffset, csize, type);
}
break;
case QCOW2_CLUSTER_NORMAL:
@ -1247,7 +1247,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
bool l1_allocated = false;
int64_t old_entry, old_l2_offset;
unsigned slice, slice_size2, n_slices;
int i, j, l1_modified = 0, nb_csectors;
int i, j, l1_modified = 0;
int ret;
assert(addend >= -1 && addend <= 1);
@ -1318,14 +1318,14 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
switch (qcow2_get_cluster_type(bs, entry)) {
case QCOW2_CLUSTER_COMPRESSED:
nb_csectors = ((entry >> s->csize_shift) &
s->csize_mask) + 1;
if (addend != 0) {
uint64_t coffset = (entry & s->cluster_offset_mask)
& QCOW2_COMPRESSED_SECTOR_MASK;
uint64_t coffset;
int csize;
qcow2_parse_compressed_l2_entry(bs, entry,
&coffset, &csize);
ret = update_refcount(
bs, coffset,
nb_csectors * QCOW2_COMPRESSED_SECTOR_SIZE,
bs, coffset, csize,
abs(addend), addend < 0,
QCOW2_DISCARD_SNAPSHOT);
if (ret < 0) {
@ -1587,6 +1587,66 @@ enum {
CHECK_FRAG_INFO = 0x2, /* update BlockFragInfo counters */
};
/*
* Fix L2 entry by making it QCOW2_CLUSTER_ZERO_PLAIN (or making all its present
* subclusters QCOW2_SUBCLUSTER_ZERO_PLAIN).
*
* This function decrements res->corruptions on success, so the caller is
* responsible to increment res->corruptions prior to the call.
*
* On failure in-memory @l2_table may be modified.
*/
static int fix_l2_entry_by_zero(BlockDriverState *bs, BdrvCheckResult *res,
uint64_t l2_offset,
uint64_t *l2_table, int l2_index, bool active,
bool *metadata_overlap)
{
BDRVQcow2State *s = bs->opaque;
int ret;
int idx = l2_index * (l2_entry_size(s) / sizeof(uint64_t));
uint64_t l2e_offset = l2_offset + (uint64_t)l2_index * l2_entry_size(s);
int ign = active ? QCOW2_OL_ACTIVE_L2 : QCOW2_OL_INACTIVE_L2;
if (has_subclusters(s)) {
uint64_t l2_bitmap = get_l2_bitmap(s, l2_table, l2_index);
/* Allocated subclusters become zero */
l2_bitmap |= l2_bitmap << 32;
l2_bitmap &= QCOW_L2_BITMAP_ALL_ZEROES;
set_l2_bitmap(s, l2_table, l2_index, l2_bitmap);
set_l2_entry(s, l2_table, l2_index, 0);
} else {
set_l2_entry(s, l2_table, l2_index, QCOW_OFLAG_ZERO);
}
ret = qcow2_pre_write_overlap_check(bs, ign, l2e_offset, l2_entry_size(s),
false);
if (metadata_overlap) {
*metadata_overlap = ret < 0;
}
if (ret < 0) {
fprintf(stderr, "ERROR: Overlap check failed\n");
goto fail;
}
ret = bdrv_pwrite_sync(bs->file, l2e_offset, &l2_table[idx],
l2_entry_size(s));
if (ret < 0) {
fprintf(stderr, "ERROR: Failed to overwrite L2 "
"table entry: %s\n", strerror(-ret));
goto fail;
}
res->corruptions--;
res->corruptions_fixed++;
return 0;
fail:
res->check_errors++;
return ret;
}
/*
* Increases the refcount in the given refcount table for the all clusters
* referenced in the L2 table. While doing so, performs some checks on L2
@ -1601,26 +1661,41 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
int flags, BdrvCheckMode fix, bool active)
{
BDRVQcow2State *s = bs->opaque;
uint64_t *l2_table, l2_entry;
uint64_t l2_entry, l2_bitmap;
uint64_t next_contiguous_offset = 0;
int i, l2_size, nb_csectors, ret;
int i, ret;
size_t l2_size_bytes = s->l2_size * l2_entry_size(s);
g_autofree uint64_t *l2_table = g_malloc(l2_size_bytes);
bool metadata_overlap;
/* Read L2 table from disk */
l2_size = s->l2_size * l2_entry_size(s);
l2_table = g_malloc(l2_size);
ret = bdrv_pread(bs->file, l2_offset, l2_table, l2_size);
ret = bdrv_pread(bs->file, l2_offset, l2_table, l2_size_bytes);
if (ret < 0) {
fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
res->check_errors++;
goto fail;
return ret;
}
/* Do the actual checks */
for(i = 0; i < s->l2_size; i++) {
l2_entry = get_l2_entry(s, l2_table, i);
for (i = 0; i < s->l2_size; i++) {
uint64_t coffset;
int csize;
QCow2ClusterType type;
switch (qcow2_get_cluster_type(bs, l2_entry)) {
l2_entry = get_l2_entry(s, l2_table, i);
l2_bitmap = get_l2_bitmap(s, l2_table, i);
type = qcow2_get_cluster_type(bs, l2_entry);
if (type != QCOW2_CLUSTER_COMPRESSED) {
/* Check reserved bits of Standard Cluster Descriptor */
if (l2_entry & L2E_STD_RESERVED_MASK) {
fprintf(stderr, "ERROR found l2 entry with reserved bits set: "
"%" PRIx64 "\n", l2_entry);
res->corruptions++;
}
}
switch (type) {
case QCOW2_CLUSTER_COMPRESSED:
/* Compressed clusters don't have QCOW_OFLAG_COPIED */
if (l2_entry & QCOW_OFLAG_COPIED) {
@ -1638,23 +1713,28 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
break;
}
if (l2_bitmap) {
fprintf(stderr, "ERROR compressed cluster %d with non-zero "
"subcluster allocation bitmap, entry=0x%" PRIx64 "\n",
i, l2_entry);
res->corruptions++;
break;
}
/* Mark cluster as used */
nb_csectors = ((l2_entry >> s->csize_shift) &
s->csize_mask) + 1;
l2_entry &= s->cluster_offset_mask;
qcow2_parse_compressed_l2_entry(bs, l2_entry, &coffset, &csize);
ret = qcow2_inc_refcounts_imrt(
bs, res, refcount_table, refcount_table_size,
l2_entry & QCOW2_COMPRESSED_SECTOR_MASK,
nb_csectors * QCOW2_COMPRESSED_SECTOR_SIZE);
bs, res, refcount_table, refcount_table_size, coffset, csize);
if (ret < 0) {
goto fail;
return ret;
}
if (flags & CHECK_FRAG_INFO) {
res->bfi.allocated_clusters++;
res->bfi.compressed_clusters++;
/* Compressed clusters are fragmented by nature. Since they
/*
* Compressed clusters are fragmented by nature. Since they
* take up sub-sector space but we only have sector granularity
* I/O we need to re-read the same sectors even for adjacent
* compressed clusters.
@ -1668,13 +1748,19 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
{
uint64_t offset = l2_entry & L2E_OFFSET_MASK;
if ((l2_bitmap >> 32) & l2_bitmap) {
res->corruptions++;
fprintf(stderr, "ERROR offset=%" PRIx64 ": Allocated "
"cluster has corrupted subcluster allocation bitmap\n",
offset);
}
/* Correct offsets are cluster aligned */
if (offset_into_cluster(s, offset)) {
bool contains_data;
res->corruptions++;
if (has_subclusters(s)) {
uint64_t l2_bitmap = get_l2_bitmap(s, l2_table, i);
contains_data = (l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC);
} else {
contains_data = !(l2_entry & QCOW_OFLAG_ZERO);
@ -1687,40 +1773,30 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR",
offset);
if (fix & BDRV_FIX_ERRORS) {
int idx = i * (l2_entry_size(s) / sizeof(uint64_t));
uint64_t l2e_offset =
l2_offset + (uint64_t)i * l2_entry_size(s);
int ign = active ? QCOW2_OL_ACTIVE_L2 :
QCOW2_OL_INACTIVE_L2;
l2_entry = has_subclusters(s) ? 0 : QCOW_OFLAG_ZERO;
set_l2_entry(s, l2_table, i, l2_entry);
ret = qcow2_pre_write_overlap_check(bs, ign,
l2e_offset, l2_entry_size(s), false);
if (ret < 0) {
fprintf(stderr, "ERROR: Overlap check failed\n");
res->check_errors++;
/* Something is seriously wrong, so abort checking
* this L2 table */
goto fail;
ret = fix_l2_entry_by_zero(bs, res, l2_offset,
l2_table, i, active,
&metadata_overlap);
if (metadata_overlap) {
/*
* Something is seriously wrong, so abort checking
* this L2 table.
*/
return ret;
}
ret = bdrv_pwrite_sync(bs->file, l2e_offset,
&l2_table[idx],
l2_entry_size(s));
if (ret < 0) {
fprintf(stderr, "ERROR: Failed to overwrite L2 "
"table entry: %s\n", strerror(-ret));
res->check_errors++;
/* Do not abort, continue checking the rest of this
* L2 table's entries */
} else {
res->corruptions--;
res->corruptions_fixed++;
/* Skip marking the cluster as used
* (it is unused now) */
if (ret == 0) {
/*
* Skip marking the cluster as used
* (it is unused now).
*/
continue;
}
/*
* Failed to fix.
* Do not abort, continue checking the rest of this
* L2 table's entries.
*/
}
} else {
fprintf(stderr, "ERROR offset=%" PRIx64 ": Data cluster is "
@ -1743,14 +1819,23 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
refcount_table_size,
offset, s->cluster_size);
if (ret < 0) {
goto fail;
return ret;
}
}
break;
}
case QCOW2_CLUSTER_ZERO_PLAIN:
/* Impossible when image has subclusters */
assert(!l2_bitmap);
break;
case QCOW2_CLUSTER_UNALLOCATED:
if (l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC) {
res->corruptions++;
fprintf(stderr, "ERROR: Unallocated "
"cluster has non-zero subcluster allocation map\n");
}
break;
default:
@ -1758,12 +1843,7 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
}
}
g_free(l2_table);
return 0;
fail:
g_free(l2_table);
return ret;
}
/*
@ -1782,71 +1862,79 @@ static int check_refcounts_l1(BlockDriverState *bs,
int flags, BdrvCheckMode fix, bool active)
{
BDRVQcow2State *s = bs->opaque;
uint64_t *l1_table = NULL, l2_offset, l1_size2;
size_t l1_size_bytes = l1_size * L1E_SIZE;
g_autofree uint64_t *l1_table = NULL;
uint64_t l2_offset;
int i, ret;
l1_size2 = l1_size * L1E_SIZE;
if (!l1_size) {
return 0;
}
/* Mark L1 table as used */
ret = qcow2_inc_refcounts_imrt(bs, res, refcount_table, refcount_table_size,
l1_table_offset, l1_size2);
l1_table_offset, l1_size_bytes);
if (ret < 0) {
goto fail;
return ret;
}
l1_table = g_try_malloc(l1_size_bytes);
if (l1_table == NULL) {
res->check_errors++;
return -ENOMEM;
}
/* Read L1 table entries from disk */
if (l1_size2 > 0) {
l1_table = g_try_malloc(l1_size2);
if (l1_table == NULL) {
ret = -ENOMEM;
res->check_errors++;
goto fail;
}
ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
if (ret < 0) {
fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
res->check_errors++;
goto fail;
}
for(i = 0;i < l1_size; i++)
be64_to_cpus(&l1_table[i]);
ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size_bytes);
if (ret < 0) {
fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
res->check_errors++;
return ret;
}
for (i = 0; i < l1_size; i++) {
be64_to_cpus(&l1_table[i]);
}
/* Do the actual checks */
for(i = 0; i < l1_size; i++) {
l2_offset = l1_table[i];
if (l2_offset) {
/* Mark L2 table as used */
l2_offset &= L1E_OFFSET_MASK;
ret = qcow2_inc_refcounts_imrt(bs, res,
refcount_table, refcount_table_size,
l2_offset, s->cluster_size);
if (ret < 0) {
goto fail;
}
for (i = 0; i < l1_size; i++) {
if (!l1_table[i]) {
continue;
}
/* L2 tables are cluster aligned */
if (offset_into_cluster(s, l2_offset)) {
fprintf(stderr, "ERROR l2_offset=%" PRIx64 ": Table is not "
"cluster aligned; L1 entry corrupted\n", l2_offset);
res->corruptions++;
}
if (l1_table[i] & L1E_RESERVED_MASK) {
fprintf(stderr, "ERROR found L1 entry with reserved bits set: "
"%" PRIx64 "\n", l1_table[i]);
res->corruptions++;
}
/* Process and check L2 entries */
ret = check_refcounts_l2(bs, res, refcount_table,
refcount_table_size, l2_offset, flags,
fix, active);
if (ret < 0) {
goto fail;
}
l2_offset = l1_table[i] & L1E_OFFSET_MASK;
/* Mark L2 table as used */
ret = qcow2_inc_refcounts_imrt(bs, res,
refcount_table, refcount_table_size,
l2_offset, s->cluster_size);
if (ret < 0) {
return ret;
}
/* L2 tables are cluster aligned */
if (offset_into_cluster(s, l2_offset)) {
fprintf(stderr, "ERROR l2_offset=%" PRIx64 ": Table is not "
"cluster aligned; L1 entry corrupted\n", l2_offset);
res->corruptions++;
}
/* Process and check L2 entries */
ret = check_refcounts_l2(bs, res, refcount_table,
refcount_table_size, l2_offset, flags,
fix, active);
if (ret < 0) {
return ret;
}
}
g_free(l1_table);
return 0;
fail:
g_free(l1_table);
return ret;
return 0;
}
/*
@ -2001,9 +2089,17 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
for(i = 0; i < s->refcount_table_size; i++) {
uint64_t offset, cluster;
offset = s->refcount_table[i];
offset = s->refcount_table[i] & REFT_OFFSET_MASK;
cluster = offset >> s->cluster_bits;
if (s->refcount_table[i] & REFT_RESERVED_MASK) {
fprintf(stderr, "ERROR refcount table entry %" PRId64 " has "
"reserved bits set\n", i);
res->corruptions++;
*rebuild = true;
continue;
}
/* Refcount blocks are cluster aligned */
if (offset_into_cluster(s, offset)) {
fprintf(stderr, "ERROR refcount block %" PRId64 " is not "

13
block/qcow2.c
View File

@ -74,7 +74,7 @@ typedef struct {
static int coroutine_fn
qcow2_co_preadv_compressed(BlockDriverState *bs,
uint64_t cluster_descriptor,
uint64_t l2_entry,
uint64_t offset,
uint64_t bytes,
QEMUIOVector *qiov,
@ -2205,7 +2205,7 @@ typedef struct Qcow2AioTask {
BlockDriverState *bs;
QCow2SubclusterType subcluster_type; /* only for read */
uint64_t host_offset; /* or full descriptor in compressed clusters */
uint64_t host_offset; /* or l2_entry for compressed read */
uint64_t offset;
uint64_t bytes;
QEMUIOVector *qiov;
@ -4693,22 +4693,19 @@ qcow2_co_pwritev_compressed_part(BlockDriverState *bs,
static int coroutine_fn
qcow2_co_preadv_compressed(BlockDriverState *bs,
uint64_t cluster_descriptor,
uint64_t l2_entry,
uint64_t offset,
uint64_t bytes,
QEMUIOVector *qiov,
size_t qiov_offset)
{
BDRVQcow2State *s = bs->opaque;
int ret = 0, csize, nb_csectors;
int ret = 0, csize;
uint64_t coffset;
uint8_t *buf, *out_buf;
int offset_in_cluster = offset_into_cluster(s, offset);
coffset = cluster_descriptor & s->cluster_offset_mask;
nb_csectors = ((cluster_descriptor >> s->csize_shift) & s->csize_mask) + 1;
csize = nb_csectors * QCOW2_COMPRESSED_SECTOR_SIZE -
(coffset & ~QCOW2_COMPRESSED_SECTOR_MASK);
qcow2_parse_compressed_l2_entry(bs, l2_entry, &coffset, &csize);
buf = g_try_malloc(csize);
if (!buf) {

7
block/qcow2.h
View File

@ -110,7 +110,6 @@
/* Defined in the qcow2 spec (compressed cluster descriptor) */
#define QCOW2_COMPRESSED_SECTOR_SIZE 512U
#define QCOW2_COMPRESSED_SECTOR_MASK (~(QCOW2_COMPRESSED_SECTOR_SIZE - 1ULL))
/* Must be at least 2 to cover COW */
#define MIN_L2_CACHE_SIZE 2 /* cache entries */
@ -587,10 +586,12 @@ typedef enum QCow2MetadataOverlap {
(QCOW2_OL_CACHED | QCOW2_OL_INACTIVE_L2)
#define L1E_OFFSET_MASK 0x00fffffffffffe00ULL
#define L1E_RESERVED_MASK 0x7f000000000001ffULL
#define L2E_OFFSET_MASK 0x00fffffffffffe00ULL
#define L2E_COMPRESSED_OFFSET_SIZE_MASK 0x3fffffffffffffffULL
#define L2E_STD_RESERVED_MASK 0x3f000000000001feULL
#define REFT_OFFSET_MASK 0xfffffffffffffe00ULL
#define REFT_RESERVED_MASK 0x1ffULL
#define INV_OFFSET (-1ULL)
@ -914,6 +915,8 @@ int qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
uint64_t offset,
int compressed_size,
uint64_t *host_offset);
void qcow2_parse_compressed_l2_entry(BlockDriverState *bs, uint64_t l2_entry,
uint64_t *coffset, int *csize);
int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m);

4
docs/tools/qemu-img.rst
View File

@ -415,7 +415,7 @@ Command description:
4
Error on reading data
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps [--skip-broken-bitmaps]] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-r RATE_LIMIT] [-m NUM_COROUTINES] [-W] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps [--skip-broken-bitmaps]] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE [-F backing_fmt]] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-r RATE_LIMIT] [-m NUM_COROUTINES] [-W] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
Convert the disk image *FILENAME* or a snapshot *SNAPSHOT_PARAM*
to disk image *OUTPUT_FILENAME* using format *OUTPUT_FMT*. It can
@ -439,7 +439,7 @@ Command description:
You can use the *BACKING_FILE* option to force the output image to be
created as a copy on write image of the specified base image; the
*BACKING_FILE* should have the same content as the input's base image,
however the path, image format, etc may differ.
however the path, image format (as given by *BACKING_FMT*), etc may differ.
If a relative path name is given, the backing file is looked up relative to
the directory containing *OUTPUT_FILENAME*.

61
include/block/block_int.h
View File

@ -34,6 +34,7 @@
#include "qemu/hbitmap.h"
#include "block/snapshot.h"
#include "qemu/throttle.h"
#include "qemu/rcu.h"
#define BLOCK_FLAG_LAZY_REFCOUNTS 8
@ -347,6 +348,15 @@ struct BlockDriver {
* clamped to bdrv_getlength() and aligned to request_alignment,
* as well as non-NULL pnum, map, and file; in turn, the driver
* must return an error or set pnum to an aligned non-zero value.
*
* Note that @bytes is just a hint on how big of a region the
* caller wants to inspect. It is not a limit on *pnum.
* Implementations are free to return larger values of *pnum if
* doing so does not incur a performance penalty.
*
* block/io.c's bdrv_co_block_status() will utilize an unclamped
* *pnum value for the block-status cache on protocol nodes, prior
* to clamping *pnum for return to its caller.
*/
int coroutine_fn (*bdrv_co_block_status)(BlockDriverState *bs,
bool want_zero, int64_t offset, int64_t bytes, int64_t *pnum,
@ -840,11 +850,23 @@ struct BdrvChild {
};
/*
* Note: the function bdrv_append() copies and swaps contents of
* BlockDriverStates, so if you add new fields to this struct, please
* inspect bdrv_append() to determine if the new fields need to be
* copied as well.
* Allows bdrv_co_block_status() to cache one data region for a
* protocol node.
*
* @valid: Whether the cache is valid (should be accessed with atomic
* functions so this can be reset by RCU readers)
* @data_start: Offset where we know (or strongly assume) is data
* @data_end: Offset where the data region ends (which is not necessarily
* the start of a zeroed region)
*/
typedef struct BdrvBlockStatusCache {
struct rcu_head rcu;
bool valid;
int64_t data_start;
int64_t data_end;
} BdrvBlockStatusCache;
struct BlockDriverState {
/* Protected by big QEMU lock or read-only after opening. No special
* locking needed during I/O...
@ -1010,6 +1032,11 @@ struct BlockDriverState {
/* BdrvChild links to this node may never be frozen */
bool never_freeze;
/* Lock for block-status cache RCU writers */
CoMutex bsc_modify_lock;
/* Always non-NULL, but must only be dereferenced under an RCU read guard */
BdrvBlockStatusCache *block_status_cache;
};
struct BlockBackendRootState {
@ -1435,4 +1462,30 @@ static inline BlockDriverState *bdrv_primary_bs(BlockDriverState *bs)
*/
void bdrv_drain_all_end_quiesce(BlockDriverState *bs);
/**
* Check whether the given offset is in the cached block-status data
* region.
*
* If it is, and @pnum is not NULL, *pnum is set to
* `bsc.data_end - offset`, i.e. how many bytes, starting from
* @offset, are data (according to the cache).
* Otherwise, *pnum is not touched.
*/
bool bdrv_bsc_is_data(BlockDriverState *bs, int64_t offset, int64_t *pnum);
/**
* If [offset, offset + bytes) overlaps with the currently cached
* block-status region, invalidate the cache.
*
* (To be used by I/O paths that cause data regions to be zero or
* holes.)
*/
void bdrv_bsc_invalidate_range(BlockDriverState *bs,
int64_t offset, int64_t bytes);
/**
* Mark the range [offset, offset + bytes) as a data region.
*/
void bdrv_bsc_fill(BlockDriverState *bs, int64_t offset, int64_t bytes);
#endif /* BLOCK_INT_H */

2
qemu-img-cmds.hx
View File

@ -46,7 +46,7 @@ SRST
ERST
DEF("convert", img_convert,
"convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-r rate_limit] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
"convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file [-F backing_fmt]] [-o options] [-l snapshot_param] [-S sparse_size] [-r rate_limit] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
SRST
.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-r RATE_LIMIT] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
ERST

18
qemu-img.c
View File

@ -2183,7 +2183,8 @@ static int img_convert(int argc, char **argv)
int c, bs_i, flags, src_flags = BDRV_O_NO_SHARE;
const char *fmt = NULL, *out_fmt = NULL, *cache = "unsafe",
*src_cache = BDRV_DEFAULT_CACHE, *out_baseimg = NULL,
*out_filename, *out_baseimg_param, *snapshot_name = NULL;
*out_filename, *out_baseimg_param, *snapshot_name = NULL,
*backing_fmt = NULL;
BlockDriver *drv = NULL, *proto_drv = NULL;
BlockDriverInfo bdi;
BlockDriverState *out_bs;
@ -2223,7 +2224,7 @@ static int img_convert(int argc, char **argv)
{"skip-broken-bitmaps", no_argument, 0, OPTION_SKIP_BROKEN},
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WUr:",
c = getopt_long(argc, argv, ":hf:O:B:CcF:o:l:S:pt:T:qnm:WUr:",
long_options, NULL);
if (c == -1) {
break;
@ -2253,6 +2254,9 @@ static int img_convert(int argc, char **argv)
case 'c':
s.compressed = true;
break;
case 'F':
backing_fmt = optarg;
break;
case 'o':
if (accumulate_options(&options, optarg) < 0) {
goto fail_getopt;
@ -2521,7 +2525,7 @@ static int img_convert(int argc, char **argv)
qemu_opt_set_number(opts, BLOCK_OPT_SIZE,
s.total_sectors * BDRV_SECTOR_SIZE, &error_abort);
ret = add_old_style_options(out_fmt, opts, out_baseimg, NULL);
ret = add_old_style_options(out_fmt, opts, out_baseimg, backing_fmt);
if (ret < 0) {
goto out;
}
@ -2628,6 +2632,14 @@ static int img_convert(int argc, char **argv)
goto out;
}
if (flags & BDRV_O_NOCACHE) {
/*
* If we open the target with O_DIRECT, it may be necessary to
* extend its size to align to the physical sector size.
*/
flags |= BDRV_O_RESIZE;
}
if (skip_create) {
s.target = img_open(tgt_image_opts, out_filename, out_fmt,
flags, writethrough, s.quiet, false);

95
scripts/simplebench/img_bench_templater.py Executable file
View File

@ -0,0 +1,95 @@
#!/usr/bin/env python3
#
# Process img-bench test templates
#
# Copyright (c) 2021 Virtuozzo International GmbH.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import sys
import subprocess
import re
import json
import simplebench
from results_to_text import results_to_text
from table_templater import Templater
def bench_func(env, case):
test = templater.gen(env['data'], case['data'])
p = subprocess.run(test, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True)
if p.returncode == 0:
try:
m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout)
return {'seconds': float(m.group(1))}
except Exception:
return {'error': f'failed to parse qemu-img output: {p.stdout}'}
else:
return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'}
if __name__ == '__main__':
if len(sys.argv) > 1:
print("""
Usage: img_bench_templater.py < path/to/test-template.sh
This script generates performance tests from a test template (example below),
runs them, and displays the results in a table. The template is read from
stdin. It must be written in bash and end with a `qemu-img bench` invocation
(whose result is parsed to get the test instance’s result).
Use the following syntax in the template to create the various different test
instances:
column templating: {var1|var2|...} - test will use different values in
different columns. You may use several {} constructions in the test, in this
case product of all choice-sets will be used.
row templating: [var1|var2|...] - similar thing to define rows (test-cases)
Test template example:
Assume you want to compare two qemu-img binaries, called qemu-img-old and
qemu-img-new in your build directory in two test-cases with 4K writes and 64K
writes. The template may look like this:
qemu_img=/path/to/qemu/build/qemu-img-{old|new}
$qemu_img create -f qcow2 /ssd/x.qcow2 1G
$qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2
When passing this to stdin of img_bench_templater.py, the resulting comparison
table will contain two columns (for two binaries) and two rows (for two
test-cases).
In addition to displaying the results, script also stores results in JSON
format into results.json file in current directory.
""")
sys.exit()
templater = Templater(sys.stdin.read())
envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns]
cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows]
result = simplebench.bench(bench_func, envs, cases, count=5,
initial_run=False)
print(results_to_text(result))
with open('results.json', 'w') as f:
json.dump(result, f, indent=4)

62
scripts/simplebench/table_templater.py Normal file
View File

@ -0,0 +1,62 @@
# Parser for test templates
#
# Copyright (c) 2021 Virtuozzo International GmbH.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import itertools
from lark import Lark
grammar = """
start: ( text | column_switch | row_switch )+
column_switch: "{" text ["|" text]+ "}"
row_switch: "[" text ["|" text]+ "]"
text: /[^|{}\[\]]+/
"""
parser = Lark(grammar)
class Templater:
def __init__(self, template):
self.tree = parser.parse(template)
c_switches = []
r_switches = []
for x in self.tree.children:
if x.data == 'column_switch':
c_switches.append([el.children[0].value for el in x.children])
elif x.data == 'row_switch':
r_switches.append([el.children[0].value for el in x.children])
self.columns = list(itertools.product(*c_switches))
self.rows = list(itertools.product(*r_switches))
def gen(self, column, row):
i = 0
j = 0
result = []
for x in self.tree.children:
if x.data == 'text':
result.append(x.children[0].value)
elif x.data == 'column_switch':
result.append(column[i])
i += 1
elif x.data == 'row_switch':
result.append(row[j])
j += 1
return ''.join(result)

2
tests/qemu-iotests/122
View File

@ -67,7 +67,7 @@ echo
_make_test_img -b "$TEST_IMG".base -F $IMGFMT
$QEMU_IO -c "write -P 0 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base -o backing_fmt=$IMGFMT \
$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base -F $IMGFMT \
"$TEST_IMG" "$TEST_IMG".orig
$QEMU_IO -c "read -P 0 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
$QEMU_IMG convert -O $IMGFMT -c -B "$TEST_IMG".base -o backing_fmt=$IMGFMT \

5
tests/qemu-iotests/271
View File

@ -893,7 +893,10 @@ EOF
}
_make_test_img -o extended_l2=on 1M
_concurrent_io | $QEMU_IO | _filter_qemu_io
# Second and third writes in _concurrent_io() are independent and may finish in
# different order. So, filter offset out to match both possible variants.
_concurrent_io | $QEMU_IO | _filter_qemu_io | \
$SED -e 's/\(20480\|40960\)/OFFSET/'
_concurrent_verify | $QEMU_IO | _filter_qemu_io
# success, all done

4
tests/qemu-iotests/271.out
View File

@ -719,8 +719,8 @@ blkdebug: Suspended request 'A'
blkdebug: Resuming request 'A'
wrote 2048/2048 bytes at offset 30720
2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 2048/2048 bytes at offset 20480
wrote 2048/2048 bytes at offset OFFSET
2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 2048/2048 bytes at offset 40960
wrote 2048/2048 bytes at offset OFFSET
2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
*** done

9
tests/qemu-iotests/297
View File

@ -29,7 +29,7 @@ import iotests
SKIP_FILES = (
'030', '040', '041', '044', '045', '055', '056', '057', '065', '093',
'096', '118', '124', '132', '136', '139', '147', '148', '149',
'151', '152', '155', '163', '165', '169', '194', '196', '199', '202',
'151', '152', '155', '163', '165', '194', '196', '202',
'203', '205', '206', '207', '208', '210', '211', '212', '213', '216',
'218', '219', '224', '228', '234', '235', '236', '237', '238',
'240', '242', '245', '246', '248', '255', '256', '257', '258', '260',
@ -46,7 +46,7 @@ def is_python_file(filename):
if filename.endswith('.py'):
return True
with open(filename) as f:
with open(filename, encoding='utf-8') as f:
try:
first_line = f.readline()
return re.match('^#!.*python', first_line) is not None
@ -55,8 +55,9 @@ def is_python_file(filename):
def run_linters():
files = [filename for filename in (set(os.listdir('.')) - set(SKIP_FILES))
if is_python_file(filename)]
named_tests = [f'tests/{entry}' for entry in os.listdir('tests')]
check_tests = set(os.listdir('.') + named_tests) - set(SKIP_FILES)
files = [filename for filename in check_tests if is_python_file(filename)]
iotests.logger.debug('Files to be checked:')
iotests.logger.debug(', '.join(sorted(files)))

12
tests/qemu-iotests/iotests.py
View File

@ -610,7 +610,7 @@ class VM(qtest.QEMUQtestMachine):
return
valgrind_filename = f"{test_dir}/{self._popen.pid}.valgrind"
if self.exitcode() == 99:
with open(valgrind_filename) as f:
with open(valgrind_filename, encoding='utf-8') as f:
print(f.read())
else:
os.remove(valgrind_filename)
@ -703,7 +703,7 @@ class VM(qtest.QEMUQtestMachine):
def flatten_qmp_object(self, obj, output=None, basestr=''):
if output is None:
output = dict()
output = {}
if isinstance(obj, list):
for i, item in enumerate(obj):
self.flatten_qmp_object(item, output, basestr + str(i) + '.')
@ -716,7 +716,7 @@ class VM(qtest.QEMUQtestMachine):
def qmp_to_opts(self, obj):
obj = self.flatten_qmp_object(obj)
output_list = list()
output_list = []
for key in obj:
output_list += [key + '=' + obj[key]]
return ','.join(output_list)
@ -1121,7 +1121,8 @@ def notrun(reason):
# Each test in qemu-iotests has a number ("seq")
seq = os.path.basename(sys.argv[0])
with open('%s/%s.notrun' % (output_dir, seq), 'w') as outfile:
with open('%s/%s.notrun' % (output_dir, seq), 'w', encoding='utf-8') \
as outfile:
outfile.write(reason + '\n')
logger.warning("%s not run: %s", seq, reason)
sys.exit(0)
@ -1135,7 +1136,8 @@ def case_notrun(reason):
# Each test in qemu-iotests has a number ("seq")
seq = os.path.basename(sys.argv[0])
with open('%s/%s.casenotrun' % (output_dir, seq), 'a') as outfile:
with open('%s/%s.casenotrun' % (output_dir, seq), 'a', encoding='utf-8') \
as outfile:
outfile.write(' [case not run] ' + reason + '\n')
def _verify_image_format(supported_fmts: Sequence[str] = (),

13
tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test
View File

@ -132,10 +132,10 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap0')
self.discards1_sha256 = result['return']['sha256']
discards1_sha256 = result['return']['sha256']
# Check, that updating the bitmap by discards works
assert self.discards1_sha256 != empty_sha256
assert discards1_sha256 != empty_sha256
# We want to calculate resulting sha256. Do it in bitmap0, so, disable
# other bitmaps
@ -148,7 +148,7 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
result = self.vm_a.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap0')
self.all_discards_sha256 = result['return']['sha256']
all_discards_sha256 = result['return']['sha256']
# Now, enable some bitmaps, to be updated during migration
for i in range(2, nb_bitmaps, 2):
@ -173,10 +173,11 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
event_resume = self.vm_b.event_wait('RESUME')
self.vm_b_events.append(event_resume)
return event_resume
return (event_resume, discards1_sha256, all_discards_sha256)
def test_postcopy_success(self):
event_resume = self.start_postcopy()
event_resume, discards1_sha256, all_discards_sha256 = \
self.start_postcopy()
# enabled bitmaps should be updated
apply_discards(self.vm_b, discards2)
@ -217,7 +218,7 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
for i in range(0, nb_bitmaps, 5):
result = self.vm_b.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap{}'.format(i))
sha = self.discards1_sha256 if i % 2 else self.all_discards_sha256
sha = discards1_sha256 if i % 2 else all_discards_sha256
self.assert_qmp(result, 'return/sha256', sha)
def test_early_shutdown_destination(self):

43
tests/qemu-iotests/tests/migrate-bitmaps-test
View File

@ -20,11 +20,10 @@
#
import os
import iotests
import time
import itertools
import operator
import re
import iotests
from iotests import qemu_img, qemu_img_create, Timeout
@ -37,6 +36,12 @@ mig_cmd = 'exec: cat > ' + mig_file
incoming_cmd = 'exec: cat ' + mig_file
def get_bitmap_hash(vm):
result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap0')
return result['return']['sha256']
class TestDirtyBitmapMigration(iotests.QMPTestCase):
def tearDown(self):
self.vm_a.shutdown()
@ -62,21 +67,16 @@ class TestDirtyBitmapMigration(iotests.QMPTestCase):
params['persistent'] = True
result = vm.qmp('block-dirty-bitmap-add', **params)
self.assert_qmp(result, 'return', {});
def get_bitmap_hash(self, vm):
result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap0')
return result['return']['sha256']
self.assert_qmp(result, 'return', {})
def check_bitmap(self, vm, sha256):
result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
node='drive0', name='bitmap0')
if sha256:
self.assert_qmp(result, 'return/sha256', sha256);
self.assert_qmp(result, 'return/sha256', sha256)
else:
self.assert_qmp(result, 'error/desc',
"Dirty bitmap 'bitmap0' not found");
"Dirty bitmap 'bitmap0' not found")
def do_test_migration_resume_source(self, persistent, migrate_bitmaps):
granularity = 512
@ -97,7 +97,7 @@ class TestDirtyBitmapMigration(iotests.QMPTestCase):
self.add_bitmap(self.vm_a, granularity, persistent)
for r in regions:
self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
sha256 = self.get_bitmap_hash(self.vm_a)
sha256 = get_bitmap_hash(self.vm_a)
result = self.vm_a.qmp('migrate', uri=mig_cmd)
while True:
@ -106,7 +106,7 @@ class TestDirtyBitmapMigration(iotests.QMPTestCase):
break
while True:
result = self.vm_a.qmp('query-status')
if (result['return']['status'] == 'postmigrate'):
if result['return']['status'] == 'postmigrate':
break
# test that bitmap is still here
@ -164,7 +164,7 @@ class TestDirtyBitmapMigration(iotests.QMPTestCase):
self.add_bitmap(self.vm_a, granularity, persistent)
for r in regions:
self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
sha256 = self.get_bitmap_hash(self.vm_a)
sha256 = get_bitmap_hash(self.vm_a)
if pre_shutdown:
self.vm_a.shutdown()
@ -214,16 +214,22 @@ class TestDirtyBitmapMigration(iotests.QMPTestCase):
self.check_bitmap(self.vm_b, sha256 if persistent else False)
def inject_test_case(klass, name, method, *args, **kwargs):
def inject_test_case(klass, suffix, method, *args, **kwargs):
mc = operator.methodcaller(method, *args, **kwargs)
setattr(klass, 'test_' + method + name, lambda self: mc(self))
# We want to add a function attribute to `klass`, so that it is
# correctly converted to a method on instantiation. The
# methodcaller object `mc` is a callable, not a function, so we
# need the lambda to turn it into a function.
# pylint: disable=unnecessary-lambda
setattr(klass, 'test_' + method + suffix, lambda self: mc(self))
for cmb in list(itertools.product((True, False), repeat=5)):
name = ('_' if cmb[0] else '_not_') + 'persistent_'
name += ('_' if cmb[1] else '_not_') + 'migbitmap_'
name += '_online' if cmb[2] else '_offline'
name += '_shared' if cmb[3] else '_nonshared'
if (cmb[4]):
if cmb[4]:
name += '__pre_shutdown'
inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration',
@ -270,7 +276,8 @@ class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
self.assert_qmp(result, 'return', {})
# Check that the bitmaps are there
for node in self.vm.qmp('query-named-block-nodes', flat=True)['return']:
nodes = self.vm.qmp('query-named-block-nodes', flat=True)['return']
for node in nodes:
if 'node0' in node['node-name']:
self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')
@ -287,7 +294,7 @@ class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
"""
Continue the source after migration.
"""
result = self.vm.qmp('migrate', uri=f'exec: cat > /dev/null')
result = self.vm.qmp('migrate', uri='exec: cat > /dev/null')
self.assert_qmp(result, 'return', {})
with Timeout(10, 'Migration timeout'):

97
tests/qemu-iotests/tests/migrate-during-backup Executable file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env python3
# group: migration
#
# Copyright (c) 2021 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import iotests
from iotests import qemu_img_create, qemu_io
disk_a = os.path.join(iotests.test_dir, 'disk_a')
disk_b = os.path.join(iotests.test_dir, 'disk_b')
size = '1M'
mig_file = os.path.join(iotests.test_dir, 'mig_file')
mig_cmd = 'exec: cat > ' + mig_file
class TestMigrateDuringBackup(iotests.QMPTestCase):
def tearDown(self):
self.vm.shutdown()
os.remove(disk_a)
os.remove(disk_b)
os.remove(mig_file)
def setUp(self):
qemu_img_create('-f', iotests.imgfmt, disk_a, size)
qemu_img_create('-f', iotests.imgfmt, disk_b, size)
qemu_io('-c', f'write 0 {size}', disk_a)
self.vm = iotests.VM().add_drive(disk_a)
self.vm.launch()
result = self.vm.qmp('blockdev-add', {
'node-name': 'target',
'driver': iotests.imgfmt,
'file': {
'driver': 'file',
'filename': disk_b
}
})
self.assert_qmp(result, 'return', {})
def test_migrate(self):
result = self.vm.qmp('blockdev-backup', device='drive0',
target='target', sync='full',
speed=1, x_perf={
'max-workers': 1,
'max-chunk': 64 * 1024
})
self.assert_qmp(result, 'return', {})
result = self.vm.qmp('job-pause', id='drive0')
self.assert_qmp(result, 'return', {})
result = self.vm.qmp('migrate-set-capabilities',
capabilities=[{'capability': 'events',
'state': True}])
self.assert_qmp(result, 'return', {})
result = self.vm.qmp('migrate', uri=mig_cmd)
self.assert_qmp(result, 'return', {})
e = self.vm.events_wait((('MIGRATION',
{'data': {'status': 'completed'}}),
('MIGRATION',
{'data': {'status': 'failed'}})))
# Don't assert that e is 'failed' now: this way we'll miss
# possible crash when backup continues :)
result = self.vm.qmp('block-job-set-speed', device='drive0',
speed=0)
self.assert_qmp(result, 'return', {})
result = self.vm.qmp('job-resume', id='drive0')
self.assert_qmp(result, 'return', {})
# For future: if something changes so that both migration
# and backup pass, let's not miss that moment, as it may
# be a bug as well as improvement.
self.assert_qmp(e, 'data/status', 'failed')
if __name__ == '__main__':
iotests.main(supported_fmts=['qcow2'],
supported_protocols=['file'])

5
tests/qemu-iotests/tests/migrate-during-backup.out Normal file
View File

@ -0,0 +1,5 @@
.
----------------------------------------------------------------------
Ran 1 tests
OK

2
tests/qemu-iotests/tests/mirror-top-perms
View File

@ -47,7 +47,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase):
def tearDown(self):
try:
self.vm.shutdown()
except qemu.machine.AbnormalShutdown:
except qemu.machine.machine.AbnormalShutdown:
pass
if self.vm_b is not None: