OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

Block patches:

Browse Source 
- Make the backup-top filter driver available for user-created block
   nodes (i.e. via blockdev-add)
 - Allow running iotests with gdb or valgrind being attached to qemu
   instances
 - Fix the raw format driver's permissions: There is no metadata, so we
   only need WRITE or RESIZE when the parent needs it
 - Basic reopen implementation for win32 files (file-win32.c) so that
   qemu-img commit can work
 - uclibc/musl build fix for the FUSE export code
 - Some iotests delinting
 - block-hmp-cmds.c refactoring
 -----BEGIN PGP SIGNATURE-----
 
 iQJGBAABCAAwFiEEy2LXoO44KeRfAE00ofpA0JgBnN8FAmEvleISHGhyZWl0ekBy
 ZWRoYXQuY29tAAoJEKH6QNCYAZzfdOEP/j4gutKzxqHEgaOxus3e1u77JnIX5OBO
 E3wr0W8IaILp3a3N8f8lcq8frw6aSvTmW8l2woKNbg/C1yuX4NFN9tyQ6jpoFAP9
 9X9GoPlU7YG5c1bEnJlO/ySt3xHRssIsZBpKWnzWwUI5nMpGUrNPem3rW8T2DaPy
 RwnRhBl2kzHYqyPXDx13lA3zKIunAISWRM9adWyKDdRo6Lqk0Us7ND+f6nRHJSG1
 uJ26uKWWXx+qYC7F8uc45vrOjesWwC0sqUn7RC/0pbBGp9L6Bgc3yWbnJWZBNEBM
 zbv47B6HsJs2tqHGj0T+EKkhqChGz3B/vMeSSw5c3dXFBfQ53Rjm4Nlr9YBzuCGV
 erMoq0j/Ytz8+T865N/kjzwdgkl+xcKWF/GIaM5rxiJ2syyCV9CY2SxD6AC+WPBk
 yCezNnZEAx2POS2ylRy+EQvJm3YdoWrXZr05Blj28TtqNLs3qCP7evG6IjH58idU
 A4YgmltwN5UdajOK9b7O7zFAFhCZCKqAVJNKI0NCTYaT3zEim5dduXfn3gHTu5Wl
 jgWvpicNgsEXC4/etp5jOVkbBXtelh66ibdDIQJEanAG1W0Br+gQIBLN84pf7gpY
 8R9BBpZRk2DzYJnYJS2FV6xRGKY+XjU4zd7yVfyq6jZYyvfUxBDhCghCB6M37EmJ
 oE64kzab7uVm
 =0tJJ
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/hreitz/tags/pull-block-2021-09-01' into staging

Block patches:
- Make the backup-top filter driver available for user-created block
  nodes (i.e. via blockdev-add)
- Allow running iotests with gdb or valgrind being attached to qemu
  instances
- Fix the raw format driver's permissions: There is no metadata, so we
  only need WRITE or RESIZE when the parent needs it
- Basic reopen implementation for win32 files (file-win32.c) so that
  qemu-img commit can work
- uclibc/musl build fix for the FUSE export code
- Some iotests delinting
- block-hmp-cmds.c refactoring

# gpg: Signature made Wed 01 Sep 2021 16:01:54 BST
# gpg:                using RSA key CB62D7A0EE3829E45F004D34A1FA40D098019CDF
# gpg:                issuer "hreitz@redhat.com"
# gpg: Good signature from "Hanna Reitz <hreitz@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: CB62 D7A0 EE38 29E4 5F00  4D34 A1FA 40D0 9801 9CDF

* remotes/hreitz/tags/pull-block-2021-09-01: (56 commits)
  block/file-win32: add reopen handlers
  block/export/fuse.c: fix fuse-lseek on uclibc or musl
  block/block-copy: block_copy_state_new(): drop extra arguments
  iotests/image-fleecing: add test-case for copy-before-write filter
  iotests/image-fleecing: prepare for adding new test-case
  iotests/image-fleecing: rename tgt_node
  iotests/image-fleecing: proper source device
  iotests.py: hmp_qemu_io: support qdev
  iotests: move 222 to tests/image-fleecing
  iotests/222: constantly use single quotes for strings
  iotests/222: fix pylint and mypy complains
  python:QEMUMachine: template typing for self returning methods
  python/qemu/machine: QEMUMachine: improve qmp() method
  python/qemu/machine.py: refactor _qemu_args()
  qapi: publish copy-before-write filter
  block/copy-before-write: make public block driver
  block/block-copy: make setting progress optional
  block/copy-before-write: initialize block-copy bitmap
  block/copy-before-write: cbw_init(): use options
  block/copy-before-write: bdrv_cbw_append(): drop unused compress arg
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2021-09-02 13:00:52 +01:00
commit 9093028dd4
37 changed files with 1179 additions and 718 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci.d/buildtest.yml
View File

@ -305,11 +305,11 @@ build-tcg-disabled:
- cd tests/qemu-iotests/
- ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048
052 063 077 086 101 104 106 113 148 150 151 152 157 159 160 163
170 171 183 184 192 194 208 221 222 226 227 236 253 277
170 171 183 184 192 194 208 221 226 227 236 253 277 image-fleecing
- ./check -qcow2 028 051 056 057 058 065 068 082 085 091 095 096 102 122
124 132 139 142 144 145 151 152 155 157 165 194 196 200 202
208 209 216 218 222 227 234 246 247 248 250 254 255 257 258
260 261 262 263 264 270 272 273 277 279
208 209 216 218 227 234 246 247 248 250 254 255 257 258
260 261 262 263 264 270 272 273 277 279 image-fleecing
build-user:
extends: .native_build_job_template

4
MAINTAINERS
View File

@ -2391,8 +2391,8 @@ F: block/mirror.c
F: qapi/job.json
F: block/block-copy.c
F: include/block/block-copy.c
F: block/backup-top.h
F: block/backup-top.c
F: block/copy-before-write.h
F: block/copy-before-write.c
F: include/block/aio_task.h
F: block/aio_task.c
F: util/qemu-co-shared-resource.c

31
block.c
View File

@ -5048,6 +5048,37 @@ out:
return ret;
}
/* Not for empty child */
int bdrv_replace_child_bs(BdrvChild *child, BlockDriverState *new_bs,
Error **errp)
{
int ret;
Transaction *tran = tran_new();
g_autoptr(GHashTable) found = NULL;
g_autoptr(GSList) refresh_list = NULL;
BlockDriverState *old_bs = child->bs;
bdrv_ref(old_bs);
bdrv_drained_begin(old_bs);
bdrv_drained_begin(new_bs);
bdrv_replace_child_tran(child, new_bs, tran);
found = g_hash_table_new(NULL, NULL);
refresh_list = bdrv_topological_dfs(refresh_list, found, old_bs);
refresh_list = bdrv_topological_dfs(refresh_list, found, new_bs);
ret = bdrv_list_refresh_perms(refresh_list, NULL, tran, errp);
tran_finalize(tran, ret);
bdrv_drained_end(old_bs);
bdrv_drained_end(new_bs);
bdrv_unref(old_bs);
return ret;
}
static void bdrv_delete(BlockDriverState *bs)
{
assert(bdrv_op_blocker_is_empty(bs));

253
block/backup-top.c
View File

@ -1,253 +0,0 @@
/*
* backup-top filter driver
*
* The driver performs Copy-Before-Write (CBW) operation: it is injected above
* some node, and before each write it copies _old_ data to the target node.
*
* Copyright (c) 2018-2019 Virtuozzo International GmbH.
*
* Author:
* Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "sysemu/block-backend.h"
#include "qemu/cutils.h"
#include "qapi/error.h"
#include "block/block_int.h"
#include "block/qdict.h"
#include "block/block-copy.h"
#include "block/backup-top.h"
typedef struct BDRVBackupTopState {
BlockCopyState *bcs;
BdrvChild *target;
int64_t cluster_size;
} BDRVBackupTopState;
static coroutine_fn int backup_top_co_preadv(
BlockDriverState *bs, uint64_t offset, uint64_t bytes,
QEMUIOVector *qiov, int flags)
{
return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
}
static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,
uint64_t bytes, BdrvRequestFlags flags)
{
BDRVBackupTopState *s = bs->opaque;
uint64_t off, end;
if (flags & BDRV_REQ_WRITE_UNCHANGED) {
return 0;
}
off = QEMU_ALIGN_DOWN(offset, s->cluster_size);
end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size);
return block_copy(s->bcs, off, end - off, true);
}
static int coroutine_fn backup_top_co_pdiscard(BlockDriverState *bs,
int64_t offset, int bytes)
{
int ret = backup_top_cbw(bs, offset, bytes, 0);
if (ret < 0) {
return ret;
}
return bdrv_co_pdiscard(bs->backing, offset, bytes);
}
static int coroutine_fn backup_top_co_pwrite_zeroes(BlockDriverState *bs,
int64_t offset, int bytes, BdrvRequestFlags flags)
{
int ret = backup_top_cbw(bs, offset, bytes, flags);
if (ret < 0) {
return ret;
}
return bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
}
static coroutine_fn int backup_top_co_pwritev(BlockDriverState *bs,
uint64_t offset,
uint64_t bytes,
QEMUIOVector *qiov, int flags)
{
int ret = backup_top_cbw(bs, offset, bytes, flags);
if (ret < 0) {
return ret;
}
return bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
}
static int coroutine_fn backup_top_co_flush(BlockDriverState *bs)
{
if (!bs->backing) {
return 0;
}
return bdrv_co_flush(bs->backing->bs);
}
static void backup_top_refresh_filename(BlockDriverState *bs)
{
if (bs->backing == NULL) {
/*
* we can be here after failed bdrv_attach_child in
* bdrv_set_backing_hd
*/
return;
}
pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
bs->backing->bs->filename);
}
static void backup_top_child_perm(BlockDriverState *bs, BdrvChild *c,
BdrvChildRole role,
BlockReopenQueue *reopen_queue,
uint64_t perm, uint64_t shared,
uint64_t *nperm, uint64_t *nshared)
{
if (!(role & BDRV_CHILD_FILTERED)) {
/*
* Target child
*
* Share write to target (child_file), to not interfere
* with guest writes to its disk which may be in target backing chain.
* Can't resize during a backup block job because we check the size
* only upfront.
*/
*nshared = BLK_PERM_ALL & ~BLK_PERM_RESIZE;
*nperm = BLK_PERM_WRITE;
} else {
/* Source child */
bdrv_default_perms(bs, c, role, reopen_queue,
perm, shared, nperm, nshared);
if (perm & BLK_PERM_WRITE) {
*nperm = *nperm | BLK_PERM_CONSISTENT_READ;
}
*nshared &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
}
}
BlockDriver bdrv_backup_top_filter = {
.format_name = "backup-top",
.instance_size = sizeof(BDRVBackupTopState),
.bdrv_co_preadv = backup_top_co_preadv,
.bdrv_co_pwritev = backup_top_co_pwritev,
.bdrv_co_pwrite_zeroes = backup_top_co_pwrite_zeroes,
.bdrv_co_pdiscard = backup_top_co_pdiscard,
.bdrv_co_flush = backup_top_co_flush,
.bdrv_refresh_filename = backup_top_refresh_filename,
.bdrv_child_perm = backup_top_child_perm,
.is_filter = true,
};
BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
BlockDriverState *target,
const char *filter_node_name,
uint64_t cluster_size,
BackupPerf *perf,
BdrvRequestFlags write_flags,
BlockCopyState **bcs,
Error **errp)
{
ERRP_GUARD();
int ret;
BDRVBackupTopState *state;
BlockDriverState *top;
bool appended = false;
assert(source->total_sectors == target->total_sectors);
top = bdrv_new_open_driver(&bdrv_backup_top_filter, filter_node_name,
BDRV_O_RDWR, errp);
if (!top) {
return NULL;
}
state = top->opaque;
top->total_sectors = source->total_sectors;
top->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |
(BDRV_REQ_FUA & source->supported_write_flags);
top->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &
source->supported_zero_flags);
bdrv_ref(target);
state->target = bdrv_attach_child(top, target, "target", &child_of_bds,
BDRV_CHILD_DATA, errp);
if (!state->target) {
bdrv_unref(target);
bdrv_unref(top);
return NULL;
}
bdrv_drained_begin(source);
ret = bdrv_append(top, source, errp);
if (ret < 0) {
error_prepend(errp, "Cannot append backup-top filter: ");
goto fail;
}
appended = true;
state->cluster_size = cluster_size;
state->bcs = block_copy_state_new(top->backing, state->target,
cluster_size, perf->use_copy_range,
write_flags, errp);
if (!state->bcs) {
error_prepend(errp, "Cannot create block-copy-state: ");
goto fail;
}
*bcs = state->bcs;
bdrv_drained_end(source);
return top;
fail:
if (appended) {
bdrv_backup_top_drop(top);
} else {
bdrv_unref(top);
}
bdrv_drained_end(source);
return NULL;
}
void bdrv_backup_top_drop(BlockDriverState *bs)
{
BDRVBackupTopState *s = bs->opaque;
bdrv_drop_filter(bs, &error_abort);
block_copy_state_free(s->bcs);
bdrv_unref(bs);
}

116
block/backup.c
View File

@ -27,13 +27,11 @@
#include "qemu/bitmap.h"
#include "qemu/error-report.h"
#include "block/backup-top.h"
#define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)
#include "block/copy-before-write.h"
typedef struct BackupBlockJob {
BlockJob common;
BlockDriverState *backup_top;
BlockDriverState *cbw;
BlockDriverState *source_bs;
BlockDriverState *target_bs;
@ -104,7 +102,7 @@ static void backup_clean(Job *job)
{
BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
block_job_remove_all_bdrv(&s->common);
bdrv_backup_top_drop(s->backup_top);
bdrv_cbw_drop(s->cbw);
}
void backup_do_checkpoint(BlockJob *job, Error **errp)
@ -235,18 +233,16 @@ static void backup_init_bcs_bitmap(BackupBlockJob *job)
BdrvDirtyBitmap *bcs_bitmap = block_copy_dirty_bitmap(job->bcs);
if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
bdrv_clear_dirty_bitmap(bcs_bitmap, NULL);
ret = bdrv_dirty_bitmap_merge_internal(bcs_bitmap, job->sync_bitmap,
NULL, true);
assert(ret);
} else {
if (job->sync_mode == MIRROR_SYNC_MODE_TOP) {
/*
* We can't hog the coroutine to initialize this thoroughly.
* Set a flag and resume work when we are able to yield safely.
*/
block_copy_set_skip_unallocated(job->bcs, true);
}
bdrv_set_dirty_bitmap(bcs_bitmap, 0, job->len);
} else if (job->sync_mode == MIRROR_SYNC_MODE_TOP) {
/*
* We can't hog the coroutine to initialize this thoroughly.
* Set a flag and resume work when we are able to yield safely.
*/
block_copy_set_skip_unallocated(job->bcs, true);
}
estimate = bdrv_get_dirty_count(bcs_bitmap);
@ -354,43 +350,6 @@ static const BlockJobDriver backup_job_driver = {
.set_speed = backup_set_speed,
};
static int64_t backup_calculate_cluster_size(BlockDriverState *target,
Error **errp)
{
int ret;
BlockDriverInfo bdi;
bool target_does_cow = bdrv_backing_chain_next(target);
/*
* If there is no backing file on the target, we cannot rely on COW if our
* backup cluster size is smaller than the target cluster size. Even for
* targets with a backing file, try to avoid COW if possible.
*/
ret = bdrv_get_info(target, &bdi);
if (ret == -ENOTSUP && !target_does_cow) {
/* Cluster size is not defined */
warn_report("The target block device doesn't provide "
"information about the block size and it doesn't have a "
"backing file. The default block size of %u bytes is "
"used. If the actual block size of the target exceeds "
"this default, the backup may be unusable",
BACKUP_CLUSTER_SIZE_DEFAULT);
return BACKUP_CLUSTER_SIZE_DEFAULT;
} else if (ret < 0 && !target_does_cow) {
error_setg_errno(errp, -ret,
"Couldn't determine the cluster size of the target image, "
"which has no backing file");
error_append_hint(errp,
"Aborting, since this may create an unusable destination image\n");
return ret;
} else if (ret < 0 && target_does_cow) {
/* Not fatal; just trudge on ahead. */
return BACKUP_CLUSTER_SIZE_DEFAULT;
}
return MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster_size);
}
BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
BlockDriverState *target, int64_t speed,
MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
@ -407,8 +366,7 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
int64_t len, target_len;
BackupBlockJob *job = NULL;
int64_t cluster_size;
BdrvRequestFlags write_flags;
BlockDriverState *backup_top = NULL;
BlockDriverState *cbw = NULL;
BlockCopyState *bcs = NULL;
assert(bs);
@ -449,11 +407,6 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
return NULL;
}
cluster_size = backup_calculate_cluster_size(target, errp);
if (cluster_size < 0) {
goto error;
}
if (perf->max_workers < 1) {
error_setg(errp, "max-workers must be greater than zero");
return NULL;
@ -465,13 +418,6 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
return NULL;
}
if (perf->max_chunk && perf->max_chunk < cluster_size) {
error_setg(errp, "Required max-chunk (%" PRIi64 ") is less than backup "
"cluster size (%" PRIi64 ")", perf->max_chunk, cluster_size);
return NULL;
}
if (sync_bitmap) {
/* If we need to write to this bitmap, check that we can: */
if (bitmap_mode != BITMAP_SYNC_MODE_NEVER &&
@ -504,39 +450,28 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
goto error;
}
/*
* If source is in backing chain of target assume that target is going to be
* used for "image fleecing", i.e. it should represent a kind of snapshot of
* source at backup-start point in time. And target is going to be read by
* somebody (for example, used as NBD export) during backup job.
*
* In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
* intersection of backup writes and third party reads from target,
* otherwise reading from target we may occasionally read already updated by
* guest data.
*
* For more information see commit f8d59dfb40bb and test
* tests/qemu-iotests/222
*/
write_flags = (bdrv_chain_contains(target, bs) ? BDRV_REQ_SERIALISING : 0) |
(compress ? BDRV_REQ_WRITE_COMPRESSED : 0),
cbw = bdrv_cbw_append(bs, target, filter_node_name, &bcs, errp);
if (!cbw) {
goto error;
}
backup_top = bdrv_backup_top_append(bs, target, filter_node_name,
cluster_size, perf,
write_flags, &bcs, errp);
if (!backup_top) {
cluster_size = block_copy_cluster_size(bcs);
if (perf->max_chunk && perf->max_chunk < cluster_size) {
error_setg(errp, "Required max-chunk (%" PRIi64 ") is less than backup "
"cluster size (%" PRIi64 ")", perf->max_chunk, cluster_size);
goto error;
}
/* job->len is fixed, so we can't allow resize */
job = block_job_create(job_id, &backup_job_driver, txn, backup_top,
job = block_job_create(job_id, &backup_job_driver, txn, cbw,
0, BLK_PERM_ALL,
speed, creation_flags, cb, opaque, errp);
if (!job) {
goto error;
}
job->backup_top = backup_top;
job->cbw = cbw;
job->source_bs = bs;
job->target_bs = target;
job->on_source_error = on_source_error;
@ -549,10 +484,11 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
job->len = len;
job->perf = *perf;
block_copy_set_copy_opts(bcs, perf->use_copy_range, compress);
block_copy_set_progress_meter(bcs, &job->common.job.progress);
block_copy_set_speed(bcs, speed);
/* Required permissions are already taken by backup-top target */
/* Required permissions are taken by copy-before-write filter target */
block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL,
&error_abort);
@ -562,8 +498,8 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
if (sync_bitmap) {
bdrv_reclaim_dirty_bitmap(sync_bitmap, NULL);
}
if (backup_top) {
bdrv_backup_top_drop(backup_top);
if (cbw) {
bdrv_cbw_drop(cbw);
}
return NULL;

8
block/block-backend.c
View File

@ -869,6 +869,14 @@ int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp)
return 0;
}
/*
* Change BlockDriverState associated with @blk.
*/
int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp)
{
return bdrv_replace_child_bs(blk->root, new_bs, errp);
}
/*
* Sets the permission bitmasks that the user of the BlockBackend needs.
*/

150
block/block-copy.c
View File

@ -21,12 +21,14 @@
#include "qemu/units.h"
#include "qemu/coroutine.h"
#include "block/aio_task.h"
#include "qemu/error-report.h"
#define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
#define BLOCK_COPY_MAX_BUFFER (1 * MiB)
#define BLOCK_COPY_MAX_MEM (128 * MiB)
#define BLOCK_COPY_MAX_WORKERS 64
#define BLOCK_COPY_SLICE_TIME 100000000ULL /* ns */
#define BLOCK_COPY_CLUSTER_SIZE_DEFAULT (1 << 16)
typedef enum {
COPY_READ_WRITE_CLUSTER,
@ -290,9 +292,11 @@ static void coroutine_fn block_copy_task_end(BlockCopyTask *task, int ret)
bdrv_set_dirty_bitmap(task->s->copy_bitmap, task->offset, task->bytes);
}
QLIST_REMOVE(task, list);
progress_set_remaining(task->s->progress,
bdrv_get_dirty_count(task->s->copy_bitmap) +
task->s->in_flight_bytes);
if (task->s->progress) {
progress_set_remaining(task->s->progress,
bdrv_get_dirty_count(task->s->copy_bitmap) +
task->s->in_flight_bytes);
}
qemu_co_queue_restart_all(&task->wait_queue);
}
@ -315,35 +319,14 @@ static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target)
target->bs->bl.max_transfer));
}
BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
int64_t cluster_size, bool use_copy_range,
BdrvRequestFlags write_flags, Error **errp)
void block_copy_set_copy_opts(BlockCopyState *s, bool use_copy_range,
bool compress)
{
BlockCopyState *s;
BdrvDirtyBitmap *copy_bitmap;
/* Keep BDRV_REQ_SERIALISING set (or not set) in block_copy_state_new() */
s->write_flags = (s->write_flags & BDRV_REQ_SERIALISING) |
(compress ? BDRV_REQ_WRITE_COMPRESSED : 0);
copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
errp);
if (!copy_bitmap) {
return NULL;
}
bdrv_disable_dirty_bitmap(copy_bitmap);
s = g_new(BlockCopyState, 1);
*s = (BlockCopyState) {
.source = source,
.target = target,
.copy_bitmap = copy_bitmap,
.cluster_size = cluster_size,
.len = bdrv_dirty_bitmap_size(copy_bitmap),
.write_flags = write_flags,
.mem = shres_create(BLOCK_COPY_MAX_MEM),
.max_transfer = QEMU_ALIGN_DOWN(
block_copy_max_transfer(source, target),
cluster_size),
};
if (s->max_transfer < cluster_size) {
if (s->max_transfer < s->cluster_size) {
/*
* copy_range does not respect max_transfer. We don't want to bother
* with requests smaller than block-copy cluster size, so fallback to
@ -351,7 +334,7 @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
* behalf).
*/
s->method = COPY_READ_WRITE_CLUSTER;
} else if (write_flags & BDRV_REQ_WRITE_COMPRESSED) {
} else if (compress) {
/* Compression supports only cluster-size writes and no copy-range. */
s->method = COPY_READ_WRITE_CLUSTER;
} else {
@ -361,6 +344,96 @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
*/
s->method = use_copy_range ? COPY_RANGE_SMALL : COPY_READ_WRITE;
}
}
static int64_t block_copy_calculate_cluster_size(BlockDriverState *target,
Error **errp)
{
int ret;
BlockDriverInfo bdi;
bool target_does_cow = bdrv_backing_chain_next(target);
/*
* If there is no backing file on the target, we cannot rely on COW if our
* backup cluster size is smaller than the target cluster size. Even for
* targets with a backing file, try to avoid COW if possible.
*/
ret = bdrv_get_info(target, &bdi);
if (ret == -ENOTSUP && !target_does_cow) {
/* Cluster size is not defined */
warn_report("The target block device doesn't provide "
"information about the block size and it doesn't have a "
"backing file. The default block size of %u bytes is "
"used. If the actual block size of the target exceeds "
"this default, the backup may be unusable",
BLOCK_COPY_CLUSTER_SIZE_DEFAULT);
return BLOCK_COPY_CLUSTER_SIZE_DEFAULT;
} else if (ret < 0 && !target_does_cow) {
error_setg_errno(errp, -ret,
"Couldn't determine the cluster size of the target image, "
"which has no backing file");
error_append_hint(errp,
"Aborting, since this may create an unusable destination image\n");
return ret;
} else if (ret < 0 && target_does_cow) {
/* Not fatal; just trudge on ahead. */
return BLOCK_COPY_CLUSTER_SIZE_DEFAULT;
}
return MAX(BLOCK_COPY_CLUSTER_SIZE_DEFAULT, bdi.cluster_size);
}
BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
Error **errp)
{
BlockCopyState *s;
int64_t cluster_size;
BdrvDirtyBitmap *copy_bitmap;
bool is_fleecing;
cluster_size = block_copy_calculate_cluster_size(target->bs, errp);
if (cluster_size < 0) {
return NULL;
}
copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
errp);
if (!copy_bitmap) {
return NULL;
}
bdrv_disable_dirty_bitmap(copy_bitmap);
/*
* If source is in backing chain of target assume that target is going to be
* used for "image fleecing", i.e. it should represent a kind of snapshot of
* source at backup-start point in time. And target is going to be read by
* somebody (for example, used as NBD export) during backup job.
*
* In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
* intersection of backup writes and third party reads from target,
* otherwise reading from target we may occasionally read already updated by
* guest data.
*
* For more information see commit f8d59dfb40bb and test
* tests/qemu-iotests/222
*/
is_fleecing = bdrv_chain_contains(target->bs, source->bs);
s = g_new(BlockCopyState, 1);
*s = (BlockCopyState) {
.source = source,
.target = target,
.copy_bitmap = copy_bitmap,
.cluster_size = cluster_size,
.len = bdrv_dirty_bitmap_size(copy_bitmap),
.write_flags = (is_fleecing ? BDRV_REQ_SERIALISING : 0),
.mem = shres_create(BLOCK_COPY_MAX_MEM),
.max_transfer = QEMU_ALIGN_DOWN(
block_copy_max_transfer(source, target),
cluster_size),
};
block_copy_set_copy_opts(s, false, false);
ratelimit_init(&s->rate_limit);
qemu_co_mutex_init(&s->lock);
@ -522,7 +595,7 @@ static coroutine_fn int block_copy_task_entry(AioTask *task)
t->call_state->ret = ret;
t->call_state->error_is_read = error_is_read;
}
} else {
} else if (s->progress) {
progress_work_done(s->progress, t->bytes);
}
}
@ -628,9 +701,11 @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
if (!ret) {
qemu_co_mutex_lock(&s->lock);
bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
progress_set_remaining(s->progress,
bdrv_get_dirty_count(s->copy_bitmap) +
s->in_flight_bytes);
if (s->progress) {
progress_set_remaining(s->progress,
bdrv_get_dirty_count(s->copy_bitmap) +
s->in_flight_bytes);
}
qemu_co_mutex_unlock(&s->lock);
}
@ -933,6 +1008,11 @@ BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s)
return s->copy_bitmap;
}
int64_t block_copy_cluster_size(BlockCopyState *s)
{
return s->cluster_size;
}
void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip)
{
qatomic_set(&s->skip_unallocated, skip);

256
block/copy-before-write.c Normal file
View File

@ -0,0 +1,256 @@
/*
* copy-before-write filter driver
*
* The driver performs Copy-Before-Write (CBW) operation: it is injected above
* some node, and before each write it copies _old_ data to the target node.
*
* Copyright (c) 2018-2021 Virtuozzo International GmbH.
*
* Author:
* Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "sysemu/block-backend.h"
#include "qemu/cutils.h"
#include "qapi/error.h"
#include "block/block_int.h"
#include "block/qdict.h"
#include "block/block-copy.h"
#include "block/copy-before-write.h"
typedef struct BDRVCopyBeforeWriteState {
BlockCopyState *bcs;
BdrvChild *target;
} BDRVCopyBeforeWriteState;
static coroutine_fn int cbw_co_preadv(
BlockDriverState *bs, uint64_t offset, uint64_t bytes,
QEMUIOVector *qiov, int flags)
{
return bdrv_co_preadv(bs->file, offset, bytes, qiov, flags);
}
static coroutine_fn int cbw_do_copy_before_write(BlockDriverState *bs,
uint64_t offset, uint64_t bytes, BdrvRequestFlags flags)
{
BDRVCopyBeforeWriteState *s = bs->opaque;
uint64_t off, end;
int64_t cluster_size = block_copy_cluster_size(s->bcs);
if (flags & BDRV_REQ_WRITE_UNCHANGED) {
return 0;
}
off = QEMU_ALIGN_DOWN(offset, cluster_size);
end = QEMU_ALIGN_UP(offset + bytes, cluster_size);
return block_copy(s->bcs, off, end - off, true);
}
static int coroutine_fn cbw_co_pdiscard(BlockDriverState *bs,
int64_t offset, int bytes)
{
int ret = cbw_do_copy_before_write(bs, offset, bytes, 0);
if (ret < 0) {
return ret;
}
return bdrv_co_pdiscard(bs->file, offset, bytes);
}
static int coroutine_fn cbw_co_pwrite_zeroes(BlockDriverState *bs,
int64_t offset, int bytes, BdrvRequestFlags flags)
{
int ret = cbw_do_copy_before_write(bs, offset, bytes, flags);
if (ret < 0) {
return ret;
}
return bdrv_co_pwrite_zeroes(bs->file, offset, bytes, flags);
}
static coroutine_fn int cbw_co_pwritev(BlockDriverState *bs,
uint64_t offset,
uint64_t bytes,
QEMUIOVector *qiov, int flags)
{
int ret = cbw_do_copy_before_write(bs, offset, bytes, flags);
if (ret < 0) {
return ret;
}
return bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags);
}
static int coroutine_fn cbw_co_flush(BlockDriverState *bs)
{
if (!bs->file) {
return 0;
}
return bdrv_co_flush(bs->file->bs);
}
static void cbw_refresh_filename(BlockDriverState *bs)
{
pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
bs->file->bs->filename);
}
static void cbw_child_perm(BlockDriverState *bs, BdrvChild *c,
BdrvChildRole role,
BlockReopenQueue *reopen_queue,
uint64_t perm, uint64_t shared,
uint64_t *nperm, uint64_t *nshared)
{
if (!(role & BDRV_CHILD_FILTERED)) {
/*
* Target child
*
* Share write to target (child_file), to not interfere
* with guest writes to its disk which may be in target backing chain.
* Can't resize during a backup block job because we check the size
* only upfront.
*/
*nshared = BLK_PERM_ALL & ~BLK_PERM_RESIZE;
*nperm = BLK_PERM_WRITE;
} else {
/* Source child */
bdrv_default_perms(bs, c, role, reopen_queue,
perm, shared, nperm, nshared);
if (!QLIST_EMPTY(&bs->parents)) {
if (perm & BLK_PERM_WRITE) {
*nperm = *nperm | BLK_PERM_CONSISTENT_READ;
}
*nshared &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
}
}
}
static int cbw_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVCopyBeforeWriteState *s = bs->opaque;
BdrvDirtyBitmap *copy_bitmap;
bs->file = bdrv_open_child(NULL, options, "file", bs, &child_of_bds,
BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY,
false, errp);
if (!bs->file) {
return -EINVAL;
}
s->target = bdrv_open_child(NULL, options, "target", bs, &child_of_bds,
BDRV_CHILD_DATA, false, errp);
if (!s->target) {
return -EINVAL;
}
bs->total_sectors = bs->file->bs->total_sectors;
bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |
(BDRV_REQ_FUA & bs->file->bs->supported_write_flags);
bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &
bs->file->bs->supported_zero_flags);
s->bcs = block_copy_state_new(bs->file, s->target, errp);
if (!s->bcs) {
error_prepend(errp, "Cannot create block-copy-state: ");
return -EINVAL;
}
copy_bitmap = block_copy_dirty_bitmap(s->bcs);
bdrv_set_dirty_bitmap(copy_bitmap, 0, bdrv_dirty_bitmap_size(copy_bitmap));
return 0;
}
static void cbw_close(BlockDriverState *bs)
{
BDRVCopyBeforeWriteState *s = bs->opaque;
block_copy_state_free(s->bcs);
s->bcs = NULL;
}
BlockDriver bdrv_cbw_filter = {
.format_name = "copy-before-write",
.instance_size = sizeof(BDRVCopyBeforeWriteState),
.bdrv_open = cbw_open,
.bdrv_close = cbw_close,
.bdrv_co_preadv = cbw_co_preadv,
.bdrv_co_pwritev = cbw_co_pwritev,
.bdrv_co_pwrite_zeroes = cbw_co_pwrite_zeroes,
.bdrv_co_pdiscard = cbw_co_pdiscard,
.bdrv_co_flush = cbw_co_flush,
.bdrv_refresh_filename = cbw_refresh_filename,
.bdrv_child_perm = cbw_child_perm,
.is_filter = true,
};
BlockDriverState *bdrv_cbw_append(BlockDriverState *source,
BlockDriverState *target,
const char *filter_node_name,
BlockCopyState **bcs,
Error **errp)
{
ERRP_GUARD();
BDRVCopyBeforeWriteState *state;
BlockDriverState *top;
QDict *opts;
assert(source->total_sectors == target->total_sectors);
opts = qdict_new();
qdict_put_str(opts, "driver", "copy-before-write");
if (filter_node_name) {
qdict_put_str(opts, "node-name", filter_node_name);
}
qdict_put_str(opts, "file", bdrv_get_node_name(source));
qdict_put_str(opts, "target", bdrv_get_node_name(target));
top = bdrv_insert_node(source, opts, BDRV_O_RDWR, errp);
if (!top) {
return NULL;
}
state = top->opaque;
*bcs = state->bcs;
return top;
}
void bdrv_cbw_drop(BlockDriverState *bs)
{
bdrv_drop_filter(bs, &error_abort);
bdrv_unref(bs);
}
static void cbw_init(void)
{
bdrv_register(&bdrv_cbw_filter);
}
block_init(cbw_init);

25
block/backup-top.h → block/copy-before-write.h
View File

@ -1,10 +1,10 @@
/*
* backup-top filter driver
* copy-before-write filter driver
*
* The driver performs Copy-Before-Write (CBW) operation: it is injected above
* some node, and before each write it copies _old_ data to the target node.
*
* Copyright (c) 2018-2019 Virtuozzo International GmbH.
* Copyright (c) 2018-2021 Virtuozzo International GmbH.
*
* Author:
* Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
@ -23,20 +23,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef BACKUP_TOP_H
#define BACKUP_TOP_H
#ifndef COPY_BEFORE_WRITE_H
#define COPY_BEFORE_WRITE_H
#include "block/block_int.h"
#include "block/block-copy.h"
BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
BlockDriverState *target,
const char *filter_node_name,
uint64_t cluster_size,
BackupPerf *perf,
BdrvRequestFlags write_flags,
BlockCopyState **bcs,
Error **errp);
void bdrv_backup_top_drop(BlockDriverState *bs);
BlockDriverState *bdrv_cbw_append(BlockDriverState *source,
BlockDriverState *target,
const char *filter_node_name,
BlockCopyState **bcs,
Error **errp);
void bdrv_cbw_drop(BlockDriverState *bs);
#endif /* BACKUP_TOP_H */
#endif /* COPY_BEFORE_WRITE_H */

3
block/export/fuse.c
View File

@ -31,6 +31,9 @@
#include <fuse.h>
#include <fuse_lowlevel.h>
#ifdef __linux__
#include <linux/fs.h>
#endif
/* Prevent overly long bounce buffer allocations */
#define FUSE_MAX_BOUNCE_BYTES (MIN(BDRV_REQUEST_MAX_BYTES, 64 * 1024 * 1024))

101
block/file-win32.c
View File

@ -58,6 +58,10 @@ typedef struct BDRVRawState {
QEMUWin32AIOState *aio;
} BDRVRawState;
typedef struct BDRVRawReopenState {
HANDLE hfile;
} BDRVRawReopenState;
/*
* Read/writes the data to/from a given linear buffer.
*
@ -392,7 +396,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags,
}
s->hfile = CreateFile(filename, access_flags,
FILE_SHARE_READ, NULL,
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
OPEN_EXISTING, overlapped, NULL);
if (s->hfile == INVALID_HANDLE_VALUE) {
int err = GetLastError();
@ -634,6 +638,97 @@ static int coroutine_fn raw_co_create_opts(BlockDriver *drv,
return raw_co_create(&options, errp);
}
static int raw_reopen_prepare(BDRVReopenState *state,
BlockReopenQueue *queue, Error **errp)
{
BDRVRawState *s = state->bs->opaque;
BDRVRawReopenState *rs;
int access_flags;
DWORD overlapped;
int ret = 0;
if (s->type != FTYPE_FILE) {
error_setg(errp, "Can only reopen files");
return -EINVAL;
}
rs = g_new0(BDRVRawReopenState, 1);
/*
* We do not support changing any options (only flags). By leaving
* all options in state->options, we tell the generic reopen code
* that we do not support changing any of them, so it will verify
* that their values did not change.
*/
raw_parse_flags(state->flags, s->aio != NULL, &access_flags, &overlapped);
rs->hfile = CreateFile(state->bs->filename, access_flags,
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
OPEN_EXISTING, overlapped, NULL);
if (rs->hfile == INVALID_HANDLE_VALUE) {
int err = GetLastError();
error_setg_win32(errp, err, "Could not reopen '%s'",
state->bs->filename);
if (err == ERROR_ACCESS_DENIED) {
ret = -EACCES;
} else {
ret = -EINVAL;
}
goto fail;
}
if (s->aio) {
ret = win32_aio_attach(s->aio, rs->hfile);
if (ret < 0) {
error_setg_errno(errp, -ret, "Could not enable AIO");
CloseHandle(rs->hfile);
goto fail;
}
}
state->opaque = rs;
return 0;
fail:
g_free(rs);
state->opaque = NULL;
return ret;
}
static void raw_reopen_commit(BDRVReopenState *state)
{
BDRVRawState *s = state->bs->opaque;
BDRVRawReopenState *rs = state->opaque;
assert(rs != NULL);
CloseHandle(s->hfile);
s->hfile = rs->hfile;
g_free(rs);
state->opaque = NULL;
}
static void raw_reopen_abort(BDRVReopenState *state)
{
BDRVRawReopenState *rs = state->opaque;
if (!rs) {
return;
}
if (rs->hfile != INVALID_HANDLE_VALUE) {
CloseHandle(rs->hfile);
}
g_free(rs);
state->opaque = NULL;
}
static QemuOptsList raw_create_opts = {
.name = "raw-create-opts",
.head = QTAILQ_HEAD_INITIALIZER(raw_create_opts.head),
@ -659,6 +754,10 @@ BlockDriver bdrv_file = {
.bdrv_co_create_opts = raw_co_create_opts,
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.bdrv_reopen_prepare = raw_reopen_prepare,
.bdrv_reopen_commit = raw_reopen_commit,
.bdrv_reopen_abort = raw_reopen_abort,
.bdrv_aio_preadv = raw_aio_preadv,
.bdrv_aio_pwritev = raw_aio_pwritev,
.bdrv_aio_flush = raw_aio_flush,

2
block/meson.build
View File

@ -4,7 +4,7 @@ block_ss.add(files(
'aio_task.c',
'amend.c',
'backup.c',
'backup-top.c',
'copy-before-write.c',
'blkdebug.c',
'blklogwrites.c',
'blkverify.c',

12
block/monitor/block-hmp-cmds.c
View File

@ -251,10 +251,10 @@ void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
if (!filename) {
error_setg(&err, QERR_MISSING_PARAMETER, "target");
hmp_handle_error(mon, err);
return;
goto end;
}
qmp_drive_mirror(&mirror, &err);
end:
hmp_handle_error(mon, err);
}
@ -281,11 +281,11 @@ void hmp_drive_backup(Monitor *mon, const QDict *qdict)
if (!filename) {
error_setg(&err, QERR_MISSING_PARAMETER, "target");
hmp_handle_error(mon, err);
return;
goto end;
}
qmp_drive_backup(&backup, &err);
end:
hmp_handle_error(mon, err);
}
@ -356,8 +356,7 @@ void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
* will be taken internally. Today it's actually required.
*/
error_setg(&err, QERR_MISSING_PARAMETER, "snapshot-file");
hmp_handle_error(mon, err);
return;
goto end;
}
mode = reuse ? NEW_IMAGE_MODE_EXISTING : NEW_IMAGE_MODE_ABSOLUTE_PATHS;
@ -365,6 +364,7 @@ void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
filename, false, NULL,
!!format, format,
true, mode, &err);
end:
hmp_handle_error(mon, err);
}

21
block/raw-format.c
View File

@ -580,6 +580,25 @@ static void raw_cancel_in_flight(BlockDriverState *bs)
bdrv_cancel_in_flight(bs->file->bs);
}
static void raw_child_perm(BlockDriverState *bs, BdrvChild *c,
BdrvChildRole role,
BlockReopenQueue *reopen_queue,
uint64_t parent_perm, uint64_t parent_shared,
uint64_t *nperm, uint64_t *nshared)
{
bdrv_default_perms(bs, c, role, reopen_queue, parent_perm,
parent_shared, nperm, nshared);
/*
* bdrv_default_perms() may add WRITE and/or RESIZE (see comment in
* bdrv_default_perms_for_storage() for an explanation) but we only need
* them if they are in parent_perm. Drop WRITE and RESIZE whenever possible
* to avoid permission conflicts.
*/
*nperm &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
*nperm |= parent_perm & (BLK_PERM_WRITE | BLK_PERM_RESIZE);
}
BlockDriver bdrv_raw = {
.format_name = "raw",
.instance_size = sizeof(BDRVRawState),
@ -588,7 +607,7 @@ BlockDriver bdrv_raw = {
.bdrv_reopen_commit = &raw_reopen_commit,
.bdrv_reopen_abort = &raw_reopen_abort,
.bdrv_open = &raw_open,
.bdrv_child_perm = bdrv_default_perms,
.bdrv_child_perm = raw_child_perm,
.bdrv_co_create_opts = &raw_co_create_opts,
.bdrv_co_preadv = &raw_co_preadv,
.bdrv_co_pwritev = &raw_co_pwritev,

29
docs/devel/testing.rst
View File

@ -224,6 +224,35 @@ another application on the host may have locked the file, possibly leading to a
test failure. If using such devices are explicitly desired, consider adding
``locking=off`` option to disable image locking.
Debugging a test case
-----------------------
The following options to the ``check`` script can be useful when debugging
a failing test:
* ``-gdb`` wraps every QEMU invocation in a ``gdbserver``, which waits for a
connection from a gdb client. The options given to ``gdbserver`` (e.g. the
address on which to listen for connections) are taken from the ``$GDB_OPTIONS``
environment variable. By default (if ``$GDB_OPTIONS`` is empty), it listens on
``localhost:12345``.
It is possible to connect to it for example with
``gdb -iex "target remote $addr"``, where ``$addr`` is the address
``gdbserver`` listens on.
If the ``-gdb`` option is not used, ``$GDB_OPTIONS`` is ignored,
regardless of whether it is set or not.
* ``-valgrind`` attaches a valgrind instance to QEMU. If it detects
warnings, it will print and save the log in
``$TEST_DIR/<valgrind_pid>.valgrind``.
The final command line will be ``valgrind --log-file=$TEST_DIR/
<valgrind_pid>.valgrind --error-exitcode=99 $QEMU ...``
* ``-d`` (debug) just increases the logging verbosity, showing
for example the QMP commands and answers.
* ``-p`` (print) redirects QEMU’s stdout and stderr to the test output,
instead of saving it into a log file in
``$TEST_DIR/qemu-machine-<random_string>``.
Test case groups
----------------

43
hw/core/qdev-properties-system.c
View File

@ -36,11 +36,11 @@
static bool check_prop_still_unset(Object *obj, const char *name,
const void *old_val, const char *new_val,
Error **errp)
bool allow_override, Error **errp)
{
const GlobalProperty *prop = qdev_find_global_prop(obj, name);
if (!old_val) {
if (!old_val || (!prop && allow_override)) {
return true;
}
@ -93,16 +93,34 @@ static void set_drive_helper(Object *obj, Visitor *v, const char *name,
BlockBackend *blk;
bool blk_created = false;
int ret;
BlockDriverState *bs;
AioContext *ctx;
if (!visit_type_str(v, name, &str, errp)) {
return;
}
/*
* TODO Should this really be an error? If no, the old value
* needs to be released before we store the new one.
*/
if (!check_prop_still_unset(obj, name, *ptr, str, errp)) {
if (!check_prop_still_unset(obj, name, *ptr, str, true, errp)) {
return;
}
if (*ptr) {
/* BlockBackend alread exists. So, we want to change attached node */
blk = *ptr;
ctx = blk_get_aio_context(blk);
bs = bdrv_lookup_bs(NULL, str, errp);
if (!bs) {
return;
}
if (ctx != bdrv_get_aio_context(bs)) {
error_setg(errp, "Different aio context is not supported for new "
"node");
}
aio_context_acquire(ctx);
blk_replace_bs(blk, bs, errp);
aio_context_release(ctx);
return;
}
@ -114,7 +132,7 @@ static void set_drive_helper(Object *obj, Visitor *v, const char *name,
blk = blk_by_name(str);
if (!blk) {
BlockDriverState *bs = bdrv_lookup_bs(NULL, str, NULL);
bs = bdrv_lookup_bs(NULL, str, NULL);
if (bs) {
/*
* If the device supports iothreads, it will make sure to move the
@ -123,8 +141,7 @@ static void set_drive_helper(Object *obj, Visitor *v, const char *name,
* aware of iothreads require their BlockBackends to be in the main
* AioContext.
*/
AioContext *ctx = iothread ? bdrv_get_aio_context(bs) :
qemu_get_aio_context();
ctx = iothread ? bdrv_get_aio_context(bs) : qemu_get_aio_context();
blk = blk_new(ctx, 0, BLK_PERM_ALL);
blk_created = true;
@ -196,6 +213,7 @@ static void release_drive(Object *obj, const char *name, void *opaque)
const PropertyInfo qdev_prop_drive = {
.name = "str",
.description = "Node name or ID of a block device to use as a backend",
.realized_set_allowed = true,
.get = get_drive,
.set = set_drive,
.release = release_drive,
@ -204,6 +222,7 @@ const PropertyInfo qdev_prop_drive = {
const PropertyInfo qdev_prop_drive_iothread = {
.name = "str",
.description = "Node name or ID of a block device to use as a backend",
.realized_set_allowed = true,
.get = get_drive,
.set = set_drive_iothread,
.release = release_drive,
@ -238,7 +257,7 @@ static void set_chr(Object *obj, Visitor *v, const char *name, void *opaque,
* TODO Should this really be an error? If no, the old value
* needs to be released before we store the new one.
*/
if (!check_prop_still_unset(obj, name, be->chr, str, errp)) {
if (!check_prop_still_unset(obj, name, be->chr, str, false, errp)) {
return;
}
@ -408,7 +427,7 @@ static void set_netdev(Object *obj, Visitor *v, const char *name,
* TODO Should this really be an error? If no, the old value
* needs to be released before we store the new one.
*/
if (!check_prop_still_unset(obj, name, ncs[i], str, errp)) {
if (!check_prop_still_unset(obj, name, ncs[i], str, false, errp)) {
goto out;
}

6
hw/core/qdev-properties.c
View File

@ -26,11 +26,11 @@ void qdev_prop_set_after_realize(DeviceState *dev, const char *name,
/* returns: true if property is allowed to be set, false otherwise */
static bool qdev_prop_allow_set(Object *obj, const char *name,
Error **errp)
const PropertyInfo *info, Error **errp)
{
DeviceState *dev = DEVICE(obj);
if (dev->realized) {
if (dev->realized && !info->realized_set_allowed) {
qdev_prop_set_after_realize(dev, name, errp);
return false;
}
@ -79,7 +79,7 @@ static void field_prop_set(Object *obj, Visitor *v, const char *name,
{
Property *prop = opaque;
if (!qdev_prop_allow_set(obj, name, errp)) {
if (!qdev_prop_allow_set(obj, name, prop->info, errp)) {
return;
}

6
include/block/block-copy.h
View File

@ -25,10 +25,11 @@ typedef struct BlockCopyState BlockCopyState;
typedef struct BlockCopyCallState BlockCopyCallState;
BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
int64_t cluster_size, bool use_copy_range,
BdrvRequestFlags write_flags,
Error **errp);
/* Function should be called prior any actual copy request */
void block_copy_set_copy_opts(BlockCopyState *s, bool use_copy_range,
bool compress);
void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm);
void block_copy_state_free(BlockCopyState *s);
@ -89,6 +90,7 @@ void block_copy_kick(BlockCopyCallState *call_state);
void block_copy_call_cancel(BlockCopyCallState *call_state);
BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s);
int64_t block_copy_cluster_size(BlockCopyState *s);
void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip);
#endif /* BLOCK_COPY_H */

2
include/block/block.h
View File

@ -361,6 +361,8 @@ int bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top,
Error **errp);
int bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
Error **errp);
int bdrv_replace_child_bs(BdrvChild *child, BlockDriverState *new_bs,
Error **errp);
BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options,
int flags, Error **errp);
int bdrv_drop_filter(BlockDriverState *bs, Error **errp);

1
include/hw/qdev-properties.h
View File

@ -32,6 +32,7 @@ struct PropertyInfo {
const char *name;
const char *description;
const QEnumLookup *enum_table;
bool realized_set_allowed; /* allow setting property on realized device */
int (*print)(Object *obj, Property *prop, char *dest, size_t len);
void (*set_default_value)(ObjectProperty *op, const Property *prop);
ObjectProperty *(*create)(ObjectClass *oc, const char *name,

1
include/sysemu/block-backend.h
View File

@ -102,6 +102,7 @@ BlockBackend *blk_by_public(BlockBackendPublic *public);
BlockDriverState *blk_bs(BlockBackend *blk);
void blk_remove_bs(BlockBackend *blk);
int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp);
int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp);
bool bdrv_has_blk(BlockDriverState *bs);
bool bdrv_is_root_node(BlockDriverState *bs);
int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,

56
python/qemu/machine/machine.py
View File

@ -36,6 +36,7 @@ from typing import (
Sequence,
Tuple,
Type,
TypeVar,
)
from qemu.qmp import ( # pylint: disable=import-error
@ -73,6 +74,9 @@ class AbnormalShutdown(QEMUMachineError):
"""
_T = TypeVar('_T', bound='QEMUMachine')
class QEMUMachine:
"""
A QEMU VM.
@ -97,7 +101,8 @@ class QEMUMachine:
sock_dir: Optional[str] = None,
drain_console: bool = False,
console_log: Optional[str] = None,
log_dir: Optional[str] = None):
log_dir: Optional[str] = None,
qmp_timer: Optional[float] = None):
'''
Initialize a QEMUMachine
@ -112,6 +117,7 @@ class QEMUMachine:
@param drain_console: (optional) True to drain console socket to buffer
@param console_log: (optional) path to console log file
@param log_dir: where to create and keep log files
@param qmp_timer: (optional) default QMP socket timeout
@note: Qemu process is not started until launch() is used.
'''
# pylint: disable=too-many-arguments
@ -121,6 +127,7 @@ class QEMUMachine:
self._binary = binary
self._args = list(args)
self._wrapper = wrapper
self._qmp_timer = qmp_timer
self._name = name or "qemu-%d" % os.getpid()
self._base_temp_dir = base_temp_dir
@ -166,7 +173,7 @@ class QEMUMachine:
self._remove_files: List[str] = []
self._user_killed = False
def __enter__(self) -> 'QEMUMachine':
def __enter__(self: _T) -> _T:
return self
def __exit__(self,
@ -182,8 +189,8 @@ class QEMUMachine:
self._args.append('-monitor')
self._args.append('null')
def add_fd(self, fd: int, fdset: int,
opaque: str, opts: str = '') -> 'QEMUMachine':
def add_fd(self: _T, fd: int, fdset: int,
opaque: str, opts: str = '') -> _T:
"""
Pass a file descriptor to the VM
"""
@ -343,7 +350,12 @@ class QEMUMachine:
def _post_launch(self) -> None:
if self._qmp_connection:
self._qmp.accept()
self._qmp.accept(self._qmp_timer)
def _close_qemu_log_file(self) -> None:
if self._qemu_log_file is not None:
self._qemu_log_file.close()
self._qemu_log_file = None
def _post_shutdown(self) -> None:
"""
@ -357,9 +369,7 @@ class QEMUMachine:
self._qmp.close()
self._qmp_connection = None
if self._qemu_log_file is not None:
self._qemu_log_file.close()
self._qemu_log_file = None
self._close_qemu_log_file()
self._load_io_log()
@ -564,22 +574,30 @@ class QEMUMachine:
return self._qmp_connection
@classmethod
def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]:
qmp_args = dict()
for key, value in args.items():
if _conv_keys:
qmp_args[key.replace('_', '-')] = value
else:
qmp_args[key] = value
return qmp_args
def _qmp_args(cls, conv_keys: bool,
args: Dict[str, Any]) -> Dict[str, object]:
if conv_keys:
return {k.replace('_', '-'): v for k, v in args.items()}
return args
def qmp(self, cmd: str,
conv_keys: bool = True,
args_dict: Optional[Dict[str, object]] = None,
conv_keys: Optional[bool] = None,
**args: Any) -> QMPMessage:
"""
Invoke a QMP command and return the response dict
"""
qmp_args = self._qmp_args(conv_keys, **args)
if args_dict is not None:
assert not args
assert conv_keys is None
args = args_dict
conv_keys = False
if conv_keys is None:
conv_keys = True
qmp_args = self._qmp_args(conv_keys, args)
return self._qmp.cmd(cmd, args=qmp_args)
def command(self, cmd: str,
@ -590,7 +608,7 @@ class QEMUMachine:
On success return the response dict.
On failure raise an exception.
"""
qmp_args = self._qmp_args(conv_keys, **args)
qmp_args = self._qmp_args(conv_keys, args)
return self._qmp.command(cmd, **qmp_args)
def get_qmp_event(self, wait: bool = False) -> Optional[QMPMessage]:

9
python/qemu/machine/qtest.py
View File

@ -112,19 +112,22 @@ class QEMUQtestMachine(QEMUMachine):
def __init__(self,
binary: str,
args: Sequence[str] = (),
wrapper: Sequence[str] = (),
name: Optional[str] = None,
base_temp_dir: str = "/var/tmp",
socket_scm_helper: Optional[str] = None,
sock_dir: Optional[str] = None):
sock_dir: Optional[str] = None,
qmp_timer: Optional[float] = None):
# pylint: disable=too-many-arguments
if name is None:
name = "qemu-%d" % os.getpid()
if sock_dir is None:
sock_dir = base_temp_dir
super().__init__(binary, args, name=name, base_temp_dir=base_temp_dir,
super().__init__(binary, args, wrapper=wrapper, name=name,
base_temp_dir=base_temp_dir,
socket_scm_helper=socket_scm_helper,
sock_dir=sock_dir)
sock_dir=sock_dir, qmp_timer=qmp_timer)
self._qtest: Optional[QEMUQtestProtocol] = None
self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock")

5
python/setup.cfg
View File

@ -105,6 +105,11 @@ good-names=i,
# Ignore imports when computing similarities.
ignore-imports=yes
# Minimum lines number of a similarity.
# TODO: Remove after we opt in to Pylint 2.8.3. See commit msg.
min-similarity-lines=6
[isort]
force_grid_wrap=4
force_sort_within_sections=True

25
qapi/block-core.json
View File

@ -2825,13 +2825,14 @@
# @blklogwrites: Since 3.0
# @blkreplay: Since 4.2
# @compress: Since 5.0
# @copy-before-write: Since 6.2
#
# Since: 2.9
##
{ 'enum': 'BlockdevDriver',
'data': [ 'blkdebug', 'blklogwrites', 'blkreplay', 'blkverify', 'bochs',
'cloop', 'compress', 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps',
'gluster',
'cloop', 'compress', 'copy-before-write', 'copy-on-read', 'dmg',
'file', 'ftp', 'ftps', 'gluster',
{'name': 'host_cdrom', 'if': 'HAVE_HOST_BLOCK_DEVICE' },
{'name': 'host_device', 'if': 'HAVE_HOST_BLOCK_DEVICE' },
'http', 'https', 'iscsi',
@ -4049,6 +4050,25 @@
'base': 'BlockdevOptionsGenericFormat',
'data': { '*bottom': 'str' } }
##
# @BlockdevOptionsCbw:
#
# Driver specific block device options for the copy-before-write driver,
# which does so called copy-before-write operations: when data is
# written to the filter, the filter first reads corresponding blocks
# from its file child and copies them to @target child. After successfully
# copying, the write request is propagated to file child. If copying
# fails, the original write request is failed too and no data is written
# to file child.
#
# @target: The target for copy-before-write operations.
#
# Since: 6.2
##
{ 'struct': 'BlockdevOptionsCbw',
'base': 'BlockdevOptionsGenericFormat',
'data': { 'target': 'BlockdevRef' } }
##
# @BlockdevOptions:
#
@ -4101,6 +4121,7 @@
'bochs': 'BlockdevOptionsGenericFormat',
'cloop': 'BlockdevOptionsGenericFormat',
'compress': 'BlockdevOptionsGenericFormat',
'copy-before-write':'BlockdevOptionsCbw',
'copy-on-read':'BlockdevOptionsCor',
'dmg': 'BlockdevOptionsGenericFormat',
'file': 'BlockdevOptionsFile',

159
tests/qemu-iotests/222
View File

@ -1,159 +0,0 @@
#!/usr/bin/env python3
# group: rw quick
#
# This test covers the basic fleecing workflow, which provides a
# point-in-time snapshot of a node that can be queried over NBD.
#
# Copyright (C) 2018 Red Hat, Inc.
# John helped, too.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Creator/Owner: John Snow <jsnow@redhat.com>
import iotests
from iotests import log, qemu_img, qemu_io, qemu_io_silent
iotests.script_initialize(
supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'],
supported_platforms=['linux'],
)
patterns = [("0x5d", "0", "64k"),
("0xd5", "1M", "64k"),
("0xdc", "32M", "64k"),
("0xcd", "0x3ff0000", "64k")] # 64M - 64K
overwrite = [("0xab", "0", "64k"), # Full overwrite
("0xad", "0x00f8000", "64k"), # Partial-left (1M-32K)
("0x1d", "0x2008000", "64k"), # Partial-right (32M+32K)
("0xea", "0x3fe0000", "64k")] # Adjacent-left (64M - 128K)
zeroes = [("0", "0x00f8000", "32k"), # Left-end of partial-left (1M-32K)
("0", "0x2010000", "32k"), # Right-end of partial-right (32M+64K)
("0", "0x3fe0000", "64k")] # overwrite[3]
remainder = [("0xd5", "0x108000", "32k"), # Right-end of partial-left [1]
("0xdc", "32M", "32k"), # Left-end of partial-right [2]
("0xcd", "0x3ff0000", "64k")] # patterns[3]
with iotests.FilePath('base.img') as base_img_path, \
iotests.FilePath('fleece.img') as fleece_img_path, \
iotests.FilePath('nbd.sock', base_dir=iotests.sock_dir) as nbd_sock_path, \
iotests.VM() as vm:
log('--- Setting up images ---')
log('')
assert qemu_img('create', '-f', iotests.imgfmt, base_img_path, '64M') == 0
assert qemu_img('create', '-f', "qcow2", fleece_img_path, '64M') == 0
for p in patterns:
qemu_io('-f', iotests.imgfmt,
'-c', 'write -P%s %s %s' % p, base_img_path)
log('Done')
log('')
log('--- Launching VM ---')
log('')
vm.add_drive(base_img_path)
vm.launch()
log('Done')
log('')
log('--- Setting up Fleecing Graph ---')
log('')
src_node = "drive0"
tgt_node = "fleeceNode"
# create tgt_node backed by src_node
log(vm.qmp("blockdev-add", **{
"driver": "qcow2",
"node-name": tgt_node,
"file": {
"driver": "file",
"filename": fleece_img_path,
},
"backing": src_node,
}))
# Establish COW from source to fleecing node
log(vm.qmp("blockdev-backup",
device=src_node,
target=tgt_node,
sync="none"))
log('')
log('--- Setting up NBD Export ---')
log('')
nbd_uri = 'nbd+unix:///%s?socket=%s' % (tgt_node, nbd_sock_path)
log(vm.qmp("nbd-server-start",
**{"addr": { "type": "unix",
"data": { "path": nbd_sock_path } } }))
log(vm.qmp("nbd-server-add", device=tgt_node))
log('')
log('--- Sanity Check ---')
log('')
for p in (patterns + zeroes):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Testing COW ---')
log('')
for p in overwrite:
cmd = "write -P%s %s %s" % p
log(cmd)
log(vm.hmp_qemu_io(src_node, cmd))
log('')
log('--- Verifying Data ---')
log('')
for p in (patterns + zeroes):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Cleanup ---')
log('')
log(vm.qmp('block-job-cancel', device=src_node))
log(vm.event_wait('BLOCK_JOB_CANCELLED'),
filters=[iotests.filter_qmp_event])
log(vm.qmp('nbd-server-stop'))
log(vm.qmp('blockdev-del', node_name=tgt_node))
vm.shutdown()
log('')
log('--- Confirming writes ---')
log('')
for p in (overwrite + remainder):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent(base_img_path, '-c', cmd) == 0
log('')
log('Done')

67
tests/qemu-iotests/222.out
View File

@ -1,67 +0,0 @@
--- Setting up images ---
Done
--- Launching VM ---
Done
--- Setting up Fleecing Graph ---
{"return": {}}
{"return": {}}
--- Setting up NBD Export ---
{"return": {}}
{"return": {}}
--- Sanity Check ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Testing COW ---
write -P0xab 0 64k
{"return": ""}
write -P0xad 0x00f8000 64k
{"return": ""}
write -P0x1d 0x2008000 64k
{"return": ""}
write -P0xea 0x3fe0000 64k
{"return": ""}
--- Verifying Data ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Cleanup ---
{"return": {}}
{"data": {"device": "drive0", "len": 67108864, "offset": 393216, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_CANCELLED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"return": {}}
{"return": {}}
--- Confirming writes ---
read -P0xab 0 64k
read -P0xad 0x00f8000 64k
read -P0x1d 0x2008000 64k
read -P0xea 0x3fe0000 64k
read -P0xd5 0x108000 32k
read -P0xdc 32M 32k
read -P0xcd 0x3ff0000 64k
Done

35
tests/qemu-iotests/283
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# group: auto quick
#
# Test for backup-top filter permission activation failure
# Test for copy-before-write filter permission conflict
#
# Copyright (c) 2019 Virtuozzo International GmbH.
#
@ -31,13 +31,13 @@ size = 1024 * 1024
""" Test description
When performing a backup, all writes on the source subtree must go through the
backup-top filter so it can copy all data to the target before it is changed.
backup-top filter is appended above source node, to achieve this thing, so all
parents of source node are handled. A configuration with side parents of source
sub-tree with write permission is unsupported (we'd have append several
backup-top filter like nodes to handle such parents). The test create an
example of such configuration and checks that a backup is then not allowed
(blockdev-backup command should fail).
copy-before-write filter so it can copy all data to the target before it is
changed. copy-before-write filter is appended above source node, to achieve
this thing, so all parents of source node are handled. A configuration with
side parents of source sub-tree with write permission is unsupported (we'd have
append several copy-before-write filter like nodes to handle such parents). The
test create an example of such configuration and checks that a backup is then
not allowed (blockdev-backup command should fail).
The configuration:
@ -57,11 +57,10 @@ The configuration:
│ base │ ◀──────────── │ other │
└─────────────┘ └───────┘
On activation (see .active field of backup-top state in block/backup-top.c),
backup-top is going to unshare write permission on its source child. Write
unsharing will be propagated to the "source->base" link and will conflict with
other node write permission. So permission update will fail and backup job will
not be started.
copy-before-write filter wants to unshare write permission on its source child.
Write unsharing will be propagated to the "source->base" link and will conflict
with other node write permission. So permission update will fail and backup job
will not be started.
Note, that the only thing which prevents backup of running on such
configuration is default permission propagation scheme. It may be altered by
@ -99,13 +98,9 @@ vm.qmp_log('blockdev-backup', sync='full', device='source', target='target')
vm.shutdown()
print('\n=== backup-top should be gone after job-finalize ===\n')
print('\n=== copy-before-write filter should be gone after job-finalize ===\n')
# Check that the backup-top node is gone after job-finalize.
#
# During finalization, the node becomes inactive and can no longer
# function. If it is still present, new parents might be attached, and
# there would be no meaningful way to handle their I/O requests.
# Check that the copy-before-write node is gone after job-finalize.
vm = iotests.VM()
vm.launch()
@ -131,7 +126,7 @@ vm.qmp_log('blockdev-backup',
vm.event_wait('BLOCK_JOB_PENDING', 5.0)
# The backup-top filter should still be present prior to finalization
# The copy-before-write filter should still be present prior to finalization
assert vm.node_info('backup-filter') is not None
vm.qmp_log('job-finalize', id='backup')

4
tests/qemu-iotests/283.out
View File

@ -5,9 +5,9 @@
{"execute": "blockdev-add", "arguments": {"driver": "blkdebug", "image": "base", "node-name": "other", "take-child-perms": ["write"]}}
{"return": {}}
{"execute": "blockdev-backup", "arguments": {"device": "source", "sync": "full", "target": "target"}}
{"error": {"class": "GenericError", "desc": "Cannot append backup-top filter: Permission conflict on node 'base': permissions 'write' are both required by node 'other' (uses node 'base' as 'image' child) and unshared by node 'source' (uses node 'base' as 'image' child)."}}
{"error": {"class": "GenericError", "desc": "Permission conflict on node 'base': permissions 'write' are both required by node 'other' (uses node 'base' as 'image' child) and unshared by node 'source' (uses node 'base' as 'image' child)."}}
=== backup-top should be gone after job-finalize ===
=== copy-before-write filter should be gone after job-finalize ===
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "source"}}
{"return": {}}

2
tests/qemu-iotests/297
View File

@ -31,7 +31,7 @@ SKIP_FILES = (
'096', '118', '124', '132', '136', '139', '147', '148', '149',
'151', '152', '155', '163', '165', '169', '194', '196', '199', '202',
'203', '205', '206', '207', '208', '210', '211', '212', '213', '216',
'218', '219', '222', '224', '228', '234', '235', '236', '237', '238',
'218', '219', '224', '228', '234', '235', '236', '237', '238',
'240', '242', '245', '246', '248', '255', '256', '257', '258', '260',
'262', '264', '266', '274', '277', '280', '281', '295', '296', '298',
'299', '302', '303', '304', '307',

15
tests/qemu-iotests/check
View File

@ -36,6 +36,15 @@ def make_argparser() -> argparse.ArgumentParser:
help='pretty print output for make check')
p.add_argument('-d', dest='debug', action='store_true', help='debug')
p.add_argument('-p', dest='print', action='store_true',
help='redirects qemu\'s stdout and stderr to the test output')
p.add_argument('-gdb', action='store_true',
help="start gdbserver with $GDB_OPTIONS options \
('localhost:12345' if $GDB_OPTIONS is empty)")
p.add_argument('-valgrind', action='store_true',
help='use valgrind, sets VALGRIND_QEMU environment '
'variable')
p.add_argument('-misalign', action='store_true',
help='misalign memory allocations')
p.add_argument('--color', choices=['on', 'off', 'auto'],
@ -85,9 +94,6 @@ def make_argparser() -> argparse.ArgumentParser:
g_bash.add_argument('-o', dest='imgopts',
help='options to pass to qemu-img create/convert, '
'sets IMGOPTS environment variable')
g_bash.add_argument('-valgrind', action='store_true',
help='use valgrind, sets VALGRIND_QEMU environment '
'variable')
g_sel = p.add_argument_group('test selecting options',
'The following options specify test set '
@ -114,7 +120,8 @@ if __name__ == '__main__':
env = TestEnv(imgfmt=args.imgfmt, imgproto=args.imgproto,
aiomode=args.aiomode, cachemode=args.cachemode,
imgopts=args.imgopts, misalign=args.misalign,
debug=args.debug, valgrind=args.valgrind)
debug=args.debug, valgrind=args.valgrind,
gdb=args.gdb, qprint=args.print)
if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':
if not args.tests:

7
tests/qemu-iotests/common.qemu
View File

@ -85,7 +85,12 @@ _timed_wait_for()
timeout=yes
QEMU_STATUS[$h]=0
while IFS= read -t ${QEMU_COMM_TIMEOUT} resp <&${QEMU_OUT[$h]}
read_timeout="-t ${QEMU_COMM_TIMEOUT}"
if [ -n "${GDB_OPTIONS}" ]; then
read_timeout=
fi
while IFS= read ${read_timeout} resp <&${QEMU_OUT[$h]}
do
if [ -n "$capture_events" ]; then
capture=0

8
tests/qemu-iotests/common.rc
View File

@ -166,8 +166,14 @@ _qemu_wrapper()
if [ -n "${QEMU_NEED_PID}" ]; then
echo $BASHPID > "${QEMU_TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
fi
GDB=""
if [ -n "${GDB_OPTIONS}" ]; then
GDB="gdbserver ${GDB_OPTIONS}"
fi
VALGRIND_QEMU="${VALGRIND_QEMU_VM}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
"$QEMU_PROG" $QEMU_OPTIONS "$@"
$GDB "$QEMU_PROG" $QEMU_OPTIONS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL

75
tests/qemu-iotests/iotests.py
View File

@ -74,6 +74,13 @@ if os.environ.get('QEMU_NBD_OPTIONS'):
qemu_prog = os.environ.get('QEMU_PROG', 'qemu')
qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ')
gdb_qemu_env = os.environ.get('GDB_OPTIONS')
qemu_gdb = []
if gdb_qemu_env:
qemu_gdb = ['gdbserver'] + gdb_qemu_env.strip().split(' ')
qemu_print = os.environ.get('PRINT_QEMU', False)
imgfmt = os.environ.get('IMGFMT', 'raw')
imgproto = os.environ.get('IMGPROTO', 'file')
output_dir = os.environ.get('OUTPUT_DIR', '.')
@ -91,6 +98,17 @@ except KeyError:
sys.stderr.write('Please run this test via the "check" script\n')
sys.exit(os.EX_USAGE)
qemu_valgrind = []
if os.environ.get('VALGRIND_QEMU') == "y" and \
os.environ.get('NO_VALGRIND') != "y":
valgrind_logfile = "--log-file=" + test_dir
# %p allows to put the valgrind process PID, since
# we don't know it a priori (subprocess.Popen is
# not yet invoked)
valgrind_logfile += "/%p.valgrind"
qemu_valgrind = ['valgrind', valgrind_logfile, '--error-exitcode=99']
socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
luks_default_secret_object = 'secret,id=keysec0,data=' + \
@ -219,18 +237,18 @@ def qemu_io_silent(*args):
default_args = qemu_io_args
args = default_args + list(args)
exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'))
if exitcode < 0:
result = subprocess.run(args, stdout=subprocess.DEVNULL, check=False)
if result.returncode < 0:
sys.stderr.write('qemu-io received signal %i: %s\n' %
(-exitcode, ' '.join(args)))
return exitcode
(-result.returncode, ' '.join(args)))
return result.returncode
def qemu_io_silent_check(*args):
'''Run qemu-io and return the true if subprocess returned 0'''
args = qemu_io_args + list(args)
exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
return exitcode == 0
result = subprocess.run(args, stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT, check=False)
return result.returncode == 0
class QemuIoInteractive:
def __init__(self, *args):
@ -472,10 +490,14 @@ class Timeout:
self.seconds = seconds
self.errmsg = errmsg
def __enter__(self):
if qemu_gdb or qemu_valgrind:
return self
signal.signal(signal.SIGALRM, self.timeout)
signal.setitimer(signal.ITIMER_REAL, self.seconds)
return self
def __exit__(self, exc_type, value, traceback):
if qemu_gdb or qemu_valgrind:
return False
signal.setitimer(signal.ITIMER_REAL, 0)
return False
def timeout(self, signum, frame):
@ -570,12 +592,35 @@ class VM(qtest.QEMUQtestMachine):
def __init__(self, path_suffix=''):
name = "qemu%s-%d" % (path_suffix, os.getpid())
super().__init__(qemu_prog, qemu_opts, name=name,
timer = 15.0 if not (qemu_gdb or qemu_valgrind) else None
if qemu_gdb and qemu_valgrind:
sys.stderr.write('gdb and valgrind are mutually exclusive\n')
sys.exit(1)
wrapper = qemu_gdb if qemu_gdb else qemu_valgrind
super().__init__(qemu_prog, qemu_opts, wrapper=wrapper,
name=name,
base_temp_dir=test_dir,
socket_scm_helper=socket_scm_helper,
sock_dir=sock_dir)
sock_dir=sock_dir, qmp_timer=timer)
self._num_drives = 0
def _post_shutdown(self) -> None:
super()._post_shutdown()
if not qemu_valgrind or not self._popen:
return
valgrind_filename = f"{test_dir}/{self._popen.pid}.valgrind"
if self.exitcode() == 99:
with open(valgrind_filename) as f:
print(f.read())
else:
os.remove(valgrind_filename)
def _pre_launch(self) -> None:
super()._pre_launch()
if qemu_print:
# set QEMU binary output to stdout
self._close_qemu_log_file()
def add_object(self, opts):
self._args.append('-object')
self._args.append(opts)
@ -651,9 +696,10 @@ class VM(qtest.QEMUQtestMachine):
self.hmp(f'qemu-io {drive} "remove_break bp_{drive}"')
def hmp_qemu_io(self, drive: str, cmd: str,
use_log: bool = False) -> QMPMessage:
use_log: bool = False, qdev: bool = False) -> QMPMessage:
"""Write to a given drive using an HMP command"""
return self.hmp(f'qemu-io {drive} "{cmd}"', use_log=use_log)
d = '-d ' if qdev else ''
return self.hmp(f'qemu-io {d}{drive} "{cmd}"', use_log=use_log)
def flatten_qmp_object(self, obj, output=None, basestr=''):
if output is None:
@ -1075,7 +1121,8 @@ def notrun(reason):
# Each test in qemu-iotests has a number ("seq")
seq = os.path.basename(sys.argv[0])
open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n')
with open('%s/%s.notrun' % (output_dir, seq), 'w') as outfile:
outfile.write(reason + '\n')
logger.warning("%s not run: %s", seq, reason)
sys.exit(0)
@ -1088,8 +1135,8 @@ def case_notrun(reason):
# Each test in qemu-iotests has a number ("seq")
seq = os.path.basename(sys.argv[0])
open('%s/%s.casenotrun' % (output_dir, seq), 'a').write(
' [case not run] ' + reason + '\n')
with open('%s/%s.casenotrun' % (output_dir, seq), 'a') as outfile:
outfile.write(' [case not run] ' + reason + '\n')
def _verify_image_format(supported_fmts: Sequence[str] = (),
unsupported_fmts: Sequence[str] = ()) -> None:

23
tests/qemu-iotests/testenv.py
View File

@ -27,6 +27,7 @@ import subprocess
import glob
from typing import List, Dict, Any, Optional, ContextManager
DEF_GDB_OPTIONS = 'localhost:12345'
def isxfile(path: str) -> bool:
return os.path.isfile(path) and os.access(path, os.X_OK)
@ -72,7 +73,8 @@ class TestEnv(ContextManager['TestEnv']):
'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO',
'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU',
'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_']
'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_',
'GDB_OPTIONS', 'PRINT_QEMU']
def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:
if self.debug:
@ -178,7 +180,9 @@ class TestEnv(ContextManager['TestEnv']):
imgopts: Optional[str] = None,
misalign: bool = False,
debug: bool = False,
valgrind: bool = False) -> None:
valgrind: bool = False,
gdb: bool = False,
qprint: bool = False) -> None:
self.imgfmt = imgfmt
self.imgproto = imgproto
self.aiomode = aiomode
@ -186,6 +190,18 @@ class TestEnv(ContextManager['TestEnv']):
self.misalign = misalign
self.debug = debug
if qprint:
self.print_qemu = 'y'
if gdb:
self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS)
if not self.gdb_options:
# cover the case 'export GDB_OPTIONS='
self.gdb_options = DEF_GDB_OPTIONS
elif 'GDB_OPTIONS' in os.environ:
# to not propagate it in prepare_subprocess()
del os.environ['GDB_OPTIONS']
if valgrind:
self.valgrind_qemu = 'y'
@ -285,6 +301,9 @@ PLATFORM -- {platform}
TEST_DIR -- {TEST_DIR}
SOCK_DIR -- {SOCK_DIR}
SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}
GDB_OPTIONS -- {GDB_OPTIONS}
VALGRIND_QEMU -- {VALGRIND_QEMU}
PRINT_QEMU_OUTPUT -- {PRINT_QEMU}
"""
args = collections.defaultdict(str, self.get_env())

192
tests/qemu-iotests/tests/image-fleecing Executable file
View File

@ -0,0 +1,192 @@
#!/usr/bin/env python3
# group: rw quick
#
# This test covers the basic fleecing workflow, which provides a
# point-in-time snapshot of a node that can be queried over NBD.
#
# Copyright (C) 2018 Red Hat, Inc.
# John helped, too.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Creator/Owner: John Snow <jsnow@redhat.com>
import iotests
from iotests import log, qemu_img, qemu_io, qemu_io_silent
iotests.script_initialize(
supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'],
supported_platforms=['linux'],
)
patterns = [('0x5d', '0', '64k'),
('0xd5', '1M', '64k'),
('0xdc', '32M', '64k'),
('0xcd', '0x3ff0000', '64k')] # 64M - 64K
overwrite = [('0xab', '0', '64k'), # Full overwrite
('0xad', '0x00f8000', '64k'), # Partial-left (1M-32K)
('0x1d', '0x2008000', '64k'), # Partial-right (32M+32K)
('0xea', '0x3fe0000', '64k')] # Adjacent-left (64M - 128K)
zeroes = [('0', '0x00f8000', '32k'), # Left-end of partial-left (1M-32K)
('0', '0x2010000', '32k'), # Right-end of partial-right (32M+64K)
('0', '0x3fe0000', '64k')] # overwrite[3]
remainder = [('0xd5', '0x108000', '32k'), # Right-end of partial-left [1]
('0xdc', '32M', '32k'), # Left-end of partial-right [2]
('0xcd', '0x3ff0000', '64k')] # patterns[3]
def do_test(use_cbw, base_img_path, fleece_img_path, nbd_sock_path, vm):
log('--- Setting up images ---')
log('')
assert qemu_img('create', '-f', iotests.imgfmt, base_img_path, '64M') == 0
assert qemu_img('create', '-f', 'qcow2', fleece_img_path, '64M') == 0
for p in patterns:
qemu_io('-f', iotests.imgfmt,
'-c', 'write -P%s %s %s' % p, base_img_path)
log('Done')
log('')
log('--- Launching VM ---')
log('')
src_node = 'source'
tmp_node = 'temp'
qom_path = '/machine/peripheral/sda'
vm.add_blockdev(f'driver={iotests.imgfmt},file.driver=file,'
f'file.filename={base_img_path},node-name={src_node}')
vm.add_device('virtio-scsi')
vm.add_device(f'scsi-hd,id=sda,drive={src_node}')
vm.launch()
log('Done')
log('')
log('--- Setting up Fleecing Graph ---')
log('')
# create tmp_node backed by src_node
log(vm.qmp('blockdev-add', {
'driver': 'qcow2',
'node-name': tmp_node,
'file': {
'driver': 'file',
'filename': fleece_img_path,
},
'backing': src_node,
}))
# Establish CBW from source to fleecing node
if use_cbw:
log(vm.qmp('blockdev-add', {
'driver': 'copy-before-write',
'node-name': 'fl-cbw',
'file': src_node,
'target': tmp_node
}))
log(vm.qmp('qom-set', path=qom_path, property='drive', value='fl-cbw'))
else:
log(vm.qmp('blockdev-backup',
job_id='fleecing',
device=src_node,
target=tmp_node,
sync='none'))
log('')
log('--- Setting up NBD Export ---')
log('')
nbd_uri = 'nbd+unix:///%s?socket=%s' % (tmp_node, nbd_sock_path)
log(vm.qmp('nbd-server-start',
{'addr': { 'type': 'unix',
'data': { 'path': nbd_sock_path } } }))
log(vm.qmp('nbd-server-add', device=tmp_node))
log('')
log('--- Sanity Check ---')
log('')
for p in patterns + zeroes:
cmd = 'read -P%s %s %s' % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Testing COW ---')
log('')
for p in overwrite:
cmd = 'write -P%s %s %s' % p
log(cmd)
log(vm.hmp_qemu_io(qom_path, cmd, qdev=True))
log('')
log('--- Verifying Data ---')
log('')
for p in patterns + zeroes:
cmd = 'read -P%s %s %s' % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Cleanup ---')
log('')
if use_cbw:
log(vm.qmp('qom-set', path=qom_path, property='drive', value=src_node))
log(vm.qmp('blockdev-del', node_name='fl-cbw'))
else:
log(vm.qmp('block-job-cancel', device='fleecing'))
e = vm.event_wait('BLOCK_JOB_CANCELLED')
assert e is not None
log(e, filters=[iotests.filter_qmp_event])
log(vm.qmp('nbd-server-stop'))
log(vm.qmp('blockdev-del', node_name=tmp_node))
vm.shutdown()
log('')
log('--- Confirming writes ---')
log('')
for p in overwrite + remainder:
cmd = 'read -P%s %s %s' % p
log(cmd)
assert qemu_io_silent(base_img_path, '-c', cmd) == 0
log('')
log('Done')
def test(use_cbw):
with iotests.FilePath('base.img') as base_img_path, \
iotests.FilePath('fleece.img') as fleece_img_path, \
iotests.FilePath('nbd.sock',
base_dir=iotests.sock_dir) as nbd_sock_path, \
iotests.VM() as vm:
do_test(use_cbw, base_img_path, fleece_img_path, nbd_sock_path, vm)
log('=== Test backup(sync=none) based fleecing ===\n')
test(False)
log('=== Test filter based fleecing ===\n')
test(True)

139
tests/qemu-iotests/tests/image-fleecing.out Normal file
View File

@ -0,0 +1,139 @@
=== Test backup(sync=none) based fleecing ===
--- Setting up images ---
Done
--- Launching VM ---
Done
--- Setting up Fleecing Graph ---
{"return": {}}
{"return": {}}
--- Setting up NBD Export ---
{"return": {}}
{"return": {}}
--- Sanity Check ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Testing COW ---
write -P0xab 0 64k
{"return": ""}
write -P0xad 0x00f8000 64k
{"return": ""}
write -P0x1d 0x2008000 64k
{"return": ""}
write -P0xea 0x3fe0000 64k
{"return": ""}
--- Verifying Data ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Cleanup ---
{"return": {}}
{"data": {"device": "fleecing", "len": 67108864, "offset": 393216, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_CANCELLED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"return": {}}
{"return": {}}
--- Confirming writes ---
read -P0xab 0 64k
read -P0xad 0x00f8000 64k
read -P0x1d 0x2008000 64k
read -P0xea 0x3fe0000 64k
read -P0xd5 0x108000 32k
read -P0xdc 32M 32k
read -P0xcd 0x3ff0000 64k
Done
=== Test filter based fleecing ===
--- Setting up images ---
Done
--- Launching VM ---
Done
--- Setting up Fleecing Graph ---
{"return": {}}
{"return": {}}
{"return": {}}
--- Setting up NBD Export ---
{"return": {}}
{"return": {}}
--- Sanity Check ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Testing COW ---
write -P0xab 0 64k
{"return": ""}
write -P0xad 0x00f8000 64k
{"return": ""}
write -P0x1d 0x2008000 64k
{"return": ""}
write -P0xea 0x3fe0000 64k
{"return": ""}
--- Verifying Data ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Cleanup ---
{"return": {}}
{"return": {}}
{"return": {}}
{"return": {}}
--- Confirming writes ---
read -P0xab 0 64k
read -P0xad 0x00f8000 64k
read -P0x1d 0x2008000 64k
read -P0xea 0x3fe0000 64k
read -P0xd5 0x108000 32k
read -P0xdc 32M 32k
read -P0xcd 0x3ff0000 64k
Done