OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity
876ab8d89d
qemu-e2k/tests/qemu-iotests/257
Eric Blake b66ff2c298 iotests: Specify explicit backing format where sensible 
There are many existing qcow2 images that specify a backing file but
no format.  This has been the source of CVEs in the past, but has
become more prominent of a problem now that libvirt has switched to
-blockdev.  With older -drive, at least the probing was always done by
qemu (so the only risk of a changed format between successive boots of
a guest was if qemu was upgraded and probed differently).  But with
newer -blockdev, libvirt must specify a format; if libvirt guesses raw
where the image was formatted, this results in data corruption visible
to the guest; conversely, if libvirt guesses qcow2 where qemu was
using raw, this can result in potential security holes, so modern
libvirt instead refuses to use images without explicit backing format.

The change in libvirt to reject images without explicit backing format
has pointed out that a number of tools have been far too reliant on
probing in the past.  It's time to set a better example in our own
iotests of properly setting this parameter.

iotest calls to create, rebase, and convert are all impacted to some
degree.  It's a bit annoying that we are inconsistent on command line
- while all of those accept -o backing_file=...,backing_fmt=..., the
shortcuts are different: create and rebase have -b and -F, while
convert has -B but no -F.  (amend has no shortcuts, but the previous
patch just deprecated the use of amend to change backing chains).

Signed-off-by: Eric Blake <eblake@redhat.com>
Message-Id: <20200706203954.341758-9-eblake@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-07-14 15:18:59 +02:00

532 lines
19 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#
# Test bitmap-sync backups (incremental, differential, and partials)
#
# Copyright (c) 2019 John Snow for Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# owner=jsnow@redhat.com
import math
import os
import iotests
from iotests import log, qemu_img
SIZE = 64 * 1024 * 1024
GRANULARITY = 64 * 1024
class Pattern:
def __init__(self, byte, offset, size=GRANULARITY):
self.byte = byte
self.offset = offset
self.size = size
def bits(self, granularity):
lower = self.offset // granularity
upper = (self.offset + self.size - 1) // granularity
return set(range(lower, upper + 1))
class PatternGroup:
"""Grouping of Pattern objects. Initialize with an iterable of Patterns."""
def __init__(self, patterns):
self.patterns = patterns
def bits(self, granularity):
"""Calculate the unique bits dirtied by this pattern grouping"""
res = set()
for pattern in self.patterns:
res |= pattern.bits(granularity)
return res
GROUPS = [
PatternGroup([
# Batch 0: 4 clusters
Pattern('0x49', 0x0000000),
Pattern('0x6c', 0x0100000), # 1M
Pattern('0x6f', 0x2000000), # 32M
Pattern('0x76', 0x3ff0000)]), # 64M - 64K
PatternGroup([
# Batch 1: 6 clusters (3 new)
Pattern('0x65', 0x0000000), # Full overwrite
Pattern('0x77', 0x00f8000), # Partial-left (1M-32K)
Pattern('0x72', 0x2008000), # Partial-right (32M+32K)
Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
PatternGroup([
# Batch 2: 7 clusters (3 new)
Pattern('0x74', 0x0010000), # Adjacent-right
Pattern('0x69', 0x00e8000), # Partial-left (1M-96K)
Pattern('0x6e', 0x2018000), # Partial-right (32M+96K)
Pattern('0x67', 0x3fe0000,
2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
PatternGroup([
# Batch 3: 8 clusters (5 new)
# Carefully chosen such that nothing re-dirties the one cluster
# that copies out successfully before failure in Group #1.
Pattern('0xaa', 0x0010000,
3*GRANULARITY), # Overwrite and 2x Adjacent-right
Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
Pattern('0xcc', 0x2028000), # Partial-right (32M+160K)
Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
]
class EmulatedBitmap:
def __init__(self, granularity=GRANULARITY):
self._bits = set()
self.granularity = granularity
def dirty_bits(self, bits):
self._bits |= set(bits)
def dirty_group(self, n):
self.dirty_bits(GROUPS[n].bits(self.granularity))
def clear(self):
self._bits = set()
def clear_bits(self, bits):
self._bits -= set(bits)
def clear_bit(self, bit):
self.clear_bits({bit})
def clear_group(self, n):
self.clear_bits(GROUPS[n].bits(self.granularity))
@property
def first_bit(self):
return sorted(self.bits)[0]
@property
def bits(self):
return self._bits
@property
def count(self):
return len(self.bits)
def compare(self, qmp_bitmap):
"""
Print a nice human-readable message checking that a bitmap as reported
by the QMP interface has as many bits set as we expect it to.
"""
name = qmp_bitmap.get('name', '(anonymous)')
log("= Checking Bitmap {:s} =".format(name))
want = self.count
have = qmp_bitmap['count'] // qmp_bitmap['granularity']
log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
want, have, "OK!" if want == have else "ERROR!"))
log('')
class Drive:
"""Represents, vaguely, a drive attached to a VM.
Includes format, graph, and device information."""
def __init__(self, path, vm=None):
self.path = path
self.vm = vm
self.fmt = None
self.size = None
self.node = None
def img_create(self, fmt, size):
self.fmt = fmt
self.size = size
iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
def create_target(self, name, fmt, size):
basename = os.path.basename(self.path)
file_node_name = "file_{}".format(basename)
vm = self.vm
log(vm.command('blockdev-create', job_id='bdc-file-job',
options={
'driver': 'file',
'filename': self.path,
'size': 0,
}))
vm.run_job('bdc-file-job')
log(vm.command('blockdev-add', driver='file',
node_name=file_node_name, filename=self.path))
log(vm.command('blockdev-create', job_id='bdc-fmt-job',
options={
'driver': fmt,
'file': file_node_name,
'size': size,
}))
vm.run_job('bdc-fmt-job')
log(vm.command('blockdev-add', driver=fmt,
node_name=name,
file=file_node_name))
self.fmt = fmt
self.size = size
self.node = name
def blockdev_backup(vm, device, target, sync, **kwargs):
# Strip any arguments explicitly nulled by the caller:
kwargs = {key: val for key, val in kwargs.items() if val is not None}
result = vm.qmp_log('blockdev-backup',
device=device,
target=target,
sync=sync,
filter_node_name='backup-top',
**kwargs)
return result
def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
target_drive = Drive(filepath, vm=drive.vm)
target_drive.create_target(target_id, drive.fmt, drive.size)
blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs)
def reference_backup(drive, n, filepath):
log("--- Reference Backup #{:d} ---\n".format(n))
target_id = "ref_target_{:d}".format(n)
job_id = "ref_backup_{:d}".format(n)
blockdev_backup_mktarget(drive, target_id, filepath, "full",
job_id=job_id)
drive.vm.run_job(job_id, auto_dismiss=True)
log('')
def backup(drive, n, filepath, sync, **kwargs):
log("--- Test Backup #{:d} ---\n".format(n))
target_id = "backup_target_{:d}".format(n)
job_id = "backup_{:d}".format(n)
kwargs.setdefault('auto-finalize', False)
blockdev_backup_mktarget(drive, target_id, filepath, sync,
job_id=job_id, **kwargs)
return job_id
def perform_writes(drive, n, filter_node_name=None):
log("--- Write #{:d} ---\n".format(n))
for pattern in GROUPS[n].patterns:
cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
pattern.byte,
pattern.offset,
pattern.size)
log(cmd)
log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd))
bitmaps = drive.vm.query_bitmaps()
log({'bitmaps': bitmaps}, indent=2)
log('')
return bitmaps
def compare_images(image, reference, baseimg=None, expected_match=True):
"""
Print a nice human-readable message comparing these images.
"""
expected_ret = 0 if expected_match else 1
if baseimg:
assert qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt,
image) == 0
ret = qemu_img("compare", image, reference)
log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
image, reference,
"Identical" if ret == 0 else "Mismatch",
"OK!" if ret == expected_ret else "ERROR!"),
filters=[iotests.filter_testfiles])
def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
"""
Test bitmap backup routines.
:param bsync_mode: Is the Bitmap Sync mode, and can be any of:
- on-success: This is the "incremental" style mode. Bitmaps are
synchronized to what was copied out only on success.
(Partial images must be discarded.)
- never: This is the "differential" style mode.
Bitmaps are never synchronized.
- always: This is a "best effort" style mode.
Bitmaps are always synchronized, regardless of failure.
(Partial images must be kept.)
:param msync_mode: The mirror sync mode to use for the first backup.
Can be any one of:
- bitmap: Backups based on bitmap manifest.
- full: Full backups.
- top: Full backups of the top layer only.
:param failure: Is the (optional) failure mode, and can be any of:
- None: No failure. Test the normative path. Default.
- simulated: Cancel the job right before it completes.
This also tests writes "during" the job.
- intermediate: This tests a job that fails mid-process and produces
an incomplete backup. Testing limitations prevent
testing competing writes.
"""
with iotests.FilePaths(['img', 'bsync1', 'bsync2',
'fbackup0', 'fbackup1', 'fbackup2']) as \
(img_path, bsync1, bsync2,
fbackup0, fbackup1, fbackup2), \
iotests.VM() as vm:
mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
preposition = "with" if failure else "without"
cond = "{:s} {:s}".format(preposition,
"{:s} failure".format(failure) if failure
else "failure")
log("\n=== {:s} {:s} ===\n".format(mode, cond))
log('--- Preparing image & VM ---\n')
drive0 = Drive(img_path, vm=vm)
drive0.img_create(iotests.imgfmt, SIZE)
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
vm.launch()
file_config = {
'driver': 'file',
'filename': drive0.path
}
if failure == 'intermediate':
file_config = {
'driver': 'blkdebug',
'image': file_config,
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}, {
'event': 'read_aio',
'state': 2,
'new_state': 3
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 3,
'immediately': False,
'once': True
}]
}
drive0.node = 'drive0'
vm.qmp_log('blockdev-add',
filters=[iotests.filter_qmp_testfiles],
node_name=drive0.node,
driver=drive0.fmt,
file=file_config)
log('')
# 0 - Writes and Reference Backup
perform_writes(drive0, 0)
reference_backup(drive0, 0, fbackup0)
log('--- Add Bitmap ---\n')
vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
name="bitmap0", granularity=GRANULARITY)
log('')
ebitmap = EmulatedBitmap()
# 1 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 1)
ebitmap.dirty_group(1)
bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)
ebitmap.compare(bitmap)
reference_backup(drive0, 1, fbackup1)
# 1 - Test Backup (w/ Optional induced failure)
if failure == 'intermediate':
# Activate blkdebug induced failure for second-to-next read
log(vm.hmp_qemu_io(drive0.node, 'flush'))
log('')
job = backup(drive0, 1, bsync1, msync_mode,
bitmap="bitmap0", bitmap_mode=bsync_mode)
def _callback():
"""Issue writes while the job is open to test bitmap divergence."""
# Note: when `failure` is 'intermediate', this isn't called.
log('')
bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top')
# Named bitmap (static, should be unchanged)
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0',
bitmaps=bitmaps))
# Anonymous bitmap (dynamic, shows new writes)
anonymous = EmulatedBitmap()
anonymous.dirty_group(2)
anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True,
bitmaps=bitmaps))
# Simulate the order in which this will happen:
# group 1 gets cleared first, then group two gets written.
if ((bsync_mode == 'on-success' and not failure) or
(bsync_mode == 'always')):
ebitmap.clear()
ebitmap.dirty_group(2)
vm.run_job(job, auto_dismiss=True, auto_finalize=False,
pre_finalize=_callback,
cancel=(failure == 'simulated'))
bitmaps = vm.query_bitmaps()
log({'bitmaps': bitmaps}, indent=2)
log('')
if bsync_mode == 'always' and failure == 'intermediate':
# TOP treats anything allocated as dirty, expect to see:
if msync_mode == 'top':
ebitmap.dirty_group(0)
# We manage to copy one sector (one bit) before the error.
ebitmap.clear_bit(ebitmap.first_bit)
# Full returns all bits set except what was copied/skipped
if msync_mode == 'full':
fail_bit = ebitmap.first_bit
ebitmap.clear()
ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
# 2 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 3)
ebitmap.dirty_group(3)
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
reference_backup(drive0, 2, fbackup2)
# 2 - Bitmap Backup (In failure modes, this is a recovery.)
job = backup(drive0, 2, bsync2, "bitmap",
bitmap="bitmap0", bitmap_mode=bsync_mode)
vm.run_job(job, auto_dismiss=True, auto_finalize=False)
bitmaps = vm.query_bitmaps()
log({'bitmaps': bitmaps}, indent=2)
log('')
if bsync_mode != 'never':
ebitmap.clear()
ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps))
log('--- Cleanup ---\n')
vm.qmp_log("block-dirty-bitmap-remove",
node=drive0.node, name="bitmap0")
bitmaps = vm.query_bitmaps()
log({'bitmaps': bitmaps}, indent=2)
vm.shutdown()
log('')
log('--- Verification ---\n')
# 'simulated' failures will actually all pass here because we canceled
# while "pending". This is actually undefined behavior,
# don't rely on this to be true!
compare_images(bsync1, fbackup1, baseimg=fbackup0,
expected_match=failure != 'intermediate')
if not failure or bsync_mode == 'always':
# Always keep the last backup on success or when using 'always'
base = bsync1
else:
base = fbackup0
compare_images(bsync2, fbackup2, baseimg=base)
compare_images(img_path, fbackup2)
log('')
def test_backup_api():
"""
Test malformed and prohibited invocations of the backup API.
"""
with iotests.FilePaths(['img', 'bsync1']) as \
(img_path, backup_path), \
iotests.VM() as vm:
log("\n=== API failure tests ===\n")
log('--- Preparing image & VM ---\n')
drive0 = Drive(img_path, vm=vm)
drive0.img_create(iotests.imgfmt, SIZE)
vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
vm.launch()
file_config = {
'driver': 'file',
'filename': drive0.path
}
drive0.node = 'drive0'
vm.qmp_log('blockdev-add',
filters=[iotests.filter_qmp_testfiles],
node_name=drive0.node,
driver=drive0.fmt,
file=file_config)
log('')
target0 = Drive(backup_path, vm=vm)
target0.create_target("backup_target", drive0.fmt, drive0.size)
log('')
vm.qmp_log("block-dirty-bitmap-add", node=drive0.node,
name="bitmap0", granularity=GRANULARITY)
log('')
log('-- Testing invalid QMP commands --\n')
error_cases = {
'incremental': {
None: ['on-success', 'always', 'never', None],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['always', 'never']
},
'bitmap': {
None: ['on-success', 'always', 'never', None],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': [None],
},
'full': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['never', None],
},
'top': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['never', None],
},
'none': {
None: ['on-success', 'always', 'never'],
'bitmap404': ['on-success', 'always', 'never', None],
'bitmap0': ['on-success', 'always', 'never', None],
}
}
# Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
log("-- Sync mode {:s} tests --\n".format(sync_mode))
for bitmap in (None, 'bitmap404', 'bitmap0'):
for policy in error_cases[sync_mode][bitmap]:
blockdev_backup(drive0.vm, drive0.node, "backup_target",
sync_mode, job_id='api_job',
bitmap=bitmap, bitmap_mode=policy)
log('')
def main():
for bsync_mode in ("never", "on-success", "always"):
for failure in ("simulated", "intermediate", None):
test_bitmap_sync(bsync_mode, "bitmap", failure)
for sync_mode in ('full', 'top'):
for bsync_mode in ('on-success', 'always'):
for failure in ('simulated', 'intermediate', None):
test_bitmap_sync(bsync_mode, sync_mode, failure)
test_backup_api()
if __name__ == '__main__':
iotests.script_main(main, supported_fmts=['qcow2'],
supported_protocols=['file'])
Reference in New Issue View Git Blame Copy Permalink