OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity
6fabae61a9
qemu-e2k/tests/qemu-iotests/124
John Snow c61b198b63 iotests: add busy/recording bit test to 124 
This adds a simple test that ensures the busy bit works for push backups,
as well as doubling as bonus test for incremental backups that get interrupted
by EIO errors.

Recording bit tests are already handled sufficiently by 236.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Tested-by: Eric Blake <eblake@redhat.com>
Message-id: 20190223000614.13894-11-jsnow@redhat.com
Signed-off-by: John Snow <jsnow@redhat.com>
2019-03-12 12:05:48 -04:00

753 lines
29 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
#
# Tests for incremental drive-backup
#
# Copyright (C) 2015 John Snow for Red Hat, Inc.
#
# Based on 056.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import iotests
def io_write_patterns(img, patterns):
for pattern in patterns:
iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
def try_remove(img):
try:
os.remove(img)
except OSError:
pass
def transaction_action(action, **kwargs):
return {
'type': action,
'data': dict((k.replace('_', '-'), v) for k, v in kwargs.items())
}
def transaction_bitmap_clear(node, name, **kwargs):
return transaction_action('block-dirty-bitmap-clear',
node=node, name=name, **kwargs)
def transaction_drive_backup(device, target, **kwargs):
return transaction_action('drive-backup', job_id=device, device=device,
target=target, **kwargs)
class Bitmap:
def __init__(self, name, drive):
self.name = name
self.drive = drive
self.num = 0
self.backups = list()
def base_target(self):
return (self.drive['backup'], None)
def new_target(self, num=None):
if num is None:
num = self.num
self.num = num + 1
base = os.path.join(iotests.test_dir,
"%s.%s." % (self.drive['id'], self.name))
suff = "%i.%s" % (num, self.drive['fmt'])
target = base + "inc" + suff
reference = base + "ref" + suff
self.backups.append((target, reference))
return (target, reference)
def last_target(self):
if self.backups:
return self.backups[-1]
return self.base_target()
def del_target(self):
for image in self.backups.pop():
try_remove(image)
self.num -= 1
def cleanup(self):
for backup in self.backups:
for image in backup:
try_remove(image)
class TestIncrementalBackupBase(iotests.QMPTestCase):
def __init__(self, *args):
super(TestIncrementalBackupBase, self).__init__(*args)
self.bitmaps = list()
self.files = list()
self.drives = list()
self.vm = iotests.VM()
self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
def setUp(self):
# Create a base image with a distinctive patterning
drive0 = self.add_node('drive0')
self.img_create(drive0['file'], drive0['fmt'])
self.vm.add_drive(drive0['file'])
self.write_default_pattern(drive0['file'])
self.vm.launch()
def write_default_pattern(self, target):
io_write_patterns(target, (('0x41', 0, 512),
('0xd5', '1M', '32k'),
('0xdc', '32M', '124k')))
def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
if path is None:
path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
if backup is None:
backup = os.path.join(iotests.test_dir,
'%s.full.backup.%s' % (node_id, fmt))
self.drives.append({
'id': node_id,
'file': path,
'backup': backup,
'fmt': fmt })
return self.drives[-1]
def img_create(self, img, fmt=iotests.imgfmt, size='64M',
parent=None, parentFormat=None, **kwargs):
optargs = []
for k,v in kwargs.items():
optargs = optargs + ['-o', '%s=%s' % (k,v)]
args = ['create', '-f', fmt] + optargs + [img, size]
if parent:
if parentFormat is None:
parentFormat = fmt
args = args + ['-b', parent, '-F', parentFormat]
iotests.qemu_img(*args)
self.files.append(img)
def do_qmp_backup(self, error='Input/output error', **kwargs):
res = self.vm.qmp('drive-backup', **kwargs)
self.assert_qmp(res, 'return', {})
return self.wait_qmp_backup(kwargs['device'], error)
def ignore_job_status_change_events(self):
while True:
e = self.vm.event_wait(name="JOB_STATUS_CHANGE")
if e['data']['status'] == 'null':
break
def wait_qmp_backup(self, device, error='Input/output error'):
event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
match={'data': {'device': device}})
self.assertNotEqual(event, None)
self.ignore_job_status_change_events()
try:
failure = self.dictpath(event, 'data/error')
except AssertionError:
# Backup succeeded.
self.assert_qmp(event, 'data/offset', event['data']['len'])
return True
else:
# Backup failed.
self.assert_qmp(event, 'data/error', error)
return False
def wait_qmp_backup_cancelled(self, device):
event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
match={'data': {'device': device}})
self.assertNotEqual(event, None)
self.ignore_job_status_change_events()
def create_anchor_backup(self, drive=None):
if drive is None:
drive = self.drives[-1]
res = self.do_qmp_backup(job_id=drive['id'],
device=drive['id'], sync='full',
format=drive['fmt'], target=drive['backup'])
self.assertTrue(res)
self.files.append(drive['backup'])
return drive['backup']
def make_reference_backup(self, bitmap=None):
if bitmap is None:
bitmap = self.bitmaps[-1]
_, reference = bitmap.last_target()
res = self.do_qmp_backup(job_id=bitmap.drive['id'],
device=bitmap.drive['id'], sync='full',
format=bitmap.drive['fmt'], target=reference)
self.assertTrue(res)
def add_bitmap(self, name, drive, **kwargs):
bitmap = Bitmap(name, drive)
self.bitmaps.append(bitmap)
result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
name=bitmap.name, **kwargs)
self.assert_qmp(result, 'return', {})
return bitmap
def prepare_backup(self, bitmap=None, parent=None):
if bitmap is None:
bitmap = self.bitmaps[-1]
if parent is None:
parent, _ = bitmap.last_target()
target, _ = bitmap.new_target()
self.img_create(target, bitmap.drive['fmt'], parent=parent)
return target
def create_incremental(self, bitmap=None, parent=None,
parentFormat=None, validate=True):
if bitmap is None:
bitmap = self.bitmaps[-1]
if parent is None:
parent, _ = bitmap.last_target()
target = self.prepare_backup(bitmap, parent)
res = self.do_qmp_backup(job_id=bitmap.drive['id'],
device=bitmap.drive['id'],
sync='incremental', bitmap=bitmap.name,
format=bitmap.drive['fmt'], target=target,
mode='existing')
if not res:
bitmap.del_target();
self.assertFalse(validate)
else:
self.make_reference_backup(bitmap)
return res
def check_backups(self):
for bitmap in self.bitmaps:
for incremental, reference in bitmap.backups:
self.assertTrue(iotests.compare_images(incremental, reference))
last = bitmap.last_target()[0]
self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
def hmp_io_writes(self, drive, patterns):
for pattern in patterns:
self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
self.vm.hmp_qemu_io(drive, 'flush')
def do_incremental_simple(self, **kwargs):
self.create_anchor_backup()
self.add_bitmap('bitmap0', self.drives[0], **kwargs)
# Sanity: Create a "hollow" incremental backup
self.create_incremental()
# Three writes: One complete overwrite, one new segment,
# and one partial overlap.
self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
self.create_incremental()
# Three more writes, one of each kind, like above
self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
self.create_incremental()
self.vm.shutdown()
self.check_backups()
def tearDown(self):
self.vm.shutdown()
for bitmap in self.bitmaps:
bitmap.cleanup()
for filename in self.files:
try_remove(filename)
class TestIncrementalBackup(TestIncrementalBackupBase):
def test_incremental_simple(self):
'''
Test: Create and verify three incremental backups.
Create a bitmap and a full backup before VM execution begins,
then create a series of three incremental backups "during execution,"
i.e.; after IO requests begin modifying the drive.
'''
return self.do_incremental_simple()
def test_small_granularity(self):
'''
Test: Create and verify backups made with a small granularity bitmap.
Perform the same test as test_incremental_simple, but with a granularity
of only 32KiB instead of the present default of 64KiB.
'''
return self.do_incremental_simple(granularity=32768)
def test_large_granularity(self):
'''
Test: Create and verify backups made with a large granularity bitmap.
Perform the same test as test_incremental_simple, but with a granularity
of 128KiB instead of the present default of 64KiB.
'''
return self.do_incremental_simple(granularity=131072)
def test_larger_cluster_target(self):
'''
Test: Create and verify backups made to a larger cluster size target.
With a default granularity of 64KiB, verify that backups made to a
larger cluster size target of 128KiB without a backing file works.
'''
drive0 = self.drives[0]
# Create a cluster_size=128k full backup / "anchor" backup
self.img_create(drive0['backup'], cluster_size='128k')
self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
format=drive0['fmt'],
target=drive0['backup'],
mode='existing'))
# Create bitmap and dirty it with some new writes.
# overwrite [32736, 32799] which will dirty bitmap clusters at
# 32M-64K and 32M. 32M+64K will be left undirtied.
bitmap0 = self.add_bitmap('bitmap0', drive0)
self.hmp_io_writes(drive0['id'],
(('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
# Check the dirty bitmap stats
result = self.vm.qmp('query-block')
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/name', 'bitmap0')
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/count', 458752)
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/granularity', 65536)
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/status', 'active')
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/persistent', False)
# Prepare a cluster_size=128k backup target without a backing file.
(target, _) = bitmap0.new_target()
self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
# Perform Incremental Backup
self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
sync='incremental',
bitmap=bitmap0.name,
format=bitmap0.drive['fmt'],
target=target,
mode='existing'))
self.make_reference_backup(bitmap0)
# Add the backing file, then compare and exit.
iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
drive0['backup'], '-F', drive0['fmt'], target)
self.vm.shutdown()
self.check_backups()
def test_incremental_transaction(self):
'''Test: Verify backups made from transactionally created bitmaps.
Create a bitmap "before" VM execution begins, then create a second
bitmap AFTER writes have already occurred. Use transactions to create
a full backup and synchronize both bitmaps to this backup.
Create an incremental backup through both bitmaps and verify that
both backups match the current drive0 image.
'''
drive0 = self.drives[0]
bitmap0 = self.add_bitmap('bitmap0', drive0)
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
bitmap1 = self.add_bitmap('bitmap1', drive0)
result = self.vm.qmp('transaction', actions=[
transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
transaction_drive_backup(drive0['id'], drive0['backup'],
sync='full', format=drive0['fmt'])
])
self.assert_qmp(result, 'return', {})
self.wait_until_completed(drive0['id'])
self.files.append(drive0['backup'])
self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
# Both bitmaps should be correctly in sync.
self.create_incremental(bitmap0)
self.create_incremental(bitmap1)
self.vm.shutdown()
self.check_backups()
def do_transaction_failure_test(self, race=False):
# Create a second drive, with pattern:
drive1 = self.add_node('drive1')
self.img_create(drive1['file'], drive1['fmt'])
io_write_patterns(drive1['file'], (('0x14', 0, 512),
('0x5d', '1M', '32k'),
('0xcd', '32M', '124k')))
# Create a blkdebug interface to this img as 'drive1'
result = self.vm.qmp('blockdev-add',
node_name=drive1['id'],
driver=drive1['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive1['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 2,
'immediately': False,
'once': True
}],
}
)
self.assert_qmp(result, 'return', {})
# Create bitmaps and full backups for both drives
drive0 = self.drives[0]
dr0bm0 = self.add_bitmap('bitmap0', drive0)
dr1bm0 = self.add_bitmap('bitmap0', drive1)
self.create_anchor_backup(drive0)
self.create_anchor_backup(drive1)
self.assert_no_active_block_jobs()
self.assertFalse(self.vm.get_qmp_events(wait=False))
# Emulate some writes
if not race:
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
('0xef', '16M', '256k'),
('0x46', '32736k', '64k')))
# Create incremental backup targets
target0 = self.prepare_backup(dr0bm0)
target1 = self.prepare_backup(dr1bm0)
# Ask for a new incremental backup per-each drive,
# expecting drive1's backup to fail. In the 'race' test,
# we expect drive1 to attempt to cancel the empty drive0 job.
transaction = [
transaction_drive_backup(drive0['id'], target0, sync='incremental',
format=drive0['fmt'], mode='existing',
bitmap=dr0bm0.name),
transaction_drive_backup(drive1['id'], target1, sync='incremental',
format=drive1['fmt'], mode='existing',
bitmap=dr1bm0.name)
]
result = self.vm.qmp('transaction', actions=transaction,
properties={'completion-mode': 'grouped'} )
self.assert_qmp(result, 'return', {})
# Observe that drive0's backup is cancelled and drive1 completes with
# an error.
self.wait_qmp_backup_cancelled(drive0['id'])
self.assertFalse(self.wait_qmp_backup(drive1['id']))
error = self.vm.event_wait('BLOCK_JOB_ERROR')
self.assert_qmp(error, 'data', {'device': drive1['id'],
'action': 'report',
'operation': 'read'})
self.assertFalse(self.vm.get_qmp_events(wait=False))
self.assert_no_active_block_jobs()
# Delete drive0's successful target and eliminate our record of the
# unsuccessful drive1 target.
dr0bm0.del_target()
dr1bm0.del_target()
if race:
# Don't re-run the transaction, we only wanted to test the race.
self.vm.shutdown()
return
# Re-run the same transaction:
target0 = self.prepare_backup(dr0bm0)
target1 = self.prepare_backup(dr1bm0)
# Re-run the exact same transaction.
result = self.vm.qmp('transaction', actions=transaction,
properties={'completion-mode':'grouped'})
self.assert_qmp(result, 'return', {})
# Both should complete successfully this time.
self.assertTrue(self.wait_qmp_backup(drive0['id']))
self.assertTrue(self.wait_qmp_backup(drive1['id']))
self.make_reference_backup(dr0bm0)
self.make_reference_backup(dr1bm0)
self.assertFalse(self.vm.get_qmp_events(wait=False))
self.assert_no_active_block_jobs()
# And the images should of course validate.
self.vm.shutdown()
self.check_backups()
def test_transaction_failure(self):
'''Test: Verify backups made from a transaction that partially fails.
Add a second drive with its own unique pattern, and add a bitmap to each
drive. Use blkdebug to interfere with the backup on just one drive and
attempt to create a coherent incremental backup across both drives.
verify a failure in one but not both, then delete the failed stubs and
re-run the same transaction.
verify that both incrementals are created successfully.
'''
self.do_transaction_failure_test()
def test_transaction_failure_race(self):
'''Test: Verify that transactions with jobs that have no data to
transfer do not cause race conditions in the cancellation of the entire
transaction job group.
'''
self.do_transaction_failure_test(race=True)
def test_sync_dirty_bitmap_missing(self):
self.assert_no_active_block_jobs()
self.files.append(self.err_img)
result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
sync='incremental', format=self.drives[0]['fmt'],
target=self.err_img)
self.assert_qmp(result, 'error/class', 'GenericError')
def test_sync_dirty_bitmap_not_found(self):
self.assert_no_active_block_jobs()
self.files.append(self.err_img)
result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
sync='incremental', bitmap='unknown',
format=self.drives[0]['fmt'], target=self.err_img)
self.assert_qmp(result, 'error/class', 'GenericError')
def test_sync_dirty_bitmap_bad_granularity(self):
'''
Test: Test what happens if we provide an improper granularity.
The granularity must always be a power of 2.
'''
self.assert_no_active_block_jobs()
self.assertRaises(AssertionError, self.add_bitmap,
'bitmap0', self.drives[0],
granularity=64000)
class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
'''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
def setUp(self):
drive0 = self.add_node('drive0')
self.img_create(drive0['file'], drive0['fmt'])
self.write_default_pattern(drive0['file'])
self.vm.launch()
def test_incremental_failure(self):
'''Test: Verify backups made after a failure are correct.
Simulate a failure during an incremental backup block job,
emulate additional writes, then create another incremental backup
afterwards and verify that the backup created is correct.
'''
drive0 = self.drives[0]
result = self.vm.qmp('blockdev-add',
node_name=drive0['id'],
driver=drive0['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive0['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 2,
'immediately': False,
'once': True
}],
}
)
self.assert_qmp(result, 'return', {})
self.create_anchor_backup(drive0)
self.add_bitmap('bitmap0', drive0)
# Note: at this point, during a normal execution,
# Assume that the VM resumes and begins issuing IO requests here.
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
result = self.create_incremental(validate=False)
self.assertFalse(result)
self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
self.create_incremental()
self.vm.shutdown()
self.check_backups()
def test_incremental_pause(self):
"""
Test an incremental backup that errors into a pause and is resumed.
"""
drive0 = self.drives[0]
# NB: The blkdebug script here looks for a "flush, read, read" pattern.
# The flush occurs in hmp_io_writes, the first read in device_add, and
# the last read during the block job.
result = self.vm.qmp('blockdev-add',
node_name=drive0['id'],
driver=drive0['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive0['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
},{
'event': 'read_aio',
'state': 2,
'new_state': 3
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 3,
'immediately': False,
'once': True
}],
})
self.assert_qmp(result, 'return', {})
self.create_anchor_backup(drive0)
bitmap = self.add_bitmap('bitmap0', drive0)
# Emulate guest activity
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
# For the purposes of query-block visibility of bitmaps, add a drive
# frontend after we've written data; otherwise we can't use hmp-io
result = self.vm.qmp("device_add",
id="device0",
drive=drive0['id'],
driver="virtio-blk")
self.assert_qmp(result, 'return', {})
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 458752)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'active')
self.assert_qmp(ret, 'busy', False)
self.assert_qmp(ret, 'recording', True)
# Start backup
parent, _ = bitmap.last_target()
target = self.prepare_backup(bitmap, parent)
res = self.vm.qmp('drive-backup',
job_id=bitmap.drive['id'],
device=bitmap.drive['id'],
sync='incremental',
bitmap=bitmap.name,
format=bitmap.drive['fmt'],
target=target,
mode='existing',
on_source_error='stop')
self.assert_qmp(res, 'return', {})
# Wait for the error
event = self.vm.event_wait(name="BLOCK_JOB_ERROR",
match={"data":{"device":bitmap.drive['id']}})
self.assert_qmp(event, 'data', {'device': bitmap.drive['id'],
'action': 'stop',
'operation': 'read'})
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 458752)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'frozen')
self.assert_qmp(ret, 'busy', True)
self.assert_qmp(ret, 'recording', True)
# Resume and check incremental backup for consistency
res = self.vm.qmp('block-job-resume', device=bitmap.drive['id'])
self.assert_qmp(res, 'return', {})
self.wait_qmp_backup(bitmap.drive['id'])
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 0)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'active')
self.assert_qmp(ret, 'busy', False)
self.assert_qmp(ret, 'recording', True)
# Finalize / Cleanup
self.make_reference_backup(bitmap)
self.vm.shutdown()
self.check_backups()
if __name__ == '__main__':
iotests.main(supported_fmts=['qcow2'])
Reference in New Issue View Git Blame Copy Permalink