OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity
0f5faca782
qemu-e2k/tests/qemu-iotests/124
Max Reitz 68474776f3 iotests: Different iterator behavior in Python 3 
In Python 3, several functions now return iterators instead of lists.
This includes range(), items(), map(), and filter().  This means that if
we really want a list, we have to wrap those instances with list().  But
then again, the two instances where this is the case for map() and
filter(), there are shorter expressions which work without either
function.

On the other hand, sometimes we do just want an iterator, in which case
we have sometimes used xrange() and iteritems() which no longer exist in
Python 3.  Just change these calls to be range() and items(), works in
both Python 2 and 3, and is really what we want in 3 (which is what
matters).  But because it is so simple to do (and to find and remove
once we completely switch to Python 3), make range() be an alias for
xrange() in the two affected tests (044 and 163).

In one instance, we only wanted the first instance of the result of a
filter() call.  Instead of using next(filter()) which would work only in
Python 3, or list(filter())[0] which would work everywhere but is a bit
weird, this instance is changed to use a generator expression with a
next() wrapped around, which works both in 2.7 and 3.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Message-Id: <20181022135307.14398-6-mreitz@redhat.com>
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
2018-10-30 21:11:52 -03:00

639 lines
24 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
#
# Tests for incremental drive-backup
#
# Copyright (C) 2015 John Snow for Red Hat, Inc.
#
# Based on 056.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import iotests
def io_write_patterns(img, patterns):
for pattern in patterns:
iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
def try_remove(img):
try:
os.remove(img)
except OSError:
pass
def transaction_action(action, **kwargs):
return {
'type': action,
'data': dict((k.replace('_', '-'), v) for k, v in kwargs.items())
}
def transaction_bitmap_clear(node, name, **kwargs):
return transaction_action('block-dirty-bitmap-clear',
node=node, name=name, **kwargs)
def transaction_drive_backup(device, target, **kwargs):
return transaction_action('drive-backup', job_id=device, device=device,
target=target, **kwargs)
class Bitmap:
def __init__(self, name, drive):
self.name = name
self.drive = drive
self.num = 0
self.backups = list()
def base_target(self):
return (self.drive['backup'], None)
def new_target(self, num=None):
if num is None:
num = self.num
self.num = num + 1
base = os.path.join(iotests.test_dir,
"%s.%s." % (self.drive['id'], self.name))
suff = "%i.%s" % (num, self.drive['fmt'])
target = base + "inc" + suff
reference = base + "ref" + suff
self.backups.append((target, reference))
return (target, reference)
def last_target(self):
if self.backups:
return self.backups[-1]
return self.base_target()
def del_target(self):
for image in self.backups.pop():
try_remove(image)
self.num -= 1
def cleanup(self):
for backup in self.backups:
for image in backup:
try_remove(image)
class TestIncrementalBackupBase(iotests.QMPTestCase):
def __init__(self, *args):
super(TestIncrementalBackupBase, self).__init__(*args)
self.bitmaps = list()
self.files = list()
self.drives = list()
self.vm = iotests.VM()
self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
def setUp(self):
# Create a base image with a distinctive patterning
drive0 = self.add_node('drive0')
self.img_create(drive0['file'], drive0['fmt'])
self.vm.add_drive(drive0['file'])
self.write_default_pattern(drive0['file'])
self.vm.launch()
def write_default_pattern(self, target):
io_write_patterns(target, (('0x41', 0, 512),
('0xd5', '1M', '32k'),
('0xdc', '32M', '124k')))
def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
if path is None:
path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
if backup is None:
backup = os.path.join(iotests.test_dir,
'%s.full.backup.%s' % (node_id, fmt))
self.drives.append({
'id': node_id,
'file': path,
'backup': backup,
'fmt': fmt })
return self.drives[-1]
def img_create(self, img, fmt=iotests.imgfmt, size='64M',
parent=None, parentFormat=None, **kwargs):
optargs = []
for k,v in kwargs.items():
optargs = optargs + ['-o', '%s=%s' % (k,v)]
args = ['create', '-f', fmt] + optargs + [img, size]
if parent:
if parentFormat is None:
parentFormat = fmt
args = args + ['-b', parent, '-F', parentFormat]
iotests.qemu_img(*args)
self.files.append(img)
def do_qmp_backup(self, error='Input/output error', **kwargs):
res = self.vm.qmp('drive-backup', **kwargs)
self.assert_qmp(res, 'return', {})
return self.wait_qmp_backup(kwargs['device'], error)
def ignore_job_status_change_events(self):
while True:
e = self.vm.event_wait(name="JOB_STATUS_CHANGE")
if e['data']['status'] == 'null':
break
def wait_qmp_backup(self, device, error='Input/output error'):
event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
match={'data': {'device': device}})
self.assertNotEqual(event, None)
self.ignore_job_status_change_events()
try:
failure = self.dictpath(event, 'data/error')
except AssertionError:
# Backup succeeded.
self.assert_qmp(event, 'data/offset', event['data']['len'])
return True
else:
# Backup failed.
self.assert_qmp(event, 'data/error', error)
return False
def wait_qmp_backup_cancelled(self, device):
event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
match={'data': {'device': device}})
self.assertNotEqual(event, None)
self.ignore_job_status_change_events()
def create_anchor_backup(self, drive=None):
if drive is None:
drive = self.drives[-1]
res = self.do_qmp_backup(job_id=drive['id'],
device=drive['id'], sync='full',
format=drive['fmt'], target=drive['backup'])
self.assertTrue(res)
self.files.append(drive['backup'])
return drive['backup']
def make_reference_backup(self, bitmap=None):
if bitmap is None:
bitmap = self.bitmaps[-1]
_, reference = bitmap.last_target()
res = self.do_qmp_backup(job_id=bitmap.drive['id'],
device=bitmap.drive['id'], sync='full',
format=bitmap.drive['fmt'], target=reference)
self.assertTrue(res)
def add_bitmap(self, name, drive, **kwargs):
bitmap = Bitmap(name, drive)
self.bitmaps.append(bitmap)
result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
name=bitmap.name, **kwargs)
self.assert_qmp(result, 'return', {})
return bitmap
def prepare_backup(self, bitmap=None, parent=None):
if bitmap is None:
bitmap = self.bitmaps[-1]
if parent is None:
parent, _ = bitmap.last_target()
target, _ = bitmap.new_target()
self.img_create(target, bitmap.drive['fmt'], parent=parent)
return target
def create_incremental(self, bitmap=None, parent=None,
parentFormat=None, validate=True):
if bitmap is None:
bitmap = self.bitmaps[-1]
if parent is None:
parent, _ = bitmap.last_target()
target = self.prepare_backup(bitmap, parent)
res = self.do_qmp_backup(job_id=bitmap.drive['id'],
device=bitmap.drive['id'],
sync='incremental', bitmap=bitmap.name,
format=bitmap.drive['fmt'], target=target,
mode='existing')
if not res:
bitmap.del_target();
self.assertFalse(validate)
else:
self.make_reference_backup(bitmap)
return res
def check_backups(self):
for bitmap in self.bitmaps:
for incremental, reference in bitmap.backups:
self.assertTrue(iotests.compare_images(incremental, reference))
last = bitmap.last_target()[0]
self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
def hmp_io_writes(self, drive, patterns):
for pattern in patterns:
self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
self.vm.hmp_qemu_io(drive, 'flush')
def do_incremental_simple(self, **kwargs):
self.create_anchor_backup()
self.add_bitmap('bitmap0', self.drives[0], **kwargs)
# Sanity: Create a "hollow" incremental backup
self.create_incremental()
# Three writes: One complete overwrite, one new segment,
# and one partial overlap.
self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
self.create_incremental()
# Three more writes, one of each kind, like above
self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
self.create_incremental()
self.vm.shutdown()
self.check_backups()
def tearDown(self):
self.vm.shutdown()
for bitmap in self.bitmaps:
bitmap.cleanup()
for filename in self.files:
try_remove(filename)
class TestIncrementalBackup(TestIncrementalBackupBase):
def test_incremental_simple(self):
'''
Test: Create and verify three incremental backups.
Create a bitmap and a full backup before VM execution begins,
then create a series of three incremental backups "during execution,"
i.e.; after IO requests begin modifying the drive.
'''
return self.do_incremental_simple()
def test_small_granularity(self):
'''
Test: Create and verify backups made with a small granularity bitmap.
Perform the same test as test_incremental_simple, but with a granularity
of only 32KiB instead of the present default of 64KiB.
'''
return self.do_incremental_simple(granularity=32768)
def test_large_granularity(self):
'''
Test: Create and verify backups made with a large granularity bitmap.
Perform the same test as test_incremental_simple, but with a granularity
of 128KiB instead of the present default of 64KiB.
'''
return self.do_incremental_simple(granularity=131072)
def test_larger_cluster_target(self):
'''
Test: Create and verify backups made to a larger cluster size target.
With a default granularity of 64KiB, verify that backups made to a
larger cluster size target of 128KiB without a backing file works.
'''
drive0 = self.drives[0]
# Create a cluster_size=128k full backup / "anchor" backup
self.img_create(drive0['backup'], cluster_size='128k')
self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
format=drive0['fmt'],
target=drive0['backup'],
mode='existing'))
# Create bitmap and dirty it with some new writes.
# overwrite [32736, 32799] which will dirty bitmap clusters at
# 32M-64K and 32M. 32M+64K will be left undirtied.
bitmap0 = self.add_bitmap('bitmap0', drive0)
self.hmp_io_writes(drive0['id'],
(('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
# Check the dirty bitmap stats
result = self.vm.qmp('query-block')
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/name', 'bitmap0')
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/count', 458752)
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/granularity', 65536)
self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/status', 'active')
# Prepare a cluster_size=128k backup target without a backing file.
(target, _) = bitmap0.new_target()
self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
# Perform Incremental Backup
self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
sync='incremental',
bitmap=bitmap0.name,
format=bitmap0.drive['fmt'],
target=target,
mode='existing'))
self.make_reference_backup(bitmap0)
# Add the backing file, then compare and exit.
iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
drive0['backup'], '-F', drive0['fmt'], target)
self.vm.shutdown()
self.check_backups()
def test_incremental_transaction(self):
'''Test: Verify backups made from transactionally created bitmaps.
Create a bitmap "before" VM execution begins, then create a second
bitmap AFTER writes have already occurred. Use transactions to create
a full backup and synchronize both bitmaps to this backup.
Create an incremental backup through both bitmaps and verify that
both backups match the current drive0 image.
'''
drive0 = self.drives[0]
bitmap0 = self.add_bitmap('bitmap0', drive0)
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
bitmap1 = self.add_bitmap('bitmap1', drive0)
result = self.vm.qmp('transaction', actions=[
transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
transaction_drive_backup(drive0['id'], drive0['backup'],
sync='full', format=drive0['fmt'])
])
self.assert_qmp(result, 'return', {})
self.wait_until_completed(drive0['id'])
self.files.append(drive0['backup'])
self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
# Both bitmaps should be correctly in sync.
self.create_incremental(bitmap0)
self.create_incremental(bitmap1)
self.vm.shutdown()
self.check_backups()
def do_transaction_failure_test(self, race=False):
# Create a second drive, with pattern:
drive1 = self.add_node('drive1')
self.img_create(drive1['file'], drive1['fmt'])
io_write_patterns(drive1['file'], (('0x14', 0, 512),
('0x5d', '1M', '32k'),
('0xcd', '32M', '124k')))
# Create a blkdebug interface to this img as 'drive1'
result = self.vm.qmp('blockdev-add',
node_name=drive1['id'],
driver=drive1['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive1['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 2,
'immediately': False,
'once': True
}],
}
)
self.assert_qmp(result, 'return', {})
# Create bitmaps and full backups for both drives
drive0 = self.drives[0]
dr0bm0 = self.add_bitmap('bitmap0', drive0)
dr1bm0 = self.add_bitmap('bitmap0', drive1)
self.create_anchor_backup(drive0)
self.create_anchor_backup(drive1)
self.assert_no_active_block_jobs()
self.assertFalse(self.vm.get_qmp_events(wait=False))
# Emulate some writes
if not race:
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
('0xef', '16M', '256k'),
('0x46', '32736k', '64k')))
# Create incremental backup targets
target0 = self.prepare_backup(dr0bm0)
target1 = self.prepare_backup(dr1bm0)
# Ask for a new incremental backup per-each drive,
# expecting drive1's backup to fail. In the 'race' test,
# we expect drive1 to attempt to cancel the empty drive0 job.
transaction = [
transaction_drive_backup(drive0['id'], target0, sync='incremental',
format=drive0['fmt'], mode='existing',
bitmap=dr0bm0.name),
transaction_drive_backup(drive1['id'], target1, sync='incremental',
format=drive1['fmt'], mode='existing',
bitmap=dr1bm0.name)
]
result = self.vm.qmp('transaction', actions=transaction,
properties={'completion-mode': 'grouped'} )
self.assert_qmp(result, 'return', {})
# Observe that drive0's backup is cancelled and drive1 completes with
# an error.
self.wait_qmp_backup_cancelled(drive0['id'])
self.assertFalse(self.wait_qmp_backup(drive1['id']))
error = self.vm.event_wait('BLOCK_JOB_ERROR')
self.assert_qmp(error, 'data', {'device': drive1['id'],
'action': 'report',
'operation': 'read'})
self.assertFalse(self.vm.get_qmp_events(wait=False))
self.assert_no_active_block_jobs()
# Delete drive0's successful target and eliminate our record of the
# unsuccessful drive1 target.
dr0bm0.del_target()
dr1bm0.del_target()
if race:
# Don't re-run the transaction, we only wanted to test the race.
self.vm.shutdown()
return
# Re-run the same transaction:
target0 = self.prepare_backup(dr0bm0)
target1 = self.prepare_backup(dr1bm0)
# Re-run the exact same transaction.
result = self.vm.qmp('transaction', actions=transaction,
properties={'completion-mode':'grouped'})
self.assert_qmp(result, 'return', {})
# Both should complete successfully this time.
self.assertTrue(self.wait_qmp_backup(drive0['id']))
self.assertTrue(self.wait_qmp_backup(drive1['id']))
self.make_reference_backup(dr0bm0)
self.make_reference_backup(dr1bm0)
self.assertFalse(self.vm.get_qmp_events(wait=False))
self.assert_no_active_block_jobs()
# And the images should of course validate.
self.vm.shutdown()
self.check_backups()
def test_transaction_failure(self):
'''Test: Verify backups made from a transaction that partially fails.
Add a second drive with its own unique pattern, and add a bitmap to each
drive. Use blkdebug to interfere with the backup on just one drive and
attempt to create a coherent incremental backup across both drives.
verify a failure in one but not both, then delete the failed stubs and
re-run the same transaction.
verify that both incrementals are created successfully.
'''
self.do_transaction_failure_test()
def test_transaction_failure_race(self):
'''Test: Verify that transactions with jobs that have no data to
transfer do not cause race conditions in the cancellation of the entire
transaction job group.
'''
self.do_transaction_failure_test(race=True)
def test_sync_dirty_bitmap_missing(self):
self.assert_no_active_block_jobs()
self.files.append(self.err_img)
result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
sync='incremental', format=self.drives[0]['fmt'],
target=self.err_img)
self.assert_qmp(result, 'error/class', 'GenericError')
def test_sync_dirty_bitmap_not_found(self):
self.assert_no_active_block_jobs()
self.files.append(self.err_img)
result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
sync='incremental', bitmap='unknown',
format=self.drives[0]['fmt'], target=self.err_img)
self.assert_qmp(result, 'error/class', 'GenericError')
def test_sync_dirty_bitmap_bad_granularity(self):
'''
Test: Test what happens if we provide an improper granularity.
The granularity must always be a power of 2.
'''
self.assert_no_active_block_jobs()
self.assertRaises(AssertionError, self.add_bitmap,
'bitmap0', self.drives[0],
granularity=64000)
class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
'''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
def setUp(self):
drive0 = self.add_node('drive0')
self.img_create(drive0['file'], drive0['fmt'])
self.write_default_pattern(drive0['file'])
self.vm.launch()
def test_incremental_failure(self):
'''Test: Verify backups made after a failure are correct.
Simulate a failure during an incremental backup block job,
emulate additional writes, then create another incremental backup
afterwards and verify that the backup created is correct.
'''
drive0 = self.drives[0]
result = self.vm.qmp('blockdev-add',
node_name=drive0['id'],
driver=drive0['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive0['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 2,
'immediately': False,
'once': True
}],
}
)
self.assert_qmp(result, 'return', {})
self.create_anchor_backup(drive0)
self.add_bitmap('bitmap0', drive0)
# Note: at this point, during a normal execution,
# Assume that the VM resumes and begins issuing IO requests here.
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
result = self.create_incremental(validate=False)
self.assertFalse(result)
self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
('0x55', '8M', '352k'),
('0x78', '15872k', '1M')))
self.create_incremental()
self.vm.shutdown()
self.check_backups()
if __name__ == '__main__':
iotests.main(supported_fmts=['qcow2'])
Reference in New Issue View Git Blame Copy Permalink