OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

Migration Pull request (20230509 vintage) take 2

Browse Source 
Hi
 
 In this take 2:
 - Change uint -> uint32_t to fix mingw32 compilation.
 
 Please apply.
 [take 1]
 In this PULL request:
 - 1st part of colo support for multifd (lukas)
 - 1st part of disabling colo option (vladimir)
 
 Please, apply.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEGJn/jt6/WMzuA0uC9IfvGFhy1yMFAmRb3dgACgkQ9IfvGFhy
 1yNLBxAAwHiAOdSPS7TqJXH2/PkBKsd42XMtWzC9UowZ6SUdQi0Q2bQUBnygJ8BA
 59yLOTPdwUhaPWk4KsyKM2znOCJ+f9MF5V4QXbyILf1WCAq6d+mtPwArnYF1TRwi
 XIewVDeRopdOO5lnWGcfAKZZ5WIDzA/bn6NiGLi+pQa5HGyk84Bk+tFa8kJI6xBL
 5CWfhNTcxDNYRFg/z/9YVirkuxIXEEL6VEeRFV+pmFuj05q9bysWJkLFoEcFNawO
 gp1foHDkU7wHmHDJ3D4AVTm3TW641ft1wdlHIHZRoOiIIu3EUOoDEVVsaCfdxrY8
 pPJZ5m37wb52GIaCJmigG8rkHxIJ8xKLk4HKu4umDqFq5jZQ2krnnj7AkQhpp7p2
 aEIOXJQQq7XCsKpuvSUIexPv4gbN5SEYKi7XKoOPe3sZ03Rkn0I5xY3KSyMQMamP
 jtk8tNlRA+9Wug82eb/FtIKDj3//4SbuQOJEdRXjKJBldd3mtWTT/FRj/8oo96/p
 hmTu/cGDrP5qgtWpz0kKI/xaBf8at1nwpDgdEzOjRw4zf6xQHFjbXgJ7tQBH/JUI
 T3A9pdiXN6QdRupcWUSV0iJsfS/5i3mOUTA/C529qGXabSnZzfMK+unL/I8N02yt
 83o7jSg22etMjaS1c+VuDmzKCAfuZloDZv2Bms/+yM/8k8Xe5S4=
 =vbqf
 -----END PGP SIGNATURE-----

Merge tag 'migration-20230509-pull-request' of https://gitlab.com/juan.quintela/qemu into staging

Migration Pull request (20230509 vintage) take 2

Hi

In this take 2:
- Change uint -> uint32_t to fix mingw32 compilation.

Please apply.
[take 1]
In this PULL request:
- 1st part of colo support for multifd (lukas)
- 1st part of disabling colo option (vladimir)

Please, apply.

# -----BEGIN PGP SIGNATURE-----
#
# iQIzBAABCAAdFiEEGJn/jt6/WMzuA0uC9IfvGFhy1yMFAmRb3dgACgkQ9IfvGFhy
# 1yNLBxAAwHiAOdSPS7TqJXH2/PkBKsd42XMtWzC9UowZ6SUdQi0Q2bQUBnygJ8BA
# 59yLOTPdwUhaPWk4KsyKM2znOCJ+f9MF5V4QXbyILf1WCAq6d+mtPwArnYF1TRwi
# XIewVDeRopdOO5lnWGcfAKZZ5WIDzA/bn6NiGLi+pQa5HGyk84Bk+tFa8kJI6xBL
# 5CWfhNTcxDNYRFg/z/9YVirkuxIXEEL6VEeRFV+pmFuj05q9bysWJkLFoEcFNawO
# gp1foHDkU7wHmHDJ3D4AVTm3TW641ft1wdlHIHZRoOiIIu3EUOoDEVVsaCfdxrY8
# pPJZ5m37wb52GIaCJmigG8rkHxIJ8xKLk4HKu4umDqFq5jZQ2krnnj7AkQhpp7p2
# aEIOXJQQq7XCsKpuvSUIexPv4gbN5SEYKi7XKoOPe3sZ03Rkn0I5xY3KSyMQMamP
# jtk8tNlRA+9Wug82eb/FtIKDj3//4SbuQOJEdRXjKJBldd3mtWTT/FRj/8oo96/p
# hmTu/cGDrP5qgtWpz0kKI/xaBf8at1nwpDgdEzOjRw4zf6xQHFjbXgJ7tQBH/JUI
# T3A9pdiXN6QdRupcWUSV0iJsfS/5i3mOUTA/C529qGXabSnZzfMK+unL/I8N02yt
# 83o7jSg22etMjaS1c+VuDmzKCAfuZloDZv2Bms/+yM/8k8Xe5S4=
# =vbqf
# -----END PGP SIGNATURE-----
# gpg: Signature made Wed 10 May 2023 07:09:28 PM BST
# gpg:                using RSA key 1899FF8EDEBF58CCEE034B82F487EF185872D723
# gpg: Good signature from "Juan Quintela <quintela@redhat.com>" [undefined]
# gpg:                 aka "Juan Quintela <quintela@trasno.org>" [undefined]
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 1899 FF8E DEBF 58CC EE03  4B82 F487 EF18 5872 D723

* tag 'migration-20230509-pull-request' of https://gitlab.com/juan.quintela/qemu:
  migration: block incoming colo when capability is disabled
  migration: disallow change capabilities in COLO state
  migration: process_incoming_migration_co: simplify code flow around ret
  migration: drop colo_incoming_thread from MigrationIncomingState
  build: move COLO under CONFIG_REPLICATION
  colo: make colo_checkpoint_notify static and provide simpler API
  block/meson.build: prefer positive condition for replication
  multifd: Add the ramblock to MultiFDRecvParams
  ram: Let colo_flush_ram_cache take the bitmap_mutex
  ram: Add public helper to set colo bitmap

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson 2023-05-11 05:55:12 +01:00
commit fff86d48a2
17 changed files with 131 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
block/meson.build
View File

@ -92,7 +92,7 @@ block_ss.add(when: 'CONFIG_WIN32', if_true: files('file-win32.c', 'win32-aio.c')
block_ss.add(when: 'CONFIG_POSIX', if_true: [files('file-posix.c'), coref, iokit])
block_ss.add(when: libiscsi, if_true: files('iscsi-opts.c'))
block_ss.add(when: 'CONFIG_LINUX', if_true: files('nvme.c'))
if not get_option('replication').disabled()
if get_option('replication').allowed()
block_ss.add(files('replication.c'))
endif
block_ss.add(when: libaio, if_true: files('linux-aio.c'))

1
docs/COLO-FT.txt
View File

@ -210,6 +210,7 @@ children.0=childs0 \
3. On Secondary VM's QEMU monitor, issue command
{"execute":"qmp_capabilities"}
{"execute": "migrate-set-capabilities", "arguments": {"capabilities": [ {"capability": "x-colo", "state": true } ] } }
{"execute": "nbd-server-start", "arguments": {"addr": {"type": "inet", "data": {"host": "0.0.0.0", "port": "9999"} } } }
{"execute": "nbd-server-add", "arguments": {"device": "parent0", "writable": true } }

2
hmp-commands.hx
View File

@ -1052,6 +1052,7 @@ SRST
migration (or once already in postcopy).
ERST
#ifdef CONFIG_REPLICATION
{
.name = "x_colo_lost_heartbeat",
.args_type = "",
@ -1060,6 +1061,7 @@ ERST
"a failover or takeover is needed.",
.cmd = hmp_x_colo_lost_heartbeat,
},
#endif
SRST
``x_colo_lost_heartbeat``

9
include/migration/colo.h
View File

@ -36,6 +36,13 @@ COLOMode get_colo_mode(void);
/* failover */
void colo_do_failover(void);
void colo_checkpoint_notify(void *opaque);
/*
* colo_checkpoint_delay_set
*
* Handles change of x-checkpoint-delay migration parameter, called from
* migrate_params_apply() to notify COLO module about the change.
*/
void colo_checkpoint_delay_set(void);
void colo_shutdown(void);
#endif

57
migration/colo.c
View File

@ -26,9 +26,7 @@
#include "qemu/rcu.h"
#include "migration/failover.h"
#include "migration/ram.h"
#ifdef CONFIG_REPLICATION
#include "block/replication.h"
#endif
#include "net/colo-compare.h"
#include "net/colo.h"
#include "block/block.h"
@ -65,10 +63,27 @@ static bool colo_runstate_is_stopped(void)
return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
}
static void colo_checkpoint_notify(void *opaque)
{
MigrationState *s = opaque;
int64_t next_notify_time;
qemu_event_set(&s->colo_checkpoint_event);
s->colo_checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
next_notify_time = s->colo_checkpoint_time + migrate_checkpoint_delay();
timer_mod(s->colo_delay_timer, next_notify_time);
}
void colo_checkpoint_delay_set(void)
{
if (migration_in_colo_state()) {
colo_checkpoint_notify(migrate_get_current());
}
}
static void secondary_vm_do_failover(void)
{
/* COLO needs enable block-replication */
#ifdef CONFIG_REPLICATION
int old_state;
MigrationIncomingState *mis = migration_incoming_get_current();
Error *local_err = NULL;
@ -133,14 +148,10 @@ static void secondary_vm_do_failover(void)
if (mis->migration_incoming_co) {
qemu_coroutine_enter(mis->migration_incoming_co);
}
#else
abort();
#endif
}
static void primary_vm_do_failover(void)
{
#ifdef CONFIG_REPLICATION
MigrationState *s = migrate_get_current();
int old_state;
Error *local_err = NULL;
@ -181,9 +192,6 @@ static void primary_vm_do_failover(void)
/* Notify COLO thread that failover work is finished */
qemu_sem_post(&s->colo_exit_sem);
#else
abort();
#endif
}
COLOMode get_colo_mode(void)
@ -217,7 +225,6 @@ void colo_do_failover(void)
}
}
#ifdef CONFIG_REPLICATION
void qmp_xen_set_replication(bool enable, bool primary,
bool has_failover, bool failover,
Error **errp)
@ -271,7 +278,6 @@ void qmp_xen_colo_do_checkpoint(Error **errp)
/* Notify all filters of all NIC to do checkpoint */
colo_notify_filters_event(COLO_EVENT_CHECKPOINT, errp);
}
#endif
COLOStatus *qmp_query_colo_status(Error **errp)
{
@ -435,15 +441,11 @@ static int colo_do_checkpoint_transaction(MigrationState *s,
}
qemu_mutex_lock_iothread();
#ifdef CONFIG_REPLICATION
replication_do_checkpoint_all(&local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
#else
abort();
#endif
colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err);
if (local_err) {
@ -561,15 +563,11 @@ static void colo_process_checkpoint(MigrationState *s)
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
#ifdef CONFIG_REPLICATION
replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
#else
abort();
#endif
vm_start();
qemu_mutex_unlock_iothread();
@ -644,17 +642,6 @@ out:
}
}
void colo_checkpoint_notify(void *opaque)
{
MigrationState *s = opaque;
int64_t next_notify_time;
qemu_event_set(&s->colo_checkpoint_event);
s->colo_checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
next_notify_time = s->colo_checkpoint_time + migrate_checkpoint_delay();
timer_mod(s->colo_delay_timer, next_notify_time);
}
void migrate_start_colo_process(MigrationState *s)
{
qemu_mutex_unlock_iothread();
@ -748,7 +735,6 @@ static void colo_incoming_process_checkpoint(MigrationIncomingState *mis,
return;
}
#ifdef CONFIG_REPLICATION
replication_get_error_all(&local_err);
if (local_err) {
error_propagate(errp, local_err);
@ -765,9 +751,6 @@ static void colo_incoming_process_checkpoint(MigrationIncomingState *mis,
qemu_mutex_unlock_iothread();
return;
}
#else
abort();
#endif
/* Notify all filters of all NIC to do checkpoint */
colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err);
@ -874,15 +857,11 @@ void *colo_process_incoming_thread(void *opaque)
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
#ifdef CONFIG_REPLICATION
replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
#else
abort();
#endif
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");

6
migration/meson.build
View File

@ -13,8 +13,6 @@ softmmu_ss.add(files(
'block-dirty-bitmap.c',
'channel.c',
'channel-block.c',
'colo-failover.c',
'colo.c',
'exec.c',
'fd.c',
'global_state.c',
@ -33,6 +31,10 @@ softmmu_ss.add(files(
'threadinfo.c',
), gnutls)
if get_option('replication').allowed()
softmmu_ss.add(files('colo-failover.c', 'colo.c'))
endif
softmmu_ss.add(when: rdma, if_true: files('rdma.c'))
if get_option('live_block_migration').allowed()
softmmu_ss.add(files('block.c'))

2
migration/migration-hmp-cmds.c
View File

@ -643,6 +643,7 @@ void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict)
hmp_handle_error(mon, err);
}
#ifdef CONFIG_REPLICATION
void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict)
{
Error *err = NULL;
@ -650,6 +651,7 @@ void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict)
qmp_x_colo_lost_heartbeat(&err);
hmp_handle_error(mon, err);
}
#endif
typedef struct HMPMigrationStatus {
QEMUTimer *timer;

35
migration/migration.c
View File

@ -392,6 +392,18 @@ void migration_incoming_disable_colo(void)
int migration_incoming_enable_colo(void)
{
#ifndef CONFIG_REPLICATION
error_report("ENABLE_COLO command come in migration stream, but COLO "
"module is not built in");
return -ENOTSUP;
#endif
if (!migrate_colo()) {
error_report("ENABLE_COLO command come in migration stream, but c-colo "
"capability is not set");
return -EINVAL;
}
if (ram_block_discard_disable(true)) {
error_report("COLO: cannot disable RAM discard");
return -EBUSY;
@ -536,8 +548,15 @@ process_incoming_migration_co(void *opaque)
/* Else if something went wrong then just fall out of the normal exit */
}
if (ret < 0) {
error_report("load of migration failed: %s", strerror(-ret));
goto fail;
}
/* we get COLO info, and know if we are in COLO mode */
if (!ret && migration_incoming_colo_enabled()) {
if (migration_incoming_colo_enabled()) {
QemuThread colo_incoming_thread;
/* Make sure all file formats throw away their mutable metadata */
bdrv_activate_all(&local_err);
if (local_err) {
@ -545,23 +564,18 @@ process_incoming_migration_co(void *opaque)
goto fail;
}
qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
qemu_thread_create(&colo_incoming_thread, "COLO incoming",
colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
mis->have_colo_incoming_thread = true;
qemu_coroutine_yield();
qemu_mutex_unlock_iothread();
/* Wait checkpoint incoming thread exit before free resource */
qemu_thread_join(&mis->colo_incoming_thread);
qemu_thread_join(&colo_incoming_thread);
qemu_mutex_lock_iothread();
/* We hold the global iothread lock, so it is safe here */
colo_release_ram_cache();
}
if (ret < 0) {
error_report("load of migration failed: %s", strerror(-ret));
goto fail;
}
mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
qemu_bh_schedule(mis->bh);
mis->migration_incoming_co = NULL;
@ -2777,10 +2791,7 @@ static void migration_iteration_finish(MigrationState *s)
runstate_set(RUN_STATE_POSTMIGRATE);
break;
case MIGRATION_STATUS_COLO:
if (!migrate_colo()) {
error_report("%s: critical error: calling COLO code without "
"COLO enabled", __func__);
}
assert(migrate_colo());
migrate_start_colo_process(s);
s->vm_was_running = true;
/* Fallthrough */

2
migration/migration.h
View File

@ -162,8 +162,6 @@ struct MigrationIncomingState {
int state;
bool have_colo_incoming_thread;
QemuThread colo_incoming_thread;
/* The coroutine we should enter (back) after failover */
Coroutine *migration_incoming_co;
QemuSemaphore colo_incoming_sem;

11
migration/multifd.c
View File

@ -281,7 +281,6 @@ static void multifd_send_fill_packet(MultiFDSendParams *p)
static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
{
MultiFDPacket_t *packet = p->packet;
RAMBlock *block;
int i;
packet->magic = be32_to_cpu(packet->magic);
@ -331,21 +330,21 @@ static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
/* make sure that ramblock is 0 terminated */
packet->ramblock[255] = 0;
block = qemu_ram_block_by_name(packet->ramblock);
if (!block) {
p->block = qemu_ram_block_by_name(packet->ramblock);
if (!p->block) {
error_setg(errp, "multifd: unknown ram block %s",
packet->ramblock);
return -1;
}
p->host = block->host;
p->host = p->block->host;
for (i = 0; i < p->normal_num; i++) {
uint64_t offset = be64_to_cpu(packet->offset[i]);
if (offset > (block->used_length - p->page_size)) {
if (offset > (p->block->used_length - p->page_size)) {
error_setg(errp, "multifd: offset too long %" PRIu64
" (max " RAM_ADDR_FMT ")",
offset, block->used_length);
offset, p->block->used_length);
return -1;
}
p->normal[i] = offset;

2
migration/multifd.h
View File

@ -175,6 +175,8 @@ typedef struct {
uint32_t next_packet_size;
/* packets sent through this channel */
uint64_t num_packets;
/* ramblock */
RAMBlock *block;
/* ramblock host address */
uint8_t *host;
/* non zero pages recv through this channel */

6
migration/options.c
View File

@ -598,7 +598,7 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
MigrationCapabilityStatusList *cap;
bool new_caps[MIGRATION_CAPABILITY__MAX];
if (migration_is_running(s->state)) {
if (migration_is_running(s->state) || migration_in_colo_state()) {
error_setg(errp, QERR_MIGRATION_ACTIVE);
return;
}
@ -1253,9 +1253,7 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
if (params->has_x_checkpoint_delay) {
s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
if (migration_in_colo_state()) {
colo_checkpoint_notify(s);
}
colo_checkpoint_delay_set();
}
if (params->has_block_incremental) {

19
migration/ram.c
View File

@ -3408,6 +3408,18 @@ static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
return ((uintptr_t)block->host + offset) & (block->page_size - 1);
}
void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
{
qemu_mutex_lock(&ram_state->bitmap_mutex);
for (int i = 0; i < pages; i++) {
ram_addr_t offset = normal[i];
ram_state->migration_dirty_pages += !test_and_set_bit(
offset >> TARGET_PAGE_BITS,
block->bmap);
}
qemu_mutex_unlock(&ram_state->bitmap_mutex);
}
static inline void *colo_cache_from_block_offset(RAMBlock *block,
ram_addr_t offset, bool record_bitmap)
{
@ -3425,9 +3437,8 @@ static inline void *colo_cache_from_block_offset(RAMBlock *block,
* It help us to decide which pages in ram cache should be flushed
* into VM's RAM later.
*/
if (record_bitmap &&
!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
ram_state->migration_dirty_pages++;
if (record_bitmap) {
colo_record_bitmap(block, &offset, 1);
}
return block->colo_cache + offset;
}
@ -3803,6 +3814,7 @@ void colo_flush_ram_cache(void)
unsigned long offset = 0;
memory_global_dirty_log_sync();
qemu_mutex_lock(&ram_state->bitmap_mutex);
WITH_RCU_READ_LOCK_GUARD() {
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
ramblock_sync_dirty_bitmap(ram_state, block);
@ -3837,6 +3849,7 @@ void colo_flush_ram_cache(void)
}
}
}
qemu_mutex_unlock(&ram_state->bitmap_mutex);
trace_colo_flush_ram_cache_end();
}

1
migration/ram.h
View File

@ -82,6 +82,7 @@ int colo_init_ram_cache(void);
void colo_flush_ram_cache(void);
void colo_release_ram_cache(void);
void colo_incoming_start_dirty_log(void);
void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages);
/* Background snapshot */
bool ram_write_tracking_available(void);

9
qapi/migration.json
View File

@ -1366,7 +1366,8 @@
# <- { "return": {} }
##
{ 'command': 'x-colo-lost-heartbeat',
'features': [ 'unstable' ] }
'features': [ 'unstable' ],
'if': 'CONFIG_REPLICATION' }
##
# @migrate_cancel:
@ -1638,7 +1639,8 @@
##
{ 'struct': 'COLOStatus',
'data': { 'mode': 'COLOMode', 'last-mode': 'COLOMode',
'reason': 'COLOExitReason' } }
'reason': 'COLOExitReason' },
'if': 'CONFIG_REPLICATION' }
##
# @query-colo-status:
@ -1655,7 +1657,8 @@
# Since: 3.1
##
{ 'command': 'query-colo-status',
'returns': 'COLOStatus' }
'returns': 'COLOStatus',
'if': 'CONFIG_REPLICATION' }
##
# @migrate-recover:

39
stubs/colo.c Normal file
View File

@ -0,0 +1,39 @@
#include "qemu/osdep.h"
#include "qemu/notify.h"
#include "net/colo-compare.h"
#include "migration/colo.h"
#include "migration/migration.h"
#include "qemu/error-report.h"
#include "qapi/qapi-commands-migration.h"
void colo_shutdown(void)
{
}
void *colo_process_incoming_thread(void *opaque)
{
error_report("Impossible happend: trying to start COLO thread when COLO "
"module is not built in");
abort();
}
void colo_checkpoint_delay_set(void)
{
}
void migrate_start_colo_process(MigrationState *s)
{
error_report("Impossible happend: trying to start COLO when COLO "
"module is not built in");
abort();
}
bool migration_in_colo_state(void)
{
return false;
}
bool migration_incoming_in_colo_state(void)
{
return false;
}

1
stubs/meson.build
View File

@ -45,6 +45,7 @@ stub_ss.add(files('target-get-monitor-def.c'))
stub_ss.add(files('target-monitor-defs.c'))
stub_ss.add(files('trace-control.c'))
stub_ss.add(files('uuid.c'))
stub_ss.add(files('colo.c'))
stub_ss.add(files('vmstate.c'))
stub_ss.add(files('vm-stop.c'))
stub_ss.add(files('win32-kbd-hook.c'))