OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

Dirtylimit and dirtyrate 20231010 patches PULL request

Browse Source 
Dirty page rate measurement optimization.
 Please apply, thanks, Yong.
 -----BEGIN PGP SIGNATURE-----
 
 iQJKBAABCAA0FiEEaF0CINwmSCgVLlfC3/Ij1rP+y5wFAmUklg0WHHlvbmcuaHVh
 bmdAc21hcnR4LmNvbQAKCRDf8iPWs/7LnIw6D/sECFML6dIDSckhKe1kRBT2oXRd
 lYz4/RRdxPJIJP0zS0yLYXd2d5vHzXdC3hETv7//QiWB2OP/UQnsZ70JCgF4DxIq
 bGL4BUHgaQmyUsyIQXceFznJJQOLs5DczDFJBR2zlQbu3YOAGeNJJmfxmVIEPfcL
 w5NK++g3nVZ3pLJBNblDBUwIW5uoOj6z85rCTc6pvddoTBcAqS0er1aTkuyx9jbB
 VsDdFNJumF6+VnE6QUITHX2knr3UmXc5dfXCJi4CLKRfx3nyK8vYydNawhPtobGD
 0G5MZAPLO3JxdM67EccKj3I/kcAXU4iHu7rV5AscSI/rlfneGjfCup2Xd0we1GCR
 TD07AVDRuW+cS76nEtvDSRj6+8KarZEa3lVbvoPaXIazoHg3GjKylIMTAcGjFlzL
 AnGornOEZSfwNxT3BRvNHFdUNdA9ICZ90sEpWjeu80UNOT2JASOB6JE2VJBFnW81
 4gaoIT74hpI8H2k/x3R8REPnw+SLMI+7VpcA2XcXuOQOdfk0+8zlvxPsJRBaKBzS
 d2es+CpUcmBxZdEQNOi905qxfCFLOhwcstJXyCvFQBp4f8l2SJfIE4liI29qpuma
 hubbOEo/EAVe8ywToHSYj2RU5hnj6gu0n3hvSeye76hS/K+bfvI+HZ3AX+rcRmP8
 vX6Vqs4wdNl5khEnNg==
 =ixu5
 -----END PGP SIGNATURE-----

Merge tag 'dirtylimit-dirtyrate-pull-request-20231010' of https://github.com/newfriday/qemu into staging

Dirtylimit and dirtyrate 20231010 patches PULL request

Dirty page rate measurement optimization.
Please apply, thanks, Yong.

# -----BEGIN PGP SIGNATURE-----
#
# iQJKBAABCAA0FiEEaF0CINwmSCgVLlfC3/Ij1rP+y5wFAmUklg0WHHlvbmcuaHVh
# bmdAc21hcnR4LmNvbQAKCRDf8iPWs/7LnIw6D/sECFML6dIDSckhKe1kRBT2oXRd
# lYz4/RRdxPJIJP0zS0yLYXd2d5vHzXdC3hETv7//QiWB2OP/UQnsZ70JCgF4DxIq
# bGL4BUHgaQmyUsyIQXceFznJJQOLs5DczDFJBR2zlQbu3YOAGeNJJmfxmVIEPfcL
# w5NK++g3nVZ3pLJBNblDBUwIW5uoOj6z85rCTc6pvddoTBcAqS0er1aTkuyx9jbB
# VsDdFNJumF6+VnE6QUITHX2knr3UmXc5dfXCJi4CLKRfx3nyK8vYydNawhPtobGD
# 0G5MZAPLO3JxdM67EccKj3I/kcAXU4iHu7rV5AscSI/rlfneGjfCup2Xd0we1GCR
# TD07AVDRuW+cS76nEtvDSRj6+8KarZEa3lVbvoPaXIazoHg3GjKylIMTAcGjFlzL
# AnGornOEZSfwNxT3BRvNHFdUNdA9ICZ90sEpWjeu80UNOT2JASOB6JE2VJBFnW81
# 4gaoIT74hpI8H2k/x3R8REPnw+SLMI+7VpcA2XcXuOQOdfk0+8zlvxPsJRBaKBzS
# d2es+CpUcmBxZdEQNOi905qxfCFLOhwcstJXyCvFQBp4f8l2SJfIE4liI29qpuma
# hubbOEo/EAVe8ywToHSYj2RU5hnj6gu0n3hvSeye76hS/K+bfvI+HZ3AX+rcRmP8
# vX6Vqs4wdNl5khEnNg==
# =ixu5
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 09 Oct 2023 20:08:45 EDT
# gpg:                using RSA key 685D0220DC264828152E57C2DFF223D6B3FECB9C
# gpg:                issuer "yong.huang@smartx.com"
# gpg: Good signature from "Yong Huang <yong.huang@smartx.com>" [unknown]
# gpg: WARNING: The key's User ID is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 685D 0220 DC26 4828 152E  57C2 DFF2 23D6 B3FE CB9C

* tag 'dirtylimit-dirtyrate-pull-request-20231010' of https://github.com/newfriday/qemu:
  migration/dirtyrate: use QEMU_CLOCK_HOST to report start-time
  migration/calc-dirty-rate: millisecond-granularity period

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
This commit is contained in:
Stefan Hajnoczi 2023-10-10 10:21:43 -04:00
commit 9a8981e68c
3 changed files with 134 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
migration/dirtyrate.c
View File

@ -189,10 +189,9 @@ retry:
return duration;
}
static bool is_sample_period_valid(int64_t sec)
static bool is_calc_time_valid(int64_t msec)
{
if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC ||
sec > MAX_FETCH_DIRTYRATE_TIME_SEC) {
if ((msec < MIN_CALC_TIME_MS) || (msec > MAX_CALC_TIME_MS)) {
return false;
}
@ -216,7 +215,39 @@ static int dirtyrate_set_state(int *state, int old_state, int new_state)
}
}
static struct DirtyRateInfo *query_dirty_rate_info(void)
/* Decimal power of given time unit relative to one second */
static int time_unit_to_power(TimeUnit time_unit)
{
switch (time_unit) {
case TIME_UNIT_SECOND:
return 0;
case TIME_UNIT_MILLISECOND:
return -3;
default:
assert(false); /* unreachable */
return 0;
}
}
static int64_t convert_time_unit(int64_t value, TimeUnit unit_from,
TimeUnit unit_to)
{
int power = time_unit_to_power(unit_from) -
time_unit_to_power(unit_to);
while (power < 0) {
value /= 10;
power += 1;
}
while (power > 0) {
value *= 10;
power -= 1;
}
return value;
}
static struct DirtyRateInfo *
query_dirty_rate_info(TimeUnit calc_time_unit)
{
int i;
int64_t dirty_rate = DirtyStat.dirty_rate;
@ -225,7 +256,10 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
info->status = CalculatingState;
info->start_time = DirtyStat.start_time;
info->calc_time = DirtyStat.calc_time;
info->calc_time = convert_time_unit(DirtyStat.calc_time_ms,
TIME_UNIT_MILLISECOND,
calc_time_unit);
info->calc_time_unit = calc_time_unit;
info->sample_pages = DirtyStat.sample_pages;
info->mode = dirtyrate_mode;
@ -259,12 +293,11 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
return info;
}
static void init_dirtyrate_stat(int64_t start_time,
struct DirtyRateConfig config)
static void init_dirtyrate_stat(struct DirtyRateConfig config)
{
DirtyStat.dirty_rate = -1;
DirtyStat.start_time = start_time;
DirtyStat.calc_time = config.sample_period_seconds;
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000;
DirtyStat.calc_time_ms = config.calc_time_ms;
DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
switch (config.mode) {
@ -574,7 +607,6 @@ static inline void dirtyrate_manual_reset_protect(void)
static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
{
int64_t msec = 0;
int64_t start_time;
DirtyPageRecord dirty_pages;
@ -600,11 +632,9 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
record_dirtypages_bitmap(&dirty_pages, true);
start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
DirtyStat.start_time = start_time / 1000;
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000;
msec = config.sample_period_seconds * 1000;
msec = dirty_stat_wait(msec, start_time);
DirtyStat.calc_time = msec / 1000;
DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms, start_time);
/*
* do two things.
@ -615,12 +645,12 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
record_dirtypages_bitmap(&dirty_pages, false);
DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, msec);
DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages,
DirtyStat.calc_time_ms);
}
static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
{
int64_t duration;
uint64_t dirtyrate = 0;
uint64_t dirtyrate_sum = 0;
int i = 0;
@ -628,15 +658,13 @@ static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
/* start log sync */
global_dirty_log_change(GLOBAL_DIRTY_DIRTY_RATE, true);
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000;
/* calculate vcpu dirtyrate */
duration = vcpu_calculate_dirtyrate(config.sample_period_seconds * 1000,
&DirtyStat.dirty_ring,
GLOBAL_DIRTY_DIRTY_RATE,
true);
DirtyStat.calc_time = duration / 1000;
DirtyStat.calc_time_ms = vcpu_calculate_dirtyrate(config.calc_time_ms,
&DirtyStat.dirty_ring,
GLOBAL_DIRTY_DIRTY_RATE,
true);
/* calculate vm dirtyrate */
for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
@ -652,27 +680,25 @@ static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
{
struct RamblockDirtyInfo *block_dinfo = NULL;
int block_count = 0;
int64_t msec = 0;
int64_t initial_time;
rcu_read_lock();
initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000;
if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) {
goto out;
}
rcu_read_unlock();
msec = config.sample_period_seconds * 1000;
msec = dirty_stat_wait(msec, initial_time);
DirtyStat.start_time = initial_time / 1000;
DirtyStat.calc_time = msec / 1000;
DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms,
initial_time);
rcu_read_lock();
if (!compare_page_hash_info(block_dinfo, block_count)) {
goto out;
}
update_dirtyrate(msec);
update_dirtyrate(DirtyStat.calc_time_ms);
out:
rcu_read_unlock();
@ -718,6 +744,8 @@ void *get_dirtyrate_thread(void *arg)
}
void qmp_calc_dirty_rate(int64_t calc_time,
bool has_calc_time_unit,
TimeUnit calc_time_unit,
bool has_sample_pages,
int64_t sample_pages,
bool has_mode,
@ -727,7 +755,6 @@ void qmp_calc_dirty_rate(int64_t calc_time,
static struct DirtyRateConfig config;
QemuThread thread;
int ret;
int64_t start_time;
/*
* If the dirty rate is already being measured, don't attempt to start.
@ -737,10 +764,15 @@ void qmp_calc_dirty_rate(int64_t calc_time,
return;
}
if (!is_sample_period_valid(calc_time)) {
error_setg(errp, "calc-time is out of range[%d, %d].",
MIN_FETCH_DIRTYRATE_TIME_SEC,
MAX_FETCH_DIRTYRATE_TIME_SEC);
int64_t calc_time_ms = convert_time_unit(
calc_time,
has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND,
TIME_UNIT_MILLISECOND
);
if (!is_calc_time_valid(calc_time_ms)) {
error_setg(errp, "Calculation time is out of range [%dms, %dms].",
MIN_CALC_TIME_MS, MAX_CALC_TIME_MS);
return;
}
@ -787,7 +819,7 @@ void qmp_calc_dirty_rate(int64_t calc_time,
return;
}
config.sample_period_seconds = calc_time;
config.calc_time_ms = calc_time_ms;
config.sample_pages_per_gigabytes = sample_pages;
config.mode = mode;
@ -799,21 +831,24 @@ void qmp_calc_dirty_rate(int64_t calc_time,
**/
dirtyrate_mode = mode;
start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
init_dirtyrate_stat(start_time, config);
init_dirtyrate_stat(config);
qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread,
(void *)&config, QEMU_THREAD_DETACHED);
}
struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
struct DirtyRateInfo *qmp_query_dirty_rate(bool has_calc_time_unit,
TimeUnit calc_time_unit,
Error **errp)
{
return query_dirty_rate_info();
return query_dirty_rate_info(
has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND);
}
void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
{
DirtyRateInfo *info = query_dirty_rate_info();
DirtyRateInfo *info = query_dirty_rate_info(TIME_UNIT_SECOND);
monitor_printf(mon, "Status: %s\n",
DirtyRateStatus_str(info->status));
@ -873,8 +908,11 @@ void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING;
}
qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true,
mode, &err);
qmp_calc_dirty_rate(sec, /* calc-time */
false, TIME_UNIT_SECOND, /* calc-time-unit */
has_sample_pages, sample_pages,
true, mode,
&err);
if (err) {
hmp_handle_error(mon, err);
return;

12
migration/dirtyrate.h
View File

@ -31,10 +31,12 @@
#define MIN_RAMBLOCK_SIZE 128
/*
* Take 1s as minimum time for calculation duration
* Allowed range for dirty page rate calculation (in milliseconds).
* Lower limit relates to the smallest realistic downtime it
* makes sense to impose on migration.
*/
#define MIN_FETCH_DIRTYRATE_TIME_SEC 1
#define MAX_FETCH_DIRTYRATE_TIME_SEC 60
#define MIN_CALC_TIME_MS 50
#define MAX_CALC_TIME_MS 60000
/*
* Take 1/16 pages in 1G as the maxmum sample page count
@ -44,7 +46,7 @@
struct DirtyRateConfig {
uint64_t sample_pages_per_gigabytes; /* sample pages per GB */
int64_t sample_period_seconds; /* time duration between two sampling */
int64_t calc_time_ms; /* desired calculation time (in milliseconds) */
DirtyRateMeasureMode mode; /* mode of dirtyrate measurement */
};
@ -73,7 +75,7 @@ typedef struct SampleVMStat {
struct DirtyRateStat {
int64_t dirty_rate; /* dirty rate in MB/s */
int64_t start_time; /* calculation start time in units of second */
int64_t calc_time; /* time duration of two sampling in units of second */
int64_t calc_time_ms; /* actual calculation time (in milliseconds) */
uint64_t sample_pages; /* sample pages per GB */
union {
SampleVMStat page_sampling;

58
qapi/migration.json
View File

@ -1836,6 +1836,21 @@
{ 'enum': 'DirtyRateMeasureMode',
'data': ['page-sampling', 'dirty-ring', 'dirty-bitmap'] }
##
# @TimeUnit:
#
# Specifies unit in which time-related value is specified.
#
# @second: value is in seconds
#
# @millisecond: value is in milliseconds
#
# Since 8.2
#
##
{ 'enum': 'TimeUnit',
'data': ['second', 'millisecond'] }
##
# @DirtyRateInfo:
#
@ -1848,8 +1863,10 @@
#
# @start-time: start time in units of second for calculation
#
# @calc-time: time period for which dirty page rate was measured
# (in seconds)
# @calc-time: time period for which dirty page rate was measured,
# expressed and rounded down to @calc-time-unit.
#
# @calc-time-unit: time unit of @calc-time (Since 8.2)
#
# @sample-pages: number of sampled pages per GiB of guest memory.
# Valid only in page-sampling mode (Since 6.1)
@ -1866,6 +1883,7 @@
'status': 'DirtyRateStatus',
'start-time': 'int64',
'calc-time': 'int64',
'calc-time-unit': 'TimeUnit',
'sample-pages': 'uint64',
'mode': 'DirtyRateMeasureMode',
'*vcpu-dirty-rate': [ 'DirtyRateVcpu' ] } }
@ -1901,12 +1919,16 @@
# This mode tracks page modification per each vCPU separately. It
# requires that KVM accelerator property "dirty-ring-size" is set.
#
# @calc-time: time period in units of second for which dirty page rate
# is calculated. Note that larger @calc-time values will
# typically result in smaller dirty page rates because page
# dirtying is a one-time event. Once some page is counted as
# dirty during @calc-time period, further writes to this page will
# not increase dirty page rate anymore.
# @calc-time: time period for which dirty page rate is calculated.
# By default it is specified in seconds, but the unit can be set
# explicitly with @calc-time-unit. Note that larger @calc-time
# values will typically result in smaller dirty page rates because
# page dirtying is a one-time event. Once some page is counted
# as dirty during @calc-time period, further writes to this page
# will not increase dirty page rate anymore.
#
# @calc-time-unit: time unit in which @calc-time is specified.
# By default it is seconds. (Since 8.2)
#
# @sample-pages: number of sampled pages per each GiB of guest memory.
# Default value is 512. For 4KiB guest pages this corresponds to
@ -1924,8 +1946,16 @@
# -> {"execute": "calc-dirty-rate", "arguments": {"calc-time": 1,
# 'sample-pages': 512} }
# <- { "return": {} }
#
# Measure dirty rate using dirty bitmap for 500 milliseconds:
#
# -> {"execute": "calc-dirty-rate", "arguments": {"calc-time": 500,
# "calc-time-unit": "millisecond", "mode": "dirty-bitmap"} }
#
# <- { "return": {} }
##
{ 'command': 'calc-dirty-rate', 'data': {'calc-time': 'int64',
'*calc-time-unit': 'TimeUnit',
'*sample-pages': 'int',
'*mode': 'DirtyRateMeasureMode'} }
@ -1934,6 +1964,9 @@
#
# Query results of the most recent invocation of @calc-dirty-rate.
#
# @calc-time-unit: time unit in which to report calculation time.
# By default it is reported in seconds. (Since 8.2)
#
# Since: 5.2
#
# Examples:
@ -1941,14 +1974,17 @@
# 1. Measurement is in progress:
#
# <- {"status": "measuring", "sample-pages": 512,
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10}
# "mode": "page-sampling", "start-time": 1693900454, "calc-time": 10,
# "calc-time-unit": "second"}
#
# 2. Measurement has been completed:
#
# <- {"status": "measured", "sample-pages": 512, "dirty-rate": 108,
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10}
# "mode": "page-sampling", "start-time": 1693900454, "calc-time": 10,
# "calc-time-unit": "second"}
##
{ 'command': 'query-dirty-rate', 'returns': 'DirtyRateInfo' }
{ 'command': 'query-dirty-rate', 'data': {'*calc-time-unit': 'TimeUnit' },
'returns': 'DirtyRateInfo' }
##
# @DirtyLimitInfo: