| Message ID | 1600137887-58739-1-git-send-email-zhengchuan@huawei.com |
|---|---|
| Headers | show |
| Series | *** A Method for evaluating dirty page rate *** | expand |
Chuan Zheng <zhengchuan@huawei.com> 于2020年9月15日周二 上午10:34写道： > > add DirtyRateStatus to denote calculating status. > > Signed-off-by: Chuan Zheng <zhengchuan@huawei.com> > Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com> Reviewed-by: Li Qiang <liq3ea@gmail.com> > --- > migration/dirtyrate.c | 26 ++++++++++++++++++++++++++ > qapi/migration.json | 17 +++++++++++++++++ > 2 files changed, 43 insertions(+) > > diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c > index bf7fd24..7bea8ff 100644 > --- a/migration/dirtyrate.c > +++ b/migration/dirtyrate.c > @@ -22,6 +22,19 @@ > #include "migration.h" > #include "dirtyrate.h" > > +static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; > + > +static int dirtyrate_set_state(int *state, int old_state, int new_state) > +{ > + assert(new_state < DIRTY_RATE_STATUS__MAX); > + if (atomic_cmpxchg(state, old_state, new_state) == old_state) { > + return 0; > + } else { > + return -1; > + } > +} > + > + > static void calculate_dirtyrate(struct DirtyRateConfig config) > { > /* todo */ > @@ -31,8 +44,21 @@ static void calculate_dirtyrate(struct DirtyRateConfig config) > void *get_dirtyrate_thread(void *arg) > { > struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; > + int ret; > + > + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED, > + DIRTY_RATE_STATUS_MEASURING); > + if (ret == -1) { > + error_report("change dirtyrate state failed."); > + return NULL; > + } > > calculate_dirtyrate(config); > > + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING, > + DIRTY_RATE_STATUS_MEASURED); > + if (ret == -1) { > + error_report("change dirtyrate state failed."); > + } > return NULL; > } > diff --git a/qapi/migration.json b/qapi/migration.json > index 5f6b061..061ff25 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -1720,3 +1720,20 @@ > ## > { 'event': 'UNPLUG_PRIMARY', > 'data': { 'device-id': 'str' } } > + > +## > +# @DirtyRateStatus: > +# > +# An enumeration of dirtyrate status. > +# > +# @unstarted: the dirtyrate thread has not been started. > +# > +# @measuring: the dirtyrate thread is measuring. > +# > +# @measured: the dirtyrate thread has measured and results are available. > +# > +# Since: 5.2 > +# > +## > +{ 'enum': 'DirtyRateStatus', > + 'data': [ 'unstarted', 'measuring', 'measured'] } > -- > 1.8.3.1 >
Chuan Zheng <zhengchuan@huawei.com> 于2020年9月15日周二 上午10:34写道： > > Record hash results for each sampled page, crc32 is taken to calculate > hash results for each sampled length in TARGET_PAGE_SIZE. > > Signed-off-by: Chuan Zheng <zhengchuan@huawei.com> > Signed-off-by: YanYing Zhuang <ann.zhuangyanying@huawei.com> > Reviewed-by: David Edmondson <david.edmondson@oracle.com> Reviewed-by: Li Qiang <liq3ea@gmail.com> > --- > migration/dirtyrate.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 109 insertions(+) > > diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c > index 7366bf3..5e6eedf 100644 > --- a/migration/dirtyrate.c > +++ b/migration/dirtyrate.c > @@ -10,6 +10,7 @@ > * See the COPYING file in the top-level directory. > */ > > +#include <zlib.h> > #include "qemu/osdep.h" > #include "qapi/error.h" > #include "cpu.h" > @@ -68,6 +69,114 @@ static void update_dirtyrate(uint64_t msec) > DirtyStat.dirty_rate = dirtyrate; > } > > +/* > + * get hash result for the sampled memory with length of TARGET_PAGE_SIZE > + * in ramblock, which starts from ramblock base address. > + */ > +static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, > + uint64_t vfn) > +{ > + uint32_t crc; > + > + crc = crc32(0, (info->ramblock_addr + > + vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); > + > + return crc; > +} > + > +static bool save_ramblock_hash(struct RamblockDirtyInfo *info) > +{ > + unsigned int sample_pages_count; > + int i; > + GRand *rand; > + > + sample_pages_count = info->sample_pages_count; > + > + /* ramblock size less than one page, return success to skip this ramblock */ > + if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) { > + return true; > + } > + > + info->hash_result = g_try_malloc0_n(sample_pages_count, > + sizeof(uint32_t)); > + if (!info->hash_result) { > + return false; > + } > + > + info->sample_page_vfn = g_try_malloc0_n(sample_pages_count, > + sizeof(uint64_t)); > + if (!info->sample_page_vfn) { > + g_free(info->hash_result); > + return false; > + } > + > + rand = g_rand_new(); > + for (i = 0; i < sample_pages_count; i++) { > + info->sample_page_vfn[i] = g_rand_int_range(rand, 0, > + info->ramblock_pages - 1); > + info->hash_result[i] = get_ramblock_vfn_hash(info, > + info->sample_page_vfn[i]); > + } > + g_rand_free(rand); > + > + return true; > +} > + > +static void get_ramblock_dirty_info(RAMBlock *block, > + struct RamblockDirtyInfo *info, > + struct DirtyRateConfig *config) > +{ > + uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes; > + > + /* Right shift 30 bits to calc ramblock size in GB */ > + info->sample_pages_count = (qemu_ram_get_used_length(block) * > + sample_pages_per_gigabytes) >> 30; > + /* Right shift TARGET_PAGE_BITS to calc page count */ > + info->ramblock_pages = qemu_ram_get_used_length(block) >> > + TARGET_PAGE_BITS; > + info->ramblock_addr = qemu_ram_get_host_addr(block); > + strcpy(info->idstr, qemu_ram_get_idstr(block)); > +} > + > +static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, > + struct DirtyRateConfig config, > + int *block_count) > +{ > + struct RamblockDirtyInfo *info = NULL; > + struct RamblockDirtyInfo *dinfo = NULL; > + RAMBlock *block = NULL; > + int total_count = 0; > + int index = 0; > + bool ret = false; > + > + RAMBLOCK_FOREACH_MIGRATABLE(block) { > + total_count++; > + } > + > + dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo)); > + if (dinfo == NULL) { > + goto out; > + } > + > + RAMBLOCK_FOREACH_MIGRATABLE(block) { > + if (index >= total_count) { > + break; > + } > + info = &dinfo[index]; > + get_ramblock_dirty_info(block, info, &config); > + if (!save_ramblock_hash(info)) { > + goto out; > + } > + index++; > + } > + ret = true; > + > +out: > + *block_count = index; > + *block_dinfo = dinfo; > + return ret; > +} > + > static void calculate_dirtyrate(struct DirtyRateConfig config) > { > /* todo */ > -- > 1.8.3.1 >
Chuan Zheng <zhengchuan@huawei.com> 于2020年9月15日周二 上午10:34写道： > > Implement calculate_dirtyrate() function. > > Signed-off-by: Chuan Zheng <zhengchuan@huawei.com> > Signed-off-by: YanYing Zhuang <ann.zhuangyanying@huawei.com> > Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com> Reviewed-by: Li Qiang <liq3ea@gmail.com> > --- > migration/dirtyrate.c | 45 +++++++++++++++++++++++++++++++++++++++++++-- > 1 file changed, 43 insertions(+), 2 deletions(-) > > diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c > index bcff10e..af02647 100644 > --- a/migration/dirtyrate.c > +++ b/migration/dirtyrate.c > @@ -162,6 +162,21 @@ static void get_ramblock_dirty_info(RAMBlock *block, > strcpy(info->idstr, qemu_ram_get_idstr(block)); > } > > +static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count) > +{ > + int i; > + > + if (!infos) { > + return; > + } > + > + for (i = 0; i < count; i++) { > + g_free(infos[i].sample_page_vfn); > + g_free(infos[i].hash_result); > + } > + g_free(infos); > +} > + > static bool skip_sample_ramblock(RAMBlock *block) > { > /* > @@ -287,8 +302,34 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info, > > static void calculate_dirtyrate(struct DirtyRateConfig config) > { > - /* todo */ > - return; > + struct RamblockDirtyInfo *block_dinfo = NULL; > + int block_count = 0; > + int64_t msec = 0; > + int64_t initial_time; > + > + rcu_register_thread(); > + reset_dirtyrate_stat(); > + rcu_read_lock(); > + initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); > + if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) { > + goto out; > + } > + rcu_read_unlock(); > + > + msec = config.sample_period_seconds * 1000; > + msec = set_sample_page_period(msec, initial_time); > + > + rcu_read_lock(); > + if (!compare_page_hash_info(block_dinfo, block_count)) { > + goto out; > + } > + > + update_dirtyrate(msec); > + > +out: > + rcu_read_unlock(); > + free_ramblock_dirty_info(block_dinfo, block_count); > + rcu_unregister_thread(); > } > > void *get_dirtyrate_thread(void *arg) > -- > 1.8.3.1 >
v8 -> v9: fix wrong index return of record_ramblock_hash_info optimize variable name according to review reset dirty_rate as -1 change returns of compare_page_hash_info to bool v7 -> v8: add atomic_read for dirtyrate status add error_report if set dirtyrate state failed change returns of save_ramblock_hash and record_ramblock_hash_info to bool alloc ramblock dirtyinfo array at one time add review-by for patches v6 -> v7: fix minior comments and coding style by review add review-by for patches v5 -> v6: fix coding style according to review use TARGET_PAGE_SIZE and TARGET_PAGE_BITS instead of self-defined macros return start-time and calc-time by qmp command v4 -> v5: fix git apply failed due to meson-build add review-by for patches in v3 v3 -> v4: use crc32 to get hash result instead of md5 add DirtyRateStatus to denote calculation status add some trace_calls to make it easier to debug fix some comments accroding to review v2 -> v3: fix size_t compile warning fix codestyle checked by checkpatch.pl v1 -> v2: use g_rand_new() to generate rand_buf move RAMBLOCK_FOREACH_MIGRATABLE into migration/ram.h add skip_sample_ramblock to filter sampled ramblock fix multi-numa vm coredump when query dirtyrate rename qapi interface and rename some structures and functions succeed to compile by appling each patch add test for migrating vm Sometimes it is neccessary to evaluate dirty page rate before migration. Users could decide whether to proceed migration based on the evaluation in case of vm performance loss due to heavy workload. Unlikey simulating dirtylog sync which could do harm on runnning vm, we provide a sample-hash method to compare hash results for samping page. In this way, it would have hardly no impact on vm performance. Evaluate the dirtypage rate both on running and migration vm. The VM specifications for migration are as follows: - VM use 4-K page; - the number of VCPU is 32; - the total memory is 32Gigabit; - use 'mempress' tool to pressurize VM(mempress 4096 1024); - migration bandwidth is 1GB/s +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ | | running | migrating | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ | no mempress | 4MB/s | 8MB/s (migrated success) | ------------------------------------------------------------------------------------------- | mempress 4096 1024 | 1060MB/s | 456MB/s ~ 1142MB/s (cpu throttle triggered) | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ | mempress 4096 4096 | 4114MB/s | 688MB/s ~ 4132MB/s (cpu throttle triggered) | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Test dirtyrate by qmp command like this: 1. virsh qemu-monitor-command [vmname] '{"execute":"calc-dirty-rate", "arguments": {"calc-time": [sleep-time]}}'; 2. sleep specific time which is a bit larger than sleep-time 3. virsh qemu-monitor-command [vmname] '{"execute":"query-dirty-rate"}' The qmp command returns like this: {"return":{"status":"measured","dirty-rate":374,"start-time":3718293,"calc-time":1},"id":"libvirt-15"} Further test dirtyrate by libvirt api like this: virsh getdirtyrate [vmname] [sleep-time] Chuan Zheng (12): migration/dirtyrate: setup up query-dirtyrate framwork migration/dirtyrate: add DirtyRateStatus to denote calculation status migration/dirtyrate: Add RamblockDirtyInfo to store sampled page info migration/dirtyrate: Add dirtyrate statistics series functions migration/dirtyrate: move RAMBLOCK_FOREACH_MIGRATABLE into ram.h migration/dirtyrate: Record hash results for each sampled page migration/dirtyrate: Compare page hash results for recorded sampled page migration/dirtyrate: skip sampling ramblock with size below MIN_RAMBLOCK_SIZE migration/dirtyrate: Implement set_sample_page_period() and is_sample_period_valid() migration/dirtyrate: Implement calculate_dirtyrate() function migration/dirtyrate: Implement qmp_cal_dirty_rate()/qmp_get_dirty_rate() function migration/dirtyrate: Add trace_calls to make it easier to debug migration/dirtyrate.c | 426 +++++++++++++++++++++++++++++++++++++++++++++++++ migration/dirtyrate.h | 70 ++++++++ migration/meson.build | 2 +- migration/ram.c | 11 +- migration/ram.h | 10 ++ migration/trace-events | 8 + qapi/migration.json | 67 ++++++++ 7 files changed, 583 insertions(+), 11 deletions(-) create mode 100644 migration/dirtyrate.c create mode 100644 migration/dirtyrate.h