diff mbox

[Xen-devel,RFC,v3,6/6] xen/arm: Implement toolstack for xl restore/save/migration

Message ID 1399583908-21755-7-git-send-email-w1.huang@samsung.com
State New
Headers show

Commit Message

Wei Huang May 8, 2014, 9:18 p.m. UTC
This patch implements xl save/restore operation in xc_arm_migrate.c
and make it compilable with existing design. The operation is also
used by migration.

The overall process of save/restore is the following: 1) save guest
parameters; 2) save memory; 3) save HVM states.

Signed-off-by: Alexey Sokolov <sokolov.a@samsung.com>
Signed-off-by: Wei Huang <w1.huang@samsung.com>
---
 config/arm32.mk              |    1 +
 config/arm64.mk              |    1 +
 tools/libxc/Makefile         |    5 +
 tools/libxc/xc_arm_migrate.c |  653 ++++++++++++++++++++++++++++++++++++++++++
 tools/libxc/xc_dom_arm.c     |    4 +-
 tools/libxc/xc_resume.c      |   20 +-
 tools/libxl/libxl.h          |    3 -
 tools/misc/Makefile          |    4 +-
 8 files changed, 677 insertions(+), 14 deletions(-)
 create mode 100644 tools/libxc/xc_arm_migrate.c

Comments

Ian Campbell May 14, 2014, 1:20 p.m. UTC | #1
On Thu, 2014-05-08 at 16:18 -0500, Wei Huang wrote:
> This patch implements xl save/restore operation in xc_arm_migrate.c
> and make it compilable with existing design. The operation is also
> used by migration.
> 
> The overall process of save/restore is the following: 1) save guest
> parameters; 2) save memory; 3) save HVM states.
> 
> Signed-off-by: Alexey Sokolov <sokolov.a@samsung.com>
> Signed-off-by: Wei Huang <w1.huang@samsung.com>

I'm going to defer reviewing this for now, it'll likely need to be
reworked to fit into Andy's new migration framework. As I mentioned
before (I think/hope!) I'd like ARM to use that from day 1.

> diff --git a/tools/misc/Makefile b/tools/misc/Makefile
> index 69b1817..f4ea7ab 100644
> --- a/tools/misc/Makefile
> +++ b/tools/misc/Makefile

This change could probably be refactored into a separate patch and
applied already.

> @@ -11,7 +11,7 @@ HDRS     = $(wildcard *.h)
>  
>  TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xencov
>  TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash xen-lowmemd xen-mfndump
> -TARGETS-$(CONFIG_MIGRATE) += xen-hptool
> +TARGETS-$(CONFIG_X86) += xen-hptool
>  TARGETS := $(TARGETS-y)
>  
>  SUBDIRS := $(SUBDIRS-y)
> @@ -23,7 +23,7 @@ INSTALL_BIN := $(INSTALL_BIN-y)
>  INSTALL_SBIN-y := xen-bugtool xen-python-path xenperf xenpm xen-tmem-list-parse gtraceview \
>  	gtracestat xenlockprof xenwatchdogd xen-ringwatch xencov
>  INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx xen-hvmcrash xen-lowmemd xen-mfndump
> -INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
> +INSTALL_SBIN-$(CONFIG_X86) += xen-hptool
>  INSTALL_SBIN := $(INSTALL_SBIN-y)
>  
>  INSTALL_PRIVBIN-y := xenpvnetboot
Andrew Cooper May 14, 2014, 1:24 p.m. UTC | #2
On 14/05/14 14:20, Ian Campbell wrote:
> On Thu, 2014-05-08 at 16:18 -0500, Wei Huang wrote:
>> This patch implements xl save/restore operation in xc_arm_migrate.c
>> and make it compilable with existing design. The operation is also
>> used by migration.
>>
>> The overall process of save/restore is the following: 1) save guest
>> parameters; 2) save memory; 3) save HVM states.
>>
>> Signed-off-by: Alexey Sokolov <sokolov.a@samsung.com>
>> Signed-off-by: Wei Huang <w1.huang@samsung.com>
> I'm going to defer reviewing this for now, it'll likely need to be
> reworked to fit into Andy's new migration framework. As I mentioned
> before (I think/hope!) I'd like ARM to use that from day 1.

With the Xen side hypercalls implemented in this series, supporting ARM
in the new framework should be as complicated as copying the x86
mostly-noop function-pointer-structure.  Should be all of 100 lines in
total or so.

~Andrew
diff mbox

Patch

diff --git a/config/arm32.mk b/config/arm32.mk
index aa79d22..01374c9 100644
--- a/config/arm32.mk
+++ b/config/arm32.mk
@@ -1,6 +1,7 @@ 
 CONFIG_ARM := y
 CONFIG_ARM_32 := y
 CONFIG_ARM_$(XEN_OS) := y
+CONFIG_MIGRATE := y
 
 CONFIG_XEN_INSTALL_SUFFIX :=
 
diff --git a/config/arm64.mk b/config/arm64.mk
index 15b57a4..7ac3b65 100644
--- a/config/arm64.mk
+++ b/config/arm64.mk
@@ -1,6 +1,7 @@ 
 CONFIG_ARM := y
 CONFIG_ARM_64 := y
 CONFIG_ARM_$(XEN_OS) := y
+CONFIG_MIGRATE := y
 
 CONFIG_XEN_INSTALL_SUFFIX :=
 
diff --git a/tools/libxc/Makefile b/tools/libxc/Makefile
index a74b19e..9aaa6ff 100644
--- a/tools/libxc/Makefile
+++ b/tools/libxc/Makefile
@@ -43,8 +43,13 @@  CTRL_SRCS-$(CONFIG_MiniOS) += xc_minios.c
 GUEST_SRCS-y :=
 GUEST_SRCS-y += xg_private.c xc_suspend.c
 ifeq ($(CONFIG_MIGRATE),y)
+ifeq ($(CONFIG_X86),y)
 GUEST_SRCS-y += xc_domain_restore.c xc_domain_save.c
 GUEST_SRCS-y += xc_offline_page.c xc_compression.c
+endif
+ifeq ($(CONFIG_ARM),y)
+GUEST_SRCS-y += xc_arm_migrate.c
+endif
 else
 GUEST_SRCS-y += xc_nomigrate.c
 endif
diff --git a/tools/libxc/xc_arm_migrate.c b/tools/libxc/xc_arm_migrate.c
new file mode 100644
index 0000000..776e373
--- /dev/null
+++ b/tools/libxc/xc_arm_migrate.c
@@ -0,0 +1,653 @@ 
+/*
+ * Copyright (c) 2014 Samsung Electronics.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <inttypes.h>
+#include <errno.h>
+#include <xenctrl.h>
+#include <xenguest.h>
+
+#include <unistd.h>
+#include <xc_private.h>
+#include <xc_dom.h>
+#include "xc_bitops.h"
+#include "xg_private.h"
+
+#define DEF_MAX_ITERS          29 /* limit us to 30 times round loop   */
+#define DEF_MAX_FACTOR         3  /* never send more than 3x p2m_size  */
+#define DEF_MIN_DIRTY_PER_ITER 50 /* dirty page count to define last iter */
+#define DEF_PROGRESS_RATE      50 /* progress bar update rate */
+
+/* Guest params to save */
+typedef struct guest_params
+{
+    unsigned long console_pfn;
+    unsigned long store_pfn;
+    uint32_t flags;
+    xen_pfn_t start_gpfn;
+    xen_pfn_t max_gpfn;
+    uint32_t max_vcpu_id;
+} guest_params_t;
+
+/***********************************/
+/*           Misc Support          */
+/***********************************/
+static int suspend_and_state(int (*suspend)(void*), void *data,
+                             xc_interface *xch, int dom)
+{
+    xc_dominfo_t info;
+
+    if ( !(*suspend)(data) )
+    {
+        PERROR("Suspend request failed");
+        return -1;
+    }
+
+    if ( (xc_domain_getinfo(xch, dom, 1, &info) != 1) || info.domid != dom ||
+         !info.shutdown || (info.shutdown_reason != SHUTDOWN_suspend) )
+    {
+        PERROR("Domain is not in suspended state after suspend attempt");
+        return -1;
+    }
+
+    return 0;
+}
+
+/***********************************/
+/*           Guest Memory          */
+/***********************************/
+static int save_guest_memory(xc_interface *xch, int io_fd, uint32_t dom,
+                             struct save_callbacks *callbacks,
+                             uint32_t max_iters, uint32_t max_factor,
+                             guest_params_t *params)
+{
+    int live =  !!(params->flags & XCFLAGS_LIVE);
+    int debug =  !!(params->flags & XCFLAGS_DEBUG);
+    xen_pfn_t i;
+    char reportbuf[80];
+    int iter = 0;
+    int last_iter = !live;
+    int total_dirty = 0;    /* total number of dirty pages */
+    int prev_iter_dirty = 0;
+    int count = 0;
+    char *page = NULL;
+    xen_pfn_t *busy_pages = 0;
+    int busy_pages_count = 0;
+    int busy_pages_max = 256;
+    DECLARE_HYPERCALL_BUFFER(unsigned long, to_send);
+    xen_pfn_t start = params->start_gpfn;
+    const xen_pfn_t end = params->max_gpfn;
+    const xen_pfn_t mem_size = end - start;    
+    int rc = -1, frc;
+
+    if ( debug )
+    {
+        IPRINTF("Saving memory: start=0x%llx end=0x%llx!\n",
+                (unsigned long long)start, (unsigned long long)end);
+    }
+
+    /* Note: to_send will be deallocated at the end of this function. */
+    to_send = xc_hypercall_buffer_alloc_pages(xch, to_send,
+                                              NRPAGES(bitmap_size(mem_size)));
+    if ( !to_send )
+    {
+        PERROR("Can't allocate to_send array!\n");
+        goto out;
+    }
+    /* send all pages on first iter */
+    memset(to_send, 0xff, bitmap_size(mem_size));
+
+    if ( live )
+    {
+        if ( xc_shadow_control(xch, dom, XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY,
+                               NULL, 0, NULL, 0, NULL) < 0 )
+        {
+            PERROR("Can't enable log-dirty mode!\n");
+            goto out;
+        }
+
+        max_iters  = max_iters  ? : DEF_MAX_ITERS;
+        max_factor = max_factor ? : DEF_MAX_FACTOR;
+    }
+
+    for ( ; ; )
+    {
+        int current_iter_dirty = 0; /* num of dirty pages in current iter */
+
+        iter++;
+        snprintf(reportbuf, sizeof(reportbuf),
+                 "Saving memory: iter %d, last sent %u", iter, 
+                 prev_iter_dirty);
+
+        xc_report_progress_start(xch, reportbuf, mem_size);
+
+        if ( (iter > 1 && prev_iter_dirty < DEF_MIN_DIRTY_PER_ITER) ||
+             (iter == max_iters) ||
+             (total_dirty >= mem_size*max_factor) )
+        {
+            last_iter = 1;
+        }
+
+        if ( last_iter )
+        {
+            if ( suspend_and_state(callbacks->suspend, callbacks->data, xch, 
+                                   dom) )
+            {
+                PERROR("Domain appears not to have suspended");
+                goto out;
+            }
+        }
+
+        if ( live && iter > 1 )
+        {
+            frc = xc_shadow_control(xch, dom, XEN_DOMCTL_SHADOW_OP_CLEAN,
+                                    HYPERCALL_BUFFER(to_send), mem_size,
+                                    NULL, 0, NULL);
+            if ( frc != mem_size )
+            {
+                PERROR("Can't peek bitmap of guest VM's dirty pages");
+                goto out;
+            }
+        }
+
+        /* busy_pages allocation */
+        busy_pages = malloc(sizeof(xen_pfn_t) * busy_pages_max);
+
+        for ( i = start; i < end; ++i )
+        {
+            if ( test_bit(i - start, to_send) )
+            {
+                page = xc_map_foreign_range(xch, dom, PAGE_SIZE, PROT_READ, i);
+                if ( !page )
+                {
+                    /* This page is mapped elsewhere, should be resent later */
+                    busy_pages[busy_pages_count] = i;
+                    busy_pages_count++;
+                    if ( busy_pages_count >= busy_pages_max )
+                    {
+                        busy_pages_max += 256;
+                        busy_pages = realloc(busy_pages, sizeof(xen_pfn_t) *
+                                             busy_pages_max);
+                    }
+                    continue;
+                }
+
+                if ( write_exact(io_fd, &i, sizeof(i)) ||
+                     write_exact(io_fd, page, PAGE_SIZE) )
+                {
+                    PERROR("Write Error: guest memory gpfn and content");
+                    free(busy_pages); /* must do here */
+                    goto out;
+                }
+
+                if ( (i % DEF_PROGRESS_RATE) == 0 )
+                    xc_report_progress_step(xch, i - start, mem_size);
+
+                count++;
+                munmap(page, PAGE_SIZE);
+                current_iter_dirty++;
+            }
+        }
+
+        while ( busy_pages_count )
+        {
+            /* Send busy pages */
+            busy_pages_count--;
+            i = busy_pages[busy_pages_count];
+            if ( test_bit(i - start, to_send) )
+            {
+                page = xc_map_foreign_range(xch, dom, PAGE_SIZE, PROT_READ, i);
+                if ( !page )
+                {
+                    IPRINTF("WARNING: 2nd attempt to save page failed at "
+                            "pfn=0x%llx", (unsigned long long)i);
+                    continue;
+                }
+
+                if ( write_exact(io_fd, &i, sizeof(i)) ||
+                     write_exact(io_fd, page, PAGE_SIZE) )
+                {
+                    PERROR("Write Error: guest memory gpfn and content");
+                    free(busy_pages); /* must do here */
+                    goto out;
+                }
+
+                count++;
+                munmap(page, PAGE_SIZE);
+                current_iter_dirty++;
+            }
+        }
+        free(busy_pages);
+
+        xc_report_progress_step(xch, mem_size, mem_size);
+
+        prev_iter_dirty = current_iter_dirty;
+        total_dirty += current_iter_dirty;
+
+        if ( last_iter )
+        {
+            if ( live )
+            {
+                if ( xc_shadow_control(xch, dom, XEN_DOMCTL_SHADOW_OP_OFF,
+                                       NULL, 0, NULL, 0, NULL) < 0 )
+                    PERROR("Can't disable log-dirty mode");
+            }
+            break;
+        }
+    }
+
+    i = (xen_pfn_t) -1; /* end page marker */
+    if ( write_exact(io_fd, &i, sizeof(i)) )
+    {
+        PERROR("Write Error: end page mark");
+        goto out;
+    }
+    /* flush last write and check for errno (must do, see fsync()). */
+    if ( fsync(io_fd) && errno != EINVAL )
+    {
+        PERROR("Write Error: flushing stream");
+        goto out;
+    }
+
+    rc = 0;
+ out:
+    /* clean up */
+    if ( page )
+        munmap(page, PAGE_SIZE);
+    if ( to_send )
+        xc_hypercall_buffer_free_pages(xch, to_send, 
+                                       NRPAGES(bitmap_size(mem_size)));
+    return rc;
+}
+
+static int restore_guest_memory(xc_interface *xch, int io_fd, uint32_t dom,
+                                guest_params_t *params)
+{
+    xen_pfn_t gpfn;
+    xen_pfn_t start = params->start_gpfn;
+    xen_pfn_t end = params->max_gpfn;
+    int count = 0;
+    char *page = NULL;
+    int rc = -1;
+
+    /* TODO: batch operation */
+    for ( gpfn = start; gpfn < end; ++gpfn )
+    {
+        if ( xc_domain_populate_physmap_exact(xch, dom, 1, 0, 0, &gpfn) )
+        {
+            PERROR("Can't populate guest physical memory");
+            goto out;
+        }
+    }
+
+    while ( 1 )
+    {
+        if ( read_exact(io_fd, &gpfn, sizeof(gpfn)) )
+        {
+            PERROR("Read Error: guest memory gpfn (count=%d)", count);
+            goto out;
+        }
+
+        /* end of guest pages */
+        if ( gpfn == (xen_pfn_t) -1 ) 
+            break;
+
+        if ( gpfn < start || gpfn >= end )
+        {
+            PERROR("gpfn 0x%llx doesn't belong to guest RAM addr space "
+                   "[0x%llx, 0x%llx]", gpfn, start, end);
+            goto out;
+        }
+
+        if ( !(page = xc_map_foreign_range(xch, dom, PAGE_SIZE, 
+                                           PROT_READ | PROT_WRITE, gpfn)) )
+        {
+            PERROR("Can't map guest page to dom0 (gpfn=0x%llx)", gpfn);
+            goto out;
+        }
+
+        if ( read_exact(io_fd, page, PAGE_SIZE) )
+        {
+            PERROR("Read Error: guest memory content (gpfn=0x%llx)", gpfn);
+            goto out;
+        }
+
+        munmap(page, PAGE_SIZE);
+        count++;
+    }
+
+    rc = 0;
+ out:
+    if ( page )
+        munmap(page, PAGE_SIZE);
+
+    return rc;
+}
+
+/***********************************/
+/*            HVM Context          */
+/***********************************/
+static int save_guest_hvm_ctxt(xc_interface *xch, int io_fd, uint32_t dom)
+{
+    uint32_t ctxt_size = 0;
+    uint8_t *hvm_ctxt = NULL;
+    uint32_t rec_size;
+    int rc = -1;
+
+    /* Figure out HVM context size and allocate a local buffer */
+    if ( (ctxt_size = xc_domain_hvm_getcontext(xch, dom, 0, 0)) == -1 )
+    {
+        PERROR("Can't get HVM context size");
+        goto out;
+    }
+
+    if ( (hvm_ctxt = malloc(ctxt_size)) == NULL )
+    {
+        PERROR("Can't allocate memory for HVM context");
+        goto out;
+    }
+
+    /* Retrieve HVM context from Xen and save it */
+    if ( (rec_size = xc_domain_hvm_getcontext(xch, dom, hvm_ctxt, 
+                                              ctxt_size)) == -1 )
+    {
+        PERROR("Can't receive HVM context");
+        goto out;
+    }
+
+    if ( write_exact(io_fd, &rec_size, sizeof(uint32_t)) || 
+         write_exact(io_fd, hvm_ctxt, rec_size) )
+    {
+        PERROR("Write Error: HVM context");
+        goto out;
+    }
+
+    rc = 0;
+out:
+    free (hvm_ctxt);
+    return rc;
+}
+
+static int restore_guest_hvm_ctxt(xc_interface *xch, int io_fd, uint32_t dom)
+{
+    uint32_t rec_size;
+    uint32_t ctxt_size = 0;
+    uint8_t *hvm_ctxt = NULL;
+    int frc = 0;
+    int rc = -1;
+
+    if ( read_exact(io_fd, &rec_size, sizeof(uint32_t)) )
+    {
+        PERROR("Read Error: HVM context size");
+        goto out;
+    }
+
+    if ( !rec_size )
+    {
+        PERROR("No HVM context");
+        goto out;
+    }
+
+    if ( (ctxt_size = xc_domain_hvm_getcontext(xch, dom, 0, 0)) != rec_size )
+    {
+        PERROR("Stored HVM context size isn't matched with current system");
+        goto out;
+    }
+
+    if ( !(hvm_ctxt = malloc(ctxt_size)) )
+    {
+        PERROR("Can't allocate memory");
+        goto out;
+    }
+
+    if ( read_exact(io_fd, hvm_ctxt, ctxt_size) )
+    {
+        PERROR("Read Error: HVM context");
+        goto out;
+    }
+
+    if ( (frc = xc_domain_hvm_setcontext(xch, dom, hvm_ctxt, ctxt_size)) )
+    {
+        PERROR("Can't set HVM context");
+        goto out;
+    }
+
+    rc = 0;
+out:
+    free (hvm_ctxt);
+    return rc;
+}
+
+/***********************************/
+/*         Guest Parameters        */
+/***********************************/
+static int save_guest_params(xc_interface *xch, int io_fd, uint32_t dom, 
+                             uint32_t flags, guest_params_t *params)
+{
+    xc_dominfo_t dom_info;
+
+    /* retrieve domain info */
+    if ( (xc_domain_getinfo(xch, dom, 1, &dom_info ) != 1) || 
+         dom_info.domid != dom)
+    {
+        PERROR("Can't get domain info for dom %d", dom);
+        return -1;
+    }
+
+    /* start and max of gpfn */
+    params->start_gpfn = (GUEST_RAM_BASE >> PAGE_SHIFT);
+    params->max_gpfn = (GUEST_RAM_BASE >> PAGE_SHIFT) + dom_info.nr_pages;
+
+    /* console pfn */
+    if ( xc_get_hvm_param(xch, dom, HVM_PARAM_CONSOLE_PFN,
+                          &params->console_pfn) )
+    {
+        PERROR("Can't get console gpfn");
+        return -1;
+    }
+
+    if ( xc_get_hvm_param(xch, dom, HVM_PARAM_STORE_PFN, &params->store_pfn) )
+    {
+        PERROR("Can't get store gpfn");
+        return -1;
+    }
+
+    params->max_vcpu_id = dom_info.max_vcpu_id;
+    params->flags = flags;
+
+    if ( write_exact(io_fd, params, sizeof(*params)) )
+    {
+        PERROR("Write Error: guest params");
+        return -1;
+    }
+
+    return 0;
+}
+
+static int restore_guest_params(xc_interface *xch, int io_fd, uint32_t dom,
+                                guest_params_t *params)
+{
+    xen_pfn_t nr_pfns;
+    unsigned int maxmemkb;
+
+    if ( read_exact(io_fd, params, sizeof(*params)) )
+    {
+        PERROR("Read Error: guest params");
+        return -1;
+    }
+
+    nr_pfns = params->max_gpfn - params->start_gpfn;
+    maxmemkb = (unsigned int) nr_pfns << (PAGE_SHIFT - 10);
+
+    if ( xc_domain_setmaxmem(xch, dom, maxmemkb) )
+    {
+        PERROR("Can't set memory map");
+        return -1;
+    }
+
+    if ( xc_domain_max_vcpus(xch, dom, params->max_vcpu_id + 1) )
+    {
+        PERROR("Can't set max vcpu number for domain");
+        return -1;
+    }
+
+    return 0;
+}
+
+/* Configure guest parameters after restore */
+static int set_guest_params(xc_interface *xch, int io_fd, uint32_t dom,
+                            guest_params_t *params, 
+                            unsigned int console_evtchn, domid_t console_domid,
+                            unsigned int store_evtchn, domid_t store_domid)
+{
+    int rc = 0;
+
+    if ( (rc = xc_clear_domain_page(xch, dom, params->console_pfn)) )
+    {
+        PERROR("Can't clear console page");
+        return rc;
+    }
+
+    if ( (rc = xc_clear_domain_page(xch, dom, params->store_pfn)) )
+    {
+        PERROR("Can't clear xenstore page");
+        return rc;
+    }
+
+    if ( (rc = xc_dom_gnttab_hvm_seed(xch, dom, params->console_pfn,
+                                      params->store_pfn, console_domid,
+                                      store_domid)) )
+    {
+        PERROR("Can't grant console and xenstore pages");
+        return rc;
+    }
+
+    if ( (rc = xc_set_hvm_param(xch, dom, HVM_PARAM_CONSOLE_PFN,
+                                params->console_pfn)) )
+    {
+        PERROR("Can't set console gpfn");
+        return rc;
+    }
+
+    if ( (rc = xc_set_hvm_param(xch, dom, HVM_PARAM_STORE_PFN,
+                                params->store_pfn)) )
+    {
+        PERROR("Can't set xenstore gpfn");
+        return rc;
+    }
+
+    if ( (rc = xc_set_hvm_param(xch, dom, HVM_PARAM_CONSOLE_EVTCHN,
+                                console_evtchn)) )
+    {
+        PERROR("Can't set console event channel");
+        return rc;
+    }
+
+    if ( (rc = xc_set_hvm_param(xch, dom, HVM_PARAM_STORE_EVTCHN,
+                                store_evtchn)) )
+    {
+        PERROR("Can't set xenstore event channel");
+        return rc;
+    }
+
+    return 0;
+}
+
+/***********************************/
+/*          Main Entries           */
+/***********************************/
+int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, 
+                   uint32_t max_iters, uint32_t max_factor, uint32_t flags,
+                   struct save_callbacks *callbacks, int hvm,
+                   unsigned long vm_generationid_addr)
+{
+    guest_params_t params;
+
+    if ( save_guest_params(xch, io_fd, dom, flags, &params) )
+    {
+        PERROR("Can't save guest params");
+        return -1;
+    }
+
+    if ( save_guest_memory(xch, io_fd, dom, callbacks, max_iters, max_factor, 
+                           &params) )
+    {
+        PERROR("Can't save guest memory");
+        return -1;
+    }
+
+    if ( save_guest_hvm_ctxt(xch, io_fd, dom) )
+    {
+        PERROR("Cant' save guest HVM context");
+        return -1;
+    }
+
+    return 0;
+}
+
+int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
+                      unsigned int store_evtchn, unsigned long *store_gpfn,
+                      domid_t store_domid, unsigned int console_evtchn,
+                      unsigned long *console_gpfn, domid_t console_domid,
+                      unsigned int hvm, unsigned int pae, int superpages,
+                      int no_incr_generationid, int checkpointed_stream,
+                      unsigned long *vm_generationid_addr,
+                      struct restore_callbacks *callbacks)
+{
+    guest_params_t params;
+
+    if ( restore_guest_params(xch, io_fd, dom, &params) )
+    {
+        PERROR("Can't restore guest params");
+        return -1;
+    }
+
+    if ( restore_guest_memory(xch, io_fd, dom, &params) )
+    {
+        PERROR("Can't restore memory");
+        return -1;
+    }
+
+    if ( set_guest_params(xch, io_fd, dom, &params, console_evtchn, 
+                          console_domid, store_evtchn, store_domid) )
+    {
+        PERROR("Can't setup guest params");
+        return -1;
+    }
+
+    /* setup console and store PFNs to caller */
+    *console_gpfn = params.console_pfn;
+    *store_gpfn = params.store_pfn;
+
+    /* restore HVM context of guest VM */
+    if ( restore_guest_hvm_ctxt(xch, io_fd, dom) )
+    {
+        PERROR("Can't restore HVM context");
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxc/xc_dom_arm.c b/tools/libxc/xc_dom_arm.c
index 60ac51a..d9e50d2 100644
--- a/tools/libxc/xc_dom_arm.c
+++ b/tools/libxc/xc_dom_arm.c
@@ -348,7 +348,9 @@  int arch_setup_meminit(struct xc_dom_image *dom)
         modbase += dtb_size;
     }
 
-    return 0;
+    return xc_domain_setmaxmem(dom->xch, dom->guest_domid,
+                               (dom->total_pages + NR_MAGIC_PAGES)
+                                << (PAGE_SHIFT - 10));
 }
 
 int arch_setup_bootearly(struct xc_dom_image *dom)
diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
index 18b4818..6d05d4e 100644
--- a/tools/libxc/xc_resume.c
+++ b/tools/libxc/xc_resume.c
@@ -18,11 +18,15 @@ 
 #include "xg_private.h"
 #include "xg_save_restore.h"
 
-#if defined(__i386__) || defined(__x86_64__)
+#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \
+    defined(__aarch64__)
 
+/* Only required for x86 */
+#if defined(__i386__) || defined(__x86_64__)
 #include <xen/foreign/x86_32.h>
 #include <xen/foreign/x86_64.h>
 #include <xen/hvm/params.h>
+#endif
 
 static int modify_returncode(xc_interface *xch, uint32_t domid)
 {
@@ -33,7 +37,7 @@  static int modify_returncode(xc_interface *xch, uint32_t domid)
     struct domain_info_context *dinfo = &_dinfo;
     int rc;
 
-    if ( xc_domain_getinfo(xch, domid, 1, &info) != 1 )
+    if ( xc_domain_getinfo(xch, domid, 1, &info) != 1 || info.domid != domid )
     {
         PERROR("Could not get domain info");
         return -1;
@@ -65,23 +69,23 @@  static int modify_returncode(xc_interface *xch, uint32_t domid)
     if ( (rc = xc_vcpu_getcontext(xch, domid, 0, &ctxt)) != 0 )
         return rc;
 
+#if defined(__i386__) || defined(__x86_64__)
     SET_FIELD(&ctxt, user_regs.eax, 1);
+#elif defined(__arm__) || defined(__aarch64__)
+    ctxt.c.user_regs.r0_usr = 1;
+#endif
 
     if ( (rc = xc_vcpu_setcontext(xch, domid, 0, &ctxt)) != 0 )
         return rc;
 
     return 0;
 }
-
-#else
-
+#else /* any architecture other than x86 or ARM */
 static int modify_returncode(xc_interface *xch, uint32_t domid)
 {
     return 0;
-
 }
-
-#endif
+#endif 
 
 static int xc_domain_resume_cooperative(xc_interface *xch, uint32_t domid)
 {
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 84f9c0e..ce22aa0 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -448,9 +448,6 @@ 
  *  - libxl_domain_resume
  *  - libxl_domain_remus_start
  */
-#if defined(__arm__) || defined(__aarch64__)
-#define LIBXL_HAVE_NO_SUSPEND_RESUME 1
-#endif
 
 /*
  * LIBXL_HAVE_DEVICE_PCI_SEIZE
diff --git a/tools/misc/Makefile b/tools/misc/Makefile
index 69b1817..f4ea7ab 100644
--- a/tools/misc/Makefile
+++ b/tools/misc/Makefile
@@ -11,7 +11,7 @@  HDRS     = $(wildcard *.h)
 
 TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xencov
 TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash xen-lowmemd xen-mfndump
-TARGETS-$(CONFIG_MIGRATE) += xen-hptool
+TARGETS-$(CONFIG_X86) += xen-hptool
 TARGETS := $(TARGETS-y)
 
 SUBDIRS := $(SUBDIRS-y)
@@ -23,7 +23,7 @@  INSTALL_BIN := $(INSTALL_BIN-y)
 INSTALL_SBIN-y := xen-bugtool xen-python-path xenperf xenpm xen-tmem-list-parse gtraceview \
 	gtracestat xenlockprof xenwatchdogd xen-ringwatch xencov
 INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx xen-hvmcrash xen-lowmemd xen-mfndump
-INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
+INSTALL_SBIN-$(CONFIG_X86) += xen-hptool
 INSTALL_SBIN := $(INSTALL_SBIN-y)
 
 INSTALL_PRIVBIN-y := xenpvnetboot