Merge ~paelzer/ubuntu/+source/qemu:fix-1587065-guest-agent-double-freeze into ubuntu/+source/qemu:ubuntu/xenial-devel

[Christian Ehrhardt ]

[Christian Ehrhardt  (paelzer)]

Testable in PPA [1] using the test steps from the bug [2]

[1]: https://launchpad.net/~ci-train-ppa-service/+archive/ubuntu/3310
[2]: https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1587065

[Andreas Hasenack (ahasenack)]

So far problem reproduced. I'm gonna need to setup nested kvm to try this on xenial.

[Andreas Hasenack (ahasenack)]

I used nested KVM, so the xenial host for the test was a kvm already, and inside it I spun the x-freeze guest. The problem was reproduced, but after I updated qemu on the fake host (not the real one), it kept happening, even after a power cycle. Maybe the actual host needs to be updated, but I don't have a bare metal machine to install xenial on and try it. Is there another way to test?

review: Needs Information

[Christian Ehrhardt  (paelzer)]

Update the innermost Guest as it is the guest tools that are changed.

[Andreas Hasenack (ahasenack)]

Confirmed working.

review: Approve

[Christian Ehrhardt  (paelzer)]

Tagged and Uploaded

diff --git a/debian/changelog b/debian/changelog
index 2aa1bf8..67ae9b8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+qemu (1:2.5+dfsg-5ubuntu10.31) xenial; urgency=medium
+
+  * d/p/ubuntu/lp-1587065-qga-ignore-EBUSY-when-freezing-a-filesystem.patch:
+    Fix qemu-guest-agent failing to freeze filesystems that were mounted
+    multiple times like bind mounts. (LP: #1587065).
+
+ -- Christian Ehrhardt <[e-mail]>  Thu, 28 Jun 2018 11:35:05 +0200
+
 qemu (1:2.5+dfsg-5ubuntu10.30) xenial-security; urgency=medium
 
   * SECURITY UPDATE: Speculative Store Bypass
diff --git a/debian/patches/series b/debian/patches/series
index 764d53c..c0a8392 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -237,3 +237,4 @@ CVE-2018-7550.patch
 CVE-2018-3639.patch
 CVE-2018-3639-2.patch
 CVE-2018-3639-3.patch
+ubuntu/lp-1587065-qga-ignore-EBUSY-when-freezing-a-filesystem.patch
diff --git a/debian/patches/ubuntu/lp-1587065-qga-ignore-EBUSY-when-freezing-a-filesystem.patch b/debian/patches/ubuntu/lp-1587065-qga-ignore-EBUSY-when-freezing-a-filesystem.patch
new file mode 100644
index 0000000..5363bd9
--- /dev/null
+++ b/debian/patches/ubuntu/lp-1587065-qga-ignore-EBUSY-when-freezing-a-filesystem.patch
@@ -0,0 +1,44 @@
+From ce2eb6c4a044d809caf4dc4e08aed77678f9760e Mon Sep 17 00:00:00 2001
+From: Peter Lieven <[e-mail]>
+Date: Tue, 31 Jan 2017 16:36:34 +0100
+Subject: [PATCH] qga: ignore EBUSY when freezing a filesystem
+
+the current implementation fails if we try to freeze an
+already frozen filesystem. This can happen if a filesystem
+is mounted more than once (e.g. with a bind mount).
+
+Suggested-by: Christian Theune <[e-mail]>
+Cc: [e-mail]
+Signed-off-by: Peter Lieven <[e-mail]>
+Reviewed-by: Paolo Bonzini <[e-mail]>
+Signed-off-by: Michael Roth <[e-mail]>
+
+Origin: backport, https://git.qemu.org/?p=qemu.git;a=commitdiff;h=ce2eb6c4a044d809c
+Bug-Ubuntu: https://bugs.launchpad.net/bugs/1587065
+Last-Update: 2018-06-28
+
+---
+ qga/commands-posix.c | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+--- a/qga/commands-posix.c
++++ b/qga/commands-posix.c
+@@ -1255,6 +1255,9 @@ int64_t qmp_guest_fsfreeze_freeze_list(b
+          * filesytems may not implement fsfreeze for less obvious reasons.
+          * these will report EOPNOTSUPP. we simply ignore these when tallying
+          * the number of frozen filesystems.
++         * if a filesystem is mounted more than once (aka bind mount) a
++         * consecutive attempt to freeze an already frozen filesystem will
++         * return EBUSY.
+          *
+          * any other error means a failure to freeze a filesystem we
+          * expect to be freezable, so return an error in those cases
+@@ -1262,7 +1265,7 @@ int64_t qmp_guest_fsfreeze_freeze_list(b
+          */
+         ret = ioctl(fd, FIFREEZE);
+         if (ret == -1) {
+-            if (errno != EOPNOTSUPP) {
++            if (errno != EOPNOTSUPP && errno != EBUSY) {
+                 error_setg_errno(errp, errno, "failed to freeze %s",
+                                  mount->dirname);
+                 close(fd);
diff --git a/dump.c b/dump.c
new file mode 100644
index 0000000..78b7d84
--- /dev/null
+++ b/dump.c
@@ -0,0 +1,1714 @@
+/*
+ * QEMU dump
+ *
+ * Copyright Fujitsu, Corp. 2011, 2012
+ *
+ * Authors:
+ *     Wen Congyang <[e-mail]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "elf.h"
+#include "cpu.h"
+#include "exec/cpu-all.h"
+#include "exec/hwaddr.h"
+#include "monitor/monitor.h"
+#include "sysemu/kvm.h"
+#include "sysemu/dump.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/memory_mapping.h"
+#include "sysemu/cpus.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qmp-commands.h"
+
+#include <zlib.h>
+#ifdef CONFIG_LZO
+#include <lzo/lzo1x.h>
+#endif
+#ifdef CONFIG_SNAPPY
+#include <snappy-c.h>
+#endif
+#ifndef ELF_MACHINE_UNAME
+#define ELF_MACHINE_UNAME "Unknown"
+#endif
+
+uint16_t cpu_to_dump16(DumpState *s, uint16_t val)
+{
+    if (s->dump_info.d_endian == ELFDATA2LSB) {
+        val = cpu_to_le16(val);
+    } else {
+        val = cpu_to_be16(val);
+    }
+
+    return val;
+}
+
+uint32_t cpu_to_dump32(DumpState *s, uint32_t val)
+{
+    if (s->dump_info.d_endian == ELFDATA2LSB) {
+        val = cpu_to_le32(val);
+    } else {
+        val = cpu_to_be32(val);
+    }
+
+    return val;
+}
+
+uint64_t cpu_to_dump64(DumpState *s, uint64_t val)
+{
+    if (s->dump_info.d_endian == ELFDATA2LSB) {
+        val = cpu_to_le64(val);
+    } else {
+        val = cpu_to_be64(val);
+    }
+
+    return val;
+}
+
+static int dump_cleanup(DumpState *s)
+{
+    guest_phys_blocks_free(&s->guest_phys_blocks);
+    memory_mapping_list_free(&s->list);
+    close(s->fd);
+    if (s->resume) {
+        vm_start();
+    }
+
+    return 0;
+}
+
+static void dump_error(DumpState *s, const char *reason, Error **errp)
+{
+    dump_cleanup(s);
+    error_setg(errp, "%s", reason);
+}
+
+static int fd_write_vmcore(const void *buf, size_t size, void *opaque)
+{
+    DumpState *s = opaque;
+    size_t written_size;
+
+    written_size = qemu_write_full(s->fd, buf, size);
+    if (written_size != size) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static void write_elf64_header(DumpState *s, Error **errp)
+{
+    Elf64_Ehdr elf_header;
+    int ret;
+
+    memset(&elf_header, 0, sizeof(Elf64_Ehdr));
+    memcpy(&elf_header, ELFMAG, SELFMAG);
+    elf_header.e_ident[EI_CLASS] = ELFCLASS64;
+    elf_header.e_ident[EI_DATA] = s->dump_info.d_endian;
+    elf_header.e_ident[EI_VERSION] = EV_CURRENT;
+    elf_header.e_type = cpu_to_dump16(s, ET_CORE);
+    elf_header.e_machine = cpu_to_dump16(s, s->dump_info.d_machine);
+    elf_header.e_version = cpu_to_dump32(s, EV_CURRENT);
+    elf_header.e_ehsize = cpu_to_dump16(s, sizeof(elf_header));
+    elf_header.e_phoff = cpu_to_dump64(s, sizeof(Elf64_Ehdr));
+    elf_header.e_phentsize = cpu_to_dump16(s, sizeof(Elf64_Phdr));
+    elf_header.e_phnum = cpu_to_dump16(s, s->phdr_num);
+    if (s->have_section) {
+        uint64_t shoff = sizeof(Elf64_Ehdr) + sizeof(Elf64_Phdr) * s->sh_info;
+
+        elf_header.e_shoff = cpu_to_dump64(s, shoff);
+        elf_header.e_shentsize = cpu_to_dump16(s, sizeof(Elf64_Shdr));
+        elf_header.e_shnum = cpu_to_dump16(s, 1);
+    }
+
+    ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write elf header", errp);
+    }
+}
+
+static void write_elf32_header(DumpState *s, Error **errp)
+{
+    Elf32_Ehdr elf_header;
+    int ret;
+
+    memset(&elf_header, 0, sizeof(Elf32_Ehdr));
+    memcpy(&elf_header, ELFMAG, SELFMAG);
+    elf_header.e_ident[EI_CLASS] = ELFCLASS32;
+    elf_header.e_ident[EI_DATA] = s->dump_info.d_endian;
+    elf_header.e_ident[EI_VERSION] = EV_CURRENT;
+    elf_header.e_type = cpu_to_dump16(s, ET_CORE);
+    elf_header.e_machine = cpu_to_dump16(s, s->dump_info.d_machine);
+    elf_header.e_version = cpu_to_dump32(s, EV_CURRENT);
+    elf_header.e_ehsize = cpu_to_dump16(s, sizeof(elf_header));
+    elf_header.e_phoff = cpu_to_dump32(s, sizeof(Elf32_Ehdr));
+    elf_header.e_phentsize = cpu_to_dump16(s, sizeof(Elf32_Phdr));
+    elf_header.e_phnum = cpu_to_dump16(s, s->phdr_num);
+    if (s->have_section) {
+        uint32_t shoff = sizeof(Elf32_Ehdr) + sizeof(Elf32_Phdr) * s->sh_info;
+
+        elf_header.e_shoff = cpu_to_dump32(s, shoff);
+        elf_header.e_shentsize = cpu_to_dump16(s, sizeof(Elf32_Shdr));
+        elf_header.e_shnum = cpu_to_dump16(s, 1);
+    }
+
+    ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write elf header", errp);
+    }
+}
+
+static void write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
+                             int phdr_index, hwaddr offset,
+                             hwaddr filesz, Error **errp)
+{
+    Elf64_Phdr phdr;
+    int ret;
+
+    memset(&phdr, 0, sizeof(Elf64_Phdr));
+    phdr.p_type = cpu_to_dump32(s, PT_LOAD);
+    phdr.p_offset = cpu_to_dump64(s, offset);
+    phdr.p_paddr = cpu_to_dump64(s, memory_mapping->phys_addr);
+    phdr.p_filesz = cpu_to_dump64(s, filesz);
+    phdr.p_memsz = cpu_to_dump64(s, memory_mapping->length);
+    phdr.p_vaddr = cpu_to_dump64(s, memory_mapping->virt_addr);
+
+    assert(memory_mapping->length >= filesz);
+
+    ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write program header table", errp);
+    }
+}
+
+static void write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
+                             int phdr_index, hwaddr offset,
+                             hwaddr filesz, Error **errp)
+{
+    Elf32_Phdr phdr;
+    int ret;
+
+    memset(&phdr, 0, sizeof(Elf32_Phdr));
+    phdr.p_type = cpu_to_dump32(s, PT_LOAD);
+    phdr.p_offset = cpu_to_dump32(s, offset);
+    phdr.p_paddr = cpu_to_dump32(s, memory_mapping->phys_addr);
+    phdr.p_filesz = cpu_to_dump32(s, filesz);
+    phdr.p_memsz = cpu_to_dump32(s, memory_mapping->length);
+    phdr.p_vaddr = cpu_to_dump32(s, memory_mapping->virt_addr);
+
+    assert(memory_mapping->length >= filesz);
+
+    ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write program header table", errp);
+    }
+}
+
+static void write_elf64_note(DumpState *s, Error **errp)
+{
+    Elf64_Phdr phdr;
+    hwaddr begin = s->memory_offset - s->note_size;
+    int ret;
+
+    memset(&phdr, 0, sizeof(Elf64_Phdr));
+    phdr.p_type = cpu_to_dump32(s, PT_NOTE);
+    phdr.p_offset = cpu_to_dump64(s, begin);
+    phdr.p_paddr = 0;
+    phdr.p_filesz = cpu_to_dump64(s, s->note_size);
+    phdr.p_memsz = cpu_to_dump64(s, s->note_size);
+    phdr.p_vaddr = 0;
+
+    ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write program header table", errp);
+    }
+}
+
+static inline int cpu_index(CPUState *cpu)
+{
+    return cpu->cpu_index + 1;
+}
+
+static void write_elf64_notes(WriteCoreDumpFunction f, DumpState *s,
+                              Error **errp)
+{
+    CPUState *cpu;
+    int ret;
+    int id;
+
+    CPU_FOREACH(cpu) {
+        id = cpu_index(cpu);
+        ret = cpu_write_elf64_note(f, cpu, id, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to write elf notes", errp);
+            return;
+        }
+    }
+
+    CPU_FOREACH(cpu) {
+        ret = cpu_write_elf64_qemunote(f, cpu, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to write CPU status", errp);
+            return;
+        }
+    }
+}
+
+static void write_elf32_note(DumpState *s, Error **errp)
+{
+    hwaddr begin = s->memory_offset - s->note_size;
+    Elf32_Phdr phdr;
+    int ret;
+
+    memset(&phdr, 0, sizeof(Elf32_Phdr));
+    phdr.p_type = cpu_to_dump32(s, PT_NOTE);
+    phdr.p_offset = cpu_to_dump32(s, begin);
+    phdr.p_paddr = 0;
+    phdr.p_filesz = cpu_to_dump32(s, s->note_size);
+    phdr.p_memsz = cpu_to_dump32(s, s->note_size);
+    phdr.p_vaddr = 0;
+
+    ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write program header table", errp);
+    }
+}
+
+static void write_elf32_notes(WriteCoreDumpFunction f, DumpState *s,
+                              Error **errp)
+{
+    CPUState *cpu;
+    int ret;
+    int id;
+
+    CPU_FOREACH(cpu) {
+        id = cpu_index(cpu);
+        ret = cpu_write_elf32_note(f, cpu, id, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to write elf notes", errp);
+            return;
+        }
+    }
+
+    CPU_FOREACH(cpu) {
+        ret = cpu_write_elf32_qemunote(f, cpu, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to write CPU status", errp);
+            return;
+        }
+    }
+}
+
+static void write_elf_section(DumpState *s, int type, Error **errp)
+{
+    Elf32_Shdr shdr32;
+    Elf64_Shdr shdr64;
+    int shdr_size;
+    void *shdr;
+    int ret;
+
+    if (type == 0) {
+        shdr_size = sizeof(Elf32_Shdr);
+        memset(&shdr32, 0, shdr_size);
+        shdr32.sh_info = cpu_to_dump32(s, s->sh_info);
+        shdr = &shdr32;
+    } else {
+        shdr_size = sizeof(Elf64_Shdr);
+        memset(&shdr64, 0, shdr_size);
+        shdr64.sh_info = cpu_to_dump32(s, s->sh_info);
+        shdr = &shdr64;
+    }
+
+    ret = fd_write_vmcore(&shdr, shdr_size, s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write section header table", errp);
+    }
+}
+
+static void write_data(DumpState *s, void *buf, int length, Error **errp)
+{
+    int ret;
+
+    ret = fd_write_vmcore(buf, length, s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to save memory", errp);
+    }
+}
+
+/* write the memory to vmcore. 1 page per I/O. */
+static void write_memory(DumpState *s, GuestPhysBlock *block, ram_addr_t start,
+                         int64_t size, Error **errp)
+{
+    int64_t i;
+    Error *local_err = NULL;
+
+    for (i = 0; i < size / TARGET_PAGE_SIZE; i++) {
+        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
+                   TARGET_PAGE_SIZE, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+    if ((size % TARGET_PAGE_SIZE) != 0) {
+        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
+                   size % TARGET_PAGE_SIZE, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+}
+
+/* get the memory's offset and size in the vmcore */
+static void get_offset_range(hwaddr phys_addr,
+                             ram_addr_t mapping_length,
+                             DumpState *s,
+                             hwaddr *p_offset,
+                             hwaddr *p_filesz)
+{
+    GuestPhysBlock *block;
+    hwaddr offset = s->memory_offset;
+    int64_t size_in_block, start;
+
+    /* When the memory is not stored into vmcore, offset will be -1 */
+    *p_offset = -1;
+    *p_filesz = 0;
+
+    if (s->has_filter) {
+        if (phys_addr < s->begin || phys_addr >= s->begin + s->length) {
+            return;
+        }
+    }
+
+    QTAILQ_FOREACH(block, &s->guest_phys_blocks.head, next) {
+        if (s->has_filter) {
+            if (block->target_start >= s->begin + s->length ||
+                block->target_end <= s->begin) {
+                /* This block is out of the range */
+                continue;
+            }
+
+            if (s->begin <= block->target_start) {
+                start = block->target_start;
+            } else {
+                start = s->begin;
+            }
+
+            size_in_block = block->target_end - start;
+            if (s->begin + s->length < block->target_end) {
+                size_in_block -= block->target_end - (s->begin + s->length);
+            }
+        } else {
+            start = block->target_start;
+            size_in_block = block->target_end - block->target_start;
+        }
+
+        if (phys_addr >= start && phys_addr < start + size_in_block) {
+            *p_offset = phys_addr - start + offset;
+
+            /* The offset range mapped from the vmcore file must not spill over
+             * the GuestPhysBlock, clamp it. The rest of the mapping will be
+             * zero-filled in memory at load time; see
+             * <http://refspecs.linuxbase.org/elf/gabi4+/ch5.pheader.html>.
+             */
+            *p_filesz = phys_addr + mapping_length <= start + size_in_block ?
+                        mapping_length :
+                        size_in_block - (phys_addr - start);
+            return;
+        }
+
+        offset += size_in_block;
+    }
+}
+
+static void write_elf_loads(DumpState *s, Error **errp)
+{
+    hwaddr offset, filesz;
+    MemoryMapping *memory_mapping;
+    uint32_t phdr_index = 1;
+    uint32_t max_index;
+    Error *local_err = NULL;
+
+    if (s->have_section) {
+        max_index = s->sh_info;
+    } else {
+        max_index = s->phdr_num;
+    }
+
+    QTAILQ_FOREACH(memory_mapping, &s->list.head, next) {
+        get_offset_range(memory_mapping->phys_addr,
+                         memory_mapping->length,
+                         s, &offset, &filesz);
+        if (s->dump_info.d_class == ELFCLASS64) {
+            write_elf64_load(s, memory_mapping, phdr_index++, offset,
+                             filesz, &local_err);
+        } else {
+            write_elf32_load(s, memory_mapping, phdr_index++, offset,
+                             filesz, &local_err);
+        }
+
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        if (phdr_index >= max_index) {
+            break;
+        }
+    }
+}
+
+/* write elf header, PT_NOTE and elf note to vmcore. */
+static void dump_begin(DumpState *s, Error **errp)
+{
+    Error *local_err = NULL;
+
+    /*
+     * the vmcore's format is:
+     *   --------------
+     *   |  elf header |
+     *   --------------
+     *   |  PT_NOTE    |
+     *   --------------
+     *   |  PT_LOAD    |
+     *   --------------
+     *   |  ......     |
+     *   --------------
+     *   |  PT_LOAD    |
+     *   --------------
+     *   |  sec_hdr    |
+     *   --------------
+     *   |  elf note   |
+     *   --------------
+     *   |  memory     |
+     *   --------------
+     *
+     * we only know where the memory is saved after we write elf note into
+     * vmcore.
+     */
+
+    /* write elf header to vmcore */
+    if (s->dump_info.d_class == ELFCLASS64) {
+        write_elf64_header(s, &local_err);
+    } else {
+        write_elf32_header(s, &local_err);
+    }
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (s->dump_info.d_class == ELFCLASS64) {
+        /* write PT_NOTE to vmcore */
+        write_elf64_note(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        /* write all PT_LOAD to vmcore */
+        write_elf_loads(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        /* write section to vmcore */
+        if (s->have_section) {
+            write_elf_section(s, 1, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+
+        /* write notes to vmcore */
+        write_elf64_notes(fd_write_vmcore, s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    } else {
+        /* write PT_NOTE to vmcore */
+        write_elf32_note(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        /* write all PT_LOAD to vmcore */
+        write_elf_loads(s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        /* write section to vmcore */
+        if (s->have_section) {
+            write_elf_section(s, 0, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+
+        /* write notes to vmcore */
+        write_elf32_notes(fd_write_vmcore, s, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+}
+
+static void dump_completed(DumpState *s)
+{
+    dump_cleanup(s);
+}
+
+static int get_next_block(DumpState *s, GuestPhysBlock *block)
+{
+    while (1) {
+        block = QTAILQ_NEXT(block, next);
+        if (!block) {
+            /* no more block */
+            return 1;
+        }
+
+        s->start = 0;
+        s->next_block = block;
+        if (s->has_filter) {
+            if (block->target_start >= s->begin + s->length ||
+                block->target_end <= s->begin) {
+                /* This block is out of the range */
+                continue;
+            }
+
+            if (s->begin > block->target_start) {
+                s->start = s->begin - block->target_start;
+            }
+        }
+
+        return 0;
+    }
+}
+
+/* write all memory to vmcore */
+static void dump_iterate(DumpState *s, Error **errp)
+{
+    GuestPhysBlock *block;
+    int64_t size;
+    Error *local_err = NULL;
+
+    do {
+        block = s->next_block;
+
+        size = block->target_end - block->target_start;
+        if (s->has_filter) {
+            size -= s->start;
+            if (s->begin + s->length < block->target_end) {
+                size -= block->target_end - (s->begin + s->length);
+            }
+        }
+        write_memory(s, block, s->start, size, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+    } while (!get_next_block(s, block));
+
+    dump_completed(s);
+}
+
+static void create_vmcore(DumpState *s, Error **errp)
+{
+    Error *local_err = NULL;
+
+    dump_begin(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    dump_iterate(s, errp);
+}
+
+static int write_start_flat_header(int fd)
+{
+    MakedumpfileHeader *mh;
+    int ret = 0;
+
+    QEMU_BUILD_BUG_ON(sizeof *mh > MAX_SIZE_MDF_HEADER);
+    mh = g_malloc0(MAX_SIZE_MDF_HEADER);
+
+    memcpy(mh->signature, MAKEDUMPFILE_SIGNATURE,
+           MIN(sizeof mh->signature, sizeof MAKEDUMPFILE_SIGNATURE));
+
+    mh->type = cpu_to_be64(TYPE_FLAT_HEADER);
+    mh->version = cpu_to_be64(VERSION_FLAT_HEADER);
+
+    size_t written_size;
+    written_size = qemu_write_full(fd, mh, MAX_SIZE_MDF_HEADER);
+    if (written_size != MAX_SIZE_MDF_HEADER) {
+        ret = -1;
+    }
+
+    g_free(mh);
+    return ret;
+}
+
+static int write_end_flat_header(int fd)
+{
+    MakedumpfileDataHeader mdh;
+
+    mdh.offset = END_FLAG_FLAT_HEADER;
+    mdh.buf_size = END_FLAG_FLAT_HEADER;
+
+    size_t written_size;
+    written_size = qemu_write_full(fd, &mdh, sizeof(mdh));
+    if (written_size != sizeof(mdh)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int write_buffer(int fd, off_t offset, const void *buf, size_t size)
+{
+    size_t written_size;
+    MakedumpfileDataHeader mdh;
+
+    mdh.offset = cpu_to_be64(offset);
+    mdh.buf_size = cpu_to_be64(size);
+
+    written_size = qemu_write_full(fd, &mdh, sizeof(mdh));
+    if (written_size != sizeof(mdh)) {
+        return -1;
+    }
+
+    written_size = qemu_write_full(fd, buf, size);
+    if (written_size != size) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int buf_write_note(const void *buf, size_t size, void *opaque)
+{
+    DumpState *s = opaque;
+
+    /* note_buf is not enough */
+    if (s->note_buf_offset + size > s->note_size) {
+        return -1;
+    }
+
+    memcpy(s->note_buf + s->note_buf_offset, buf, size);
+
+    s->note_buf_offset += size;
+
+    return 0;
+}
+
+/* write common header, sub header and elf note to vmcore */
+static void create_header32(DumpState *s, Error **errp)
+{
+    DiskDumpHeader32 *dh = NULL;
+    KdumpSubHeader32 *kh = NULL;
+    size_t size;
+    uint32_t block_size;
+    uint32_t sub_hdr_size;
+    uint32_t bitmap_blocks;
+    uint32_t status = 0;
+    uint64_t offset_note;
+    Error *local_err = NULL;
+
+    /* write common header, the version of kdump-compressed format is 6th */
+    size = sizeof(DiskDumpHeader32);
+    dh = g_malloc0(size);
+
+    strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
+    dh->header_version = cpu_to_dump32(s, 6);
+    block_size = TARGET_PAGE_SIZE;
+    dh->block_size = cpu_to_dump32(s, block_size);
+    sub_hdr_size = sizeof(struct KdumpSubHeader32) + s->note_size;
+    sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
+    dh->sub_hdr_size = cpu_to_dump32(s, sub_hdr_size);
+    /* dh->max_mapnr may be truncated, full 64bit is in kh.max_mapnr_64 */
+    dh->max_mapnr = cpu_to_dump32(s, MIN(s->max_mapnr, UINT_MAX));
+    dh->nr_cpus = cpu_to_dump32(s, s->nr_cpus);
+    bitmap_blocks = DIV_ROUND_UP(s->len_dump_bitmap, block_size) * 2;
+    dh->bitmap_blocks = cpu_to_dump32(s, bitmap_blocks);
+    strncpy(dh->utsname.machine, ELF_MACHINE_UNAME, sizeof(dh->utsname.machine));
+
+    if (s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) {
+        status |= DUMP_DH_COMPRESSED_ZLIB;
+    }
+#ifdef CONFIG_LZO
+    if (s->flag_compress & DUMP_DH_COMPRESSED_LZO) {
+        status |= DUMP_DH_COMPRESSED_LZO;
+    }
+#endif
+#ifdef CONFIG_SNAPPY
+    if (s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) {
+        status |= DUMP_DH_COMPRESSED_SNAPPY;
+    }
+#endif
+    dh->status = cpu_to_dump32(s, status);
+
+    if (write_buffer(s->fd, 0, dh, size) < 0) {
+        dump_error(s, "dump: failed to write disk dump header", errp);
+        goto out;
+    }
+
+    /* write sub header */
+    size = sizeof(KdumpSubHeader32);
+    kh = g_malloc0(size);
+
+    /* 64bit max_mapnr_64 */
+    kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
+    kh->phys_base = cpu_to_dump32(s, PHYS_BASE);
+    kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
+
+    offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
+    kh->offset_note = cpu_to_dump64(s, offset_note);
+    kh->note_size = cpu_to_dump32(s, s->note_size);
+
+    if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
+                     block_size, kh, size) < 0) {
+        dump_error(s, "dump: failed to write kdump sub header", errp);
+        goto out;
+    }
+
+    /* write note */
+    s->note_buf = g_malloc0(s->note_size);
+    s->note_buf_offset = 0;
+
+    /* use s->note_buf to store notes temporarily */
+    write_elf32_notes(buf_write_note, s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto out;
+    }
+    if (write_buffer(s->fd, offset_note, s->note_buf,
+                     s->note_size) < 0) {
+        dump_error(s, "dump: failed to write notes", errp);
+        goto out;
+    }
+
+    /* get offset of dump_bitmap */
+    s->offset_dump_bitmap = (DISKDUMP_HEADER_BLOCKS + sub_hdr_size) *
+                             block_size;
+
+    /* get offset of page */
+    s->offset_page = (DISKDUMP_HEADER_BLOCKS + sub_hdr_size + bitmap_blocks) *
+                     block_size;
+
+out:
+    g_free(dh);
+    g_free(kh);
+    g_free(s->note_buf);
+}
+
+/* write common header, sub header and elf note to vmcore */
+static void create_header64(DumpState *s, Error **errp)
+{
+    DiskDumpHeader64 *dh = NULL;
+    KdumpSubHeader64 *kh = NULL;
+    size_t size;
+    uint32_t block_size;
+    uint32_t sub_hdr_size;
+    uint32_t bitmap_blocks;
+    uint32_t status = 0;
+    uint64_t offset_note;
+    Error *local_err = NULL;
+
+    /* write common header, the version of kdump-compressed format is 6th */
+    size = sizeof(DiskDumpHeader64);
+    dh = g_malloc0(size);
+
+    strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
+    dh->header_version = cpu_to_dump32(s, 6);
+    block_size = TARGET_PAGE_SIZE;
+    dh->block_size = cpu_to_dump32(s, block_size);
+    sub_hdr_size = sizeof(struct KdumpSubHeader64) + s->note_size;
+    sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
+    dh->sub_hdr_size = cpu_to_dump32(s, sub_hdr_size);
+    /* dh->max_mapnr may be truncated, full 64bit is in kh.max_mapnr_64 */
+    dh->max_mapnr = cpu_to_dump32(s, MIN(s->max_mapnr, UINT_MAX));
+    dh->nr_cpus = cpu_to_dump32(s, s->nr_cpus);
+    bitmap_blocks = DIV_ROUND_UP(s->len_dump_bitmap, block_size) * 2;
+    dh->bitmap_blocks = cpu_to_dump32(s, bitmap_blocks);
+    strncpy(dh->utsname.machine, ELF_MACHINE_UNAME, sizeof(dh->utsname.machine));
+
+    if (s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) {
+        status |= DUMP_DH_COMPRESSED_ZLIB;
+    }
+#ifdef CONFIG_LZO
+    if (s->flag_compress & DUMP_DH_COMPRESSED_LZO) {
+        status |= DUMP_DH_COMPRESSED_LZO;
+    }
+#endif
+#ifdef CONFIG_SNAPPY
+    if (s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) {
+        status |= DUMP_DH_COMPRESSED_SNAPPY;
+    }
+#endif
+    dh->status = cpu_to_dump32(s, status);
+
+    if (write_buffer(s->fd, 0, dh, size) < 0) {
+        dump_error(s, "dump: failed to write disk dump header", errp);
+        goto out;
+    }
+
+    /* write sub header */
+    size = sizeof(KdumpSubHeader64);
+    kh = g_malloc0(size);
+
+    /* 64bit max_mapnr_64 */
+    kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
+    kh->phys_base = cpu_to_dump64(s, PHYS_BASE);
+    kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
+
+    offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
+    kh->offset_note = cpu_to_dump64(s, offset_note);
+    kh->note_size = cpu_to_dump64(s, s->note_size);
+
+    if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
+                     block_size, kh, size) < 0) {
+        dump_error(s, "dump: failed to write kdump sub header", errp);
+        goto out;
+    }
+
+    /* write note */
+    s->note_buf = g_malloc0(s->note_size);
+    s->note_buf_offset = 0;
+
+    /* use s->note_buf to store notes temporarily */
+    write_elf64_notes(buf_write_note, s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto out;
+    }
+
+    if (write_buffer(s->fd, offset_note, s->note_buf,
+                     s->note_size) < 0) {
+        dump_error(s, "dump: failed to write notes", errp);
+        goto out;
+    }
+
+    /* get offset of dump_bitmap */
+    s->offset_dump_bitmap = (DISKDUMP_HEADER_BLOCKS + sub_hdr_size) *
+                             block_size;
+
+    /* get offset of page */
+    s->offset_page = (DISKDUMP_HEADER_BLOCKS + sub_hdr_size + bitmap_blocks) *
+                     block_size;
+
+out:
+    g_free(dh);
+    g_free(kh);
+    g_free(s->note_buf);
+}
+
+static void write_dump_header(DumpState *s, Error **errp)
+{
+     Error *local_err = NULL;
+
+    if (s->dump_info.d_class == ELFCLASS32) {
+        create_header32(s, &local_err);
+    } else {
+        create_header64(s, &local_err);
+    }
+    if (local_err) {
+        error_propagate(errp, local_err);
+    }
+}
+
+/*
+ * set dump_bitmap sequencely. the bit before last_pfn is not allowed to be
+ * rewritten, so if need to set the first bit, set last_pfn and pfn to 0.
+ * set_dump_bitmap will always leave the recently set bit un-sync. And setting
+ * (last bit + sizeof(buf) * 8) to 0 will do flushing the content in buf into
+ * vmcore, ie. synchronizing un-sync bit into vmcore.
+ */
+static int set_dump_bitmap(uint64_t last_pfn, uint64_t pfn, bool value,
+                           uint8_t *buf, DumpState *s)
+{
+    off_t old_offset, new_offset;
+    off_t offset_bitmap1, offset_bitmap2;
+    uint32_t byte, bit;
+
+    /* should not set the previous place */
+    assert(last_pfn <= pfn);
+
+    /*
+     * if the bit needed to be set is not cached in buf, flush the data in buf
+     * to vmcore firstly.
+     * making new_offset be bigger than old_offset can also sync remained data
+     * into vmcore.
+     */
+    old_offset = BUFSIZE_BITMAP * (last_pfn / PFN_BUFBITMAP);
+    new_offset = BUFSIZE_BITMAP * (pfn / PFN_BUFBITMAP);
+
+    while (old_offset < new_offset) {
+        /* calculate the offset and write dump_bitmap */
+        offset_bitmap1 = s->offset_dump_bitmap + old_offset;
+        if (write_buffer(s->fd, offset_bitmap1, buf,
+                         BUFSIZE_BITMAP) < 0) {
+            return -1;
+        }
+
+        /* dump level 1 is chosen, so 1st and 2nd bitmap are same */
+        offset_bitmap2 = s->offset_dump_bitmap + s->len_dump_bitmap +
+                         old_offset;
+        if (write_buffer(s->fd, offset_bitmap2, buf,
+                         BUFSIZE_BITMAP) < 0) {
+            return -1;
+        }
+
+        memset(buf, 0, BUFSIZE_BITMAP);
+        old_offset += BUFSIZE_BITMAP;
+    }
+
+    /* get the exact place of the bit in the buf, and set it */
+    byte = (pfn % PFN_BUFBITMAP) / CHAR_BIT;
+    bit = (pfn % PFN_BUFBITMAP) % CHAR_BIT;
+    if (value) {
+        buf[byte] |= 1u << bit;
+    } else {
+        buf[byte] &= ~(1u << bit);
+    }
+
+    return 0;
+}
+
+/*
+ * exam every page and return the page frame number and the address of the page.
+ * bufptr can be NULL. note: the blocks here is supposed to reflect guest-phys
+ * blocks, so block->target_start and block->target_end should be interal
+ * multiples of the target page size.
+ */
+static bool get_next_page(GuestPhysBlock **blockptr, uint64_t *pfnptr,
+                          uint8_t **bufptr, DumpState *s)
+{
+    GuestPhysBlock *block = *blockptr;
+    hwaddr addr;
+    uint8_t *buf;
+
+    /* block == NULL means the start of the iteration */
+    if (!block) {
+        block = QTAILQ_FIRST(&s->guest_phys_blocks.head);
+        *blockptr = block;
+        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
+        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
+        *pfnptr = paddr_to_pfn(block->target_start);
+        if (bufptr) {
+            *bufptr = block->host_addr;
+        }
+        return true;
+    }
+
+    *pfnptr = *pfnptr + 1;
+    addr = pfn_to_paddr(*pfnptr);
+
+    if ((addr >= block->target_start) &&
+        (addr + TARGET_PAGE_SIZE <= block->target_end)) {
+        buf = block->host_addr + (addr - block->target_start);
+    } else {
+        /* the next page is in the next block */
+        block = QTAILQ_NEXT(block, next);
+        *blockptr = block;
+        if (!block) {
+            return false;
+        }
+        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
+        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
+        *pfnptr = paddr_to_pfn(block->target_start);
+        buf = block->host_addr;
+    }
+
+    if (bufptr) {
+        *bufptr = buf;
+    }
+
+    return true;
+}
+
+static void write_dump_bitmap(DumpState *s, Error **errp)
+{
+    int ret = 0;
+    uint64_t last_pfn, pfn;
+    void *dump_bitmap_buf;
+    size_t num_dumpable;
+    GuestPhysBlock *block_iter = NULL;
+
+    /* dump_bitmap_buf is used to store dump_bitmap temporarily */
+    dump_bitmap_buf = g_malloc0(BUFSIZE_BITMAP);
+
+    num_dumpable = 0;
+    last_pfn = 0;
+
+    /*
+     * exam memory page by page, and set the bit in dump_bitmap corresponded
+     * to the existing page.
+     */
+    while (get_next_page(&block_iter, &pfn, NULL, s)) {
+        ret = set_dump_bitmap(last_pfn, pfn, true, dump_bitmap_buf, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to set dump_bitmap", errp);
+            goto out;
+        }
+
+        last_pfn = pfn;
+        num_dumpable++;
+    }
+
+    /*
+     * set_dump_bitmap will always leave the recently set bit un-sync. Here we
+     * set last_pfn + PFN_BUFBITMAP to 0 and those set but un-sync bit will be
+     * synchronized into vmcore.
+     */
+    if (num_dumpable > 0) {
+        ret = set_dump_bitmap(last_pfn, last_pfn + PFN_BUFBITMAP, false,
+                              dump_bitmap_buf, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to sync dump_bitmap", errp);
+            goto out;
+        }
+    }
+
+    /* number of dumpable pages that will be dumped later */
+    s->num_dumpable = num_dumpable;
+
+out:
+    g_free(dump_bitmap_buf);
+}
+
+static void prepare_data_cache(DataCache *data_cache, DumpState *s,
+                               off_t offset)
+{
+    data_cache->fd = s->fd;
+    data_cache->data_size = 0;
+    data_cache->buf_size = BUFSIZE_DATA_CACHE;
+    data_cache->buf = g_malloc0(BUFSIZE_DATA_CACHE);
+    data_cache->offset = offset;
+}
+
+static int write_cache(DataCache *dc, const void *buf, size_t size,
+                       bool flag_sync)
+{
+    /*
+     * dc->buf_size should not be less than size, otherwise dc will never be
+     * enough
+     */
+    assert(size <= dc->buf_size);
+
+    /*
+     * if flag_sync is set, synchronize data in dc->buf into vmcore.
+     * otherwise check if the space is enough for caching data in buf, if not,
+     * write the data in dc->buf to dc->fd and reset dc->buf
+     */
+    if ((!flag_sync && dc->data_size + size > dc->buf_size) ||
+        (flag_sync && dc->data_size > 0)) {
+        if (write_buffer(dc->fd, dc->offset, dc->buf, dc->data_size) < 0) {
+            return -1;
+        }
+
+        dc->offset += dc->data_size;
+        dc->data_size = 0;
+    }
+
+    if (!flag_sync) {
+        memcpy(dc->buf + dc->data_size, buf, size);
+        dc->data_size += size;
+    }
+
+    return 0;
+}
+
+static void free_data_cache(DataCache *data_cache)
+{
+    g_free(data_cache->buf);
+}
+
+static size_t get_len_buf_out(size_t page_size, uint32_t flag_compress)
+{
+    switch (flag_compress) {
+    case DUMP_DH_COMPRESSED_ZLIB:
+        return compressBound(page_size);
+
+    case DUMP_DH_COMPRESSED_LZO:
+        /*
+         * LZO will expand incompressible data by a little amount. Please check
+         * the following URL to see the expansion calculation:
+         * http://www.oberhumer.com/opensource/lzo/lzofaq.php
+         */
+        return page_size + page_size / 16 + 64 + 3;
+
+#ifdef CONFIG_SNAPPY
+    case DUMP_DH_COMPRESSED_SNAPPY:
+        return snappy_max_compressed_length(page_size);
+#endif
+    }
+    return 0;
+}
+
+/*
+ * check if the page is all 0
+ */
+static inline bool is_zero_page(const uint8_t *buf, size_t page_size)
+{
+    return buffer_is_zero(buf, page_size);
+}
+
+static void write_dump_pages(DumpState *s, Error **errp)
+{
+    int ret = 0;
+    DataCache page_desc, page_data;
+    size_t len_buf_out, size_out;
+#ifdef CONFIG_LZO
+    lzo_bytep wrkmem = NULL;
+#endif
+    uint8_t *buf_out = NULL;
+    off_t offset_desc, offset_data;
+    PageDescriptor pd, pd_zero;
+    uint8_t *buf;
+    GuestPhysBlock *block_iter = NULL;
+    uint64_t pfn_iter;
+
+    /* get offset of page_desc and page_data in dump file */
+    offset_desc = s->offset_page;
+    offset_data = offset_desc + sizeof(PageDescriptor) * s->num_dumpable;
+
+    prepare_data_cache(&page_desc, s, offset_desc);
+    prepare_data_cache(&page_data, s, offset_data);
+
+    /* prepare buffer to store compressed data */
+    len_buf_out = get_len_buf_out(TARGET_PAGE_SIZE, s->flag_compress);
+    assert(len_buf_out != 0);
+
+#ifdef CONFIG_LZO
+    wrkmem = g_malloc(LZO1X_1_MEM_COMPRESS);
+#endif
+
+    buf_out = g_malloc(len_buf_out);
+
+    /*
+     * init zero page's page_desc and page_data, because every zero page
+     * uses the same page_data
+     */
+    pd_zero.size = cpu_to_dump32(s, TARGET_PAGE_SIZE);
+    pd_zero.flags = cpu_to_dump32(s, 0);
+    pd_zero.offset = cpu_to_dump64(s, offset_data);
+    pd_zero.page_flags = cpu_to_dump64(s, 0);
+    buf = g_malloc0(TARGET_PAGE_SIZE);
+    ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+    g_free(buf);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write page data (zero page)", errp);
+        goto out;
+    }
+
+    offset_data += TARGET_PAGE_SIZE;
+
+    /*
+     * dump memory to vmcore page by page. zero page will all be resided in the
+     * first page of page section
+     */
+    while (get_next_page(&block_iter, &pfn_iter, &buf, s)) {
+        /* check zero page */
+        if (is_zero_page(buf, TARGET_PAGE_SIZE)) {
+            ret = write_cache(&page_desc, &pd_zero, sizeof(PageDescriptor),
+                              false);
+            if (ret < 0) {
+                dump_error(s, "dump: failed to write page desc", errp);
+                goto out;
+            }
+        } else {
+            /*
+             * not zero page, then:
+             * 1. compress the page
+             * 2. write the compressed page into the cache of page_data
+             * 3. get page desc of the compressed page and write it into the
+             *    cache of page_desc
+             *
+             * only one compression format will be used here, for
+             * s->flag_compress is set. But when compression fails to work,
+             * we fall back to save in plaintext.
+             */
+             size_out = len_buf_out;
+             if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
+                    (compress2(buf_out, (uLongf *)&size_out, buf,
+                               TARGET_PAGE_SIZE, Z_BEST_SPEED) == Z_OK) &&
+                    (size_out < TARGET_PAGE_SIZE)) {
+                pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_ZLIB);
+                pd.size  = cpu_to_dump32(s, size_out);
+
+                ret = write_cache(&page_data, buf_out, size_out, false);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data", errp);
+                    goto out;
+                }
+#ifdef CONFIG_LZO
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
+                    (lzo1x_1_compress(buf, TARGET_PAGE_SIZE, buf_out,
+                    (lzo_uint *)&size_out, wrkmem) == LZO_E_OK) &&
+                    (size_out < TARGET_PAGE_SIZE)) {
+                pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_LZO);
+                pd.size  = cpu_to_dump32(s, size_out);
+
+                ret = write_cache(&page_data, buf_out, size_out, false);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data", errp);
+                    goto out;
+                }
+#endif
+#ifdef CONFIG_SNAPPY
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
+                    (snappy_compress((char *)buf, TARGET_PAGE_SIZE,
+                    (char *)buf_out, &size_out) == SNAPPY_OK) &&
+                    (size_out < TARGET_PAGE_SIZE)) {
+                pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_SNAPPY);
+                pd.size  = cpu_to_dump32(s, size_out);
+
+                ret = write_cache(&page_data, buf_out, size_out, false);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data", errp);
+                    goto out;
+                }
+#endif
+            } else {
+                /*
+                 * fall back to save in plaintext, size_out should be
+                 * assigned TARGET_PAGE_SIZE
+                 */
+                pd.flags = cpu_to_dump32(s, 0);
+                size_out = TARGET_PAGE_SIZE;
+                pd.size = cpu_to_dump32(s, size_out);
+
+                ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data", errp);
+                    goto out;
+                }
+            }
+
+            /* get and write page desc here */
+            pd.page_flags = cpu_to_dump64(s, 0);
+            pd.offset = cpu_to_dump64(s, offset_data);
+            offset_data += size_out;
+
+            ret = write_cache(&page_desc, &pd, sizeof(PageDescriptor), false);
+            if (ret < 0) {
+                dump_error(s, "dump: failed to write page desc", errp);
+                goto out;
+            }
+        }
+    }
+
+    ret = write_cache(&page_desc, NULL, 0, true);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_desc", errp);
+        goto out;
+    }
+    ret = write_cache(&page_data, NULL, 0, true);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_data", errp);
+        goto out;
+    }
+
+out:
+    free_data_cache(&page_desc);
+    free_data_cache(&page_data);
+
+#ifdef CONFIG_LZO
+    g_free(wrkmem);
+#endif
+
+    g_free(buf_out);
+}
+
+static void create_kdump_vmcore(DumpState *s, Error **errp)
+{
+    int ret;
+    Error *local_err = NULL;
+
+    /*
+     * the kdump-compressed format is:
+     *                                               File offset
+     *  +------------------------------------------+ 0x0
+     *  |    main header (struct disk_dump_header) |
+     *  |------------------------------------------+ block 1
+     *  |    sub header (struct kdump_sub_header)  |
+     *  |------------------------------------------+ block 2
+     *  |            1st-dump_bitmap               |
+     *  |------------------------------------------+ block 2 + X blocks
+     *  |            2nd-dump_bitmap               | (aligned by block)
+     *  |------------------------------------------+ block 2 + 2 * X blocks
+     *  |  page desc for pfn 0 (struct page_desc)  | (aligned by block)
+     *  |  page desc for pfn 1 (struct page_desc)  |
+     *  |                    :                     |
+     *  |------------------------------------------| (not aligned by block)
+     *  |         page data (pfn 0)                |
+     *  |         page data (pfn 1)                |
+     *  |                    :                     |
+     *  +------------------------------------------+
+     */
+
+    ret = write_start_flat_header(s->fd);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write start flat header", errp);
+        return;
+    }
+
+    write_dump_header(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    write_dump_bitmap(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    write_dump_pages(s, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    ret = write_end_flat_header(s->fd);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to write end flat header", errp);
+        return;
+    }
+
+    dump_completed(s);
+}
+
+static ram_addr_t get_start_block(DumpState *s)
+{
+    GuestPhysBlock *block;
+
+    if (!s->has_filter) {
+        s->next_block = QTAILQ_FIRST(&s->guest_phys_blocks.head);
+        return 0;
+    }
+
+    QTAILQ_FOREACH(block, &s->guest_phys_blocks.head, next) {
+        if (block->target_start >= s->begin + s->length ||
+            block->target_end <= s->begin) {
+            /* This block is out of the range */
+            continue;
+        }
+
+        s->next_block = block;
+        if (s->begin > block->target_start) {
+            s->start = s->begin - block->target_start;
+        } else {
+            s->start = 0;
+        }
+        return s->start;
+    }
+
+    return -1;
+}
+
+static void get_max_mapnr(DumpState *s)
+{
+    GuestPhysBlock *last_block;
+
+    last_block = QTAILQ_LAST(&s->guest_phys_blocks.head, GuestPhysBlockHead);
+    s->max_mapnr = paddr_to_pfn(last_block->target_end);
+}
+
+static void dump_init(DumpState *s, int fd, bool has_format,
+                      DumpGuestMemoryFormat format, bool paging, bool has_filter,
+                      int64_t begin, int64_t length, Error **errp)
+{
+    CPUState *cpu;
+    int nr_cpus;
+    Error *err = NULL;
+    int ret;
+
+    /* kdump-compressed is conflict with paging and filter */
+    if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+        assert(!paging && !has_filter);
+    }
+
+    if (runstate_is_running()) {
+        vm_stop(RUN_STATE_SAVE_VM);
+        s->resume = true;
+    } else {
+        s->resume = false;
+    }
+
+    /* If we use KVM, we should synchronize the registers before we get dump
+     * info or physmap info.
+     */
+    cpu_synchronize_all_states();
+    nr_cpus = 0;
+    CPU_FOREACH(cpu) {
+        nr_cpus++;
+    }
+
+    s->fd = fd;
+    s->has_filter = has_filter;
+    s->begin = begin;
+    s->length = length;
+
+    memory_mapping_list_init(&s->list);
+
+    guest_phys_blocks_init(&s->guest_phys_blocks);
+    guest_phys_blocks_append(&s->guest_phys_blocks);
+
+    s->start = get_start_block(s);
+    if (s->start == -1) {
+        error_setg(errp, QERR_INVALID_PARAMETER, "begin");
+        goto cleanup;
+    }
+
+    /* get dump info: endian, class and architecture.
+     * If the target architecture is not supported, cpu_get_dump_info() will
+     * return -1.
+     */
+    ret = cpu_get_dump_info(&s->dump_info, &s->guest_phys_blocks);
+    if (ret < 0) {
+        error_setg(errp, QERR_UNSUPPORTED);
+        goto cleanup;
+    }
+
+    s->note_size = cpu_get_note_size(s->dump_info.d_class,
+                                     s->dump_info.d_machine, nr_cpus);
+    if (s->note_size < 0) {
+        error_setg(errp, QERR_UNSUPPORTED);
+        goto cleanup;
+    }
+
+    /* get memory mapping */
+    if (paging) {
+        qemu_get_guest_memory_mapping(&s->list, &s->guest_phys_blocks, &err);
+        if (err != NULL) {
+            error_propagate(errp, err);
+            goto cleanup;
+        }
+    } else {
+        qemu_get_guest_simple_memory_mapping(&s->list, &s->guest_phys_blocks);
+    }
+
+    s->nr_cpus = nr_cpus;
+
+    get_max_mapnr(s);
+
+    uint64_t tmp;
+    tmp = DIV_ROUND_UP(DIV_ROUND_UP(s->max_mapnr, CHAR_BIT), TARGET_PAGE_SIZE);
+    s->len_dump_bitmap = tmp * TARGET_PAGE_SIZE;
+
+    /* init for kdump-compressed format */
+    if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+        switch (format) {
+        case DUMP_GUEST_MEMORY_FORMAT_KDUMP_ZLIB:
+            s->flag_compress = DUMP_DH_COMPRESSED_ZLIB;
+            break;
+
+        case DUMP_GUEST_MEMORY_FORMAT_KDUMP_LZO:
+#ifdef CONFIG_LZO
+            if (lzo_init() != LZO_E_OK) {
+                error_setg(errp, "failed to initialize the LZO library");
+                goto cleanup;
+            }
+#endif
+            s->flag_compress = DUMP_DH_COMPRESSED_LZO;
+            break;
+
+        case DUMP_GUEST_MEMORY_FORMAT_KDUMP_SNAPPY:
+            s->flag_compress = DUMP_DH_COMPRESSED_SNAPPY;
+            break;
+
+        default:
+            s->flag_compress = 0;
+        }
+
+        return;
+    }
+
+    if (s->has_filter) {
+        memory_mapping_filter(&s->list, s->begin, s->length);
+    }
+
+    /*
+     * calculate phdr_num
+     *
+     * the type of ehdr->e_phnum is uint16_t, so we should avoid overflow
+     */
+    s->phdr_num = 1; /* PT_NOTE */
+    if (s->list.num < UINT16_MAX - 2) {
+        s->phdr_num += s->list.num;
+        s->have_section = false;
+    } else {
+        s->have_section = true;
+        s->phdr_num = PN_XNUM;
+        s->sh_info = 1; /* PT_NOTE */
+
+        /* the type of shdr->sh_info is uint32_t, so we should avoid overflow */
+        if (s->list.num <= UINT32_MAX - 1) {
+            s->sh_info += s->list.num;
+        } else {
+            s->sh_info = UINT32_MAX;
+        }
+    }
+
+    if (s->dump_info.d_class == ELFCLASS64) {
+        if (s->have_section) {
+            s->memory_offset = sizeof(Elf64_Ehdr) +
+                               sizeof(Elf64_Phdr) * s->sh_info +
+                               sizeof(Elf64_Shdr) + s->note_size;
+        } else {
+            s->memory_offset = sizeof(Elf64_Ehdr) +
+                               sizeof(Elf64_Phdr) * s->phdr_num + s->note_size;
+        }
+    } else {
+        if (s->have_section) {
+            s->memory_offset = sizeof(Elf32_Ehdr) +
+                               sizeof(Elf32_Phdr) * s->sh_info +
+                               sizeof(Elf32_Shdr) + s->note_size;
+        } else {
+            s->memory_offset = sizeof(Elf32_Ehdr) +
+                               sizeof(Elf32_Phdr) * s->phdr_num + s->note_size;
+        }
+    }
+
+    return;
+
+cleanup:
+    dump_cleanup(s);
+}
+
+void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
+                           int64_t begin, bool has_length,
+                           int64_t length, bool has_format,
+                           DumpGuestMemoryFormat format, Error **errp)
+{
+    const char *p;
+    int fd = -1;
+    DumpState *s;
+    Error *local_err = NULL;
+
+    /*
+     * kdump-compressed format need the whole memory dumped, so paging or
+     * filter is not supported here.
+     */
+    if ((has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) &&
+        (paging || has_begin || has_length)) {
+        error_setg(errp, "kdump-compressed format doesn't support paging or "
+                         "filter");
+        return;
+    }
+    if (has_begin && !has_length) {
+        error_setg(errp, QERR_MISSING_PARAMETER, "length");
+        return;
+    }
+    if (!has_begin && has_length) {
+        error_setg(errp, QERR_MISSING_PARAMETER, "begin");
+        return;
+    }
+
+    /* check whether lzo/snappy is supported */
+#ifndef CONFIG_LZO
+    if (has_format && format == DUMP_GUEST_MEMORY_FORMAT_KDUMP_LZO) {
+        error_setg(errp, "kdump-lzo is not available now");
+        return;
+    }
+#endif
+
+#ifndef CONFIG_SNAPPY
+    if (has_format && format == DUMP_GUEST_MEMORY_FORMAT_KDUMP_SNAPPY) {
+        error_setg(errp, "kdump-snappy is not available now");
+        return;
+    }
+#endif
+
+#if !defined(WIN32)
+    if (strstart(file, "fd:", &p)) {
+        fd = monitor_get_fd(cur_mon, p, errp);
+        if (fd == -1) {
+            return;
+        }
+    }
+#endif
+
+    if  (strstart(file, "file:", &p)) {
+        fd = qemu_open(p, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR);
+        if (fd < 0) {
+            error_setg_file_open(errp, errno, p);
+            return;
+        }
+    }
+
+    if (fd == -1) {
+        error_setg(errp, QERR_INVALID_PARAMETER, "protocol");
+        return;
+    }
+
+    s = g_malloc0(sizeof(DumpState));
+
+    dump_init(s, fd, has_format, format, paging, has_begin,
+              begin, length, &local_err);
+    if (local_err) {
+        g_free(s);
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+        create_kdump_vmcore(s, errp);
+    } else {
+        create_vmcore(s, errp);
+    }
+
+    g_free(s);
+}
+
+DumpGuestMemoryCapability *qmp_query_dump_guest_memory_capability(Error **errp)
+{
+    DumpGuestMemoryFormatList *item;
+    DumpGuestMemoryCapability *cap =
+                                  g_malloc0(sizeof(DumpGuestMemoryCapability));
+
+    /* elf is always available */
+    item = g_malloc0(sizeof(DumpGuestMemoryFormatList));
+    cap->formats = item;
+    item->value = DUMP_GUEST_MEMORY_FORMAT_ELF;
+
+    /* kdump-zlib is always available */
+    item->next = g_malloc0(sizeof(DumpGuestMemoryFormatList));
+    item = item->next;
+    item->value = DUMP_GUEST_MEMORY_FORMAT_KDUMP_ZLIB;
+
+    /* add new item if kdump-lzo is available */
+#ifdef CONFIG_LZO
+    item->next = g_malloc0(sizeof(DumpGuestMemoryFormatList));
+    item = item->next;
+    item->value = DUMP_GUEST_MEMORY_FORMAT_KDUMP_LZO;
+#endif
+
+    /* add new item if kdump-snappy is available */
+#ifdef CONFIG_SNAPPY
+    item->next = g_malloc0(sizeof(DumpGuestMemoryFormatList));
+    item = item->next;
+    item->value = DUMP_GUEST_MEMORY_FORMAT_KDUMP_SNAPPY;
+#endif
+
+    return cap;
+}
diff --git a/exec.c b/exec.c
new file mode 100644
index 0000000..0bf0a6e
--- /dev/null
+++ b/exec.c
@@ -0,0 +1,3618 @@
+/*
+ *  Virtual page mapping
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "config.h"
+#ifndef _WIN32
+#include <sys/types.h>
+#include <sys/mman.h>
+#endif
+
+#include "qemu-common.h"
+#include "cpu.h"
+#include "tcg.h"
+#include "hw/hw.h"
+#if !defined(CONFIG_USER_ONLY)
+#include "hw/boards.h"
+#endif
+#include "hw/qdev.h"
+#include "qemu/osdep.h"
+#include "sysemu/kvm.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/xen.h"
+#include "qemu/timer.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "exec/memory.h"
+#include "sysemu/dma.h"
+#include "exec/address-spaces.h"
+#if defined(CONFIG_USER_ONLY)
+#include <qemu.h>
+#else /* !CONFIG_USER_ONLY */
+#include "sysemu/xen-mapcache.h"
+#include "trace.h"
+#endif
+#include "exec/cpu-all.h"
+#include "qemu/rcu_queue.h"
+#include "qemu/main-loop.h"
+#include "translate-all.h"
+#include "sysemu/replay.h"
+
+#include "exec/memory-internal.h"
+#include "exec/ram_addr.h"
+
+#include "qemu/range.h"
+#ifndef _WIN32
+#include "qemu/mmap-alloc.h"
+#endif
+
+//#define DEBUG_SUBPAGE
+
+#if !defined(CONFIG_USER_ONLY)
+/* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
+ * are protected by the ramlist lock.
+ */
+RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
+
+static MemoryRegion *system_memory;
+static MemoryRegion *system_io;
+
+AddressSpace address_space_io;
+AddressSpace address_space_memory;
+
+MemoryRegion io_mem_rom, io_mem_notdirty;
+static MemoryRegion io_mem_unassigned;
+
+/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
+#define RAM_PREALLOC   (1 << 0)
+
+/* RAM is mmap-ed with MAP_SHARED */
+#define RAM_SHARED     (1 << 1)
+
+/* Only a portion of RAM (used_length) is actually used, and migrated.
+ * This used_length size can change across reboots.
+ */
+#define RAM_RESIZEABLE (1 << 2)
+
+/* RAM is backed by an mmapped file.
+ */
+#define RAM_FILE (1 << 3)
+#endif
+
+struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
+/* current CPU in the current thread. It is only valid inside
+   cpu_exec() */
+__thread CPUState *current_cpu;
+/* 0 = Do not count executed instructions.
+   1 = Precise instruction counting.
+   2 = Adaptive rate instruction counting.  */
+int use_icount;
+
+#if !defined(CONFIG_USER_ONLY)
+
+typedef struct PhysPageEntry PhysPageEntry;
+
+struct PhysPageEntry {
+    /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
+    uint32_t skip : 6;
+     /* index into phys_sections (!skip) or phys_map_nodes (skip) */
+    uint32_t ptr : 26;
+};
+
+#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
+
+/* Size of the L2 (and L3, etc) page tables.  */
+#define ADDR_SPACE_BITS 64
+
+#define P_L2_BITS 9
+#define P_L2_SIZE (1 << P_L2_BITS)
+
+#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
+
+typedef PhysPageEntry Node[P_L2_SIZE];
+
+typedef struct PhysPageMap {
+    struct rcu_head rcu;
+
+    unsigned sections_nb;
+    unsigned sections_nb_alloc;
+    unsigned nodes_nb;
+    unsigned nodes_nb_alloc;
+    Node *nodes;
+    MemoryRegionSection *sections;
+} PhysPageMap;
+
+struct AddressSpaceDispatch {
+    struct rcu_head rcu;
+
+    /* This is a multi-level map on the physical address space.
+     * The bottom level has pointers to MemoryRegionSections.
+     */
+    PhysPageEntry phys_map;
+    PhysPageMap map;
+    AddressSpace *as;
+};
+
+#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
+typedef struct subpage_t {
+    MemoryRegion iomem;
+    AddressSpace *as;
+    hwaddr base;
+    uint16_t sub_section[TARGET_PAGE_SIZE];
+} subpage_t;
+
+#define PHYS_SECTION_UNASSIGNED 0
+#define PHYS_SECTION_NOTDIRTY 1
+#define PHYS_SECTION_ROM 2
+#define PHYS_SECTION_WATCH 3
+
+static void io_mem_init(void);
+static void memory_map_init(void);
+static void tcg_commit(MemoryListener *listener);
+
+static MemoryRegion io_mem_watch;
+
+/**
+ * CPUAddressSpace: all the information a CPU needs about an AddressSpace
+ * @cpu: the CPU whose AddressSpace this is
+ * @as: the AddressSpace itself
+ * @memory_dispatch: its dispatch pointer (cached, RCU protected)
+ * @tcg_as_listener: listener for tracking changes to the AddressSpace
+ */
+struct CPUAddressSpace {
+    CPUState *cpu;
+    AddressSpace *as;
+    struct AddressSpaceDispatch *memory_dispatch;
+    MemoryListener tcg_as_listener;
+};
+
+#endif
+
+#if !defined(CONFIG_USER_ONLY)
+
+static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
+{
+    if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
+        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
+        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
+        map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
+    }
+}
+
+static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
+{
+    unsigned i;
+    uint32_t ret;
+    PhysPageEntry e;
+    PhysPageEntry *p;
+
+    ret = map->nodes_nb++;
+    p = map->nodes[ret];
+    assert(ret != PHYS_MAP_NODE_NIL);
+    assert(ret != map->nodes_nb_alloc);
+
+    e.skip = leaf ? 0 : 1;
+    e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
+    for (i = 0; i < P_L2_SIZE; ++i) {
+        memcpy(&p[i], &e, sizeof(e));
+    }
+    return ret;
+}
+
+static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
+                                hwaddr *index, hwaddr *nb, uint16_t leaf,
+                                int level)
+{
+    PhysPageEntry *p;
+    hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
+
+    if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
+        lp->ptr = phys_map_node_alloc(map, level == 0);
+    }
+    p = map->nodes[lp->ptr];
+    lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
+
+    while (*nb && lp < &p[P_L2_SIZE]) {
+        if ((*index & (step - 1)) == 0 && *nb >= step) {
+            lp->skip = 0;
+            lp->ptr = leaf;
+            *index += step;
+            *nb -= step;
+        } else {
+            phys_page_set_level(map, lp, index, nb, leaf, level - 1);
+        }
+        ++lp;
+    }
+}
+
+static void phys_page_set(AddressSpaceDispatch *d,
+                          hwaddr index, hwaddr nb,
+                          uint16_t leaf)
+{
+    /* Wildly overreserve - it doesn't matter much. */
+    phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
+
+    phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
+}
+
+/* Compact a non leaf page entry. Simply detect that the entry has a single child,
+ * and update our entry so we can skip it and go directly to the destination.
+ */
+static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
+{
+    unsigned valid_ptr = P_L2_SIZE;
+    int valid = 0;
+    PhysPageEntry *p;
+    int i;
+
+    if (lp->ptr == PHYS_MAP_NODE_NIL) {
+        return;
+    }
+
+    p = nodes[lp->ptr];
+    for (i = 0; i < P_L2_SIZE; i++) {
+        if (p[i].ptr == PHYS_MAP_NODE_NIL) {
+            continue;
+        }
+
+        valid_ptr = i;
+        valid++;
+        if (p[i].skip) {
+            phys_page_compact(&p[i], nodes, compacted);
+        }
+    }
+
+    /* We can only compress if there's only one child. */
+    if (valid != 1) {
+        return;
+    }
+
+    assert(valid_ptr < P_L2_SIZE);
+
+    /* Don't compress if it won't fit in the # of bits we have. */
+    if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
+        return;
+    }
+
+    lp->ptr = p[valid_ptr].ptr;
+    if (!p[valid_ptr].skip) {
+        /* If our only child is a leaf, make this a leaf. */
+        /* By design, we should have made this node a leaf to begin with so we
+         * should never reach here.
+         * But since it's so simple to handle this, let's do it just in case we
+         * change this rule.
+         */
+        lp->skip = 0;
+    } else {
+        lp->skip += p[valid_ptr].skip;
+    }
+}
+
+static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
+{
+    DECLARE_BITMAP(compacted, nodes_nb);
+
+    if (d->phys_map.skip) {
+        phys_page_compact(&d->phys_map, d->map.nodes, compacted);
+    }
+}
+
+static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
+                                           Node *nodes, MemoryRegionSection *sections)
+{
+    PhysPageEntry *p;
+    hwaddr index = addr >> TARGET_PAGE_BITS;
+    int i;
+
+    for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
+        if (lp.ptr == PHYS_MAP_NODE_NIL) {
+            return &sections[PHYS_SECTION_UNASSIGNED];
+        }
+        p = nodes[lp.ptr];
+        lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
+    }
+
+    if (sections[lp.ptr].size.hi ||
+        range_covers_byte(sections[lp.ptr].offset_within_address_space,
+                          sections[lp.ptr].size.lo, addr)) {
+        return &sections[lp.ptr];
+    } else {
+        return &sections[PHYS_SECTION_UNASSIGNED];
+    }
+}
+
+bool memory_region_is_unassigned(MemoryRegion *mr)
+{
+    return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
+        && mr != &io_mem_watch;
+}
+
+/* Called from RCU critical section */
+static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
+                                                        hwaddr addr,
+                                                        bool resolve_subpage)
+{
+    MemoryRegionSection *section;
+    subpage_t *subpage;
+
+    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
+    if (resolve_subpage && section->mr->subpage) {
+        subpage = container_of(section->mr, subpage_t, iomem);
+        section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
+    }
+    return section;
+}
+
+/* Called from RCU critical section */
+static MemoryRegionSection *
+address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
+                                 hwaddr *plen, bool resolve_subpage)
+{
+    MemoryRegionSection *section;
+    MemoryRegion *mr;
+    Int128 diff;
+
+    section = address_space_lookup_region(d, addr, resolve_subpage);
+    /* Compute offset within MemoryRegionSection */
+    addr -= section->offset_within_address_space;
+
+    /* Compute offset within MemoryRegion */
+    *xlat = addr + section->offset_within_region;
+
+    mr = section->mr;
+
+    /* MMIO registers can be expected to perform full-width accesses based only
+     * on their address, without considering adjacent registers that could
+     * decode to completely different MemoryRegions.  When such registers
+     * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
+     * regions overlap wildly.  For this reason we cannot clamp the accesses
+     * here.
+     *
+     * If the length is small (as is the case for address_space_ldl/stl),
+     * everything works fine.  If the incoming length is large, however,
+     * the caller really has to do the clamping through memory_access_size.
+     */
+    if (memory_region_is_ram(mr)) {
+        diff = int128_sub(section->size, int128_make64(addr));
+        *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
+    }
+    return section;
+}
+
+static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
+{
+    if (memory_region_is_ram(mr)) {
+        return !(is_write && mr->readonly);
+    }
+    if (memory_region_is_romd(mr)) {
+        return !is_write;
+    }
+
+    return false;
+}
+
+/* Called from RCU critical section */
+MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
+                                      hwaddr *xlat, hwaddr *plen,
+                                      bool is_write)
+{
+    IOMMUTLBEntry iotlb;
+    MemoryRegionSection *section;
+    MemoryRegion *mr;
+
+    for (;;) {
+        AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
+        section = address_space_translate_internal(d, addr, &addr, plen, true);
+        mr = section->mr;
+
+        if (!mr->iommu_ops) {
+            break;
+        }
+
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
+        addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
+                | (addr & iotlb.addr_mask));
+        *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
+        if (!(iotlb.perm & (1 << is_write))) {
+            mr = &io_mem_unassigned;
+            break;
+        }
+
+        as = iotlb.target_as;
+    }
+
+    if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
+        hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
+        *plen = MIN(page, *plen);
+    }
+
+    *xlat = addr;
+    return mr;
+}
+
+/* Called from RCU critical section */
+MemoryRegionSection *
+address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
+                                  hwaddr *xlat, hwaddr *plen)
+{
+    MemoryRegionSection *section;
+    section = address_space_translate_internal(cpu->cpu_ases[0].memory_dispatch,
+                                               addr, xlat, plen, false);
+
+    assert(!section->mr->iommu_ops);
+    return section;
+}
+#endif
+
+#if !defined(CONFIG_USER_ONLY)
+
+static int cpu_common_post_load(void *opaque, int version_id)
+{
+    CPUState *cpu = opaque;
+
+    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
+       version_id is increased. */
+    cpu->interrupt_request &= ~0x01;
+    tlb_flush(cpu, 1);
+
+    return 0;
+}
+
+static int cpu_common_pre_load(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    cpu->exception_index = -1;
+
+    return 0;
+}
+
+static bool cpu_common_exception_index_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return tcg_enabled() && cpu->exception_index != -1;
+}
+
+static const VMStateDescription vmstate_cpu_common_exception_index = {
+    .name = "cpu_common/exception_index",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = cpu_common_exception_index_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(exception_index, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool cpu_common_crash_occurred_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return cpu->crash_occurred;
+}
+
+static const VMStateDescription vmstate_cpu_common_crash_occurred = {
+    .name = "cpu_common/crash_occurred",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = cpu_common_crash_occurred_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(crash_occurred, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+const VMStateDescription vmstate_cpu_common = {
+    .name = "cpu_common",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_load = cpu_common_pre_load,
+    .post_load = cpu_common_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(halted, CPUState),
+        VMSTATE_UINT32(interrupt_request, CPUState),
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_cpu_common_exception_index,
+        &vmstate_cpu_common_crash_occurred,
+        NULL
+    }
+};
+
+#endif
+
+CPUState *qemu_get_cpu(int index)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu) {
+        if (cpu->cpu_index == index) {
+            return cpu;
+        }
+    }
+
+    return NULL;
+}
+
+#if !defined(CONFIG_USER_ONLY)
+void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
+{
+    /* We only support one address space per cpu at the moment.  */
+    assert(cpu->as == as);
+
+    if (cpu->cpu_ases) {
+        /* We've already registered the listener for our only AS */
+        return;
+    }
+
+    cpu->cpu_ases = g_new0(CPUAddressSpace, 1);
+    cpu->cpu_ases[0].cpu = cpu;
+    cpu->cpu_ases[0].as = as;
+    cpu->cpu_ases[0].tcg_as_listener.commit = tcg_commit;
+    memory_listener_register(&cpu->cpu_ases[0].tcg_as_listener, as);
+}
+#endif
+
+#ifndef CONFIG_USER_ONLY
+static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
+
+static int cpu_get_free_index(Error **errp)
+{
+    int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
+
+    if (cpu >= MAX_CPUMASK_BITS) {
+        error_setg(errp, "Trying to use more CPUs than max of %d",
+                   MAX_CPUMASK_BITS);
+        return -1;
+    }
+
+    bitmap_set(cpu_index_map, cpu, 1);
+    return cpu;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+    if (cpu->cpu_index == -1) {
+        /* cpu_index was never allocated by this @cpu or was already freed. */
+        return;
+    }
+
+    bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
+    cpu->cpu_index = -1;
+}
+#else
+
+static int cpu_get_free_index(Error **errp)
+{
+    CPUState *some_cpu;
+    int cpu_index = 0;
+
+    CPU_FOREACH(some_cpu) {
+        cpu_index++;
+    }
+    return cpu_index;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+}
+#endif
+
+void cpu_exec_init(CPUState *cpu, Error **errp)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    int cpu_index;
+    Error *local_err = NULL;
+
+#ifndef CONFIG_USER_ONLY
+    cpu->as = &address_space_memory;
+    cpu->thread_id = qemu_get_thread_id();
+#endif
+
+#if defined(CONFIG_USER_ONLY)
+    cpu_list_lock();
+#endif
+    cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+#if defined(CONFIG_USER_ONLY)
+        cpu_list_unlock();
+#endif
+        return;
+    }
+    QTAILQ_INSERT_TAIL(&cpus, cpu, node);
+#if defined(CONFIG_USER_ONLY)
+    cpu_list_unlock();
+#endif
+    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
+        vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
+    }
+#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
+    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
+                    cpu_save, cpu_load, cpu->env_ptr);
+    assert(cc->vmsd == NULL);
+    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
+#endif
+    if (cc->vmsd != NULL) {
+        vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
+    }
+}
+
+#if defined(CONFIG_USER_ONLY)
+static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
+{
+    tb_invalidate_phys_page_range(pc, pc + 1, 0);
+}
+#else
+static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
+{
+    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
+    if (phys != -1) {
+        tb_invalidate_phys_addr(cpu->as,
+                                phys | (pc & ~TARGET_PAGE_MASK));
+    }
+}
+#endif
+
+#if defined(CONFIG_USER_ONLY)
+void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
+
+{
+}
+
+int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags)
+{
+    return -ENOSYS;
+}
+
+void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
+{
+}
+
+int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags, CPUWatchpoint **watchpoint)
+{
+    return -ENOSYS;
+}
+#else
+/* Add a watchpoint.  */
+int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags, CPUWatchpoint **watchpoint)
+{
+    CPUWatchpoint *wp;
+
+    /* forbid ranges which are empty or run off the end of the address space */
+    if (len == 0 || (addr + len - 1) < addr) {
+        error_report("tried to set invalid watchpoint at %"
+                     VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
+        return -EINVAL;
+    }
+    wp = g_malloc(sizeof(*wp));
+
+    wp->vaddr = addr;
+    wp->len = len;
+    wp->flags = flags;
+
+    /* keep all GDB-injected watchpoints in front */
+    if (flags & BP_GDB) {
+        QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
+    } else {
+        QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
+    }
+
+    tlb_flush_page(cpu, addr);
+
+    if (watchpoint)
+        *watchpoint = wp;
+    return 0;
+}
+
+/* Remove a specific watchpoint.  */
+int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags)
+{
+    CPUWatchpoint *wp;
+
+    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
+        if (addr == wp->vaddr && len == wp->len
+                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
+            cpu_watchpoint_remove_by_ref(cpu, wp);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+/* Remove a specific watchpoint by reference.  */
+void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
+{
+    QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
+
+    tlb_flush_page(cpu, watchpoint->vaddr);
+
+    g_free(watchpoint);
+}
+
+/* Remove all matching watchpoints.  */
+void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
+{
+    CPUWatchpoint *wp, *next;
+
+    QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
+        if (wp->flags & mask) {
+            cpu_watchpoint_remove_by_ref(cpu, wp);
+        }
+    }
+}
+
+/* Return true if this watchpoint address matches the specified
+ * access (ie the address range covered by the watchpoint overlaps
+ * partially or completely with the address range covered by the
+ * access).
+ */
+static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
+                                                  vaddr addr,
+                                                  vaddr len)
+{
+    /* We know the lengths are non-zero, but a little caution is
+     * required to avoid errors in the case where the range ends
+     * exactly at the top of the address space and so addr + len
+     * wraps round to zero.
+     */
+    vaddr wpend = wp->vaddr + wp->len - 1;
+    vaddr addrend = addr + len - 1;
+
+    return !(addr > wpend || wp->vaddr > addrend);
+}
+
+#endif
+
+/* Add a breakpoint.  */
+int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
+                          CPUBreakpoint **breakpoint)
+{
+    CPUBreakpoint *bp;
+
+    bp = g_malloc(sizeof(*bp));
+
+    bp->pc = pc;
+    bp->flags = flags;
+
+    /* keep all GDB-injected breakpoints in front */
+    if (flags & BP_GDB) {
+        QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
+    } else {
+        QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
+    }
+
+    breakpoint_invalidate(cpu, pc);
+
+    if (breakpoint) {
+        *breakpoint = bp;
+    }
+    return 0;
+}
+
+/* Remove a specific breakpoint.  */
+int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
+{
+    CPUBreakpoint *bp;
+
+    QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
+        if (bp->pc == pc && bp->flags == flags) {
+            cpu_breakpoint_remove_by_ref(cpu, bp);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+/* Remove a specific breakpoint by reference.  */
+void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
+{
+    QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
+
+    breakpoint_invalidate(cpu, breakpoint->pc);
+
+    g_free(breakpoint);
+}
+
+/* Remove all matching breakpoints. */
+void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
+{
+    CPUBreakpoint *bp, *next;
+
+    QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
+        if (bp->flags & mask) {
+            cpu_breakpoint_remove_by_ref(cpu, bp);
+        }
+    }
+}
+
+/* enable or disable single step mode. EXCP_DEBUG is returned by the
+   CPU loop after each instruction */
+void cpu_single_step(CPUState *cpu, int enabled)
+{
+    if (cpu->singlestep_enabled != enabled) {
+        cpu->singlestep_enabled = enabled;
+        if (kvm_enabled()) {
+            kvm_update_guest_debug(cpu, 0);
+        } else {
+            /* must flush all the translated code to avoid inconsistencies */
+            /* XXX: only flush what is necessary */
+            tb_flush(cpu);
+        }
+    }
+}
+
+void cpu_abort(CPUState *cpu, const char *fmt, ...)
+{
+    va_list ap;
+    va_list ap2;
+
+    va_start(ap, fmt);
+    va_copy(ap2, ap);
+    fprintf(stderr, "qemu: fatal: ");
+    vfprintf(stderr, fmt, ap);
+    fprintf(stderr, "\n");
+    cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
+    if (qemu_log_enabled()) {
+        qemu_log("qemu: fatal: ");
+        qemu_log_vprintf(fmt, ap2);
+        qemu_log("\n");
+        log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
+        qemu_log_flush();
+        qemu_log_close();
+    }
+    va_end(ap2);
+    va_end(ap);
+    replay_finish();
+#if defined(CONFIG_USER_ONLY)
+    {
+        struct sigaction act;
+        sigfillset(&act.sa_mask);
+        act.sa_handler = SIG_DFL;
+        sigaction(SIGABRT, &act, NULL);
+    }
+#endif
+    abort();
+}
+
+#if !defined(CONFIG_USER_ONLY)
+/* Called from RCU critical section */
+static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
+{
+    RAMBlock *block;
+
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && addr - block->offset < block->max_length) {
+        return block;
+    }
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (addr - block->offset < block->max_length) {
+            goto found;
+        }
+    }
+
+    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
+    abort();
+
+found:
+    /* It is safe to write mru_block outside the iothread lock.  This
+     * is what happens:
+     *
+     *     mru_block = xxx
+     *     rcu_read_unlock()
+     *                                        xxx removed from list
+     *                  rcu_read_lock()
+     *                  read mru_block
+     *                                        mru_block = NULL;
+     *                                        call_rcu(reclaim_ramblock, xxx);
+     *                  rcu_read_unlock()
+     *
+     * atomic_rcu_set is not needed here.  The block was already published
+     * when it was placed into the list.  Here we're just making an extra
+     * copy of the pointer.
+     */
+    ram_list.mru_block = block;
+    return block;
+}
+
+static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
+{
+    CPUState *cpu;
+    ram_addr_t start1;
+    RAMBlock *block;
+    ram_addr_t end;
+
+    end = TARGET_PAGE_ALIGN(start + length);
+    start &= TARGET_PAGE_MASK;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(start);
+    assert(block == qemu_get_ram_block(end - 1));
+    start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
+    CPU_FOREACH(cpu) {
+        tlb_reset_dirty(cpu, start1, length);
+    }
+    rcu_read_unlock();
+}
+
+/* Note: start and end must be within the same ram block.  */
+bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
+                                              ram_addr_t length,
+                                              unsigned client)
+{
+    unsigned long end, page;
+    bool dirty;
+
+    if (length == 0) {
+        return false;
+    }
+
+    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+    page = start >> TARGET_PAGE_BITS;
+    dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
+                                         page, end - page);
+
+    if (dirty && tcg_enabled()) {
+        tlb_reset_dirty_range_all(start, length);
+    }
+
+    return dirty;
+}
+
+/* Called from RCU critical section */
+hwaddr memory_region_section_get_iotlb(CPUState *cpu,
+                                       MemoryRegionSection *section,
+                                       target_ulong vaddr,
+                                       hwaddr paddr, hwaddr xlat,
+                                       int prot,
+                                       target_ulong *address)
+{
+    hwaddr iotlb;
+    CPUWatchpoint *wp;
+
+    if (memory_region_is_ram(section->mr)) {
+        /* Normal RAM.  */
+        iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
+            + xlat;
+        if (!section->readonly) {
+            iotlb |= PHYS_SECTION_NOTDIRTY;
+        } else {
+            iotlb |= PHYS_SECTION_ROM;
+        }
+    } else {
+        AddressSpaceDispatch *d;
+
+        d = atomic_rcu_read(&section->address_space->dispatch);
+        iotlb = section - d->map.sections;
+        iotlb += xlat;
+    }
+
+    /* Make accesses to pages with watchpoints go via the
+       watchpoint trap routines.  */
+    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
+        if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
+            /* Avoid trapping reads of pages with a write breakpoint. */
+            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
+                iotlb = PHYS_SECTION_WATCH + paddr;
+                *address |= TLB_MMIO;
+                break;
+            }
+        }
+    }
+
+    return iotlb;
+}
+#endif /* defined(CONFIG_USER_ONLY) */
+
+#if !defined(CONFIG_USER_ONLY)
+
+static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
+                             uint16_t section);
+static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
+
+static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
+                               qemu_anon_ram_alloc;
+
+/*
+ * Set a custom physical guest memory alloator.
+ * Accelerators with unusual needs may need this.  Hopefully, we can
+ * get rid of it eventually.
+ */
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
+{
+    phys_mem_alloc = alloc;
+}
+
+static uint16_t phys_section_add(PhysPageMap *map,
+                                 MemoryRegionSection *section)
+{
+    /* The physical section number is ORed with a page-aligned
+     * pointer to produce the iotlb entries.  Thus it should
+     * never overflow into the page-aligned value.
+     */
+    assert(map->sections_nb < TARGET_PAGE_SIZE);
+
+    if (map->sections_nb == map->sections_nb_alloc) {
+        map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
+        map->sections = g_renew(MemoryRegionSection, map->sections,
+                                map->sections_nb_alloc);
+    }
+    map->sections[map->sections_nb] = *section;
+    memory_region_ref(section->mr);
+    return map->sections_nb++;
+}
+
+static void phys_section_destroy(MemoryRegion *mr)
+{
+    bool have_sub_page = mr->subpage;
+
+    memory_region_unref(mr);
+
+    if (have_sub_page) {
+        subpage_t *subpage = container_of(mr, subpage_t, iomem);
+        object_unref(OBJECT(&subpage->iomem));
+        g_free(subpage);
+    }
+}
+
+static void phys_sections_free(PhysPageMap *map)
+{
+    while (map->sections_nb > 0) {
+        MemoryRegionSection *section = &map->sections[--map->sections_nb];
+        phys_section_destroy(section->mr);
+    }
+    g_free(map->sections);
+    g_free(map->nodes);
+}
+
+static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
+{
+    subpage_t *subpage;
+    hwaddr base = section->offset_within_address_space
+        & TARGET_PAGE_MASK;
+    MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
+                                                   d->map.nodes, d->map.sections);
+    MemoryRegionSection subsection = {
+        .offset_within_address_space = base,
+        .size = int128_make64(TARGET_PAGE_SIZE),
+    };
+    hwaddr start, end;
+
+    assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
+
+    if (!(existing->mr->subpage)) {
+        subpage = subpage_init(d->as, base);
+        subsection.address_space = d->as;
+        subsection.mr = &subpage->iomem;
+        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
+                      phys_section_add(&d->map, &subsection));
+    } else {
+        subpage = container_of(existing->mr, subpage_t, iomem);
+    }
+    start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
+    end = start + int128_get64(section->size) - 1;
+    subpage_register(subpage, start, end,
+                     phys_section_add(&d->map, section));
+}
+
+
+static void register_multipage(AddressSpaceDispatch *d,
+                               MemoryRegionSection *section)
+{
+    hwaddr start_addr = section->offset_within_address_space;
+    uint16_t section_index = phys_section_add(&d->map, section);
+    uint64_t num_pages = int128_get64(int128_rshift(section->size,
+                                                    TARGET_PAGE_BITS));
+
+    assert(num_pages);
+    phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
+}
+
+static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
+{
+    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
+    AddressSpaceDispatch *d = as->next_dispatch;
+    MemoryRegionSection now = *section, remain = *section;
+    Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
+
+    if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
+        uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
+                       - now.offset_within_address_space;
+
+        now.size = int128_min(int128_make64(left), now.size);
+        register_subpage(d, &now);
+    } else {
+        now.size = int128_zero();
+    }
+    while (int128_ne(remain.size, now.size)) {
+        remain.size = int128_sub(remain.size, now.size);
+        remain.offset_within_address_space += int128_get64(now.size);
+        remain.offset_within_region += int128_get64(now.size);
+        now = remain;
+        if (int128_lt(remain.size, page_size)) {
+            register_subpage(d, &now);
+        } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
+            now.size = page_size;
+            register_subpage(d, &now);
+        } else {
+            now.size = int128_and(now.size, int128_neg(page_size));
+            register_multipage(d, &now);
+        }
+    }
+}
+
+void qemu_flush_coalesced_mmio_buffer(void)
+{
+    if (kvm_enabled())
+        kvm_flush_coalesced_mmio_buffer();
+}
+
+void qemu_mutex_lock_ramlist(void)
+{
+    qemu_mutex_lock(&ram_list.mutex);
+}
+
+void qemu_mutex_unlock_ramlist(void)
+{
+    qemu_mutex_unlock(&ram_list.mutex);
+}
+
+#ifdef __linux__
+
+#include <sys/vfs.h>
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+static long gethugepagesize(const char *path, Error **errp)
+{
+    struct statfs fs;
+    int ret;
+
+    do {
+        ret = statfs(path, &fs);
+    } while (ret != 0 && errno == EINTR);
+
+    if (ret != 0) {
+        error_setg_errno(errp, errno, "failed to get page size of file %s",
+                         path);
+        return 0;
+    }
+
+    return fs.f_bsize;
+}
+
+static void *file_ram_alloc(RAMBlock *block,
+                            ram_addr_t memory,
+                            const char *path,
+                            Error **errp)
+{
+    struct stat st;
+    char *filename;
+    char *sanitized_name;
+    char *c;
+    void *area;
+    int fd;
+    uint64_t hpagesize;
+    Error *local_err = NULL;
+
+    hpagesize = gethugepagesize(path, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+    block->mr->align = hpagesize;
+
+    if (memory < hpagesize) {
+        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
+                   "or larger than huge page size 0x%" PRIx64,
+                   memory, hpagesize);
+        goto error;
+    }
+
+    if (kvm_enabled() && !kvm_has_sync_mmu()) {
+        error_setg(errp,
+                   "host lacks kvm mmu notifiers, -mem-path unsupported");
+        goto error;
+    }
+
+    if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
+        /* Make name safe to use with mkstemp by replacing '/' with '_'. */
+        sanitized_name = g_strdup(memory_region_name(block->mr));
+        for (c = sanitized_name; *c != '\0'; c++) {
+            if (*c == '/') {
+                *c = '_';
+            }
+        }
+
+        filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
+                                   sanitized_name);
+        g_free(sanitized_name);
+
+        fd = mkstemp(filename);
+        if (fd >= 0) {
+            unlink(filename);
+        }
+        g_free(filename);
+    } else {
+        fd = open(path, O_RDWR | O_CREAT, 0644);
+    }
+
+    if (fd < 0) {
+        error_setg_errno(errp, errno,
+                         "unable to create backing store for hugepages");
+        goto error;
+    }
+
+    memory = ROUND_UP(memory, hpagesize);
+
+    /*
+     * ftruncate is not supported by hugetlbfs in older
+     * hosts, so don't bother bailing out on errors.
+     * If anything goes wrong with it under other filesystems,
+     * mmap will fail.
+     */
+    if (ftruncate(fd, memory)) {
+        perror("ftruncate");
+    }
+
+    area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
+    if (area == MAP_FAILED) {
+        error_setg_errno(errp, errno,
+                         "unable to map backing store for hugepages");
+        close(fd);
+        goto error;
+    }
+
+    if (mem_prealloc) {
+        os_mem_prealloc(fd, area, memory);
+    }
+
+    block->fd = fd;
+    return area;
+
+error:
+    return NULL;
+}
+#endif
+
+/* Called with the ramlist lock held.  */
+static ram_addr_t find_ram_offset(ram_addr_t size)
+{
+    RAMBlock *block, *next_block;
+    ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
+
+    assert(size != 0); /* it would hand out same offset multiple times */
+
+    if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
+        return 0;
+    }
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        ram_addr_t end, next = RAM_ADDR_MAX;
+
+        end = block->offset + block->max_length;
+
+        QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
+            if (next_block->offset >= end) {
+                next = MIN(next, next_block->offset);
+            }
+        }
+        if (next - end >= size && next - end < mingap) {
+            offset = end;
+            mingap = next - end;
+        }
+    }
+
+    if (offset == RAM_ADDR_MAX) {
+        fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
+                (uint64_t)size);
+        abort();
+    }
+
+    return offset;
+}
+
+ram_addr_t last_ram_offset(void)
+{
+    RAMBlock *block;
+    ram_addr_t last = 0;
+
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last = MAX(last, block->offset + block->max_length);
+    }
+    rcu_read_unlock();
+    return last;
+}
+
+static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
+{
+    int ret;
+
+    /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
+    if (!machine_dump_guest_core(current_machine)) {
+        ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
+        if (ret) {
+            perror("qemu_madvise");
+            fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
+                            "but dump_guest_core=off specified\n");
+        }
+    }
+}
+
+/* Called within an RCU critical section, or while the ramlist lock
+ * is held.
+ */
+static RAMBlock *find_ram_block(ram_addr_t addr)
+{
+    RAMBlock *block;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (block->offset == addr) {
+            return block;
+        }
+    }
+
+    return NULL;
+}
+
+const char *qemu_ram_get_idstr(RAMBlock *rb)
+{
+    return rb->idstr;
+}
+
+/* Called with iothread lock held.  */
+void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
+{
+    RAMBlock *new_block, *block;
+
+    rcu_read_lock();
+    new_block = find_ram_block(addr);
+    assert(new_block);
+    assert(!new_block->idstr[0]);
+
+    if (dev) {
+        char *id = qdev_get_dev_path(dev);
+        if (id) {
+            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
+            g_free(id);
+        }
+    }
+    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
+            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
+                    new_block->idstr);
+            abort();
+        }
+    }
+    rcu_read_unlock();
+}
+
+/* Called with iothread lock held.  */
+void qemu_ram_unset_idstr(ram_addr_t addr)
+{
+    RAMBlock *block;
+
+    /* FIXME: arch_init.c assumes that this is not called throughout
+     * migration.  Ignore the problem since hot-unplug during migration
+     * does not work anyway.
+     */
+
+    rcu_read_lock();
+    block = find_ram_block(addr);
+    if (block) {
+        memset(block->idstr, 0, sizeof(block->idstr));
+    }
+    rcu_read_unlock();
+}
+
+static int memory_try_enable_merging(void *addr, size_t len)
+{
+    if (!machine_mem_merge(current_machine)) {
+        /* disabled by the user */
+        return 0;
+    }
+
+    return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
+}
+
+/* Only legal before guest might have detected the memory size: e.g. on
+ * incoming migration, or right after reset.
+ *
+ * As memory core doesn't know how is memory accessed, it is up to
+ * resize callback to update device state and/or add assertions to detect
+ * misuse, if necessary.
+ */
+int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
+{
+    RAMBlock *block = find_ram_block(base);
+
+    assert(block);
+
+    newsize = HOST_PAGE_ALIGN(newsize);
+
+    if (block->used_length == newsize) {
+        return 0;
+    }
+
+    if (!(block->flags & RAM_RESIZEABLE)) {
+        error_setg_errno(errp, EINVAL,
+                         "Length mismatch: %s: 0x" RAM_ADDR_FMT
+                         " in != 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->used_length);
+        return -EINVAL;
+    }
+
+    if (block->max_length < newsize) {
+        error_setg_errno(errp, EINVAL,
+                         "Length too large: %s: 0x" RAM_ADDR_FMT
+                         " > 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->max_length);
+        return -EINVAL;
+    }
+
+    cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
+    block->used_length = newsize;
+    cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
+                                        DIRTY_CLIENTS_ALL);
+    memory_region_set_size(block->mr, newsize);
+    if (block->resized) {
+        block->resized(block->idstr, newsize, block->host);
+    }
+    return 0;
+}
+
+static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
+{
+    RAMBlock *block;
+    RAMBlock *last_block = NULL;
+    ram_addr_t old_ram_size, new_ram_size;
+
+    old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    qemu_mutex_lock_ramlist();
+    new_block->offset = find_ram_offset(new_block->max_length);
+
+    if (!new_block->host) {
+        if (xen_enabled()) {
+            xen_ram_alloc(new_block->offset, new_block->max_length,
+                          new_block->mr);
+        } else {
+            new_block->host = phys_mem_alloc(new_block->max_length,
+                                             &new_block->mr->align);
+            if (!new_block->host) {
+                error_setg_errno(errp, errno,
+                                 "cannot set up guest memory '%s'",
+                                 memory_region_name(new_block->mr));
+                qemu_mutex_unlock_ramlist();
+                return -1;
+            }
+            memory_try_enable_merging(new_block->host, new_block->max_length);
+        }
+    }
+
+    new_ram_size = MAX(old_ram_size,
+              (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
+    if (new_ram_size > old_ram_size) {
+        migration_bitmap_extend(old_ram_size, new_ram_size);
+    }
+    /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
+     * QLIST (which has an RCU-friendly variant) does not have insertion at
+     * tail, so save the last element in last_block.
+     */
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last_block = block;
+        if (block->max_length < new_block->max_length) {
+            break;
+        }
+    }
+    if (block) {
+        QLIST_INSERT_BEFORE_RCU(block, new_block, next);
+    } else if (last_block) {
+        QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
+    } else { /* list is empty */
+        QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
+    }
+    ram_list.mru_block = NULL;
+
+    /* Write list before version */
+    smp_wmb();
+    ram_list.version++;
+    qemu_mutex_unlock_ramlist();
+
+    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    if (new_ram_size > old_ram_size) {
+        int i;
+
+        /* ram_list.dirty_memory[] is protected by the iothread lock.  */
+        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+            ram_list.dirty_memory[i] =
+                bitmap_zero_extend(ram_list.dirty_memory[i],
+                                   old_ram_size, new_ram_size);
+       }
+    }
+    cpu_physical_memory_set_dirty_range(new_block->offset,
+                                        new_block->used_length,
+                                        DIRTY_CLIENTS_ALL);
+
+    if (new_block->host) {
+        qemu_ram_setup_dump(new_block->host, new_block->max_length);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
+        if (kvm_enabled()) {
+            kvm_setup_guest_memory(new_block->host, new_block->max_length);
+        }
+    }
+
+    return new_block->offset;
+}
+
+#ifdef __linux__
+ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
+                                    bool share, const char *mem_path,
+                                    Error **errp)
+{
+    RAMBlock *new_block;
+    ram_addr_t addr;
+    Error *local_err = NULL;
+
+    if (xen_enabled()) {
+        error_setg(errp, "-mem-path not supported with Xen");
+        return -1;
+    }
+
+    if (phys_mem_alloc != qemu_anon_ram_alloc) {
+        /*
+         * file_ram_alloc() needs to allocate just like
+         * phys_mem_alloc, but we haven't bothered to provide
+         * a hook there.
+         */
+        error_setg(errp,
+                   "-mem-path not supported with this accelerator");
+        return -1;
+    }
+
+    size = HOST_PAGE_ALIGN(size);
+    new_block = g_malloc0(sizeof(*new_block));
+    new_block->mr = mr;
+    new_block->used_length = size;
+    new_block->max_length = size;
+    new_block->flags = share ? RAM_SHARED : 0;
+    new_block->flags |= RAM_FILE;
+    new_block->host = file_ram_alloc(new_block, size,
+                                     mem_path, errp);
+    if (!new_block->host) {
+        g_free(new_block);
+        return -1;
+    }
+
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;
+}
+#endif
+
+static
+ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
+                                   void (*resized)(const char*,
+                                                   uint64_t length,
+                                                   void *host),
+                                   void *host, bool resizeable,
+                                   MemoryRegion *mr, Error **errp)
+{
+    RAMBlock *new_block;
+    ram_addr_t addr;
+    Error *local_err = NULL;
+
+    size = HOST_PAGE_ALIGN(size);
+    max_size = HOST_PAGE_ALIGN(max_size);
+    new_block = g_malloc0(sizeof(*new_block));
+    new_block->mr = mr;
+    new_block->resized = resized;
+    new_block->used_length = size;
+    new_block->max_length = max_size;
+    assert(max_size >= size);
+    new_block->fd = -1;
+    new_block->host = host;
+    if (host) {
+        new_block->flags |= RAM_PREALLOC;
+    }
+    if (resizeable) {
+        new_block->flags |= RAM_RESIZEABLE;
+    }
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;
+}
+
+ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+                                   MemoryRegion *mr, Error **errp)
+{
+    return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
+}
+
+ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
+{
+    return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
+}
+
+ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
+                                     void (*resized)(const char*,
+                                                     uint64_t length,
+                                                     void *host),
+                                     MemoryRegion *mr, Error **errp)
+{
+    return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
+}
+
+void qemu_ram_free_from_ptr(ram_addr_t addr)
+{
+    RAMBlock *block;
+
+    qemu_mutex_lock_ramlist();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (addr == block->offset) {
+            QLIST_REMOVE_RCU(block, next);
+            ram_list.mru_block = NULL;
+            /* Write list before version */
+            smp_wmb();
+            ram_list.version++;
+            g_free_rcu(block, rcu);
+            break;
+        }
+    }
+    qemu_mutex_unlock_ramlist();
+}
+
+static void reclaim_ramblock(RAMBlock *block)
+{
+    if (block->flags & RAM_PREALLOC) {
+        ;
+    } else if (xen_enabled()) {
+        xen_invalidate_map_cache_entry(block->host);
+#ifndef _WIN32
+    } else if (block->fd >= 0) {
+        if (block->flags & RAM_FILE) {
+            qemu_ram_munmap(block->host, block->max_length);
+        } else {
+            munmap(block->host, block->max_length);
+        }
+        close(block->fd);
+#endif
+    } else {
+        qemu_anon_ram_free(block->host, block->max_length);
+    }
+    g_free(block);
+}
+
+void qemu_ram_free(ram_addr_t addr)
+{
+    RAMBlock *block;
+
+    qemu_mutex_lock_ramlist();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (addr == block->offset) {
+            QLIST_REMOVE_RCU(block, next);
+            ram_list.mru_block = NULL;
+            /* Write list before version */
+            smp_wmb();
+            ram_list.version++;
+            call_rcu(block, reclaim_ramblock, rcu);
+            break;
+        }
+    }
+    qemu_mutex_unlock_ramlist();
+}
+
+#ifndef _WIN32
+void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
+{
+    RAMBlock *block;
+    ram_addr_t offset;
+    int flags;
+    void *area, *vaddr;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        offset = addr - block->offset;
+        if (offset < block->max_length) {
+            vaddr = ramblock_ptr(block, offset);
+            if (block->flags & RAM_PREALLOC) {
+                ;
+            } else if (xen_enabled()) {
+                abort();
+            } else {
+                flags = MAP_FIXED;
+                if (block->fd >= 0) {
+                    flags |= (block->flags & RAM_SHARED ?
+                              MAP_SHARED : MAP_PRIVATE);
+                    area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
+                                flags, block->fd, offset);
+                } else {
+                    /*
+                     * Remap needs to match alloc.  Accelerators that
+                     * set phys_mem_alloc never remap.  If they did,
+                     * we'd need a remap hook here.
+                     */
+                    assert(phys_mem_alloc == qemu_anon_ram_alloc);
+
+                    flags |= MAP_PRIVATE | MAP_ANONYMOUS;
+                    area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
+                                flags, -1, 0);
+                }
+                if (area != vaddr) {
+                    fprintf(stderr, "Could not remap addr: "
+                            RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
+                            length, addr);
+                    exit(1);
+                }
+                memory_try_enable_merging(vaddr, length);
+                qemu_ram_setup_dump(vaddr, length);
+            }
+        }
+    }
+}
+#endif /* !_WIN32 */
+
+int qemu_get_ram_fd(ram_addr_t addr)
+{
+    RAMBlock *block;
+    int fd;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+    fd = block->fd;
+    rcu_read_unlock();
+    return fd;
+}
+
+void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
+{
+    RAMBlock *block;
+    void *ptr;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+    ptr = ramblock_ptr(block, 0);
+    rcu_read_unlock();
+    return ptr;
+}
+
+/* Return a host pointer to ram allocated with qemu_ram_alloc.
+ * This should not be used for general purpose DMA.  Use address_space_map
+ * or address_space_rw instead. For local memory (e.g. video ram) that the
+ * device owns, use memory_region_get_ram_ptr.
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */
+void *qemu_get_ram_ptr(ram_addr_t addr)
+{
+    RAMBlock *block;
+    void *ptr;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+
+    if (xen_enabled() && block->host == NULL) {
+        /* We need to check if the requested address is in the RAM
+         * because we don't want to map the entire memory in QEMU.
+         * In that case just map until the end of the page.
+         */
+        if (block->offset == 0) {
+            ptr = xen_map_cache(addr, 0, 0);
+            goto unlock;
+        }
+
+        block->host = xen_map_cache(block->offset, block->max_length, 1);
+    }
+    ptr = ramblock_ptr(block, addr - block->offset);
+
+unlock:
+    rcu_read_unlock();
+    return ptr;
+}
+
+/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
+ * but takes a size argument.
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */
+static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
+{
+    void *ptr;
+    if (*size == 0) {
+        return NULL;
+    }
+    if (xen_enabled()) {
+        return xen_map_cache(addr, *size, 1);
+    } else {
+        RAMBlock *block;
+        rcu_read_lock();
+        QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+            if (addr - block->offset < block->max_length) {
+                if (addr - block->offset + *size > block->max_length)
+                    *size = block->max_length - addr + block->offset;
+                ptr = ramblock_ptr(block, addr - block->offset);
+                rcu_read_unlock();
+                return ptr;
+            }
+        }
+
+        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
+        abort();
+    }
+}
+
+/*
+ * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
+ * in that RAMBlock.
+ *
+ * ptr: Host pointer to look up
+ * round_offset: If true round the result offset down to a page boundary
+ * *ram_addr: set to result ram_addr
+ * *offset: set to result offset within the RAMBlock
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *ram_addr,
+                                   ram_addr_t *offset)
+{
+    RAMBlock *block;
+    uint8_t *host = ptr;
+
+    if (xen_enabled()) {
+        rcu_read_lock();
+        *ram_addr = xen_ram_addr_from_mapcache(ptr);
+        block = qemu_get_ram_block(*ram_addr);
+        if (block) {
+            *offset = (host - block->host);
+        }
+        rcu_read_unlock();
+        return block;
+    }
+
+    rcu_read_lock();
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && block->host && host - block->host < block->max_length) {
+        goto found;
+    }
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        /* This case append when the block is not mapped. */
+        if (block->host == NULL) {
+            continue;
+        }
+        if (host - block->host < block->max_length) {
+            goto found;
+        }
+    }
+
+    rcu_read_unlock();
+    return NULL;
+
+found:
+    *offset = (host - block->host);
+    if (round_offset) {
+        *offset &= TARGET_PAGE_MASK;
+    }
+    *ram_addr = block->offset + *offset;
+    rcu_read_unlock();
+    return block;
+}
+
+/*
+ * Finds the named RAMBlock
+ *
+ * name: The name of RAMBlock to find
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ */
+RAMBlock *qemu_ram_block_by_name(const char *name)
+{
+    RAMBlock *block;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (!strcmp(name, block->idstr)) {
+            return block;
+        }
+    }
+
+    return NULL;
+}
+
+/* Some of the softmmu routines need to translate from a host pointer
+   (typically a TLB entry) back to a ram offset.  */
+MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+{
+    RAMBlock *block;
+    ram_addr_t offset; /* Not used */
+
+    block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
+
+    if (!block) {
+        return NULL;
+    }
+
+    return block->mr;
+}
+
+static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
+                               uint64_t val, unsigned size)
+{
+    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_page_fast(ram_addr, size);
+    }
+    switch (size) {
+    case 1:
+        stb_p(qemu_get_ram_ptr(ram_addr), val);
+        break;
+    case 2:
+        stw_p(qemu_get_ram_ptr(ram_addr), val);
+        break;
+    case 4:
+        stl_p(qemu_get_ram_ptr(ram_addr), val);
+        break;
+    default:
+        abort();
+    }
+    /* Set both VGA and migration bits for simplicity and to remove
+     * the notdirty callback faster.
+     */
+    cpu_physical_memory_set_dirty_range(ram_addr, size,
+                                        DIRTY_CLIENTS_NOCODE);
+    /* we remove the notdirty callback only if the code has been
+       flushed */
+    if (!cpu_physical_memory_is_clean(ram_addr)) {
+        tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
+    }
+}
+
+static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
+                                 unsigned size, bool is_write)
+{
+    return is_write;
+}
+
+static const MemoryRegionOps notdirty_mem_ops = {
+    .write = notdirty_mem_write,
+    .valid.accepts = notdirty_mem_accepts,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+/* Generate a debug exception if a watchpoint has been hit.  */
+static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
+{
+    CPUState *cpu = current_cpu;
+    CPUArchState *env = cpu->env_ptr;
+    target_ulong pc, cs_base;
+    target_ulong vaddr;
+    CPUWatchpoint *wp;
+    int cpu_flags;
+
+    if (cpu->watchpoint_hit) {
+        /* We re-entered the check after replacing the TB. Now raise
+         * the debug interrupt so that is will trigger after the
+         * current instruction. */
+        cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
+        return;
+    }
+    vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
+    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
+        if (cpu_watchpoint_address_matches(wp, vaddr, len)
+            && (wp->flags & flags)) {
+            if (flags == BP_MEM_READ) {
+                wp->flags |= BP_WATCHPOINT_HIT_READ;
+            } else {
+                wp->flags |= BP_WATCHPOINT_HIT_WRITE;
+            }
+            wp->hitaddr = vaddr;
+            wp->hitattrs = attrs;
+            if (!cpu->watchpoint_hit) {
+                cpu->watchpoint_hit = wp;
+                tb_check_watchpoint(cpu);
+                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
+                    cpu->exception_index = EXCP_DEBUG;
+                    cpu_loop_exit(cpu);
+                } else {
+                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
+                    tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
+                    cpu_resume_from_signal(cpu, NULL);
+                }
+            }
+        } else {
+            wp->flags &= ~BP_WATCHPOINT_HIT;
+        }
+    }
+}
+
+/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
+   so these check for a hit then pass through to the normal out-of-line
+   phys routines.  */
+static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
+                                  unsigned size, MemTxAttrs attrs)
+{
+    MemTxResult res;
+    uint64_t data;
+
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
+    switch (size) {
+    case 1:
+        data = address_space_ldub(&address_space_memory, addr, attrs, &res);
+        break;
+    case 2:
+        data = address_space_lduw(&address_space_memory, addr, attrs, &res);
+        break;
+    case 4:
+        data = address_space_ldl(&address_space_memory, addr, attrs, &res);
+        break;
+    default: abort();
+    }
+    *pdata = data;
+    return res;
+}
+
+static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
+                                   uint64_t val, unsigned size,
+                                   MemTxAttrs attrs)
+{
+    MemTxResult res;
+
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
+    switch (size) {
+    case 1:
+        address_space_stb(&address_space_memory, addr, val, attrs, &res);
+        break;
+    case 2:
+        address_space_stw(&address_space_memory, addr, val, attrs, &res);
+        break;
+    case 4:
+        address_space_stl(&address_space_memory, addr, val, attrs, &res);
+        break;
+    default: abort();
+    }
+    return res;
+}
+
+static const MemoryRegionOps watch_mem_ops = {
+    .read_with_attrs = watch_mem_read,
+    .write_with_attrs = watch_mem_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
+                                unsigned len, MemTxAttrs attrs)
+{
+    subpage_t *subpage = opaque;
+    uint8_t buf[8];
+    MemTxResult res;
+
+#if defined(DEBUG_SUBPAGE)
+    printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
+           subpage, len, addr);
+#endif
+    res = address_space_read(subpage->as, addr + subpage->base,
+                             attrs, buf, len);
+    if (res) {
+        return res;
+    }
+    switch (len) {
+    case 1:
+        *data = ldub_p(buf);
+        return MEMTX_OK;
+    case 2:
+        *data = lduw_p(buf);
+        return MEMTX_OK;
+    case 4:
+        *data = ldl_p(buf);
+        return MEMTX_OK;
+    case 8:
+        *data = ldq_p(buf);
+        return MEMTX_OK;
+    default:
+        abort();
+    }
+}
+
+static MemTxResult subpage_write(void *opaque, hwaddr addr,
+                                 uint64_t value, unsigned len, MemTxAttrs attrs)
+{
+    subpage_t *subpage = opaque;
+    uint8_t buf[8];
+
+#if defined(DEBUG_SUBPAGE)
+    printf("%s: subpage %p len %u addr " TARGET_FMT_plx
+           " value %"PRIx64"\n",
+           __func__, subpage, len, addr, value);
+#endif
+    switch (len) {
+    case 1:
+        stb_p(buf, value);
+        break;
+    case 2:
+        stw_p(buf, value);
+        break;
+    case 4:
+        stl_p(buf, value);
+        break;
+    case 8:
+        stq_p(buf, value);
+        break;
+    default:
+        abort();
+    }
+    return address_space_write(subpage->as, addr + subpage->base,
+                               attrs, buf, len);
+}
+
+static bool subpage_accepts(void *opaque, hwaddr addr,
+                            unsigned len, bool is_write)
+{
+    subpage_t *subpage = opaque;
+#if defined(DEBUG_SUBPAGE)
+    printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
+           __func__, subpage, is_write ? 'w' : 'r', len, addr);
+#endif
+
+    return address_space_access_valid(subpage->as, addr + subpage->base,
+                                      len, is_write);
+}
+
+static const MemoryRegionOps subpage_ops = {
+    .read_with_attrs = subpage_read,
+    .write_with_attrs = subpage_write,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 8,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 8,
+    .valid.accepts = subpage_accepts,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
+                             uint16_t section)
+{
+    int idx, eidx;
+
+    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
+        return -1;
+    idx = SUBPAGE_IDX(start);
+    eidx = SUBPAGE_IDX(end);
+#if defined(DEBUG_SUBPAGE)
+    printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
+           __func__, mmio, start, end, idx, eidx, section);
+#endif
+    for (; idx <= eidx; idx++) {
+        mmio->sub_section[idx] = section;
+    }
+
+    return 0;
+}
+
+static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
+{
+    subpage_t *mmio;
+
+    mmio = g_malloc0(sizeof(subpage_t));
+
+    mmio->as = as;
+    mmio->base = base;
+    memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
+                          NULL, TARGET_PAGE_SIZE);
+    mmio->iomem.subpage = true;
+#if defined(DEBUG_SUBPAGE)
+    printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
+           mmio, base, TARGET_PAGE_SIZE);
+#endif
+    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
+
+    return mmio;
+}
+
+static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
+                              MemoryRegion *mr)
+{
+    assert(as);
+    MemoryRegionSection section = {
+        .address_space = as,
+        .mr = mr,
+        .offset_within_address_space = 0,
+        .offset_within_region = 0,
+        .size = int128_2_64(),
+    };
+
+    return phys_section_add(map, &section);
+}
+
+MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
+{
+    CPUAddressSpace *cpuas = &cpu->cpu_ases[0];
+    AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
+    MemoryRegionSection *sections = d->map.sections;
+
+    return sections[index & ~TARGET_PAGE_MASK].mr;
+}
+
+static void io_mem_init(void)
+{
+    memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
+    memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
+                          NULL, UINT64_MAX);
+    memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
+                          NULL, UINT64_MAX);
+    memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
+                          NULL, UINT64_MAX);
+}
+
+static void mem_begin(MemoryListener *listener)
+{
+    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
+    AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
+    uint16_t n;
+
+    n = dummy_section(&d->map, as, &io_mem_unassigned);
+    assert(n == PHYS_SECTION_UNASSIGNED);
+    n = dummy_section(&d->map, as, &io_mem_notdirty);
+    assert(n == PHYS_SECTION_NOTDIRTY);
+    n = dummy_section(&d->map, as, &io_mem_rom);
+    assert(n == PHYS_SECTION_ROM);
+    n = dummy_section(&d->map, as, &io_mem_watch);
+    assert(n == PHYS_SECTION_WATCH);
+
+    d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
+    d->as = as;
+    as->next_dispatch = d;
+}
+
+static void address_space_dispatch_free(AddressSpaceDispatch *d)
+{
+    phys_sections_free(&d->map);
+    g_free(d);
+}
+
+static void mem_commit(MemoryListener *listener)
+{
+    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
+    AddressSpaceDispatch *cur = as->dispatch;
+    AddressSpaceDispatch *next = as->next_dispatch;
+
+    phys_page_compact_all(next, next->map.nodes_nb);
+
+    atomic_rcu_set(&as->dispatch, next);
+    if (cur) {
+        call_rcu(cur, address_space_dispatch_free, rcu);
+    }
+}
+
+static void tcg_commit(MemoryListener *listener)
+{
+    CPUAddressSpace *cpuas;
+    AddressSpaceDispatch *d;
+
+    /* since each CPU stores ram addresses in its TLB cache, we must
+       reset the modified entries */
+    cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
+    cpu_reloading_memory_map();
+    /* The CPU and TLB are protected by the iothread lock.
+     * We reload the dispatch pointer now because cpu_reloading_memory_map()
+     * may have split the RCU critical section.
+     */
+    d = atomic_rcu_read(&cpuas->as->dispatch);
+    cpuas->memory_dispatch = d;
+    tlb_flush(cpuas->cpu, 1);
+}
+
+void address_space_init_dispatch(AddressSpace *as)
+{
+    as->dispatch = NULL;
+    as->dispatch_listener = (MemoryListener) {
+        .begin = mem_begin,
+        .commit = mem_commit,
+        .region_add = mem_add,
+        .region_nop = mem_add,
+        .priority = 0,
+    };
+    memory_listener_register(&as->dispatch_listener, as);
+}
+
+void address_space_unregister(AddressSpace *as)
+{
+    memory_listener_unregister(&as->dispatch_listener);
+}
+
+void address_space_destroy_dispatch(AddressSpace *as)
+{
+    AddressSpaceDispatch *d = as->dispatch;
+
+    atomic_rcu_set(&as->dispatch, NULL);
+    if (d) {
+        call_rcu(d, address_space_dispatch_free, rcu);
+    }
+}
+
+static void memory_map_init(void)
+{
+    system_memory = g_malloc(sizeof(*system_memory));
+
+    memory_region_init(system_memory, NULL, "system", UINT64_MAX);
+    address_space_init(&address_space_memory, system_memory, "memory");
+
+    system_io = g_malloc(sizeof(*system_io));
+    memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
+                          65536);
+    address_space_init(&address_space_io, system_io, "I/O");
+}
+
+MemoryRegion *get_system_memory(void)
+{
+    return system_memory;
+}
+
+MemoryRegion *get_system_io(void)
+{
+    return system_io;
+}
+
+#endif /* !defined(CONFIG_USER_ONLY) */
+
+/* physical memory access (slow version, mainly for debug) */
+#if defined(CONFIG_USER_ONLY)
+int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
+                        uint8_t *buf, int len, int is_write)
+{
+    int l, flags;
+    target_ulong page;
+    void * p;
+
+    while (len > 0) {
+        page = addr & TARGET_PAGE_MASK;
+        l = (page + TARGET_PAGE_SIZE) - addr;
+        if (l > len)
+            l = len;
+        flags = page_get_flags(page);
+        if (!(flags & PAGE_VALID))
+            return -1;
+        if (is_write) {
+            if (!(flags & PAGE_WRITE))
+                return -1;
+            /* XXX: this code should not depend on lock_user */
+            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
+                return -1;
+            memcpy(p, buf, l);
+            unlock_user(p, addr, l);
+        } else {
+            if (!(flags & PAGE_READ))
+                return -1;
+            /* XXX: this code should not depend on lock_user */
+            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
+                return -1;
+            memcpy(buf, p, l);
+            unlock_user(p, addr, 0);
+        }
+        len -= l;
+        buf += l;
+        addr += l;
+    }
+    return 0;
+}
+
+#else
+
+static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
+                                     hwaddr length)
+{
+    uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
+    /* No early return if dirty_log_mask is or becomes 0, because
+     * cpu_physical_memory_set_dirty_range will still call
+     * xen_modified_memory.
+     */
+    if (dirty_log_mask) {
+        dirty_log_mask =
+            cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
+    }
+    if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_range(addr, addr + length);
+        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
+    }
+    cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
+}
+
+static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
+{
+    unsigned access_size_max = mr->ops->valid.max_access_size;
+
+    /* Regions are assumed to support 1-4 byte accesses unless
+       otherwise specified.  */
+    if (access_size_max == 0) {
+        access_size_max = 4;
+    }
+
+    /* Bound the maximum access by the alignment of the address.  */
+    if (!mr->ops->impl.unaligned) {
+        unsigned align_size_max = addr & -addr;
+        if (align_size_max != 0 && align_size_max < access_size_max) {
+            access_size_max = align_size_max;
+        }
+    }
+
+    /* Don't attempt accesses larger than the maximum.  */
+    if (l > access_size_max) {
+        l = access_size_max;
+    }
+    l = pow2floor(l);
+
+    return l;
+}
+
+static bool prepare_mmio_access(MemoryRegion *mr)
+{
+    bool unlocked = !qemu_mutex_iothread_locked();
+    bool release_lock = false;
+
+    if (unlocked && mr->global_locking) {
+        qemu_mutex_lock_iothread();
+        unlocked = false;
+        release_lock = true;
+    }
+    if (mr->flush_coalesced_mmio) {
+        if (unlocked) {
+            qemu_mutex_lock_iothread();
+        }
+        qemu_flush_coalesced_mmio_buffer();
+        if (unlocked) {
+            qemu_mutex_unlock_iothread();
+        }
+    }
+
+    return release_lock;
+}
+
+MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                             uint8_t *buf, int len, bool is_write)
+{
+    hwaddr l;
+    uint8_t *ptr;
+    uint64_t val;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+    bool release_lock = false;
+
+    rcu_read_lock();
+    while (len > 0) {
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, is_write);
+
+        if (is_write) {
+            if (!memory_access_is_direct(mr, is_write)) {
+                release_lock |= prepare_mmio_access(mr);
+                l = memory_access_size(mr, l, addr1);
+                /* XXX: could force current_cpu to NULL to avoid
+                   potential bugs */
+                switch (l) {
+                case 8:
+                    /* 64 bit write access */
+                    val = ldq_p(buf);
+                    result |= memory_region_dispatch_write(mr, addr1, val, 8,
+                                                           attrs);
+                    break;
+                case 4:
+                    /* 32 bit write access */
+                    val = ldl_p(buf);
+                    result |= memory_region_dispatch_write(mr, addr1, val, 4,
+                                                           attrs);
+                    break;
+                case 2:
+                    /* 16 bit write access */
+                    val = lduw_p(buf);
+                    result |= memory_region_dispatch_write(mr, addr1, val, 2,
+                                                           attrs);
+                    break;
+                case 1:
+                    /* 8 bit write access */
+                    val = ldub_p(buf);
+                    result |= memory_region_dispatch_write(mr, addr1, val, 1,
+                                                           attrs);
+                    break;
+                default:
+                    abort();
+                }
+            } else {
+                addr1 += memory_region_get_ram_addr(mr);
+                /* RAM case */
+                ptr = qemu_get_ram_ptr(addr1);
+                memcpy(ptr, buf, l);
+                invalidate_and_set_dirty(mr, addr1, l);
+            }
+        } else {
+            if (!memory_access_is_direct(mr, is_write)) {
+                /* I/O case */
+                release_lock |= prepare_mmio_access(mr);
+                l = memory_access_size(mr, l, addr1);
+                switch (l) {
+                case 8:
+                    /* 64 bit read access */
+                    result |= memory_region_dispatch_read(mr, addr1, &val, 8,
+                                                          attrs);
+                    stq_p(buf, val);
+                    break;
+                case 4:
+                    /* 32 bit read access */
+                    result |= memory_region_dispatch_read(mr, addr1, &val, 4,
+                                                          attrs);
+                    stl_p(buf, val);
+                    break;
+                case 2:
+                    /* 16 bit read access */
+                    result |= memory_region_dispatch_read(mr, addr1, &val, 2,
+                                                          attrs);
+                    stw_p(buf, val);
+                    break;
+                case 1:
+                    /* 8 bit read access */
+                    result |= memory_region_dispatch_read(mr, addr1, &val, 1,
+                                                          attrs);
+                    stb_p(buf, val);
+                    break;
+                default:
+                    abort();
+                }
+            } else {
+                /* RAM case */
+                ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
+                memcpy(buf, ptr, l);
+            }
+        }
+
+        if (release_lock) {
+            qemu_mutex_unlock_iothread();
+            release_lock = false;
+        }
+
+        len -= l;
+        buf += l;
+        addr += l;
+    }
+    rcu_read_unlock();
+
+    return result;
+}
+
+MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                                const uint8_t *buf, int len)
+{
+    return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
+}
+
+MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                               uint8_t *buf, int len)
+{
+    return address_space_rw(as, addr, attrs, buf, len, false);
+}
+
+
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
+                            int len, int is_write)
+{
+    address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
+                     buf, len, is_write);
+}
+
+enum write_rom_type {
+    WRITE_DATA,
+    FLUSH_CACHE,
+};
+
+static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
+    hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
+{
+    hwaddr l;
+    uint8_t *ptr;
+    hwaddr addr1;
+    MemoryRegion *mr;
+
+    rcu_read_lock();
+    while (len > 0) {
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
+
+        if (!(memory_region_is_ram(mr) ||
+              memory_region_is_romd(mr))) {
+            l = memory_access_size(mr, l, addr1);
+        } else {
+            addr1 += memory_region_get_ram_addr(mr);
+            /* ROM/RAM case */
+            ptr = qemu_get_ram_ptr(addr1);
+            switch (type) {
+            case WRITE_DATA:
+                memcpy(ptr, buf, l);
+                invalidate_and_set_dirty(mr, addr1, l);
+                break;
+            case FLUSH_CACHE:
+                flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
+                break;
+            }
+        }
+        len -= l;
+        buf += l;
+        addr += l;
+    }
+    rcu_read_unlock();
+}
+
+/* used for ROM loading : can write in RAM and ROM */
+void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
+                                   const uint8_t *buf, int len)
+{
+    cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
+}
+
+void cpu_flush_icache_range(hwaddr start, int len)
+{
+    /*
+     * This function should do the same thing as an icache flush that was
+     * triggered from within the guest. For TCG we are always cache coherent,
+     * so there is no need to flush anything. For KVM / Xen we need to flush
+     * the host's instruction cache at least.
+     */
+    if (tcg_enabled()) {
+        return;
+    }
+
+    cpu_physical_memory_write_rom_internal(&address_space_memory,
+                                           start, NULL, len, FLUSH_CACHE);
+}
+
+typedef struct {
+    MemoryRegion *mr;
+    void *buffer;
+    hwaddr addr;
+    hwaddr len;
+    bool in_use;
+} BounceBuffer;
+
+static BounceBuffer bounce;
+
+typedef struct MapClient {
+    QEMUBH *bh;
+    QLIST_ENTRY(MapClient) link;
+} MapClient;
+
+QemuMutex map_client_list_lock;
+static QLIST_HEAD(map_client_list, MapClient) map_client_list
+    = QLIST_HEAD_INITIALIZER(map_client_list);
+
+static void cpu_unregister_map_client_do(MapClient *client)
+{
+    QLIST_REMOVE(client, link);
+    g_free(client);
+}
+
+static void cpu_notify_map_clients_locked(void)
+{
+    MapClient *client;
+
+    while (!QLIST_EMPTY(&map_client_list)) {
+        client = QLIST_FIRST(&map_client_list);
+        qemu_bh_schedule(client->bh);
+        cpu_unregister_map_client_do(client);
+    }
+}
+
+void cpu_register_map_client(QEMUBH *bh)
+{
+    MapClient *client = g_malloc(sizeof(*client));
+
+    qemu_mutex_lock(&map_client_list_lock);
+    client->bh = bh;
+    QLIST_INSERT_HEAD(&map_client_list, client, link);
+    if (!atomic_read(&bounce.in_use)) {
+        cpu_notify_map_clients_locked();
+    }
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
+void cpu_exec_init_all(void)
+{
+    qemu_mutex_init(&ram_list.mutex);
+    io_mem_init();
+    memory_map_init();
+    qemu_mutex_init(&map_client_list_lock);
+}
+
+void cpu_unregister_map_client(QEMUBH *bh)
+{
+    MapClient *client;
+
+    qemu_mutex_lock(&map_client_list_lock);
+    QLIST_FOREACH(client, &map_client_list, link) {
+        if (client->bh == bh) {
+            cpu_unregister_map_client_do(client);
+            break;
+        }
+    }
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
+static void cpu_notify_map_clients(void)
+{
+    qemu_mutex_lock(&map_client_list_lock);
+    cpu_notify_map_clients_locked();
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
+bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
+{
+    MemoryRegion *mr;
+    hwaddr l, xlat;
+
+    rcu_read_lock();
+    while (len > 0) {
+        l = len;
+        mr = address_space_translate(as, addr, &xlat, &l, is_write);
+        if (!memory_access_is_direct(mr, is_write)) {
+            l = memory_access_size(mr, l, addr);
+            if (!memory_region_access_valid(mr, xlat, l, is_write)) {
+                return false;
+            }
+        }
+
+        len -= l;
+        addr += l;
+    }
+    rcu_read_unlock();
+    return true;
+}
+
+/* Map a physical memory region into a host virtual address.
+ * May map a subset of the requested range, given by and returned in *plen.
+ * May return NULL if resources needed to perform the mapping are exhausted.
+ * Use only for reads OR writes - not for read-modify-write operations.
+ * Use cpu_register_map_client() to know when retrying the map operation is
+ * likely to succeed.
+ */
+void *address_space_map(AddressSpace *as,
+                        hwaddr addr,
+                        hwaddr *plen,
+                        bool is_write)
+{
+    hwaddr len = *plen;
+    hwaddr done = 0;
+    hwaddr l, xlat, base;
+    MemoryRegion *mr, *this_mr;
+    ram_addr_t raddr;
+
+    if (len == 0) {
+        return NULL;
+    }
+
+    l = len;
+    rcu_read_lock();
+    mr = address_space_translate(as, addr, &xlat, &l, is_write);
+
+    if (!memory_access_is_direct(mr, is_write)) {
+        if (atomic_xchg(&bounce.in_use, true)) {
+            rcu_read_unlock();
+            return NULL;
+        }
+        /* Avoid unbounded allocations */
+        l = MIN(l, TARGET_PAGE_SIZE);
+        bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
+        bounce.addr = addr;
+        bounce.len = l;
+
+        memory_region_ref(mr);
+        bounce.mr = mr;
+        if (!is_write) {
+            address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
+                               bounce.buffer, l);
+        }
+
+        rcu_read_unlock();
+        *plen = l;
+        return bounce.buffer;
+    }
+
+    base = xlat;
+    raddr = memory_region_get_ram_addr(mr);
+
+    for (;;) {
+        len -= l;
+        addr += l;
+        done += l;
+        if (len == 0) {
+            break;
+        }
+
+        l = len;
+        this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
+        if (this_mr != mr || xlat != base + done) {
+            break;
+        }
+    }
+
+    memory_region_ref(mr);
+    rcu_read_unlock();
+    *plen = done;
+    return qemu_ram_ptr_length(raddr + base, plen);
+}
+
+/* Unmaps a memory region previously mapped by address_space_map().
+ * Will also mark the memory as dirty if is_write == 1.  access_len gives
+ * the amount of memory that was actually read or written by the caller.
+ */
+void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
+                         int is_write, hwaddr access_len)
+{
+    if (buffer != bounce.buffer) {
+        MemoryRegion *mr;
+        ram_addr_t addr1;
+
+        mr = qemu_ram_addr_from_host(buffer, &addr1);
+        assert(mr != NULL);
+        if (is_write) {
+            invalidate_and_set_dirty(mr, addr1, access_len);
+        }
+        if (xen_enabled()) {
+            xen_invalidate_map_cache_entry(buffer);
+        }
+        memory_region_unref(mr);
+        return;
+    }
+    if (is_write) {
+        address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
+                            bounce.buffer, access_len);
+    }
+    qemu_vfree(bounce.buffer);
+    bounce.buffer = NULL;
+    memory_region_unref(bounce.mr);
+    atomic_mb_set(&bounce.in_use, false);
+    cpu_notify_map_clients();
+}
+
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
+                              int is_write)
+{
+    return address_space_map(&address_space_memory, addr, plen, is_write);
+}
+
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len)
+{
+    return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
+}
+
+/* warning: addr must be aligned */
+static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
+                                                  MemTxAttrs attrs,
+                                                  MemTxResult *result,
+                                                  enum device_endian endian)
+{
+    uint8_t *ptr;
+    uint64_t val;
+    MemoryRegion *mr;
+    hwaddr l = 4;
+    hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
+
+    rcu_read_lock();
+    mr = address_space_translate(as, addr, &addr1, &l, false);
+    if (l < 4 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
+        /* I/O case */
+        r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
+#if defined(TARGET_WORDS_BIGENDIAN)
+        if (endian == DEVICE_LITTLE_ENDIAN) {
+            val = bswap32(val);
+        }
+#else
+        if (endian == DEVICE_BIG_ENDIAN) {
+            val = bswap32(val);
+        }
+#endif
+    } else {
+        /* RAM case */
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+                                & TARGET_PAGE_MASK)
+                               + addr1);
+        switch (endian) {
+        case DEVICE_LITTLE_ENDIAN:
+            val = ldl_le_p(ptr);
+            break;
+        case DEVICE_BIG_ENDIAN:
+            val = ldl_be_p(ptr);
+            break;
+        default:
+            val = ldl_p(ptr);
+            break;
+        }
+        r = MEMTX_OK;
+    }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+    return val;
+}
+
+uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_NATIVE_ENDIAN);
+}
+
+uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
+                              MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_LITTLE_ENDIAN);
+}
+
+uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
+                              MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_BIG_ENDIAN);
+}
+
+uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+/* warning: addr must be aligned */
+static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
+                                                  MemTxAttrs attrs,
+                                                  MemTxResult *result,
+                                                  enum device_endian endian)
+{
+    uint8_t *ptr;
+    uint64_t val;
+    MemoryRegion *mr;
+    hwaddr l = 8;
+    hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
+
+    rcu_read_lock();
+    mr = address_space_translate(as, addr, &addr1, &l,
+                                 false);
+    if (l < 8 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
+        /* I/O case */
+        r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
+#if defined(TARGET_WORDS_BIGENDIAN)
+        if (endian == DEVICE_LITTLE_ENDIAN) {
+            val = bswap64(val);
+        }
+#else
+        if (endian == DEVICE_BIG_ENDIAN) {
+            val = bswap64(val);
+        }
+#endif
+    } else {
+        /* RAM case */
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+                                & TARGET_PAGE_MASK)
+                               + addr1);
+        switch (endian) {
+        case DEVICE_LITTLE_ENDIAN:
+            val = ldq_le_p(ptr);
+            break;
+        case DEVICE_BIG_ENDIAN:
+            val = ldq_be_p(ptr);
+            break;
+        default:
+            val = ldq_p(ptr);
+            break;
+        }
+        r = MEMTX_OK;
+    }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+    return val;
+}
+
+uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_NATIVE_ENDIAN);
+}
+
+uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_LITTLE_ENDIAN);
+}
+
+uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_BIG_ENDIAN);
+}
+
+uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+/* XXX: optimize */
+uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result)
+{
+    uint8_t val;
+    MemTxResult r;
+
+    r = address_space_rw(as, addr, attrs, &val, 1, 0);
+    if (result) {
+        *result = r;
+    }
+    return val;
+}
+
+uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
+/* warning: addr must be aligned */
+static inline uint32_t address_space_lduw_internal(AddressSpace *as,
+                                                   hwaddr addr,
+                                                   MemTxAttrs attrs,
+                                                   MemTxResult *result,
+                                                   enum device_endian endian)
+{
+    uint8_t *ptr;
+    uint64_t val;
+    MemoryRegion *mr;
+    hwaddr l = 2;
+    hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
+
+    rcu_read_lock();
+    mr = address_space_translate(as, addr, &addr1, &l,
+                                 false);
+    if (l < 2 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
+        /* I/O case */
+        r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
+#if defined(TARGET_WORDS_BIGENDIAN)
+        if (endian == DEVICE_LITTLE_ENDIAN) {
+            val = bswap16(val);
+        }
+#else
+        if (endian == DEVICE_BIG_ENDIAN) {
+            val = bswap16(val);
+        }
+#endif
+    } else {
+        /* RAM case */
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+                                & TARGET_PAGE_MASK)
+                               + addr1);
+        switch (endian) {
+        case DEVICE_LITTLE_ENDIAN:
+            val = lduw_le_p(ptr);
+            break;
+        case DEVICE_BIG_ENDIAN:
+            val = lduw_be_p(ptr);
+            break;
+        default:
+            val = lduw_p(ptr);
+            break;
+        }
+        r = MEMTX_OK;
+    }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+    return val;
+}
+
+uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)