Merge remote-tracking branch 'pmaydell/target-arm.for-upstream' into staging

# By Peter Maydell
# Via Peter Maydell
* pmaydell/target-arm.for-upstream:
  target-arm: Make LPAE feature imply V7MP
  target-arm: Use tuple list to sync cp regs with KVM
  target-arm: Reinitialize all KVM VCPU registers on reset
  target-arm: Initialize cpreg list from KVM when using KVM
  target-arm: Convert TCG to using (index,value) list for cp migration
  target-arm: mark up cpregs for no-migrate or raw access
  target-arm: Add raw_readfn and raw_writefn to ARMCPRegInfo
  target-arm: Allow special cpregs to have flags set

Message-id: 1372181592-32170-1-git-send-email-peter.maydell@linaro.org
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

9 files changed, 760 insertions, 167 deletions

diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index d89b57c114..4a6e52e528 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -1,5 +1,6 @@
 obj-y += arm-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
+obj-$(CONFIG_NO_KVM) += kvm-stub.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index 12fcefe0c6..25239b8952 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -62,6 +62,29 @@ typedef struct ARMCPU {
 
     /* Coprocessor information */
     GHashTable *cp_regs;
+    /* For marshalling (mostly coprocessor) register state between the
+     * kernel and QEMU (for KVM) and between two QEMUs (for migration),
+     * we use these arrays.
+     */
+    /* List of register indexes managed via these arrays; (full KVM style
+     * 64 bit indexes, not CPRegInfo 32 bit indexes)
+     */
+    uint64_t *cpreg_indexes;
+    /* Values of the registers (cpreg_indexes[i]'s value is cpreg_values[i]) */
+    uint64_t *cpreg_values;
+    /* When using KVM, keeps a copy of the initial state of the VCPU,
+     * so that on reset we can feed the reset values back into the kernel.
+     */
+    uint64_t *cpreg_reset_values;
+    /* Length of the indexes, values, reset_values arrays */
+    int32_t cpreg_array_len;
+    /* These are used only for migration: incoming data arrives in
+     * these fields and is sanity checked in post_load before copying
+     * to the working data structures above.
+     */
+    uint64_t *cpreg_vmstate_indexes;
+    uint64_t *cpreg_vmstate_values;
+    int32_t cpreg_vmstate_array_len;
 
     /* The instance init functions for implementation-specific subclasses
      * set these fields to specify the implementation-dependent values of
@@ -116,6 +139,7 @@ extern const struct VMStateDescription vmstate_arm_cpu;
 #endif
 
 void register_cp_regs_for_features(ARMCPU *cpu);
+void init_cpreg_list(ARMCPU *cpu);
 
 void arm_cpu_do_interrupt(CPUState *cpu);
 void arm_v7m_cpu_do_interrupt(CPUState *cpu);
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index 496a59f5c0..2371f48057 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -198,12 +198,15 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
         set_feature(env, ARM_FEATURE_VFP);
     }
     if (arm_feature(env, ARM_FEATURE_LPAE)) {
+        set_feature(env, ARM_FEATURE_V7MP);
         set_feature(env, ARM_FEATURE_PXN);
     }
 
     register_cp_regs_for_features(cpu);
     arm_cpu_register_gdb_regs_for_features(cpu);
 
+    init_cpreg_list(cpu);
+
     cpu_reset(CPU(cpu));
     qemu_init_vcpu(env);
 
@@ -571,7 +574,6 @@ static void cortex_a15_initfn(Object *obj)
     set_feature(&cpu->env, ARM_FEATURE_NEON);
     set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
     set_feature(&cpu->env, ARM_FEATURE_ARM_DIV);
-    set_feature(&cpu->env, ARM_FEATURE_V7MP);
     set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER);
     set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
     set_feature(&cpu->env, ARM_FEATURE_LPAE);
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 54384446b4..abcc0b4b1c 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -424,6 +424,43 @@ void armv7m_nvic_complete_irq(void *opaque, int irq);
     (((cp) << 16) | ((is64) << 15) | ((crn) << 11) |    \
      ((crm) << 7) | ((opc1) << 3) | (opc2))
 
+/* Note that these must line up with the KVM/ARM register
+ * ID field definitions (kvm.c will check this, but we
+ * can't just use the KVM defines here as the kvm headers
+ * are unavailable to non-KVM-specific files)
+ */
+#define CP_REG_SIZE_SHIFT 52
+#define CP_REG_SIZE_MASK       0x00f0000000000000ULL
+#define CP_REG_SIZE_U32        0x0020000000000000ULL
+#define CP_REG_SIZE_U64        0x0030000000000000ULL
+#define CP_REG_ARM             0x4000000000000000ULL
+
+/* Convert a full 64 bit KVM register ID to the truncated 32 bit
+ * version used as a key for the coprocessor register hashtable
+ */
+static inline uint32_t kvm_to_cpreg_id(uint64_t kvmid)
+{
+    uint32_t cpregid = kvmid;
+    if ((kvmid & CP_REG_SIZE_MASK) == CP_REG_SIZE_U64) {
+        cpregid |= (1 << 15);
+    }
+    return cpregid;
+}
+
+/* Convert a truncated 32 bit hashtable key into the full
+ * 64 bit KVM register ID.
+ */
+static inline uint64_t cpreg_to_kvm_id(uint32_t cpregid)
+{
+    uint64_t kvmid = cpregid & ~(1 << 15);
+    if (cpregid & (1 << 15)) {
+        kvmid |= CP_REG_SIZE_U64 | CP_REG_ARM;
+    } else {
+        kvmid |= CP_REG_SIZE_U32 | CP_REG_ARM;
+    }
+    return kvmid;
+}
+
 /* ARMCPRegInfo type field bits. If the SPECIAL bit is set this is a
  * special-behaviour cp reg and bits [15..8] indicate what behaviour
  * it has. Otherwise it is a simple cp reg, where CONST indicates that
@@ -434,19 +471,22 @@ void armv7m_nvic_complete_irq(void *opaque, int irq);
  * a register definition to override a previous definition for the
  * same (cp, is64, crn, crm, opc1, opc2) tuple: either the new or the
  * old must have the OVERRIDE bit set.
+ * NO_MIGRATE indicates that this register should be ignored for migration;
+ * (eg because any state is accessed via some other coprocessor register).
  */
 #define ARM_CP_SPECIAL 1
 #define ARM_CP_CONST 2
 #define ARM_CP_64BIT 4
 #define ARM_CP_SUPPRESS_TB_END 8
 #define ARM_CP_OVERRIDE 16
+#define ARM_CP_NO_MIGRATE 32
 #define ARM_CP_NOP (ARM_CP_SPECIAL | (1 << 8))
 #define ARM_CP_WFI (ARM_CP_SPECIAL | (2 << 8))
 #define ARM_LAST_SPECIAL ARM_CP_WFI
 /* Used only as a terminator for ARMCPRegInfo lists */
 #define ARM_CP_SENTINEL 0xffff
 /* Mask of only the flag bits in a type field */
-#define ARM_CP_FLAG_MASK 0x1f
+#define ARM_CP_FLAG_MASK 0x3f
 
 /* Return true if cptype is a valid type field. This is used to try to
  * catch errors where the sentinel has been accidentally left off the end
@@ -456,7 +496,7 @@ static inline bool cptype_valid(int cptype)
 {
     return ((cptype & ~ARM_CP_FLAG_MASK) == 0)
         || ((cptype & ARM_CP_SPECIAL) &&
-            (cptype <= ARM_LAST_SPECIAL));
+            ((cptype & ~ARM_CP_FLAG_MASK) <= ARM_LAST_SPECIAL));
 }
 
 /* Access rights:
@@ -562,6 +602,19 @@ struct ARMCPRegInfo {
      * by fieldoffset.
      */
     CPWriteFn *writefn;
+    /* Function for doing a "raw" read; used when we need to copy
+     * coprocessor state to the kernel for KVM or out for
+     * migration. This only needs to be provided if there is also a
+     * readfn and it makes an access permission check.
+     */
+    CPReadFn *raw_readfn;
+    /* Function for doing a "raw" write; used when we need to copy KVM
+     * kernel coprocessor state into userspace, or for inbound
+     * migration. This only needs to be provided if there is also a
+     * writefn and it makes an access permission check or masks out
+     * "unwritable" bits or has write-one-to-clear or similar behaviour.
+     */
+    CPWriteFn *raw_writefn;
     /* Function for resetting the register. If NULL, then reset will be done
      * by writing resetvalue to the field specified in fieldoffset. If
      * fieldoffset is 0 then no reset will be done.
@@ -605,6 +658,38 @@ static inline bool cp_access_ok(CPUARMState *env,
     return (ri->access >> ((arm_current_pl(env) * 2) + isread)) & 1;
 }
 
+/**
+ * write_list_to_cpustate
+ * @cpu: ARMCPU
+ *
+ * For each register listed in the ARMCPU cpreg_indexes list, write
+ * its value from the cpreg_values list into the ARMCPUState structure.
+ * This updates TCG's working data structures from KVM data or
+ * from incoming migration state.
+ *
+ * Returns: true if all register values were updated correctly,
+ * false if some register was unknown or could not be written.
+ * Note that we do not stop early on failure -- we will attempt
+ * writing all registers in the list.
+ */
+bool write_list_to_cpustate(ARMCPU *cpu);
+
+/**
+ * write_cpustate_to_list:
+ * @cpu: ARMCPU
+ *
+ * For each register listed in the ARMCPU cpreg_indexes list, write
+ * its value from the ARMCPUState structure into the cpreg_values list.
+ * This is used to copy info from TCG's working data structures into
+ * KVM or for outbound migration.
+ *
+ * Returns: true if all register values were read correctly,
+ * false if some register was unknown or could not be read.
+ * Note that we do not stop early on failure -- we will attempt
+ * reading all registers in the list.
+ */
+bool write_cpustate_to_list(ARMCPU *cpu);
+
 /* Does the core conform to the the "MicroController" profile. e.g. Cortex-M3.
    Note the M in older cores (eg. ARM7TDMI) stands for Multiply. These are
    conventional cores (ie. Application or Realtime profile).  */
diff --git a/target-arm/helper.c b/target-arm/helper.c
index fd055e89f2..5f639fdeb8 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -64,6 +64,194 @@ static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
     return 0;
 }
 
+static int raw_read(CPUARMState *env, const ARMCPRegInfo *ri,
+                    uint64_t *value)
+{
+    *value = CPREG_FIELD32(env, ri);
+    return 0;
+}
+
+static int raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                     uint64_t value)
+{
+    CPREG_FIELD32(env, ri) = value;
+    return 0;
+}
+
+static bool read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
+                            uint64_t *v)
+{
+    /* Raw read of a coprocessor register (as needed for migration, etc)
+     * return true on success, false if the read is impossible for some reason.
+     */
+    if (ri->type & ARM_CP_CONST) {
+        *v = ri->resetvalue;
+    } else if (ri->raw_readfn) {
+        return (ri->raw_readfn(env, ri, v) == 0);
+    } else if (ri->readfn) {
+        return (ri->readfn(env, ri, v) == 0);
+    } else {
+        if (ri->type & ARM_CP_64BIT) {
+            *v = CPREG_FIELD64(env, ri);
+        } else {
+            *v = CPREG_FIELD32(env, ri);
+        }
+    }
+    return true;
+}
+
+static bool write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
+                             int64_t v)
+{
+    /* Raw write of a coprocessor register (as needed for migration, etc).
+     * Return true on success, false if the write is impossible for some reason.
+     * Note that constant registers are treated as write-ignored; the
+     * caller should check for success by whether a readback gives the
+     * value written.
+     */
+    if (ri->type & ARM_CP_CONST) {
+        return true;
+    } else if (ri->raw_writefn) {
+        return (ri->raw_writefn(env, ri, v) == 0);
+    } else if (ri->writefn) {
+        return (ri->writefn(env, ri, v) == 0);
+    } else {
+        if (ri->type & ARM_CP_64BIT) {
+            CPREG_FIELD64(env, ri) = v;
+        } else {
+            CPREG_FIELD32(env, ri) = v;
+        }
+    }
+    return true;
+}
+
+bool write_cpustate_to_list(ARMCPU *cpu)
+{
+    /* Write the coprocessor state from cpu->env to the (index,value) list. */
+    int i;
+    bool ok = true;
+
+    for (i = 0; i < cpu->cpreg_array_len; i++) {
+        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
+        const ARMCPRegInfo *ri;
+        uint64_t v;
+        ri = get_arm_cp_reginfo(cpu, regidx);
+        if (!ri) {
+            ok = false;
+            continue;
+        }
+        if (ri->type & ARM_CP_NO_MIGRATE) {
+            continue;
+        }
+        if (!read_raw_cp_reg(&cpu->env, ri, &v)) {
+            ok = false;
+            continue;
+        }
+        cpu->cpreg_values[i] = v;
+    }
+    return ok;
+}
+
+bool write_list_to_cpustate(ARMCPU *cpu)
+{
+    int i;
+    bool ok = true;
+
+    for (i = 0; i < cpu->cpreg_array_len; i++) {
+        uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
+        uint64_t v = cpu->cpreg_values[i];
+        uint64_t readback;
+        const ARMCPRegInfo *ri;
+
+        ri = get_arm_cp_reginfo(cpu, regidx);
+        if (!ri) {
+            ok = false;
+            continue;
+        }
+        if (ri->type & ARM_CP_NO_MIGRATE) {
+            continue;
+        }
+        /* Write value and confirm it reads back as written
+         * (to catch read-only registers and partially read-only
+         * registers where the incoming migration value doesn't match)
+         */
+        if (!write_raw_cp_reg(&cpu->env, ri, v) ||
+            !read_raw_cp_reg(&cpu->env, ri, &readback) ||
+            readback != v) {
+            ok = false;
+        }
+    }
+    return ok;
+}
+
+static void add_cpreg_to_list(gpointer key, gpointer opaque)
+{
+    ARMCPU *cpu = opaque;
+    uint64_t regidx;
+    const ARMCPRegInfo *ri;
+
+    regidx = *(uint32_t *)key;
+    ri = get_arm_cp_reginfo(cpu, regidx);
+
+    if (!(ri->type & ARM_CP_NO_MIGRATE)) {
+        cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
+        /* The value array need not be initialized at this point */
+        cpu->cpreg_array_len++;
+    }
+}
+
+static void count_cpreg(gpointer key, gpointer opaque)
+{
+    ARMCPU *cpu = opaque;
+    uint64_t regidx;
+    const ARMCPRegInfo *ri;
+
+    regidx = *(uint32_t *)key;
+    ri = get_arm_cp_reginfo(cpu, regidx);
+
+    if (!(ri->type & ARM_CP_NO_MIGRATE)) {
+        cpu->cpreg_array_len++;
+    }
+}
+
+static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
+{
+    uint32_t aidx = *(uint32_t *)a;
+    uint32_t bidx = *(uint32_t *)b;
+
+    return aidx - bidx;
+}
+
+void init_cpreg_list(ARMCPU *cpu)
+{
+    /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
+     * Note that we require cpreg_tuples[] to be sorted by key ID.
+     */
+    GList *keys;
+    int arraylen;
+
+    keys = g_hash_table_get_keys(cpu->cp_regs);
+    keys = g_list_sort(keys, cpreg_key_compare);
+
+    cpu->cpreg_array_len = 0;
+
+    g_list_foreach(keys, count_cpreg, cpu);
+
+    arraylen = cpu->cpreg_array_len;
+    cpu->cpreg_indexes = g_new(uint64_t, arraylen);
+    cpu->cpreg_values = g_new(uint64_t, arraylen);
+    cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
+    cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
+    cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
+    cpu->cpreg_array_len = 0;
+
+    g_list_foreach(keys, add_cpreg_to_list, cpu);
+
+    assert(cpu->cpreg_array_len == arraylen);
+
+    g_list_free(keys);
+}
+
 static int dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     env->cp15.c3 = value;
@@ -139,13 +327,13 @@ static const ARMCPRegInfo cp_reginfo[] = {
     { .name = "DACR", .cp = 15,
       .crn = 3, .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
-      .resetvalue = 0, .writefn = dacr_write },
+      .resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
     { .name = "FCSEIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
-      .resetvalue = 0, .writefn = fcse_write },
+      .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
     { .name = "CONTEXTIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
-      .resetvalue = 0, .writefn = contextidr_write },
+      .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
     /* ??? This covers not just the impdef TLB lockdown registers but also
      * some v7VMSA registers relating to TEX remap, so it is overly broad.
      */
@@ -155,13 +343,17 @@ static const ARMCPRegInfo cp_reginfo[] = {
      * the unified TLB ops but also the dside/iside/inner-shareable variants.
      */
     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write, },
+      .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
+      .type = ARM_CP_NO_MIGRATE },
     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write, },
+      .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
+      .type = ARM_CP_NO_MIGRATE },
     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write, },
+      .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
+      .type = ARM_CP_NO_MIGRATE },
     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
-      .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write, },
+      .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
+      .type = ARM_CP_NO_MIGRATE },
     /* Cache maintenance ops; some of this space may be overridden later. */
     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
@@ -196,7 +388,8 @@ static const ARMCPRegInfo not_v7_cp_reginfo[] = {
       .resetvalue = 0 },
     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
-      .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
@@ -235,6 +428,7 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
+
 static int pmreg_read(CPUARMState *env, const ARMCPRegInfo *ri,
                       uint64_t *value)
 {
@@ -366,13 +560,16 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
       .access = PL0_RW, .resetvalue = 0,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
-      .readfn = pmreg_read, .writefn = pmcntenset_write },
+      .readfn = pmreg_read, .writefn = pmcntenset_write,
+      .raw_readfn = raw_read, .raw_writefn = raw_write },
     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
-      .readfn = pmreg_read, .writefn = pmcntenclr_write },
+      .readfn = pmreg_read, .writefn = pmcntenclr_write,
+      .type = ARM_CP_NO_MIGRATE },
     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
-      .readfn = pmreg_read, .writefn = pmovsr_write },
+      .readfn = pmreg_read, .writefn = pmovsr_write,
+      .raw_readfn = raw_read, .raw_writefn = raw_write },
     /* Unimplemented so WI. Strictly speaking write accesses in PL0 should
      * respect PMUSERENR.
      */
@@ -389,7 +586,8 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
       .access = PL0_RW,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
-      .readfn = pmreg_read, .writefn = pmxevtyper_write },
+      .readfn = pmreg_read, .writefn = pmxevtyper_write,
+      .raw_readfn = raw_read, .raw_writefn = raw_write },
     /* Unimplemented, RAZ/WI. XXX PMUSERENR */
     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -397,22 +595,21 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .access = PL0_R | PL1_RW,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
       .resetvalue = 0,
-      .writefn = pmuserenr_write },
+      .writefn = pmuserenr_write, .raw_writefn = raw_write },
     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
       .resetvalue = 0,
-      .writefn = pmintenset_write },
+      .writefn = pmintenset_write, .raw_writefn = raw_write },
     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
-      .resetvalue = 0,
-      .writefn = pmintenclr_write },
+      .resetvalue = 0, .writefn = pmintenclr_write, },
     { .name = "SCR", .cp = 15, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_scr),
       .resetvalue = 0, },
     { .name = "CCSIDR", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
-      .access = PL1_R, .readfn = ccsidr_read },
+      .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_MIGRATE },
     { .name = "CSSELR", .cp = 15, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c0_cssel),
       .writefn = csselr_write, .resetvalue = 0 },
@@ -461,7 +658,7 @@ static const ARMCPRegInfo t2ee_cp_reginfo[] = {
       .writefn = teecr_write },
     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
-      .resetvalue = 0,
+      .resetvalue = 0, .raw_readfn = raw_read, .raw_writefn = raw_write,
       .readfn = teehbr_read, .writefn = teehbr_write },
     REGINFO_SENTINEL
 };
@@ -486,7 +683,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     /* Dummy implementation: RAZ/WI the whole crn=14 space */
     { .name = "GENERIC_TIMER", .cp = 15, .crn = 14,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
-      .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
@@ -579,7 +777,7 @@ static const ARMCPRegInfo vapa_cp_reginfo[] = {
       .writefn = par_write },
 #ifndef CONFIG_USER_ONLY
     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
-      .access = PL1_W, .writefn = ats_write },
+      .access = PL1_W, .writefn = ats_write, .type = ARM_CP_NO_MIGRATE },
 #endif
     REGINFO_SENTINEL
 };
@@ -664,11 +862,11 @@ static int arm946_prbs_write(CPUARMState *env, const ARMCPRegInfo *ri,
 
 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
       .fieldoffset = offsetof(CPUARMState, cp15.c5_data), .resetvalue = 0,
       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
       .fieldoffset = offsetof(CPUARMState, cp15.c5_insn), .resetvalue = 0,
       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
@@ -690,15 +888,11 @@ static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
-static int vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                            uint64_t value)
+static int vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                uint64_t value)
 {
     if (arm_feature(env, ARM_FEATURE_LPAE)) {
         value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
-        /* With LPAE the TTBCR could result in a change of ASID
-         * via the TTBCR.A1 bit, so do a TLB flush.
-         */
-        tlb_flush(env, 1);
     } else {
         value &= 7;
     }
@@ -713,6 +907,18 @@ static int vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     return 0;
 }
 
+static int vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                            uint64_t value)
+{
+    if (arm_feature(env, ARM_FEATURE_LPAE)) {
+        /* With LPAE the TTBCR could result in a change of ASID
+         * via the TTBCR.A1 bit, so do a TLB flush.
+         */
+        tlb_flush(env, 1);
+    }
+    return vmsa_ttbcr_raw_write(env, ri, value);
+}
+
 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     env->cp15.c2_base_mask = 0xffffc000u;
@@ -735,7 +941,7 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = {
       .fieldoffset = offsetof(CPUARMState, cp15.c2_base1), .resetvalue = 0, },
     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW, .writefn = vmsa_ttbcr_write,
-      .resetfn = vmsa_ttbcr_reset,
+      .resetfn = vmsa_ttbcr_reset, .raw_writefn = vmsa_ttbcr_raw_write,
       .fieldoffset = offsetof(CPUARMState, cp15.c2_control) },
     { .name = "DFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c6_data),
@@ -801,6 +1007,7 @@ static const ARMCPRegInfo omap_cp_reginfo[] = {
       .writefn = omap_threadid_write },
     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
+      .type = ARM_CP_NO_MIGRATE,
       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
     /* TODO: Peripheral port remap register:
      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
@@ -808,7 +1015,8 @@ static const ARMCPRegInfo omap_cp_reginfo[] = {
      * when MMU is off.
      */
     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
-      .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W, .type = ARM_CP_OVERRIDE,
+      .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
+      .type = ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE,
       .writefn = omap_cachemaint_write },
     { .name = "C9", .cp = 15, .crn = 9,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
@@ -848,21 +1056,24 @@ static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
      */
     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
-      .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
     /* Cache status: RAZ because we have no cache so it's always clean */
     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
-      .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
     /* We never have a a block transfer operation in progress */
     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = 0 },
     /* The cache ops themselves: these all NOP for QEMU */
     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
@@ -884,9 +1095,11 @@ static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
      * to indicate that there are no dirty cache lines.
      */
     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = (1 << 30) },
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = (1 << 30) },
     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = (1 << 30) },
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .resetvalue = (1 << 30) },
     REGINFO_SENTINEL
 };
 
@@ -894,8 +1107,8 @@ static const ARMCPRegInfo strongarm_cp_reginfo[] = {
     /* Ignore ReadBuffer accesses */
     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
-      .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
-      .resetvalue = 0 },
+      .access = PL1_RW, .resetvalue = 0,
+      .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE },
     REGINFO_SENTINEL
 };
 
@@ -921,7 +1134,7 @@ static int mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri,
 
 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
     { .name = "MPIDR", .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
-      .access = PL1_R, .readfn = mpidr_read },
+      .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_MIGRATE },
     REGINFO_SENTINEL
 };
 
@@ -951,14 +1164,20 @@ static int ttbr064_read(CPUARMState *env, const ARMCPRegInfo *ri,
     return 0;
 }
 
-static int ttbr064_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                         uint64_t value)
+static int ttbr064_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
 {
     env->cp15.c2_base0_hi = value >> 32;
     env->cp15.c2_base0 = value;
+    return 0;
+}
+
+static int ttbr064_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
     /* Writes to the 64 bit format TTBRs may change the ASID */
     tlb_flush(env, 1);
-    return 0;
+    return ttbr064_raw_write(env, ri, value);
 }
 
 static void ttbr064_reset(CPUARMState *env, const ARMCPRegInfo *ri)
@@ -1008,7 +1227,8 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
       .readfn = par64_read, .writefn = par64_write, .resetfn = par64_reset },
     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
       .access = PL1_RW, .type = ARM_CP_64BIT, .readfn = ttbr064_read,
-      .writefn = ttbr064_write, .resetfn = ttbr064_reset },
+      .writefn = ttbr064_write, .raw_writefn = ttbr064_raw_write,
+      .resetfn = ttbr064_reset },
     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
       .access = PL1_RW, .type = ARM_CP_64BIT, .readfn = ttbr164_read,
       .writefn = ttbr164_write, .resetfn = ttbr164_reset },
@@ -1104,7 +1324,8 @@ void register_cp_regs_for_features(ARMCPU *cpu)
             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
             .access = PL0_RW, .resetvalue = cpu->midr & 0xff000000,
             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
-            .readfn = pmreg_read, .writefn = pmcr_write
+            .readfn = pmreg_read, .writefn = pmcr_write,
+            .raw_readfn = raw_read, .raw_writefn = raw_write,
         };
         ARMCPRegInfo clidr = {
             .name = "CLIDR", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
@@ -1176,7 +1397,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
             { .name = "MIDR",
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
               .access = PL1_R, .resetvalue = cpu->midr,
-              .writefn = arm_cp_write_ignore,
+              .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid) },
             { .name = "CTR",
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
@@ -1245,7 +1466,8 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         ARMCPRegInfo sctlr = {
             .name = "SCTLR", .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
             .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_sys),
-            .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr
+            .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
+            .raw_writefn = raw_write,
         };
         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
             /* Normally we would always end the TB on an SCTLR write, but Linux
@@ -1392,6 +1614,19 @@ void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
                 r2->crm = crm;
                 r2->opc1 = opc1;
                 r2->opc2 = opc2;
+                /* By convention, for wildcarded registers only the first
+                 * entry is used for migration; the others are marked as
+                 * NO_MIGRATE so we don't try to transfer the register
+                 * multiple times. Special registers (ie NOP/WFI) are
+                 * never migratable.
+                 */
+                if ((r->type & ARM_CP_SPECIAL) ||
+                    ((r->crm == CP_ANY) && crm != 0) ||
+                    ((r->opc1 == CP_ANY) && opc1 != 0) ||
+                    ((r->opc2 == CP_ANY) && opc2 != 0)) {
+                    r2->type |= ARM_CP_NO_MIGRATE;
+                }
+
                 /* Overriding of an existing definition must be explicitly
                  * requested.
                  */
diff --git a/target-arm/kvm-stub.c b/target-arm/kvm-stub.c
new file mode 100644
index 0000000000..cd1849f72c
--- /dev/null
+++ b/target-arm/kvm-stub.c
@@ -0,0 +1,23 @@
+/*
+ * QEMU KVM ARM specific function stubs
+ *
+ * Copyright Linaro Limited 2013
+ *
+ * Author: Peter Maydell <peter.maydell@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include "qemu-common.h"
+#include "kvm_arm.h"
+
+bool write_kvmstate_to_list(ARMCPU *cpu)
+{
+    abort();
+}
+
+bool write_list_to_kvmstate(ARMCPU *cpu)
+{
+    abort();
+}
diff --git a/target-arm/kvm.c b/target-arm/kvm.c
index 27dcab93f6..d3937a260f 100644
--- a/target-arm/kvm.c
+++ b/target-arm/kvm.c
@@ -23,6 +23,15 @@
 #include "cpu.h"
 #include "hw/arm/arm.h"
 
+/* Check that cpu.h's idea of coprocessor fields matches KVM's */
+#if (CP_REG_SIZE_SHIFT != KVM_REG_SIZE_SHIFT) || \
+    (CP_REG_SIZE_MASK != KVM_REG_SIZE_MASK) ||   \
+    (CP_REG_SIZE_U32 != KVM_REG_SIZE_U32) || \
+    (CP_REG_SIZE_U64 != KVM_REG_SIZE_U64) || \
+    (CP_REG_ARM != KVM_REG_ARM)
+#error mismatch between cpu.h and KVM header definitions
+#endif
+
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
@@ -41,12 +50,35 @@ unsigned long kvm_arch_vcpu_id(CPUState *cpu)
     return cpu->cpu_index;
 }
 
+static bool reg_syncs_via_tuple_list(uint64_t regidx)
+{
+    /* Return true if the regidx is a register we should synchronize
+     * via the cpreg_tuples array (ie is not a core reg we sync by
+     * hand in kvm_arch_get/put_registers())
+     */
+    switch (regidx & KVM_REG_ARM_COPROC_MASK) {
+    case KVM_REG_ARM_CORE:
+    case KVM_REG_ARM_VFP:
+        return false;
+    default:
+        return true;
+    }
+}
+
+static int compare_u64(const void *a, const void *b)
+{
+    return *(uint64_t *)a - *(uint64_t *)b;
+}
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct kvm_vcpu_init init;
-    int ret;
+    int i, ret, arraylen;
     uint64_t v;
     struct kvm_one_reg r;
+    struct kvm_reg_list rl;
+    struct kvm_reg_list *rlp;
+    ARMCPU *cpu = ARM_CPU(cs);
 
     init.target = KVM_ARM_TARGET_CORTEX_A15;
     memset(init.features, 0, sizeof(init.features));
@@ -65,6 +97,80 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (ret == -ENOENT) {
         return -EINVAL;
     }
+
+    /* Populate the cpreg list based on the kernel's idea
+     * of what registers exist (and throw away the TCG-created list).
+     */
+    rl.n = 0;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
+    if (ret != -E2BIG) {
+        return ret;
+    }
+    rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
+    rlp->n = rl.n;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
+    if (ret) {
+        goto out;
+    }
+    /* Sort the list we get back from the kernel, since cpreg_tuples
+     * must be in strictly ascending order.
+     */
+    qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
+
+    for (i = 0, arraylen = 0; i < rlp->n; i++) {
+        if (!reg_syncs_via_tuple_list(rlp->reg[i])) {
+            continue;
+        }
+        switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
+        case KVM_REG_SIZE_U32:
+        case KVM_REG_SIZE_U64:
+            break;
+        default:
+            fprintf(stderr, "Can't handle size of register in kernel list\n");
+            ret = -EINVAL;
+            goto out;
+        }
+
+        arraylen++;
+    }
+
+    cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
+    cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
+    cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
+                                         arraylen);
+    cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
+                                        arraylen);
+    cpu->cpreg_array_len = arraylen;
+    cpu->cpreg_vmstate_array_len = arraylen;
+
+    for (i = 0, arraylen = 0; i < rlp->n; i++) {
+        uint64_t regidx = rlp->reg[i];
+        if (!reg_syncs_via_tuple_list(regidx)) {
+            continue;
+        }
+        cpu->cpreg_indexes[arraylen] = regidx;
+        arraylen++;
+    }
+    assert(cpu->cpreg_array_len == arraylen);
+
+    if (!write_kvmstate_to_list(cpu)) {
+        /* Shouldn't happen unless kernel is inconsistent about
+         * what registers exist.
+         */
+        fprintf(stderr, "Initial read of kernel register state failed\n");
+        ret = -EINVAL;
+        goto out;
+    }
+
+    /* Save a copy of the initial register values so that we can
+     * feed it back to the kernel on VCPU reset.
+     */
+    cpu->cpreg_reset_values = g_memdup(cpu->cpreg_values,
+                                       cpu->cpreg_array_len *
+                                       sizeof(cpu->cpreg_values[0]));
+
+out:
+    g_free(rlp);
     return ret;
 }
 
@@ -154,6 +260,78 @@ void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid)
     QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
 }
 
+bool write_kvmstate_to_list(ARMCPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    int i;
+    bool ok = true;
+
+    for (i = 0; i < cpu->cpreg_array_len; i++) {
+        struct kvm_one_reg r;
+        uint64_t regidx = cpu->cpreg_indexes[i];
+        uint32_t v32;
+        int ret;
+
+        r.id = regidx;
+
+        switch (regidx & KVM_REG_SIZE_MASK) {
+        case KVM_REG_SIZE_U32:
+            r.addr = (uintptr_t)&v32;
+            ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
+            if (!ret) {
+                cpu->cpreg_values[i] = v32;
+            }
+            break;
+        case KVM_REG_SIZE_U64:
+            r.addr = (uintptr_t)(cpu->cpreg_values + i);
+            ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
+            break;
+        default:
+            abort();
+        }
+        if (ret) {
+            ok = false;
+        }
+    }
+    return ok;
+}
+
+bool write_list_to_kvmstate(ARMCPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    int i;
+    bool ok = true;
+
+    for (i = 0; i < cpu->cpreg_array_len; i++) {
+        struct kvm_one_reg r;
+        uint64_t regidx = cpu->cpreg_indexes[i];
+        uint32_t v32;
+        int ret;
+
+        r.id = regidx;
+        switch (regidx & KVM_REG_SIZE_MASK) {
+        case KVM_REG_SIZE_U32:
+            v32 = cpu->cpreg_values[i];
+            r.addr = (uintptr_t)&v32;
+            break;
+        case KVM_REG_SIZE_U64:
+            r.addr = (uintptr_t)(cpu->cpreg_values + i);
+            break;
+        default:
+            abort();
+        }
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
+        if (ret) {
+            /* We might fail for "unknown register" and also for
+             * "you tried to set a register which is constant with
+             * a different value from what it actually contains".
+             */
+            ok = false;
+        }
+    }
+    return ok;
+}
+
 typedef struct Reg {
     uint64_t id;
     int offset;
@@ -166,17 +344,6 @@ typedef struct Reg {
         offsetof(CPUARMState, QEMUFIELD)                     \
     }
 
-#define CP15REG(CRN, CRM, OPC1, OPC2, QEMUFIELD) \
-    {                                            \
-        KVM_REG_ARM | KVM_REG_SIZE_U32 |         \
-        (15 << KVM_REG_ARM_COPROC_SHIFT) |       \
-        ((CRN) << KVM_REG_ARM_32_CRN_SHIFT) |    \
-        ((CRM) << KVM_REG_ARM_CRM_SHIFT) |       \
-        ((OPC1) << KVM_REG_ARM_OPC1_SHIFT) |     \
-        ((OPC2) << KVM_REG_ARM_32_OPC2_SHIFT),   \
-        offsetof(CPUARMState, QEMUFIELD)         \
-    }
-
 #define VFPSYSREG(R)                                       \
     {                                                      \
         KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP | \
@@ -225,12 +392,6 @@ static const Reg regs[] = {
     COREREG(fiq_regs[7], banked_spsr[5]),
     /* R15 */
     COREREG(usr_regs.uregs[15], regs[15]),
-    /* A non-comprehensive set of cp15 registers.
-     * TODO: drive this from the cp_regs hashtable instead.
-     */
-    CP15REG(1, 0, 0, 0, cp15.c1_sys), /* SCTLR */
-    CP15REG(2, 0, 0, 2, cp15.c2_control), /* TTBCR */
-    CP15REG(3, 0, 0, 0, cp15.c3), /* DACR */
     /* VFP system registers */
     VFPSYSREG(FPSID),
     VFPSYSREG(MVFR1),
@@ -248,7 +409,6 @@ int kvm_arch_put_registers(CPUState *cs, int level)
     int mode, bn;
     int ret, i;
     uint32_t cpsr, fpscr;
-    uint64_t ttbr;
 
     /* Make sure the banked regs are properly set */
     mode = env->uncached_cpsr & CPSR_M;
@@ -282,26 +442,6 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         return ret;
     }
 
-    /* TTBR0: cp15 crm=2 opc1=0 */
-    ttbr = ((uint64_t)env->cp15.c2_base0_hi << 32) | env->cp15.c2_base0;
-    r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
-        (2 << KVM_REG_ARM_CRM_SHIFT) | (0 << KVM_REG_ARM_OPC1_SHIFT);
-    r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
-    if (ret) {
-        return ret;
-    }
-
-    /* TTBR1: cp15 crm=2 opc1=1 */
-    ttbr = ((uint64_t)env->cp15.c2_base1_hi << 32) | env->cp15.c2_base1;
-    r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
-        (2 << KVM_REG_ARM_CRM_SHIFT) | (1 << KVM_REG_ARM_OPC1_SHIFT);
-    r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
-    if (ret) {
-        return ret;
-    }
-
     /* VFP registers */
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
     for (i = 0; i < 32; i++) {
@@ -318,6 +458,31 @@ int kvm_arch_put_registers(CPUState *cs, int level)
     fpscr = vfp_get_fpscr(env);
     r.addr = (uintptr_t)&fpscr;
     ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
+    if (ret) {
+        return ret;
+    }
+
+    /* Note that we do not call write_cpustate_to_list()
+     * here, so we are only writing the tuple list back to
+     * KVM. This is safe because nothing can change the
+     * CPUARMState cp15 fields (in particular gdb accesses cannot)
+     * and so there are no changes to sync. In fact syncing would
+     * be wrong at this point: for a constant register where TCG and
+     * KVM disagree about its value, the preceding write_list_to_cpustate()
+     * would not have had any effect on the CPUARMState value (since the
+     * register is read-only), and a write_cpustate_to_list() here would
+     * then try to write the TCG value back into KVM -- this would either
+     * fail or incorrectly change the value the guest sees.
+     *
+     * If we ever want to allow the user to modify cp15 registers via
+     * the gdb stub, we would need to be more clever here (for instance
+     * tracking the set of registers kvm_arch_get_registers() successfully
+     * managed to update the CPUARMState with, and only allowing those
+     * to be written back up into the kernel).
+     */
+    if (!write_list_to_kvmstate(cpu)) {
+        return EINVAL;
+    }
 
     return ret;
 }
@@ -330,7 +495,6 @@ int kvm_arch_get_registers(CPUState *cs)
     int mode, bn;
     int ret, i;
     uint32_t cpsr, fpscr;
-    uint64_t ttbr;
 
     for (i = 0; i < ARRAY_SIZE(regs); i++) {
         r.id = regs[i].id;
@@ -351,28 +515,6 @@ int kvm_arch_get_registers(CPUState *cs)
     }
     cpsr_write(env, cpsr, 0xffffffff);
 
-    /* TTBR0: cp15 crm=2 opc1=0 */
-    r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
-        (2 << KVM_REG_ARM_CRM_SHIFT) | (0 << KVM_REG_ARM_OPC1_SHIFT);
-    r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
-    if (ret) {
-        return ret;
-    }
-    env->cp15.c2_base0_hi = ttbr >> 32;
-    env->cp15.c2_base0 = ttbr;
-
-    /* TTBR1: cp15 crm=2 opc1=1 */
-    r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
-        (2 << KVM_REG_ARM_CRM_SHIFT) | (1 << KVM_REG_ARM_OPC1_SHIFT);
-    r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
-    if (ret) {
-        return ret;
-    }
-    env->cp15.c2_base1_hi = ttbr >> 32;
-    env->cp15.c2_base1 = ttbr;
-
     /* Make sure the current mode regs are properly set */
     mode = env->uncached_cpsr & CPSR_M;
     bn = bank_number(mode);
@@ -385,15 +527,6 @@ int kvm_arch_get_registers(CPUState *cs)
     env->regs[14] = env->banked_r14[bn];
     env->spsr = env->banked_spsr[bn];
 
-    /* The main GET_ONE_REG loop above set c2_control, but we need to
-     * update some extra cached precomputed values too.
-     * When this is driven from the cp_regs hashtable then this ugliness
-     * can disappear because we'll use the access function which sets
-     * these values automatically.
-     */
-    env->cp15.c2_mask = ~(0xffffffffu >> env->cp15.c2_control);
-    env->cp15.c2_base_mask = ~(0x3fffu >> env->cp15.c2_control);
-
     /* VFP registers */
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
     for (i = 0; i < 32; i++) {
@@ -414,6 +547,14 @@ int kvm_arch_get_registers(CPUState *cs)
     }
     vfp_set_fpscr(env, fpscr);
 
+    if (!write_kvmstate_to_list(cpu)) {
+        return EINVAL;
+    }
+    /* Note that it's OK to have registers which aren't in CPUState,
+     * so we can ignore a failure return here.
+     */
+    write_list_to_cpustate(cpu);
+
     return 0;
 }
 
@@ -432,6 +573,15 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 
 void kvm_arch_reset_vcpu(CPUState *cs)
 {
+    /* Feed the kernel back its initial register state */
+    ARMCPU *cpu = ARM_CPU(cs);
+
+    memmove(cpu->cpreg_values, cpu->cpreg_reset_values,
+            cpu->cpreg_array_len * sizeof(cpu->cpreg_values[0]));
+
+    if (!write_list_to_kvmstate(cpu)) {
+        abort();
+    }
 }
 
 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
diff --git a/target-arm/kvm_arm.h b/target-arm/kvm_arm.h
index b1c54ffb5d..5d14887e66 100644
--- a/target-arm/kvm_arm.h
+++ b/target-arm/kvm_arm.h
@@ -29,4 +29,37 @@
  */
 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid);
 
+/**
+ * write_list_to_kvmstate:
+ * @cpu: ARMCPU
+ *
+ * For each register listed in the ARMCPU cpreg_indexes list, write
+ * its value from the cpreg_values list into the kernel (via ioctl).
+ * This updates KVM's working data structures from TCG data or
+ * from incoming migration state.
+ *
+ * Returns: true if all register values were updated correctly,
+ * false if some register was unknown to the kernel or could not
+ * be written (eg constant register with the wrong value).
+ * Note that we do not stop early on failure -- we will attempt
+ * writing all registers in the list.
+ */
+bool write_list_to_kvmstate(ARMCPU *cpu);
+
+/**
+ * write_kvmstate_to_list:
+ * @cpu: ARMCPU
+ *
+ * For each register listed in the ARMCPU cpreg_indexes list, write
+ * its value from the kernel into the cpreg_values list. This is used to
+ * copy info from KVM's working data structures into TCG or
+ * for outbound migration.
+ *
+ * Returns: true if all register values were read correctly,
+ * false if some register was unknown or could not be read.
+ * Note that we do not stop early on failure -- we will attempt
+ * reading all registers in the list.
+ */
+bool write_kvmstate_to_list(ARMCPU *cpu);
+
 #endif
diff --git a/target-arm/machine.c b/target-arm/machine.c
index 4dd057c488..6d4c2d4ed0 100644
--- a/target-arm/machine.c
+++ b/target-arm/machine.c
@@ -1,5 +1,7 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
 
 static bool vfp_needed(void *opaque)
 {
@@ -148,11 +150,83 @@ static const VMStateInfo vmstate_cpsr = {
     .put = put_cpsr,
 };
 
+static void cpu_pre_save(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    if (kvm_enabled()) {
+        if (!write_kvmstate_to_list(cpu)) {
+            /* This should never fail */
+            abort();
+        }
+    } else {
+        if (!write_cpustate_to_list(cpu)) {
+            /* This should never fail. */
+            abort();
+        }
+    }
+
+    cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
+    memcpy(cpu->cpreg_vmstate_indexes, cpu->cpreg_indexes,
+           cpu->cpreg_array_len * sizeof(uint64_t));
+    memcpy(cpu->cpreg_vmstate_values, cpu->cpreg_values,
+           cpu->cpreg_array_len * sizeof(uint64_t));
+}
+
+static int cpu_post_load(void *opaque, int version_id)
+{
+    ARMCPU *cpu = opaque;
+    int i, v;
+
+    /* Update the values list from the incoming migration data.
+     * Anything in the incoming data which we don't know about is
+     * a migration failure; anything we know about but the incoming
+     * data doesn't specify retains its current (reset) value.
+     * The indexes list remains untouched -- we only inspect the
+     * incoming migration index list so we can match the values array
+     * entries with the right slots in our own values array.
+     */
+
+    for (i = 0, v = 0; i < cpu->cpreg_array_len
+             && v < cpu->cpreg_vmstate_array_len; i++) {
+        if (cpu->cpreg_vmstate_indexes[v] > cpu->cpreg_indexes[i]) {
+            /* register in our list but not incoming : skip it */
+            continue;
+        }
+        if (cpu->cpreg_vmstate_indexes[v] < cpu->cpreg_indexes[i]) {
+            /* register in their list but not ours: fail migration */
+            return -1;
+        }
+        /* matching register, copy the value over */
+        cpu->cpreg_values[i] = cpu->cpreg_vmstate_values[v];
+        v++;
+    }
+
+    if (kvm_enabled()) {
+        if (!write_list_to_kvmstate(cpu)) {
+            return -1;
+        }
+        /* Note that it's OK for the TCG side not to know about
+         * every register in the list; KVM is authoritative if
+         * we're using it.
+         */
+        write_list_to_cpustate(cpu);
+    } else {
+        if (!write_list_to_cpustate(cpu)) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 const VMStateDescription vmstate_arm_cpu = {
     .name = "cpu",
-    .version_id = 11,
-    .minimum_version_id = 11,
-    .minimum_version_id_old = 11,
+    .version_id = 12,
+    .minimum_version_id = 12,
+    .minimum_version_id_old = 12,
+    .pre_save = cpu_pre_save,
+    .post_load = cpu_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32_ARRAY(env.regs, ARMCPU, 16),
         {
@@ -169,50 +243,16 @@ const VMStateDescription vmstate_arm_cpu = {
         VMSTATE_UINT32_ARRAY(env.banked_r14, ARMCPU, 6),
         VMSTATE_UINT32_ARRAY(env.usr_regs, ARMCPU, 5),
         VMSTATE_UINT32_ARRAY(env.fiq_regs, ARMCPU, 5),
-        VMSTATE_UINT32(env.cp15.c0_cpuid, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c0_cssel, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c1_sys, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c1_coproc, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c1_xscaleauxcr, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c1_scr, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_base0, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_base0_hi, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_base1, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_base1_hi, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_control, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_mask, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_base_mask, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_data, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c2_insn, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c3, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c5_insn, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c5_data, ARMCPU),
-        VMSTATE_UINT32_ARRAY(env.cp15.c6_region, ARMCPU, 8),
-        VMSTATE_UINT32(env.cp15.c6_insn, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c6_data, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c7_par, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c7_par_hi, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_insn, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_data, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pmcr, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pmcnten, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pmovsr, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pmxevtyper, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pmuserenr, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c9_pminten, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c13_fcse, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c13_context, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c13_tls1, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c13_tls2, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c13_tls3, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_cpar, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_ticonfig, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_i_max, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_i_min, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_threadid, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_power_control, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_diagnostic, ARMCPU),
-        VMSTATE_UINT32(env.cp15.c15_power_diagnostic, ARMCPU),
+        /* The length-check must come before the arrays to avoid
+         * incoming data possibly overflowing the array.
+         */
+        VMSTATE_INT32_LE(cpreg_vmstate_array_len, ARMCPU),
+        VMSTATE_VARRAY_INT32(cpreg_vmstate_indexes, ARMCPU,
+                             cpreg_vmstate_array_len,
+                             0, vmstate_info_uint64, uint64_t),
+        VMSTATE_VARRAY_INT32(cpreg_vmstate_values, ARMCPU,
+                             cpreg_vmstate_array_len,
+                             0, vmstate_info_uint64, uint64_t),
         VMSTATE_UINT32(env.exclusive_addr, ARMCPU),
         VMSTATE_UINT32(env.exclusive_val, ARMCPU),
         VMSTATE_UINT32(env.exclusive_high, ARMCPU),