[RFC 3/3] use migration code for cpu and device save/restore

[Richard Liu]

Reused device migration code for cpu and device state snapshots. In this
initial version, I used several hacks to get the device code working.

vm_stop doesn't have the intended effect (for qemu_save_device_state)
unless called outside the vcpu thread. I trick the function into
thinking it is outside the vcpu thread by temporarily setting
`current_cpu` to be null.

The restore code (qemu_loadvm_state in particular) needs to be called
in a bottom half or a coroutine. I am not sure why.

Signed-off-by: Richard Liu <richy.liu.2002@gmail.com>
---
 hw/misc/snapshot.c |  6 ++++
 migration/savevm.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++
 migration/savevm.h |  3 ++
 3 files changed, 93 insertions(+)

diff --git a/hw/misc/snapshot.c b/hw/misc/snapshot.c
index 510bf59dce..afdc5b7f15 100644
--- a/hw/misc/snapshot.c
+++ b/hw/misc/snapshot.c
@@ -55,6 +55,9 @@ static void save_snapshot(struct SnapshotState *s) {
     // map as MAP_PRIVATE to avoid carrying writes back to the saved file
     fd = open(filepath, O_RDONLY);
     mmap(guest_mem, guest_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | 
MAP_FIXED, fd, 0);
+
+    // save cpu and device state
+    s->ioc = qemu_snapshot_save_cpu_state();
 }
 
 static void restore_snapshot(struct SnapshotState *s) {
@@ -73,6 +76,9 @@ static void restore_snapshot(struct SnapshotState *s) {
     fd = open(filepath, O_RDONLY);
     mmap(guest_mem, guest_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | 
MAP_FIXED, fd, 0);
     close(fd);
+
+    // restore cpu and device state
+    qemu_snapshot_load_cpu_state(s->ioc);
 }
 
 static uint64_t snapshot_mmio_read(void *opaque, hwaddr addr, unsigned size)
diff --git a/migration/savevm.c b/migration/savevm.c
index 48e85c052c..62e5e5b564 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -3309,3 +3309,87 @@ void qmp_snapshot_delete(const char *job_id,
 
     job_start(&s->common);
 }
+
+// saves the cpu and devices state
+QIOChannelBuffer* qemu_snapshot_save_cpu_state(void)
+{
+    QEMUFile *f;
+    QIOChannelBuffer *ioc;
+    MigrationState *ms = migrate_get_current();
+    int ret;
+
+    /* This is a hack to trick vm_stop() into thinking it is not in vcpu 
thread.
+     * This is needed to properly stop the VM for a snapshot.
+     */
+    CPUState *cpu = current_cpu;
+    current_cpu = NULL;
+    vm_stop(RUN_STATE_SAVE_VM);
+    current_cpu = cpu;
+
+    global_state_store_running();
+
+    ioc = qio_channel_buffer_new(0x10000);
+    qio_channel_set_name(QIO_CHANNEL(ioc), "snapshot-buffer");
+    f = qemu_file_new_output(QIO_CHANNEL(ioc));
+
+    /* We need to initialize migration otherwise qemu_save_device_state() will
+     * complain.
+     */
+    migrate_init(ms);
+    ms->state = MIGRATION_STATUS_NONE;
+    ms->send_configuration = false;
+
+    cpu_synchronize_all_states();
+
+    ret = qemu_save_device_state(f);
+    if (ret < 0) {
+        fprintf(stderr, "[QEMU] save device err: %d\n", ret);
+    }
+
+    // clean up and restart vm
+    qemu_fflush(f);
+    g_free(f);
+
+    vm_start();
+
+    /* Needed so qemu_loadvm_state does not error with:
+     * qemu-system-x86_64: Expected vmdescription section, but got 0
+     */
+    ms->state = MIGRATION_STATUS_POSTCOPY_ACTIVE;
+
+    return ioc;
+}
+
+// loads the cpu and devices state
+static void do_snapshot_load(void* opaque) {
+    QIOChannelBuffer *ioc = opaque;
+    QEMUFile *f;
+    int ret;
+
+    vm_stop(RUN_STATE_RESTORE_VM);
+
+    // seek back to beginning of file
+    qio_channel_io_seek(QIO_CHANNEL(ioc), 0, 0, NULL);
+    f = qemu_file_new_input(QIO_CHANNEL(ioc));
+
+    ret = qemu_loadvm_state(f);
+    if (ret < 0) {
+        fprintf(stderr, "[QEMU] loadvm err: %d\n", ret);
+    }
+
+    vm_start();
+
+    g_free(f);
+
+    // print time to debug speed
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    fprintf(stderr, "loaded snapshot at %ld.%ld\n", ts.tv_sec, ts.tv_nsec);
+}
+
+void qemu_snapshot_load_cpu_state(QIOChannelBuffer *ioc) {
+    /* Run in a bh because otherwise qemu_loadvm_state won't work
+     */
+    QEMUBH *bh = qemu_bh_new(do_snapshot_load, ioc);
+    qemu_bh_schedule(bh);
+}
diff --git a/migration/savevm.h b/migration/savevm.h
index 6461342cb4..990bcddd2f 100644
--- a/migration/savevm.h
+++ b/migration/savevm.h
@@ -68,4 +68,7 @@ int qemu_load_device_state(QEMUFile *f);
 int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f,
         bool in_postcopy, bool inactivate_disks);
 
+QIOChannelBuffer* qemu_snapshot_save_cpu_state(void);
+void qemu_snapshot_load_cpu_state(QIOChannelBuffer *ioc);
+
 #endif
-- 
2.35.1