Re: [Qemu-devel] balloon config change seems to break live migration from 3.0.1 to 4.0

[Dr. David Alan Gilbert]

* Wolfgang Bumiller (address@hidden) wrote:
> While testing with 4.0 we've run into issues with live migration from
> 3.0.1 to 4.0 when a balloon device was involved.
> 
> We'd see the following error on the destination:
>   qemu-system-x86_64: get_pci_config_device: Bad config data: i=0x10 read: a1 
> device: 1 cmask: ff wmask: c0 w1cmask:0 
>   qemu-system-x86_64: Failed to load PCIDevice:config 
>   qemu-system-x86_64: Failed to load virtio-balloon:virtio 
>   qemu-system-x86_64: error while loading state for instance 0x0 of device 
> '0000:00:03.0/virtio-balloon' 
>   qemu-system-x86_64: load of migration failed: Invalid argument
> 
> After looking through the commits I noticed that the pci config sent for
> the balloon device comes from
> include/standard-headers/linux/virtio_balloon.h and changed size between
> 3.1 and 4.0.
> As a "guess" I tried reverting that change (commented out the two last
> fields (and access to it in hw/virtio/virtio-balloon.c's
> virtio_balloon_get_config()), and then the migration seems to go through
> successfully.
> 
> I've since also rebuilt qemu without our patches (tags v3.0.1 and v4.0.0)
> and also tried with master (since dgilbert mentioned on irc remembering
> the issue and that there may have been a fix around), but got the same
> result.
> 
> Posting here now as dgilbert requested on irc.

Apologies for the delay, I was out last week.

Thanks for the command lines; I can confirm I can recreate it here using
a simplified version of your command line.

Dave

> Here are the commands used to start qemu:
>   Source:
>     /usr/bin/kvm \
>       -name randomclone \
>       -chardev 
> 'socket,id=qmp,path=/var/run/qemu-server/101.qmp,server,nowait' \
>       -mon 'chardev=qmp,mode=control' \
>       -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
>       -mon 'chardev=qmp-event,mode=control' \
>       -pidfile /var/run/qemu-server/101.pid \
>       -daemonize \
>       -smbios 'type=1,uuid=f3ab31f6-ca7d-469c-bf51-547fd9bbd2d9' \
>       -smp '4,sockets=1,cores=4,maxcpus=4' \
>       -nodefaults \
>       -boot 
> 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg'
>  \
>       -vnc unix:/var/run/qemu-server/101.vnc,password \
>       -cpu host,+pcid,+spec-ctrl,+ssbd,+pdpe1gb,+kvm_pv_unhalt,+kvm_pv_eoi \
>       -m 4096 \
>       -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
>       -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
>       -device 'vmgenid,guid=fb282779-7056-4f1d-96bb-70f578294e45' \
>       -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
>       -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
>       -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
>       -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
>       -iscsi 'initiator-name=iqn.1993-08.org.debian:01:856d32b504d' \
>       -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \
>       -device 
> 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
>       -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
>       -drive 
> 'file=rbd:rbd/vm-101-disk-0:conf=/etc/pve/ceph.conf:id=admin:keyring=/etc/pve/priv/ceph/rbd.keyring,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap'
>  \
>       -device 
> 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,rotation_rate=1,bootindex=100'
>  \
>       -netdev 
> 'type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on'
>  \
>       -device 
> 'virtio-net-pci,mac=4E:5D:50:75:4D:ED,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300'
>  \
>       -machine 'type=pc' \
>       -enable-kvm
> 
>   Destination:
>     /usr/bin/kvm \
>       -name randomclone \
>       -chardev socket,id=qmp,path=/var/run/qemu-server/101.qmp,server,nowait \
>       -mon chardev=qmp,mode=control \
>       -chardev socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5 \
>       -mon chardev=qmp-event,mode=control \
>       -pidfile /var/run/qemu-server/101.pid \
>       -smbios type=1,uuid=f3ab31f6-ca7d-469c-bf51-547fd9bbd2d9 \
>       -smp 4,sockets=1,cores=4,maxcpus=4 \
>       -nodefaults \
>       -boot 
> menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg
>  \
>       -vnc unix:/var/run/qemu-server/101.vnc,password \
>       -cpu host,+pcid,+spec-ctrl,+ssbd,+pdpe1gb,+kvm_pv_unhalt,+kvm_pv_eoi \
>       -m 4096 \
>       -device pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e \
>       -device pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f \
>       -device vmgenid,guid=fb282779-7056-4f1d-96bb-70f578294e45 \
>       -device piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2 \
>       -device usb-tablet,id=tablet,bus=uhci.0,port=1 \
>       -device VGA,id=vga,bus=pci.0,addr=0x2 \
>       -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \
>       -iscsi initiator-name=iqn.1993-08.org.debian:01:ee4e4a566b \
>       -drive if=none,id=drive-ide2,media=cdrom,aio=threads \
>       -device ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200 \
>       -device virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5 \
>       -drive 
> file=rbd:rbd/vm-101-disk-0:conf=/etc/pve/ceph.conf:id=admin:keyring=/etc/pve/priv/ceph/rbd.keyring,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap
>  \
>       -device 
> scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,rotation_rate=1,bootindex=100
>  \
>       -netdev 
> type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on
>  \
>       -device 
> virtio-net-pci,mac=4E:5D:50:75:4D:ED,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300
>  \
>       -machine type=pc-i440fx-3.0 \
>       -enable-kvm \
>       -incoming tcp:10.9.2.106:9989 \
>       -S
> 
> This is the exact test-change I made which seems to work around it, but
> a proper fix would be nicer. Not sure how, though.
> 
> ---8<---
> diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c
> index d96e4aa96f..8d631d67a8 100644
> --- a/hw/virtio/virtio-balloon.c
> +++ b/hw/virtio/virtio-balloon.c
> @@ -623,16 +623,16 @@ static void virtio_balloon_get_config(VirtIODevice 
> *vdev, uint8_t *config_data)
>      config.num_pages = cpu_to_le32(dev->num_pages);
>      config.actual = cpu_to_le32(dev->actual);
>  
> -    if (dev->free_page_report_status == FREE_PAGE_REPORT_S_REQUESTED) {
> -        config.free_page_report_cmd_id =
> -                       cpu_to_le32(dev->free_page_report_cmd_id);
> -    } else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_STOP) {
> -        config.free_page_report_cmd_id =
> -                       cpu_to_le32(VIRTIO_BALLOON_CMD_ID_STOP);
> -    } else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_DONE) {
> -        config.free_page_report_cmd_id =
> -                       cpu_to_le32(VIRTIO_BALLOON_CMD_ID_DONE);
> -    }
> +    //if (dev->free_page_report_status == FREE_PAGE_REPORT_S_REQUESTED) {
> +    //    config.free_page_report_cmd_id =
> +    //                   cpu_to_le32(dev->free_page_report_cmd_id);
> +    //} else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_STOP) {
> +    //    config.free_page_report_cmd_id =
> +    //                   cpu_to_le32(VIRTIO_BALLOON_CMD_ID_STOP);
> +    //} else if (dev->free_page_report_status == FREE_PAGE_REPORT_S_DONE) {
> +    //    config.free_page_report_cmd_id =
> +    //                   cpu_to_le32(VIRTIO_BALLOON_CMD_ID_DONE);
> +    //}
>  
>      trace_virtio_balloon_get_config(config.num_pages, config.actual);
>      memcpy(config_data, &config, sizeof(struct virtio_balloon_config));
> diff --git a/include/standard-headers/linux/virtio_balloon.h 
> b/include/standard-headers/linux/virtio_balloon.h
> index 9375ca2a70..86aca75972 100644
> --- a/include/standard-headers/linux/virtio_balloon.h
> +++ b/include/standard-headers/linux/virtio_balloon.h
> @@ -48,9 +48,9 @@ struct virtio_balloon_config {
>       /* Number of pages we've actually got in balloon. */
>       uint32_t actual;
>       /* Free page report command id, readonly by guest */
> -     uint32_t free_page_report_cmd_id;
> -     /* Stores PAGE_POISON if page poisoning is in use */
> -     uint32_t poison_val;
> +     //uint32_t free_page_report_cmd_id;
> +     ///* Stores PAGE_POISON if page poisoning is in use */
> +     //uint32_t poison_val;
>  };
>  
>  #define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
> 
--
Dr. David Alan Gilbert / address@hidden / Manchester, UK