Re: [PULL v3 12/55] virtio-pci: add support for configure interrupt

[Volker Rümelin]

Am 09.01.22 um 19:54 schrieb Volker Rümelin:
> Am 09.01.22 um 19:01 schrieb Michael S. Tsirkin:
> > On Sun, Jan 09, 2022 at 06:52:28PM +0100, Volker Rümelin wrote:
> > > Am 09.01.22 um 17:11 schrieb Michael S. Tsirkin:
> > > > On Sun, Jan 09, 2022 at 07:17:30AM +0100, Volker Rümelin wrote:
> > > > > Hi,
> > > > >
> > > > > > From: Cindy Lu <lulu@redhat.com>
> > > > > >
> > > > > > Add support for configure interrupt, The process is used 
> > > > > > kvm_irqfd_assign
> > > > > > to set the gsi to kernel. When the configure notifier was signal by
> > > > > > host, qemu will inject a msix interrupt to guest
> > > > > >
> > > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > > Message-Id: <20211104164827.21911-11-lulu@redhat.com>
> > > > > > Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
> > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > > > > > ---
> > > > > >     hw/virtio/virtio-pci.h |  4 +-
> > > > > >     hw/virtio/virtio-pci.c | 92 
> > > > > > ++++++++++++++++++++++++++++++++++++------
> > > > > >     2 files changed, 83 insertions(+), 13 deletions(-)
> > > > > Since this commit I see the following warnings.
> > > > >
> > > > > With -drive
> > > > > if=virtio,id=disk1,file=/srv/cdimg/Linux/images/opensuse.qcow2,discard=unmap 
> > > > >
> > > > >
> > > > > qemu-system-x86_64: virtio-blk failed to set guest notifier (-16), 
> > > > > ensure
> > > > > -accel kvm is set.
> > > > > qemu-system-x86_64: virtio_bus_start_ioeventfd: failed. Fallback to
> > > > > userspace (slower).
> > > > >
> > > > > With libvirt
> > > > >       <controller type='pci' index='1' model='pcie-root-port'>
> > > > >         <address type='pci' domain='0x0000' bus='0x00' slot='0x1c'
> > > > > function='0'
> > > > >          multifunction='on'/>
> > > > >       </controller>
> > > > >       <controller type='pci' index='2' model='pcie-root-port'>
> > > > >         <address type='pci' domain='0x0000' bus='0x00' slot='0x1c'
> > > > > function='1'/>
> > > > >       </controller>
> > > > >       <controller type='scsi' index='0' model='virtio-scsi'>
> > > > >         <driver queues='4'/>
> > > > >         <address type='pci' domain='0x0000' bus='0x01' slot='0x00'
> > > > > function='0'/>
> > > > >       </controller>
> > > > >       <disk type='block' device='disk'>
> > > > >         <driver name='qemu' type='raw' cache='none' discard='unmap'
> > > > > io='io_uring'/>
> > > > >         <source dev='/dev/vgtmp/lnxpowerm1'/>
> > > > >         <target dev='sda' bus='scsi'/>
> > > > >         <address type='drive' controller='0' bus='0' unit='0'/>
> > > > >         <boot order='1'/>
> > > > >       </disk>
> > > > >
> > > > > 2022-01-08T17:45:26.911491Z qemu-system-x86_64: virtio-scsi: 
> > > > > Failed to set
> > > > > guest notifiers (-16), ensure -accel kvm is set.
> > > > > 2022-01-08T17:45:26.911505Z qemu-system-x86_64: 
> > > > > virtio_bus_start_ioeventfd:
> > > > > failed. Fallback to userspace (slower).
> > > > >
> > > > > The messages appear around the time the Linux guest initializes 
> > > > > the drivers.
> > > > >
> > > > > With best regards,
> > > > > Volker
> > > > I guess it's a host that has an oldish kernel?
> > > It's an openSUSE 5.3.18 frankenstein kernel.
> > >
> > > > Does the following help?
> > > No.
> > >
> > > > diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> > > > index 98fb5493ae..b77cd69f97 100644
> > > > --- a/hw/virtio/virtio-pci.c
> > > > +++ b/hw/virtio/virtio-pci.c
> > > > @@ -1130,15 +1130,15 @@ static int 
> > > > virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
> > > >                proxy->vector_irqfd =
> > > >                    g_malloc0(sizeof(*proxy->vector_irqfd) *
> > > > msix_nr_vectors_allocated(&proxy->pci_dev));
> > > > +            r = kvm_virtio_pci_vector_config_use(proxy);
> > > > +            if (r < 0) {
> > > > +                goto config_error;
> > > > +            }
> > > >                r = kvm_virtio_pci_vector_use(proxy, nvqs);
> > > >                if (r < 0) {
> > > >                    goto config_assign_error;
> > > >                }
> > > >            }
> > > > -        r = kvm_virtio_pci_vector_config_use(proxy);
> > > > -        if (r < 0) {
> > > > -            goto config_error;
> > > > -        }
> > > >            r = msix_set_vector_notifiers(&proxy->pci_dev, 
> > > > virtio_pci_vector_unmask,
> > > > virtio_pci_vector_mask,
> > > > virtio_pci_vector_poll);
> > > With and without this patch msix_set_vector_notifiers() returns -16.
> >
> > EBUSY
> >
> > strace? did a syscall return this?
>
> I hope I got that right. I used
>
> strace ./qemu-system-x86_64 ... 2>strace.txt
>
> There is no EBUSY in strace.txt.
>
> I will sprinkle a few fprintfs to find the first function returning -16.

This is the call sequence:

virtio_pci_set_guest_notifiers: call msix_set_vector_notifiers()
msix_set_vector_notifiers: call msix_set_notifier_for_vector(dev, 0);
msix_set_notifier_for_vector: call msix_vector_use_notifier(dev, 0, msg);
virtio_pci_vector_unmask: call virtio_pci_one_vector_unmask(proxy, 
VIRTIO_CONFIG_IRQ_IDX, 0, msg, n)
virtio_pci_one_vector_unmask: call kvm_irqchip_update_msi_route()
kvm_irqchip_update_msi_route ret 0
virtio_pci_one_vector_unmask: call kvm_virtio_pci_irqfd_use(proxy, n, 0)
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret 0
kvm_virtio_pci_irqfd_use ret 0
virtio_pci_one_vector_unmask ret 0
msix_vector_use_notifier ret 0
msix_set_notifier_for_vector ret 0
msix_set_vector_notifiers: call msix_set_notifier_for_vector(dev, 1);
msix_set_notifier_for_vector: call msix_vector_use_notifier(dev, 1, msg);
virtio_pci_vector_unmask: call virtio_pci_one_vector_unmask(proxy, 0, 1, 
msg, n)
virtio_pci_one_vector_unmask: call kvm_irqchip_update_msi_route()
kvm_irqchip_update_msi_route ret 0
virtio_pci_one_vector_unmask: call kvm_virtio_pci_irqfd_use(proxy, n, 1)
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret 0
kvm_virtio_pci_irqfd_use ret 0
virtio_pci_one_vector_unmask ret 0
virtio_pci_vector_unmask: call virtio_pci_one_vector_unmask(proxy, 
VIRTIO_CONFIG_IRQ_IDX, 1, msg, n)
virtio_pci_one_vector_unmask: call kvm_irqchip_update_msi_route()
kvm_irqchip_update_msi_route ret 0
virtio_pci_one_vector_unmask: call kvm_virtio_pci_irqfd_use(proxy, n, 1)
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret -16
kvm_virtio_pci_irqfd_use ret -16
virtio_pci_one_vector_unmask ret -16
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret 0
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret 0
msix_vector_use_notifier ret -16
msix_set_notifier_for_vector ret -16
kvm_irqchip_assign_irqfd: call kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
kvm_vm_ioctl ret 0
msix_set_vector_notifiers r -16
qemu-system-x86_64: virtio-blk failed to set guest notifier (-16), 
ensure -accel kvm is set.
qemu-system-x86_64: virtio_bus_start_ioeventfd: failed. Fallback to 
userspace (slower).

> > > > @@ -1155,7 +1155,9 @@ notifiers_error:
> > > >            kvm_virtio_pci_vector_release(proxy, nvqs);
> > > >        }
> > > >    config_error:
> > > > -    kvm_virtio_pci_vector_config_release(proxy);
> > > > +    if (with_irqfd) {
> > > > +        kvm_virtio_pci_vector_config_release(proxy);
> > > > +    }
> > > >    config_assign_error:
> > > >        virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, 
> > > > !assign,
> > > >                                      with_irqfd);