Re: [PATCH v2 2/3] s390x/pci: Honor DMA limits set by vfio

[Matthew Rosato]

On 9/15/20 7:28 AM, Cornelia Huck wrote:
> On Mon, 14 Sep 2020 18:29:29 -0400
> Matthew Rosato <mjrosato@linux.ibm.com> wrote:
>
> > When an s390 guest is using lazy unmapping, it can result in a very
> > large number of oustanding DMA requests, far beyond the default
> > limit configured for vfio.  Let's track DMA usage similar to vfio
> > in the host, and trigger the guest to flush their DMA mappings
> > before vfio runs out.
> >
> > Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com>
> > ---
> >   hw/s390x/s390-pci-bus.c  | 99 +++++++++++++++++++++++++++++++++++++++++++++---
> >   hw/s390x/s390-pci-bus.h  |  9 +++++
> >   hw/s390x/s390-pci-inst.c | 29 +++++++++++---
> >   hw/s390x/s390-pci-inst.h |  3 ++
> >   4 files changed, 129 insertions(+), 11 deletions(-)
>
> (...)
>
> > @@ -737,6 +742,82 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus 
> > *bus, int32_t devfn)
> >       object_unref(OBJECT(iommu));
> >   }
> >   
> > +static bool s390_sync_dma_avail(int fd, unsigned int *avail)
>
> Not sure I like the name. It sounds like the function checks whether
> "sync dma" is available. Maybe s390_update_dma_avail()?

Sounds fine to me.

> > +{
> > +    struct vfio_iommu_type1_info *info;
> > +    uint32_t argsz;
> > +    bool rval = false;
> > +    int ret;
> > +
> > +    if (avail == NULL) {
> > +        return false;
> > +    }
> > +
> > +    argsz = sizeof(struct vfio_iommu_type1_info);
> > +    info = g_malloc0(argsz);
> > +    info->argsz = argsz;
> > +    /*
> > +     * If the specified argsz is not large enough to contain all
> > +     * capabilities it will be updated upon return.  In this case
> > +     * use the updated value to get the entire capability chain.
> > +     */
> > +    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> > +    if (argsz != info->argsz) {
> > +        argsz = info->argsz;
> > +        info = g_realloc(info, argsz);
> > +        info->argsz = argsz;
> > +        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> > +    }
> > +
> > +    if (ret) {
> > +        goto out;
> > +    }
> > +
> > +    /* If the capability exists, update with the current value */
> > +    rval = vfio_get_info_dma_avail(info, avail);
>
> Adding vfio specific things into the generic s390 pci emulation code
> looks a bit ugly... I'd prefer to factor that out into a separate file,
> especially if you plan to add more vfio-specific things in the future.

Fair.   hw/s390x/s390-pci-vfio.* ?

> > +
> > +out:
> > +    g_free(info);
> > +    return rval;
> > +}
> > +
>
> (...)
>
> > diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
> > index 2f7a7d7..6af9af4 100644
> > --- a/hw/s390x/s390-pci-inst.c
> > +++ b/hw/s390x/s390-pci-inst.c
> > @@ -32,6 +32,9 @@
> >           }                                                          \
> >       } while (0)
> >   
> > +#define INC_DMA_AVAIL(iommu) if (iommu->dma_limit) iommu->dma_limit->avail++;
> > +#define DEC_DMA_AVAIL(iommu) if (iommu->dma_limit) iommu->dma_limit->avail--;
>
> Hm... maybe lowercase inline functions might be better here?

OK

> > +
> >   static void s390_set_status_code(CPUS390XState *env,
> >                                    uint8_t r, uint64_t status_code)
> >   {
>
> (...)
>
> > @@ -620,6 +629,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t 
> > r2, uintptr_t ra)
> >       S390PCIIOMMU *iommu;
> >       S390IOTLBEntry entry;
> >       hwaddr start, end;
> > +    uint32_t dma_avail;
> >   
> >       if (env->psw.mask & PSW_MASK_PSTATE) {
> >           s390_program_interrupt(env, PGM_PRIVILEGED, ra);
> > @@ -675,8 +685,9 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t 
> > r2, uintptr_t ra)
> >           }
> >   
> >           start += entry.len;
> > -        while (entry.iova < start && entry.iova < end) {
> > -            s390_pci_update_iotlb(iommu, &entry);
> > +        dma_avail = 1; /* Assume non-zero dma_avail to start */
> > +        while (entry.iova < start && entry.iova < end && dma_avail > 0) {
> > +            dma_avail = s390_pci_update_iotlb(iommu, &entry);
> >               entry.iova += PAGE_SIZE;
> >               entry.translated_addr += PAGE_SIZE;
> >           }
> > @@ -689,7 +700,13 @@ err:
> >           s390_pci_generate_error_event(error, pbdev->fh, pbdev->fid, start, 0);
> >       } else {
> >           pbdev->fmb.counter[ZPCI_FMB_CNT_RPCIT]++;
> > -        setcc(cpu, ZPCI_PCI_LS_OK);
> > +        if (dma_avail > 0) {
>
> When I compile this (with a headers update), the compiler moans here
> about an uninitialized variable.

D'oh.  Obviously dma_avail needs to be initialized outside of the 
if/else -- I'll double-check the logic here and fix.

> > +            setcc(cpu, ZPCI_PCI_LS_OK);
> > +        } else {
> > +            /* vfio DMA mappings are exhausted, trigger a RPCIT */
> > +            setcc(cpu, ZPCI_PCI_LS_ERR);
> > +            s390_set_status_code(env, r1, ZPCI_RPCIT_ST_INSUFF_RES);
> > +        }
> >       }
> >       return 0;
> >   }
>
> (...)