[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for devic
From: |
Greg Kurz |
Subject: |
Re: [RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for device tree creation |
Date: |
Thu, 26 Mar 2020 12:57:38 +0100 |
On Thu, 26 Mar 2020 16:40:06 +1100
David Gibson <address@hidden> wrote:
> Currently spapr_phb_nvgpu_populate_pcidev_dt() works a little cryptically.
> It steps through all the NVLink2 GPUs and NPUs and if they match the device
> we're called for, we generate the relevant device tree information.
>
> Make this a little more obvious by introducing helpers to determine it a
... to determine if a
> given PCI device is an NVLink2 GPU or NPU, returning the NVLink2 slot and
> link number information as well.
>
> Signed-off-by: David Gibson <address@hidden>
> ---
LGTM
Reviewed-by: Greg Kurz <address@hidden>
> hw/ppc/spapr_pci_nvlink2.c | 115 +++++++++++++++++++++++++------------
> 1 file changed, 79 insertions(+), 36 deletions(-)
>
> diff --git a/hw/ppc/spapr_pci_nvlink2.c b/hw/ppc/spapr_pci_nvlink2.c
> index 8332d5694e..7d3a685421 100644
> --- a/hw/ppc/spapr_pci_nvlink2.c
> +++ b/hw/ppc/spapr_pci_nvlink2.c
> @@ -390,13 +390,12 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState
> *sphb, void *fdt)
>
> }
>
> -void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int
> offset,
> - SpaprPhbState *sphb)
> +static bool is_nvgpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot)
> {
> - int i, j;
> + int i;
>
> if (!sphb->nvgpus) {
> - return;
> + return false;
> }
>
> for (i = 0; i < sphb->nvgpus->num; ++i) {
> @@ -406,47 +405,91 @@ void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev,
> void *fdt, int offset,
> if (!nvslot->gpdev) {
> continue;
> }
> +
> if (dev == nvslot->gpdev) {
> - uint32_t npus[nvslot->linknum];
> + if (slot) {
> + *slot = i;
> + }
> + return true;
> + }
> + }
>
> - for (j = 0; j < nvslot->linknum; ++j) {
> - PCIDevice *npdev = nvslot->links[j].npdev;
> + return false;
> +}
>
> - npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
> - }
> - _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> - j * sizeof(npus[0])));
> - _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> - PHANDLE_PCIDEV(sphb, dev))));
> +static bool is_nvnpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot, int
> *link)
> +{
> + int i, j;
> +
> + if (!sphb->nvgpus) {
> + return false;
> + }
> +
> + for (i = 0; i < sphb->nvgpus->num; ++i) {
> + SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
> +
> + /* Skip "slot" without attached GPU */
> + if (!nvslot->gpdev) {
> continue;
> }
>
> for (j = 0; j < nvslot->linknum; ++j) {
> - if (dev != nvslot->links[j].npdev) {
> - continue;
> + if (dev == nvslot->links[j].npdev) {
> + if (slot) {
> + *slot = i;
> + }
> + if (link) {
> + *link = j;
> + }
> + return true;
> }
> + }
> + }
>
> - _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> - PHANDLE_PCIDEV(sphb, dev))));
> - _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> - PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> - _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> - PHANDLE_NVLINK(sphb, i, j))));
> - /*
> - * If we ever want to emulate GPU RAM at the same location as on
> - * the host - here is the encoding GPA->TGT:
> - *
> - * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> - * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> - * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> - * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> - */
> - _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> - PHANDLE_GPURAM(sphb, i)));
> - _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> - nvslot->tgt));
> - _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> - nvslot->links[j].link_speed));
> + return false;
> +}
> +
> +void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int
> offset,
> + SpaprPhbState *sphb)
> +{
> + int slot, link;
> +
> + if (is_nvgpu(dev, sphb, &slot)) {
> + SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> + uint32_t npus[nvslot->linknum];
> +
> + for (link = 0; link < nvslot->linknum; ++link) {
> + PCIDevice *npdev = nvslot->links[link].npdev;
> +
> + npus[link] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
> }
> + _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> + link * sizeof(npus[0])));
> + _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> + PHANDLE_PCIDEV(sphb, dev))));
> + } else if (is_nvnpu(dev, sphb, &slot, &link)) {
> + SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> +
> + _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> + PHANDLE_PCIDEV(sphb, dev))));
> + _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> + PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> + _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> + PHANDLE_NVLINK(sphb, slot, link))));
> + /*
> + * If we ever want to emulate GPU RAM at the same location as
> + * on the host - here is the encoding GPA->TGT:
> + *
> + * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> + * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> + * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> + * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> + */
> + _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> + PHANDLE_GPURAM(sphb, slot)));
> + _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> + nvslot->tgt));
> + _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> + nvslot->links[link].link_speed));
> }
> }
- Re: [RFC for-5.1 2/4] spapr: Helper to determine if a device is NVLink2 related, (continued)