From patchwork Tue Jun 9 10:45:08 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Laurentiu Tudor X-Patchwork-Id: 241997 List-Id: U-Boot discussion From: laurentiu.tudor at nxp.com (laurentiu.tudor at nxp.com) Date: Tue, 9 Jun 2020 13:45:08 +0300 Subject: [PATCH 1/3] pci: layerscape: move per-pci device fdt fixup in a function In-Reply-To: <20200609104510.19781-1-laurentiu.tudor@nxp.com> References: <20200609104510.19781-1-laurentiu.tudor@nxp.com> Message-ID: <20200609104510.19781-2-laurentiu.tudor@nxp.com> From: Laurentiu Tudor Move the pci device related fdt fixup in a function in order to re-use it in a following patch. While at it, improve the error handling. Signed-off-by: Laurentiu Tudor --- drivers/pci/pcie_layerscape_fixup.c | 58 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/drivers/pci/pcie_layerscape_fixup.c b/drivers/pci/pcie_layerscape_fixup.c index 25a3c3870c..12ee5e3f20 100644 --- a/drivers/pci/pcie_layerscape_fixup.c +++ b/drivers/pci/pcie_layerscape_fixup.c @@ -167,12 +167,40 @@ static void fdt_pcie_set_iommu_map_entry_ls(void *blob, struct ls_pcie *pcie, } } +static int fdt_fixup_pcie_device_ls(void *blob, pci_dev_t bdf, + struct ls_pcie *pcie) +{ + int streamid, index; + + streamid = pcie_next_streamid(pcie->stream_id_cur, pcie->idx); + if (streamid < 0) { + printf("ERROR: out of stream ids for BDF %d.%d.%d\n", + PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf)); + return -ENOENT; + } + pcie->stream_id_cur++; + + index = ls_pcie_next_lut_index(pcie); + if (index < 0) { + printf("ERROR: out of LUT indexes for BDF %d.%d.%d\n", + PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf)); + return -ENOENT; + } + + /* map PCI b.d.f to streamID in LUT */ + ls_pcie_lut_set_mapping(pcie, index, bdf >> 8, streamid); + /* update msi-map in device tree */ + fdt_pcie_set_msi_map_entry_ls(blob, pcie, bdf >> 8, streamid); + /* update iommu-map in device tree */ + fdt_pcie_set_iommu_map_entry_ls(blob, pcie, bdf >> 8, streamid); + + return 0; +} + static void fdt_fixup_pcie_ls(void *blob) { struct udevice *dev, *bus; struct ls_pcie *pcie; - int streamid; - int index; pci_dev_t bdf; /* Scan all known buses */ @@ -183,31 +211,11 @@ static void fdt_fixup_pcie_ls(void *blob) bus = bus->parent; pcie = dev_get_priv(bus); - streamid = pcie_next_streamid(pcie->stream_id_cur, pcie->idx); - if (streamid < 0) { - debug("ERROR: no stream ids free\n"); - continue; - } else { - pcie->stream_id_cur++; - } - - index = ls_pcie_next_lut_index(pcie); - if (index < 0) { - debug("ERROR: no LUT indexes free\n"); - continue; - } - /* the DT fixup must be relative to the hose first_busno */ bdf = dm_pci_get_bdf(dev) - PCI_BDF(bus->seq, 0, 0); - /* map PCI b.d.f to streamID in LUT */ - ls_pcie_lut_set_mapping(pcie, index, bdf >> 8, - streamid); - /* update msi-map in device tree */ - fdt_pcie_set_msi_map_entry_ls(blob, pcie, bdf >> 8, - streamid); - /* update iommu-map in device tree */ - fdt_pcie_set_iommu_map_entry_ls(blob, pcie, bdf >> 8, - streamid); + + if (fdt_fixup_pcie_device_ls(blob, bdf, pcie) < 0) + break; } pcie_board_fix_fdt(blob); } From patchwork Tue Jun 9 10:45:09 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Laurentiu Tudor X-Patchwork-Id: 241998 List-Id: U-Boot discussion From: laurentiu.tudor at nxp.com (laurentiu.tudor at nxp.com) Date: Tue, 9 Jun 2020 13:45:09 +0300 Subject: [PATCH 2/3] pci: layerscape: move pci node search in a common function In-Reply-To: <20200609104510.19781-1-laurentiu.tudor@nxp.com> References: <20200609104510.19781-1-laurentiu.tudor@nxp.com> Message-ID: <20200609104510.19781-3-laurentiu.tudor@nxp.com> From: Laurentiu Tudor Fix duplication of this code by placing it in a common function. Furthermore, the resulting function will be re-used in upcoming patches. Signed-off-by: Laurentiu Tudor --- drivers/pci/pcie_layerscape_fixup.c | 87 ++++++++++------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/drivers/pci/pcie_layerscape_fixup.c b/drivers/pci/pcie_layerscape_fixup.c index 12ee5e3f20..64738453e1 100644 --- a/drivers/pci/pcie_layerscape_fixup.c +++ b/drivers/pci/pcie_layerscape_fixup.c @@ -53,19 +53,8 @@ static void ls_pcie_lut_set_mapping(struct ls_pcie *pcie, int index, u32 devid, lut_writel(pcie, streamid | PCIE_LUT_ENABLE, PCIE_LUT_LDR(index)); } -/* - * An msi-map is a property to be added to the pci controller - * node. It is a table, where each entry consists of 4 fields - * e.g.: - * - * msi-map = <[devid] [phandle-to-msi-ctrl] [stream-id] [count] - * [devid] [phandle-to-msi-ctrl] [stream-id] [count]>; - */ -static void fdt_pcie_set_msi_map_entry_ls(void *blob, struct ls_pcie *pcie, - u32 devid, u32 streamid) +static int fdt_pcie_get_nodeoffset(void *blob, struct ls_pcie *pcie) { - u32 *prop; - u32 phandle; int nodeoffset; uint svr; char *compat = NULL; @@ -75,7 +64,7 @@ static void fdt_pcie_set_msi_map_entry_ls(void *blob, struct ls_pcie *pcie, pcie->dbi_res.start); if (nodeoffset < 0) { #ifdef CONFIG_FSL_PCIE_COMPAT /* Compatible with older version of dts node */ - svr = (get_svr() >> SVR_VAR_PER_SHIFT) & 0xFFFFFE; + svr = SVR_SOC_VER(get_svr()); if (svr == SVR_LS2088A || svr == SVR_LS2084A || svr == SVR_LS2048A || svr == SVR_LS2044A || svr == SVR_LS2081A || svr == SVR_LS2041A) @@ -86,10 +75,30 @@ static void fdt_pcie_set_msi_map_entry_ls(void *blob, struct ls_pcie *pcie, nodeoffset = fdt_node_offset_by_compat_reg(blob, compat, pcie->dbi_res.start); #endif - if (nodeoffset < 0) - return; } + return nodeoffset; +} + +/* + * An msi-map is a property to be added to the pci controller + * node. It is a table, where each entry consists of 4 fields + * e.g.: + * + * msi-map = <[devid] [phandle-to-msi-ctrl] [stream-id] [count] + * [devid] [phandle-to-msi-ctrl] [stream-id] [count]>; + */ +static void fdt_pcie_set_msi_map_entry_ls(void *blob, struct ls_pcie *pcie, + u32 devid, u32 streamid) +{ + u32 *prop; + u32 phandle; + int nodeoffset; + + nodeoffset = fdt_pcie_get_nodeoffset(blob, pcie); + if (nodeoffset < 0) + return; + /* get phandle to MSI controller */ prop = (u32 *)fdt_getprop(blob, nodeoffset, "msi-parent", 0); if (prop == NULL) { @@ -121,29 +130,10 @@ static void fdt_pcie_set_iommu_map_entry_ls(void *blob, struct ls_pcie *pcie, u32 iommu_map[4]; int nodeoffset; int lenp; - uint svr; - char *compat = NULL; - /* find pci controller node */ - nodeoffset = fdt_node_offset_by_compat_reg(blob, "fsl,ls-pcie", - pcie->dbi_res.start); - if (nodeoffset < 0) { -#ifdef CONFIG_FSL_PCIE_COMPAT /* Compatible with older version of dts node */ - svr = (get_svr() >> SVR_VAR_PER_SHIFT) & 0xFFFFFE; - if (svr == SVR_LS2088A || svr == SVR_LS2084A || - svr == SVR_LS2048A || svr == SVR_LS2044A || - svr == SVR_LS2081A || svr == SVR_LS2041A) - compat = "fsl,ls2088a-pcie"; - else - compat = CONFIG_FSL_PCIE_COMPAT; - - if (compat) - nodeoffset = fdt_node_offset_by_compat_reg(blob, - compat, pcie->dbi_res.start); -#endif - if (nodeoffset < 0) - return; - } + nodeoffset = fdt_pcie_get_nodeoffset(blob, pcie); + if (nodeoffset < 0) + return; /* get phandle to iommu controller */ prop = fdt_getprop_w(blob, nodeoffset, "iommu-map", &lenp); @@ -224,27 +214,10 @@ static void fdt_fixup_pcie_ls(void *blob) static void ft_pcie_rc_fix(void *blob, struct ls_pcie *pcie) { int off; - uint svr; - char *compat = NULL; - off = fdt_node_offset_by_compat_reg(blob, "fsl,ls-pcie", - pcie->dbi_res.start); - if (off < 0) { -#ifdef CONFIG_FSL_PCIE_COMPAT /* Compatible with older version of dts node */ - svr = (get_svr() >> SVR_VAR_PER_SHIFT) & 0xFFFFFE; - if (svr == SVR_LS2088A || svr == SVR_LS2084A || - svr == SVR_LS2048A || svr == SVR_LS2044A || - svr == SVR_LS2081A || svr == SVR_LS2041A) - compat = "fsl,ls2088a-pcie"; - else - compat = CONFIG_FSL_PCIE_COMPAT; - if (compat) - off = fdt_node_offset_by_compat_reg(blob, - compat, pcie->dbi_res.start); -#endif - if (off < 0) - return; - } + off = fdt_pcie_get_nodeoffset(blob, pcie); + if (off < 0) + return; if (pcie->enabled && pcie->mode == PCI_HEADER_TYPE_BRIDGE) fdt_set_node_status(blob, off, FDT_STATUS_OKAY, 0); From patchwork Tue Jun 9 10:45:10 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Laurentiu Tudor X-Patchwork-Id: 241999 List-Id: U-Boot discussion From: laurentiu.tudor at nxp.com (laurentiu.tudor at nxp.com) Date: Tue, 9 Jun 2020 13:45:10 +0300 Subject: [PATCH 3/3] pci: layerscape: add a way of specifying additional iommu mappings In-Reply-To: <20200609104510.19781-1-laurentiu.tudor@nxp.com> References: <20200609104510.19781-1-laurentiu.tudor@nxp.com> Message-ID: <20200609104510.19781-4-laurentiu.tudor@nxp.com> From: Laurentiu Tudor In the current implementation, u-boot creates iommu mappings only for PCI devices enumarated at boot time thus does not take into account more dynamic scenarios such as SR-IOV or PCI hot-plug. Add an u-boot env var and a device tree property (to be used for example in more static scenarios such as hardwired PCI endpoints that get initialized later in the system setup) that would allow two things: - for a SRIOV capable PCI EP identified by its B.D.F specify the maximum number of VFs that will ever be created for it - for hot-plug case, specify the B.D.F with which the device will show up on the PCI bus The env var consists of a list of , pairs for a certain pci bus identified by its controller's base register address, as defined in the "reg" property in the device tree. pci_iommu_extra = pci@,,,,, pci@,,,,,... where: is the register base address of the pci controller for which the subsequent , pairs apply identifies to which B.D.F the action applies to can be: - "vfs=" to specify that for the PCI EP identified previously by the to include mappings for of VFs - "hp" to specify that on this there will be a hot-plugged device so it needs a mapping The device tree property must be placed under the correct pci controller node and only the bdf and action pairs need to be specified, like this: pci-iommu-extra = ",,,,..."; For example, given this configuration on bus 6: => pci 6 Scanning PCI devices on bus 6 BusDevFun VendorId DeviceId Device Class Sub-Class diff --git a/arch/arm/cpu/armv8/fsl-layerscape/doc/README.pci_iommu_extra b/arch/arm/cpu/armv8/fsl-layerscape/doc/README.pci_iommu_extra new file mode 100644 index 0000000000..cb1388796b --- /dev/null +++ b/arch/arm/cpu/armv8/fsl-layerscape/doc/README.pci_iommu_extra @@ -0,0 +1,68 @@ +# +# Copyright 2020 NXP +# +# SPDX-License-Identifier: GPL-2.0+ +# + +Specifying extra IOMMU mappings for PCI controllers + +This feature can be enabled through the PCI_IOMMU_EXTRA_MAPPINGS +Kconfig option. + +The "pci_iommu_extra" env var or "pci-iommu-extra" device tree +property (to be used for example in more static scenarios such +as hardwired PCI endpoints that get initialized later in the system +setup) allows two things: + - for a SRIOV capable PCI EP identified by its B.D.F specify + the maximum number of VFs that will ever be created for it + - for hot-plug case, specify the B.D.F with which the device + will show up on the PCI bus + +The env var consists of a list of , pairs for a certain +pci bus identified by its controller's base register address, as +defined in the "reg" property in the device tree. + +pci_iommu_extra = pci@,,,,, + pci@,,,,,... + +where: + is the register base address of the pci controller for which +the subsequent , pairs apply + identifies to which B.D.F the action applies to + can be: + - "vfs=" to specify that for the PCI EP identified + previously by the to include mappings for of VFs + - "hp" to specify that on this there will be a hot-plugged + device so it needs a mapping +The device tree property must be placed under the correct pci +controller node and only the bdf and action pairs need to be specified, +like this: + +pci-iommu-extra = ",,,,..."; + +For example, given this configuration on bus 6: + +=> pci 6 +Scanning PCI devices on bus 6 +BusDevFun VendorId DeviceId Device Class Sub-Class +_____________________________________________________________ +06.00.00 0x8086 0x1572 Network controller 0x00 +06.00.01 0x8086 0x1572 Network controller 0x00 + +The following u-boot env var will create iommu mappings for 3 VFs +for each PF: + +=> setenv pci_iommu_extra pci at 0x3800000,6.0.0,vfs=3,6.0.1,vfs=3 + +For the device tree case, this would be specified like this: + +pci-iommu-extra = "6.0.0,vfs=3,6.0.1,vfs=3"; + +To add an iommu mapping for a hot-plugged device, please see +following example: + +=> setenv pci_iommu_extra pci at 0x3800000,6.2.0,hp + +For the device tree case, this would be specified like this: + +pci-iommu-extra = "6.2.0,hp"; diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 6d8c22aacf..2697879dec 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -135,6 +135,18 @@ config PCIE_LAYERSCAPE PCIe controllers. The PCIe may works in RC or EP mode according to RCW[HOST_AGT_PEX] setting. +config PCI_IOMMU_EXTRA_MAPPINGS + bool "Support for specifying extra IOMMU mappings for PCI" + depends on PCIE_LAYERSCAPE + help + Enable support for specifying extra IOMMU mappings for PCI + controllers through a special env var called "pci_iommu_extra" or + through a device tree property named "pci-iommu-extra" placed in + the node describing the PCI controller. + The intent is to cover SR-IOV scenarios which need mappings for VFs + and PCI hot-plug scenarios. More documentation can be found under: + arch/arm/cpu/armv8/fsl-layerscape/doc/README.pci_iommu_extra + config PCIE_LAYERSCAPE_GEN4 bool "Layerscape Gen4 PCIe support" depends on DM_PCI diff --git a/drivers/pci/pcie_layerscape_fixup.c b/drivers/pci/pcie_layerscape_fixup.c index 64738453e1..72233eee3d 100644 --- a/drivers/pci/pcie_layerscape_fixup.c +++ b/drivers/pci/pcie_layerscape_fixup.c @@ -18,6 +18,7 @@ #ifdef CONFIG_ARM #include #endif +#include #include "pcie_layerscape.h" #include "pcie_layerscape_fixup_common.h" @@ -187,11 +188,119 @@ static int fdt_fixup_pcie_device_ls(void *blob, pci_dev_t bdf, return 0; } +#ifdef CONFIG_PCI_IOMMU_EXTRA_MAPPINGS +struct extra_iommu_entry { + int action; + pci_dev_t bdf; + int num_vfs; +}; + +#define EXTRA_IOMMU_ENTRY_HOTPLUG 1 +#define EXTRA_IOMMU_ENTRY_VFS 2 + +static struct extra_iommu_entry *get_extra_iommu_ents(void *blob, + int nodeoffset, + phys_addr_t addr, + int *cnt) +{ + const char *s, *p, *tok; + struct extra_iommu_entry *entries; + int i = 0, b, d, f; + + s = env_get("pci_iommu_extra"); + if (!s) { + s = fdt_getprop(blob, nodeoffset, "pci-iommu-extra", NULL); + } else { + phys_addr_t pci_base; + char *endp; + + tok = s; + p = strchrnul(s + 1, ','); + s = NULL; + do { + if (!strncmp(tok, "pci", 3)) { + pci_base = simple_strtoul(tok + 4, &endp, 0); + if (pci_base == addr) { + s = endp + 1; + break; + } + } + p = strchrnul(p + 1, ','); + tok = p + 1; + } while (*p); + } + + if (!s) + return NULL; + + *cnt = 0; + p = s; + while (*p && strncmp(p, "pci", 3)) { + if (*p == ',') + (*cnt)++; + p++; + } + if (!(*p)) + (*cnt)++; + + if (!(*cnt) || (*cnt) % 2) { + printf("ERROR: invalid or odd extra iommu token count %d\n", + *cnt); + return NULL; + } + *cnt = (*cnt) / 2; + + entries = malloc((*cnt) * sizeof(*entries)); + if (!entries) { + printf("ERROR: fail to allocate extra iommu entries\n"); + return NULL; + } + + p = s; + while (p) { + b = simple_strtoul(p, (char **)&p, 0); p++; + d = simple_strtoul(p, (char **)&p, 0); p++; + f = simple_strtoul(p, (char **)&p, 0); p++; + entries[i].bdf = PCI_BDF(b, d, f); + + if (!strncmp(p, "hp", 2)) { + entries[i].action = EXTRA_IOMMU_ENTRY_HOTPLUG; + p += 3; + } else if (!strncmp(p, "vfs", 3)) { + entries[i].action = EXTRA_IOMMU_ENTRY_VFS; + + p = strchr(p, '='); + entries[i].num_vfs = simple_strtoul(p + 1, (char **)&p, + 0); + if (*p) + p++; + } else { + printf("ERROR: invalid action in extra iommu entry\n"); + free(entries); + + return NULL; + } + + if (!(*p) || !strncmp(p, "pci", 3)) + break; + + i++; + } + + return entries; +} +#endif /* CONFIG_PCI_IOMMU_EXTRA_MAPPINGS */ + static void fdt_fixup_pcie_ls(void *blob) { struct udevice *dev, *bus; struct ls_pcie *pcie; pci_dev_t bdf; +#ifdef CONFIG_PCI_IOMMU_EXTRA_MAPPINGS + struct extra_iommu_entry *entries; + unsigned short vf_offset, vf_stride; + int i, j, cnt, sriov_pos, nodeoffset; +#endif /* Scan all known buses */ for (pci_find_first_device(&dev); @@ -207,6 +316,75 @@ static void fdt_fixup_pcie_ls(void *blob) if (fdt_fixup_pcie_device_ls(blob, bdf, pcie) < 0) break; } + +#ifdef CONFIG_PCI_IOMMU_EXTRA_MAPPINGS + list_for_each_entry(pcie, &ls_pcie_list, list) { + nodeoffset = fdt_pcie_get_nodeoffset(blob, pcie); + if (nodeoffset < 0) { + printf("ERROR: couldn't find pci node\n"); + continue; + } + + entries = get_extra_iommu_ents(blob, nodeoffset, + pcie->dbi_res.start, &cnt); + if (!entries) + continue; + + for (i = 0; i < cnt; i++) { + if (entries[i].action == EXTRA_IOMMU_ENTRY_HOTPLUG) { + bdf = entries[i].bdf - + PCI_BDF(pcie->bus->seq + 1, 0, 0); + printf("Added iommu map for hotplug %d.%d.%d\n", + PCI_BUS(entries[i].bdf), + PCI_DEV(entries[i].bdf), + PCI_FUNC(entries[i].bdf)); + if (fdt_fixup_pcie_device_ls(blob, + bdf, pcie) < 0) { + free(entries); + return; + } + continue; + } + + /* EXTRA_IOMMU_ENTRY_VFS case */ + if (dm_pci_bus_find_bdf(entries[i].bdf, &dev)) { + printf("ERROR: BDF %d.%d.%d not found\n", + PCI_BUS(entries[i].bdf), + PCI_DEV(entries[i].bdf), + PCI_FUNC(entries[i].bdf)); + continue; + } + sriov_pos = dm_pci_find_ext_capability + (dev, PCI_EXT_CAP_ID_SRIOV); + if (!sriov_pos) { + printf("WARN: setting VFs on non-SRIOV dev\n"); + continue; + } + dm_pci_read_config16(dev, sriov_pos + 0x14, + &vf_offset); + dm_pci_read_config16(dev, sriov_pos + 0x16, + &vf_stride); + + bdf = entries[i].bdf - + PCI_BDF(pcie->bus->seq + 1, 0, 0) + + (vf_offset << 8); + printf("Added %d iommu VF mappings for PF %d.%d.%d\n", + entries[i].num_vfs, PCI_BUS(entries[i].bdf), + PCI_DEV(entries[i].bdf), + PCI_FUNC(entries[i].bdf)); + for (j = 0; j < entries[i].num_vfs; j++) { + if (fdt_fixup_pcie_device_ls(blob, + bdf, pcie) < 0) { + free(entries); + return; + } + bdf += vf_stride << 8; + } + } + free(entries); + } +#endif /* CONFIG_PCI_IOMMU_EXTRA_MAPPINGS */ + pcie_board_fix_fdt(blob); } #endif