From 5bc1cf1466f635682c3e26eb4179541feeefb1be Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Mon, 18 May 2020 22:16:56 +0800 Subject: [PATCH 01/76] iommu/qcom: add optional 'tbu' clock for TLB invalidate On some SoCs like MSM8939 with A405 adreno, there is a gfx_tbu clock needs to be on while doing TLB invalidate. Otherwise, TLBSYNC status will not be correctly reflected, causing the system to go into a bad state. Add it as an optional clock, so that platforms that have this clock can pass it over DT. While adding the third clock, let's switch to bulk clk API to simplify the enable/disable calls. clk_bulk_get() cannot used because the existing two clocks are required while the new one is optional. Signed-off-by: Shawn Guo Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20200518141656.26284-1-shawn.guo@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/qcom_iommu.c | 62 ++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 0e2a96467767..116d8188f87f 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -37,14 +37,20 @@ #define SMMU_INTR_SEL_NS 0x2000 +enum qcom_iommu_clk { + CLK_IFACE, + CLK_BUS, + CLK_TBU, + CLK_NUM, +}; + struct qcom_iommu_ctx; struct qcom_iommu_dev { /* IOMMU core code handle */ struct iommu_device iommu; struct device *dev; - struct clk *iface_clk; - struct clk *bus_clk; + struct clk_bulk_data clks[CLK_NUM]; void __iomem *local_base; u32 sec_id; u8 num_ctxs; @@ -626,32 +632,6 @@ static const struct iommu_ops qcom_iommu_ops = { .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, }; -static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu) -{ - int ret; - - ret = clk_prepare_enable(qcom_iommu->iface_clk); - if (ret) { - dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n"); - return ret; - } - - ret = clk_prepare_enable(qcom_iommu->bus_clk); - if (ret) { - dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n"); - clk_disable_unprepare(qcom_iommu->iface_clk); - return ret; - } - - return 0; -} - -static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu) -{ - clk_disable_unprepare(qcom_iommu->bus_clk); - clk_disable_unprepare(qcom_iommu->iface_clk); -} - static int qcom_iommu_sec_ptbl_init(struct device *dev) { size_t psize = 0; @@ -808,6 +788,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) struct qcom_iommu_dev *qcom_iommu; struct device *dev = &pdev->dev; struct resource *res; + struct clk *clk; int ret, max_asid = 0; /* find the max asid (which is 1:1 to ctx bank idx), so we know how @@ -827,17 +808,26 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) if (res) qcom_iommu->local_base = devm_ioremap_resource(dev, res); - qcom_iommu->iface_clk = devm_clk_get(dev, "iface"); - if (IS_ERR(qcom_iommu->iface_clk)) { + clk = devm_clk_get(dev, "iface"); + if (IS_ERR(clk)) { dev_err(dev, "failed to get iface clock\n"); - return PTR_ERR(qcom_iommu->iface_clk); + return PTR_ERR(clk); } + qcom_iommu->clks[CLK_IFACE].clk = clk; - qcom_iommu->bus_clk = devm_clk_get(dev, "bus"); - if (IS_ERR(qcom_iommu->bus_clk)) { + clk = devm_clk_get(dev, "bus"); + if (IS_ERR(clk)) { dev_err(dev, "failed to get bus clock\n"); - return PTR_ERR(qcom_iommu->bus_clk); + return PTR_ERR(clk); } + qcom_iommu->clks[CLK_BUS].clk = clk; + + clk = devm_clk_get_optional(dev, "tbu"); + if (IS_ERR(clk)) { + dev_err(dev, "failed to get tbu clock\n"); + return PTR_ERR(clk); + } + qcom_iommu->clks[CLK_TBU].clk = clk; if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id", &qcom_iommu->sec_id)) { @@ -909,14 +899,14 @@ static int __maybe_unused qcom_iommu_resume(struct device *dev) { struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev); - return qcom_iommu_enable_clocks(qcom_iommu); + return clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks); } static int __maybe_unused qcom_iommu_suspend(struct device *dev) { struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev); - qcom_iommu_disable_clocks(qcom_iommu); + clk_bulk_disable_unprepare(CLK_NUM, qcom_iommu->clks); return 0; } From d3e3d2be688b4b5864538de61e750721a311e4fc Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 2 Jun 2020 14:08:18 +0100 Subject: [PATCH 02/76] iommu/iova: Don't BUG on invalid PFNs Unlike the other instances which represent a complete loss of consistency within the rcache mechanism itself, or a fundamental and obvious misconfiguration by an IOMMU driver, the BUG_ON() in iova_magazine_free_pfns() can be provoked at more or less any time in a "spooky action-at-a-distance" manner by any old device driver passing nonsense to dma_unmap_*() which then propagates through to queue_iova(). Not only is this well outside the IOVA layer's control, it's also nowhere near fatal enough to justify panicking anyway - all that really achieves is to make debugging the offending driver more difficult. Let's simply WARN and otherwise ignore bogus PFNs. Reported-by: Prakash Gupta Signed-off-by: Robin Murphy Reviewed-by: Prakash Gupta Link: https://lore.kernel.org/r/acbd2d092b42738a03a21b417ce64e27f8c91c86.1591103298.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 49fc01f2a28d..45a251da5453 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -811,7 +811,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) for (i = 0 ; i < mag->size; ++i) { struct iova *iova = private_find_iova(iovad, mag->pfns[i]); - BUG_ON(!iova); + if (WARN_ON(!iova)) + continue; + private_free_iova(iovad, iova); } From f5e383ac8b58f421ac0a005f6df4f2e9eefeb93f Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Thu, 4 Jun 2020 15:37:09 +0300 Subject: [PATCH 03/76] iommu/pamu: Use kzfree() in fsl_pamu_probe() Use kzfree() instead of opencoded memset with 0 followed by kfree(). Null check is not required since kzfree() checks for NULL internally. Signed-off-by: Denis Efremov Link: https://lore.kernel.org/r/20200604123709.96561-1-efremov@linux.com Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index cde281b97afa..099a11a35fb9 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -1174,10 +1174,7 @@ error: if (irq != NO_IRQ) free_irq(irq, data); - if (data) { - memset(data, 0, sizeof(struct pamu_isr_data)); - kfree(data); - } + kzfree(data); if (pamu_regs) iounmap(pamu_regs); From e725a00a8f2e1ae866c29f956d637a889f561882 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Sat, 6 Jun 2020 12:16:17 -0700 Subject: [PATCH 04/76] iommu/qcom: Change CONFIG_BIG_ENDIAN to CONFIG_CPU_BIG_ENDIAN CONFIG_BIG_ENDIAN does not exist as a Kconfig symbol. Signed-off-by: Joe Perches Reviewed-by: Rob Clark Link: https://lore.kernel.org/r/5a663096b489b86472fe3bfbd5138c411d669bad.camel@perches.com Signed-off-by: Joerg Roedel --- drivers/iommu/qcom_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 801768cf86c6..cca39b209158 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -310,7 +310,7 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, ARM_SMMU_SCTLR_M | ARM_SMMU_SCTLR_S1_ASIDPNE | ARM_SMMU_SCTLR_CFCFG; - if (IS_ENABLED(CONFIG_BIG_ENDIAN)) + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) reg |= ARM_SMMU_SCTLR_E; iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg); From 215c224f4dfa89f1d97500f419092c0110da3c2e Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 11 Jun 2020 20:10:29 +0900 Subject: [PATCH 05/76] dt-bindings: iommu: renesas,ipmmu-vmsa: add r8a77961 support Add support for r8a77961 (R-Car M3-W+). Signed-off-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Acked-by: Rob Herring Link: https://lore.kernel.org/r/1591873830-10128-2-git-send-email-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml index 39675cf4ed71..e9d28a4060fa 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml @@ -35,6 +35,7 @@ properties: - renesas,ipmmu-r8a774c0 # RZ/G2E - renesas,ipmmu-r8a7795 # R-Car H3 - renesas,ipmmu-r8a7796 # R-Car M3-W + - renesas,ipmmu-r8a77961 # R-Car M3-W+ - renesas,ipmmu-r8a77965 # R-Car M3-N - renesas,ipmmu-r8a77970 # R-Car V3M - renesas,ipmmu-r8a77980 # R-Car V3H From 17fe16181639801bfeba647a1e452a75efe651b4 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 11 Jun 2020 20:10:30 +0900 Subject: [PATCH 06/76] iommu/renesas: Add support for r8a77961 Add support for r8a77961 (R-Car M3-W+). Signed-off-by: Yoshihiro Shimoda Link: https://lore.kernel.org/r/1591873830-10128-3-git-send-email-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 4c2972f3153b..b57b1f213a48 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -3,7 +3,7 @@ * IOMMU API for Renesas VMSA-compatible IPMMU * Author: Laurent Pinchart * - * Copyright (C) 2014 Renesas Electronics Corporation + * Copyright (C) 2014-2020 Renesas Electronics Corporation */ #include @@ -753,6 +753,7 @@ static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { { .soc_id = "r8a774b1", }, { .soc_id = "r8a774c0", }, { .soc_id = "r8a7795", .revision = "ES3.*" }, + { .soc_id = "r8a77961", }, { .soc_id = "r8a77965", }, { .soc_id = "r8a77990", }, { .soc_id = "r8a77995", }, @@ -969,6 +970,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a7796", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a77961", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a77965", .data = &ipmmu_features_rcar_gen3, From 970471914c67b70df24def6b2a30cc42acbebded Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Tue, 16 Jun 2020 16:47:14 +0200 Subject: [PATCH 07/76] iommu: Allow page responses without PASID Some PCIe devices do not expect a PASID value in PRI Page Responses. If the "PRG Response PASID Required" bit in the PRI capability is zero, then the OS should not set the PASID field. Similarly on Arm SMMU, responses to stall events do not have a PASID. Currently iommu_page_response() systematically checks that the PASID in the page response corresponds to the one in the page request. This can't work with virtualization because a page response coming from a guest OS won't have a PASID if the passed-through device does not require one. Add a flag to page requests that declares whether the corresponding response needs to have a PASID. When this flag isn't set, allow page responses without PASID. Reported-by: Shameerali Kolothum Thodi Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20200616144712.748818-1-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 25 ++++++++++++++++++------- include/uapi/linux/iommu.h | 6 +++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d43120eb1dc5..1ed1e14a1f0c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1185,11 +1185,12 @@ EXPORT_SYMBOL_GPL(iommu_report_device_fault); int iommu_page_response(struct device *dev, struct iommu_page_response *msg) { - bool pasid_valid; + bool needs_pasid; int ret = -EINVAL; struct iommu_fault_event *evt; struct iommu_fault_page_request *prm; struct dev_iommu *param = dev->iommu; + bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; struct iommu_domain *domain = iommu_get_domain_for_dev(dev); if (!domain || !domain->ops->page_response) @@ -1214,14 +1215,24 @@ int iommu_page_response(struct device *dev, */ list_for_each_entry(evt, ¶m->fault_param->faults, list) { prm = &evt->fault.prm; - pasid_valid = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; - - if ((pasid_valid && prm->pasid != msg->pasid) || - prm->grpid != msg->grpid) + if (prm->grpid != msg->grpid) continue; - /* Sanitize the reply */ - msg->flags = pasid_valid ? IOMMU_PAGE_RESP_PASID_VALID : 0; + /* + * If the PASID is required, the corresponding request is + * matched using the group ID, the PASID valid bit and the PASID + * value. Otherwise only the group ID matches request and + * response. + */ + needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) + continue; + + if (!needs_pasid && has_pasid) { + /* No big deal, just clear it. */ + msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; + msg->pasid = 0; + } ret = domain->ops->page_response(dev, evt, msg); list_del(&evt->list); diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h index e907b7091a46..c2b2caf9ed41 100644 --- a/include/uapi/linux/iommu.h +++ b/include/uapi/linux/iommu.h @@ -81,7 +81,10 @@ struct iommu_fault_unrecoverable { /** * struct iommu_fault_page_request - Page Request data * @flags: encodes whether the corresponding fields are valid and whether this - * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values) + * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). + * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response + * must have the same PASID value as the page request. When it is clear, + * the page response should not have a PASID. * @pasid: Process Address Space ID * @grpid: Page Request Group Index * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) @@ -92,6 +95,7 @@ struct iommu_fault_page_request { #define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) #define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) #define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) +#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) __u32 flags; __u32 pasid; __u32 grpid; From 9a295ff0ffc94e1be60d604fee0cce4fbb3b8964 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Wed, 17 Jun 2020 00:04:20 +0200 Subject: [PATCH 08/76] iommu/amd: Print extended features in one line to fix divergent log levels Currently, Linux logs the two messages below. [ 0.979142] pci 0000:00:00.2: AMD-Vi: Extended features (0xf77ef22294ada): [ 0.979546] PPR NX GT IA GA PC GA_vAPIC The log level of these lines differs though. The first one has level *info*, while the second has level *warn*, which is confusing. $ dmesg -T --level=info | grep "Extended features" [Tue Jun 16 21:46:58 2020] pci 0000:00:00.2: AMD-Vi: Extended features (0xf77ef22294ada): $ dmesg -T --level=warn | grep "PPR" [Tue Jun 16 21:46:58 2020] PPR NX GT IA GA PC GA_vAPIC The problem is, that commit 3928aa3f57 ("iommu/amd: Detect and enable guest vAPIC support") introduced a newline, causing `pr_cont()`, used to print the features, to default back to the default log level. /** * pr_cont - Continues a previous log message in the same line. * @fmt: format string * @...: arguments for the format string * * This macro expands to a printk with KERN_CONT loglevel. It should only be * used when continuing a log message with no newline ('\n') enclosed. Otherwise * it defaults back to KERN_DEFAULT loglevel. */ #define pr_cont(fmt, ...) \ printk(KERN_CONT fmt, ##__VA_ARGS__) So, remove the line break, so only one line is logged. Fixes: 3928aa3f57 ("iommu/amd: Detect and enable guest vAPIC support") Signed-off-by: Paul Menzel Reviewed-by: Suravee Suthikulpanit Cc: Suravee Suthikulpanit Cc: iommu@lists.linux-foundation.org Link: https://lore.kernel.org/r/20200616220420.19466-1-pmenzel@molgen.mpg.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 6ebd4825e320..232683ea10e0 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1842,7 +1842,7 @@ static void print_iommu_info(void) pci_info(pdev, "Found IOMMU cap 0x%hx\n", iommu->cap_ptr); if (iommu->cap & (1 << IOMMU_CAP_EFR)) { - pci_info(pdev, "Extended features (%#llx):\n", + pci_info(pdev, "Extended features (%#llx):", iommu->features); for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { if (iommu_feature(iommu, (1ULL << i))) From 70fcd3592b05fde4d95938cd5a20e996b4ef4e15 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 26 Jun 2020 10:05:46 +0200 Subject: [PATCH 09/76] iommu/amd: Add helper functions to update domain->pt_root Do not call atomic64_set() directly to update the domain page-table root and use two new helper functions. This makes it easier to implement additional work necessary when the page-table is updated. Signed-off-by: Joerg Roedel Link: https://lore.kernel.org/r/20200626080547.24865-2-joro@8bytes.org --- drivers/iommu/amd/iommu.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 74cca1757172..5286ddcfc2f9 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -162,7 +162,18 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ } -static u64 amd_iommu_domain_encode_pgtable(u64 *root, int mode) +static void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root) +{ + atomic64_set(&domain->pt_root, root); +} + +static void amd_iommu_domain_clr_pt_root(struct protection_domain *domain) +{ + amd_iommu_domain_set_pt_root(domain, 0); +} + +static void amd_iommu_domain_set_pgtable(struct protection_domain *domain, + u64 *root, int mode) { u64 pt_root; @@ -170,7 +181,7 @@ static u64 amd_iommu_domain_encode_pgtable(u64 *root, int mode) pt_root = mode & 7; pt_root |= (u64)root; - return pt_root; + amd_iommu_domain_set_pt_root(domain, pt_root); } static struct iommu_dev_data *alloc_dev_data(u16 devid) @@ -1410,7 +1421,7 @@ static bool increase_address_space(struct protection_domain *domain, struct domain_pgtable pgtable; unsigned long flags; bool ret = true; - u64 *pte, root; + u64 *pte; spin_lock_irqsave(&domain->lock, flags); @@ -1438,8 +1449,7 @@ static bool increase_address_space(struct protection_domain *domain, * Device Table needs to be updated and flushed before the new root can * be published. */ - root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode); - atomic64_set(&domain->pt_root, root); + amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode); ret = true; @@ -2319,7 +2329,7 @@ static void protection_domain_free(struct protection_domain *domain) domain_id_free(domain->id); amd_iommu_domain_get_pgtable(domain, &pgtable); - atomic64_set(&domain->pt_root, 0); + amd_iommu_domain_clr_pt_root(domain); free_pagetable(&pgtable); kfree(domain); @@ -2327,7 +2337,7 @@ static void protection_domain_free(struct protection_domain *domain) static int protection_domain_init(struct protection_domain *domain, int mode) { - u64 *pt_root = NULL, root; + u64 *pt_root = NULL; BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL); @@ -2343,8 +2353,7 @@ static int protection_domain_init(struct protection_domain *domain, int mode) return -ENOMEM; } - root = amd_iommu_domain_encode_pgtable(pt_root, mode); - atomic64_set(&domain->pt_root, root); + amd_iommu_domain_set_pgtable(domain, pt_root, mode); return 0; } @@ -2713,8 +2722,8 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom) /* First save pgtable configuration*/ amd_iommu_domain_get_pgtable(domain, &pgtable); - /* Update data structure */ - atomic64_set(&domain->pt_root, 0); + /* Remove page-table from domain */ + amd_iommu_domain_clr_pt_root(domain); /* Make changes visible to IOMMUs */ update_domain(domain); From 0f45b04da18305726c18d4b6169c4083f301d91c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:24 +0200 Subject: [PATCH 10/76] iommu/exynos: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Acked-by: Marek Szyprowski Link: https://lore.kernel.org/r/20200625130836.1916-2-joro@8bytes.org --- drivers/iommu/exynos-iommu.c | 20 +++++++++---------- .../media/platform/s5p-mfc/s5p_mfc_iommu.h | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 60c8a56e4a3f..6a9b67302369 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -173,7 +173,7 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_V5_FAULT_AR_VA 0x070 #define REG_V5_FAULT_AW_VA 0x080 -#define has_sysmmu(dev) (dev->archdata.iommu != NULL) +#define has_sysmmu(dev) (dev_iommu_priv_get(dev) != NULL) static struct device *dma_dev; static struct kmem_cache *lv2table_kmem_cache; @@ -226,7 +226,7 @@ static const struct sysmmu_fault_info sysmmu_v5_faults[] = { }; /* - * This structure is attached to dev.archdata.iommu of the master device + * This structure is attached to dev->iommu->priv of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, * which are bound to given master device. It is usually referenced by 'owner' * pointer. @@ -670,7 +670,7 @@ static int __maybe_unused exynos_sysmmu_suspend(struct device *dev) struct device *master = data->master; if (master) { - struct exynos_iommu_owner *owner = master->archdata.iommu; + struct exynos_iommu_owner *owner = dev_iommu_priv_get(master); mutex_lock(&owner->rpm_lock); if (data->domain) { @@ -688,7 +688,7 @@ static int __maybe_unused exynos_sysmmu_resume(struct device *dev) struct device *master = data->master; if (master) { - struct exynos_iommu_owner *owner = master->archdata.iommu; + struct exynos_iommu_owner *owner = dev_iommu_priv_get(master); mutex_lock(&owner->rpm_lock); if (data->domain) { @@ -837,8 +837,8 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, struct device *dev) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); phys_addr_t pagetable = virt_to_phys(domain->pgtable); struct sysmmu_drvdata *data, *next; unsigned long flags; @@ -875,8 +875,8 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); struct sysmmu_drvdata *data; phys_addr_t pagetable = virt_to_phys(domain->pgtable); unsigned long flags; @@ -1237,7 +1237,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, static struct iommu_device *exynos_iommu_probe_device(struct device *dev) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); struct sysmmu_drvdata *data; if (!has_sysmmu(dev)) @@ -1263,7 +1263,7 @@ static struct iommu_device *exynos_iommu_probe_device(struct device *dev) static void exynos_iommu_release_device(struct device *dev) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); struct sysmmu_drvdata *data; if (!has_sysmmu(dev)) @@ -1287,8 +1287,8 @@ static void exynos_iommu_release_device(struct device *dev) static int exynos_iommu_of_xlate(struct device *dev, struct of_phandle_args *spec) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; struct platform_device *sysmmu = of_find_device_by_node(spec->np); + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); struct sysmmu_drvdata *data, *entry; if (!sysmmu) @@ -1305,7 +1305,7 @@ static int exynos_iommu_of_xlate(struct device *dev, INIT_LIST_HEAD(&owner->controllers); mutex_init(&owner->rpm_lock); - dev->archdata.iommu = owner; + dev_iommu_priv_set(dev, owner); } list_for_each_entry(entry, &owner->controllers, owner_node) diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h b/drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h index 152a713fff78..1a32266b7ddc 100644 --- a/drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h +++ b/drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h @@ -9,9 +9,11 @@ #if defined(CONFIG_EXYNOS_IOMMU) +#include + static inline bool exynos_is_iommu_available(struct device *dev) { - return dev->archdata.iommu != NULL; + return dev_iommu_priv_get(dev) != NULL; } #else From 01b9d4e21148c89fdbab3d6b3705f9791314b631 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:25 +0200 Subject: [PATCH 11/76] iommu/vt-d: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20200625130836.1916-3-joro@8bytes.org --- .../gpu/drm/i915/selftests/mock_gem_device.c | 10 ++++++++-- drivers/iommu/intel/iommu.c | 18 +++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 9b105b811f1f..e08601905a64 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -24,6 +24,7 @@ #include #include +#include #include @@ -118,6 +119,9 @@ struct drm_i915_private *mock_gem_device(void) { struct drm_i915_private *i915; struct pci_dev *pdev; +#if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU) + struct dev_iommu iommu; +#endif int err; pdev = kzalloc(sizeof(*pdev), GFP_KERNEL); @@ -136,8 +140,10 @@ struct drm_i915_private *mock_gem_device(void) dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); #if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU) - /* hack to disable iommu for the fake device; force identity mapping */ - pdev->dev.archdata.iommu = (void *)-1; + /* HACK HACK HACK to disable iommu for the fake device; force identity mapping */ + memset(&iommu, 0, sizeof(iommu)); + iommu.priv = (void *)-1; + pdev->dev.iommu = &iommu; #endif pci_set_drvdata(pdev, i915); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index d759e7234e98..2ce490c2eab8 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -372,7 +372,7 @@ struct device_domain_info *get_domain_info(struct device *dev) if (!dev) return NULL; - info = dev->archdata.iommu; + info = dev_iommu_priv_get(dev); if (unlikely(info == DUMMY_DEVICE_DOMAIN_INFO || info == DEFER_DEVICE_DOMAIN_INFO)) return NULL; @@ -743,12 +743,12 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, static int iommu_dummy(struct device *dev) { - return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO; + return dev_iommu_priv_get(dev) == DUMMY_DEVICE_DOMAIN_INFO; } static bool attach_deferred(struct device *dev) { - return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO; + return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO; } /** @@ -2420,7 +2420,7 @@ static inline void unlink_domain_info(struct device_domain_info *info) list_del(&info->link); list_del(&info->global); if (info->dev) - info->dev->archdata.iommu = NULL; + dev_iommu_priv_set(info->dev, NULL); } static void domain_remove_dev_info(struct dmar_domain *domain) @@ -2453,7 +2453,7 @@ static void do_deferred_attach(struct device *dev) { struct iommu_domain *domain; - dev->archdata.iommu = NULL; + dev_iommu_priv_set(dev, NULL); domain = iommu_get_domain_for_dev(dev); if (domain) intel_iommu_attach_device(domain, dev); @@ -2599,7 +2599,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, list_add(&info->link, &domain->devices); list_add(&info->global, &device_domain_list); if (dev) - dev->archdata.iommu = info; + dev_iommu_priv_set(dev, info); spin_unlock_irqrestore(&device_domain_lock, flags); /* PASID table is mandatory for a PCI device in scalable mode. */ @@ -4004,7 +4004,7 @@ static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev) if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) { pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"); add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); - pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + dev_iommu_priv_set(&pdev->dev, DUMMY_DEVICE_DOMAIN_INFO); } } DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu); @@ -4043,7 +4043,7 @@ static void __init init_no_remapping_devices(void) drhd->ignored = 1; for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) - dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + dev_iommu_priv_set(dev, DUMMY_DEVICE_DOMAIN_INFO); } } } @@ -5665,7 +5665,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) return ERR_PTR(-ENODEV); if (translation_pre_enabled(iommu)) - dev->archdata.iommu = DEFER_DEVICE_DOMAIN_INFO; + dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO); return &iommu->iommu; } From 4bbe0c7ccc431dcf06242860a7900f92800724c0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:26 +0200 Subject: [PATCH 12/76] iommu/msm: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-4-joro@8bytes.org --- drivers/iommu/msm_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 3d8a63555c25..f773cc85f311 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -593,14 +593,14 @@ static void insert_iommu_master(struct device *dev, struct msm_iommu_dev **iommu, struct of_phandle_args *spec) { - struct msm_iommu_ctx_dev *master = dev->archdata.iommu; + struct msm_iommu_ctx_dev *master = dev_iommu_priv_get(dev); int sid; if (list_empty(&(*iommu)->ctx_list)) { master = kzalloc(sizeof(*master), GFP_ATOMIC); master->of_node = dev->of_node; list_add(&master->list, &(*iommu)->ctx_list); - dev->archdata.iommu = master; + dev_iommu_priv_set(dev, master); } for (sid = 0; sid < master->num_mids; sid++) From 97ea1202601a7f936c9c9a86f900ec001cb589d3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:27 +0200 Subject: [PATCH 13/76] iommu/omap: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-5-joro@8bytes.org --- drivers/iommu/omap-iommu.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index c8282cc212cb..e84ead6fb234 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -71,7 +71,7 @@ static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom) **/ void omap_iommu_save_ctx(struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); struct omap_iommu *obj; u32 *p; int i; @@ -101,7 +101,7 @@ EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); **/ void omap_iommu_restore_ctx(struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); struct omap_iommu *obj; u32 *p; int i; @@ -1398,7 +1398,7 @@ static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, static int omap_iommu_count(struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); int count = 0; while (arch_data->iommu_dev) { @@ -1459,8 +1459,8 @@ static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain) static int omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); struct omap_iommu_domain *omap_domain = to_omap_domain(domain); - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; struct omap_iommu_device *iommu; struct omap_iommu *oiommu; int ret = 0; @@ -1524,7 +1524,7 @@ out: static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); struct omap_iommu_device *iommu = omap_domain->iommus; struct omap_iommu *oiommu; int i; @@ -1650,7 +1650,7 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev) int num_iommus, i; /* - * Allocate the archdata iommu structure for DT-based devices. + * Allocate the per-device iommu structure for DT-based devices. * * TODO: Simplify this when removing non-DT support completely from the * IOMMU users. @@ -1698,7 +1698,7 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev) of_node_put(np); } - dev->archdata.iommu = arch_data; + dev_iommu_priv_set(dev, arch_data); /* * use the first IOMMU alone for the sysfs device linking. @@ -1712,19 +1712,19 @@ static struct iommu_device *omap_iommu_probe_device(struct device *dev) static void omap_iommu_release_device(struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); if (!dev->of_node || !arch_data) return; - dev->archdata.iommu = NULL; + dev_iommu_priv_set(dev, NULL); kfree(arch_data); } static struct iommu_group *omap_iommu_device_group(struct device *dev) { - struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; + struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev); struct iommu_group *group = ERR_PTR(-EINVAL); if (!arch_data) From 8b9cc3b71bfda8e7c273b8f0fd80a8860ed077ad Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:28 +0200 Subject: [PATCH 14/76] iommu/rockchip: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-6-joro@8bytes.org --- drivers/iommu/rockchip-iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index d25c2486ca07..e5d86b7177de 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -836,7 +836,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, static struct rk_iommu *rk_iommu_from_dev(struct device *dev) { - struct rk_iommudata *data = dev->archdata.iommu; + struct rk_iommudata *data = dev_iommu_priv_get(dev); return data ? data->iommu : NULL; } @@ -1059,7 +1059,7 @@ static struct iommu_device *rk_iommu_probe_device(struct device *dev) struct rk_iommudata *data; struct rk_iommu *iommu; - data = dev->archdata.iommu; + data = dev_iommu_priv_get(dev); if (!data) return ERR_PTR(-ENODEV); @@ -1073,7 +1073,7 @@ static struct iommu_device *rk_iommu_probe_device(struct device *dev) static void rk_iommu_release_device(struct device *dev) { - struct rk_iommudata *data = dev->archdata.iommu; + struct rk_iommudata *data = dev_iommu_priv_get(dev); device_link_del(data->link); } @@ -1100,7 +1100,7 @@ static int rk_iommu_of_xlate(struct device *dev, iommu_dev = of_find_device_by_node(args->np); data->iommu = platform_get_drvdata(iommu_dev); - dev->archdata.iommu = data; + dev_iommu_priv_set(dev, data); platform_device_put(iommu_dev); From a5616e24609a2bb3dc6ea1ab360207fa1198c976 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:29 +0200 Subject: [PATCH 15/76] iommu/tegra: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-7-joro@8bytes.org --- drivers/iommu/tegra-gart.c | 8 ++++---- drivers/iommu/tegra-smmu.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index 5fbdff6ff41a..fac720273889 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -113,8 +113,8 @@ static int gart_iommu_attach_dev(struct iommu_domain *domain, if (gart->active_domain && gart->active_domain != domain) { ret = -EBUSY; - } else if (dev->archdata.iommu != domain) { - dev->archdata.iommu = domain; + } else if (dev_iommu_priv_get(dev) != domain) { + dev_iommu_priv_set(dev, domain); gart->active_domain = domain; gart->active_devices++; } @@ -131,8 +131,8 @@ static void gart_iommu_detach_dev(struct iommu_domain *domain, spin_lock(&gart->dom_lock); - if (dev->archdata.iommu == domain) { - dev->archdata.iommu = NULL; + if (dev_iommu_priv_get(dev) == domain) { + dev_iommu_priv_set(dev, NULL); if (--gart->active_devices == 0) gart->active_domain = NULL; diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 7426b7666e2b..124c8848ab7e 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -465,7 +465,7 @@ static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, static int tegra_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct tegra_smmu *smmu = dev->archdata.iommu; + struct tegra_smmu *smmu = dev_iommu_priv_get(dev); struct tegra_smmu_as *as = to_smmu_as(domain); struct device_node *np = dev->of_node; struct of_phandle_args args; @@ -780,7 +780,7 @@ static struct iommu_device *tegra_smmu_probe_device(struct device *dev) * supported by the Linux kernel, so abort after the * first match. */ - dev->archdata.iommu = smmu; + dev_iommu_priv_set(dev, smmu); break; } @@ -797,7 +797,7 @@ static struct iommu_device *tegra_smmu_probe_device(struct device *dev) static void tegra_smmu_release_device(struct device *dev) { - dev->archdata.iommu = NULL; + dev_iommu_priv_set(dev, NULL); } static const struct tegra_smmu_group_soc * @@ -856,7 +856,7 @@ static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, static struct iommu_group *tegra_smmu_device_group(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct tegra_smmu *smmu = dev->archdata.iommu; + struct tegra_smmu *smmu = dev_iommu_priv_get(dev); struct iommu_group *group; group = tegra_smmu_group_get(smmu, fwspec->ids[0]); From 2263d818bceff516254bfce4091632530720505a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:30 +0200 Subject: [PATCH 16/76] iommu/pamu: Use dev_iommu_priv_get/set() Remove the use of dev->archdata.iommu_domain and use the private per-device pointer provided by IOMMU core code instead. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-8-joro@8bytes.org --- drivers/iommu/fsl_pamu_domain.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 928d37771ece..b2110767caf4 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -323,7 +323,7 @@ static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) pamu_disable_liodn(info->liodn); spin_unlock_irqrestore(&iommu_lock, flags); spin_lock_irqsave(&device_domain_lock, flags); - info->dev->archdata.iommu_domain = NULL; + dev_iommu_priv_set(info->dev, NULL); kmem_cache_free(iommu_devinfo_cache, info); spin_unlock_irqrestore(&device_domain_lock, flags); } @@ -352,7 +352,7 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d * Check here if the device is already attached to domain or not. * If the device is already attached to a domain detach it. */ - old_domain_info = dev->archdata.iommu_domain; + old_domain_info = dev_iommu_priv_get(dev); if (old_domain_info && old_domain_info->domain != dma_domain) { spin_unlock_irqrestore(&device_domain_lock, flags); detach_device(dev, old_domain_info->domain); @@ -371,8 +371,8 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d * the info for the first LIODN as all * LIODNs share the same domain */ - if (!dev->archdata.iommu_domain) - dev->archdata.iommu_domain = info; + if (!dev_iommu_priv_get(dev)) + dev_iommu_priv_set(dev, info); spin_unlock_irqrestore(&device_domain_lock, flags); } From 589601720d9d0366670eb415a4a62692d7af3713 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:31 +0200 Subject: [PATCH 17/76] iommu/mediatek: Do no use dev->archdata.iommu The iommu private pointer is already used in the Mediatek IOMMU v1 driver, so move the dma_iommu_mapping pointer into 'struct mtk_iommu_data' and do not use dev->archdata.iommu anymore. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-9-joro@8bytes.org --- drivers/iommu/mtk_iommu.h | 2 ++ drivers/iommu/mtk_iommu_v1.c | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index ea949a324e33..1682406e98dc 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -62,6 +62,8 @@ struct mtk_iommu_data { struct iommu_device iommu; const struct mtk_iommu_plat_data *plat_data; + struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct list_head list; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; }; diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index c9d79cff4d17..82ddfe9170d4 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -269,7 +269,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, int ret; /* Only allow the domain created internally. */ - mtk_mapping = data->dev->archdata.iommu; + mtk_mapping = data->mapping; if (mtk_mapping->domain != domain) return 0; @@ -369,7 +369,6 @@ static int mtk_iommu_create_mapping(struct device *dev, struct mtk_iommu_data *data; struct platform_device *m4updev; struct dma_iommu_mapping *mtk_mapping; - struct device *m4udev; int ret; if (args->args_count != 1) { @@ -401,8 +400,7 @@ static int mtk_iommu_create_mapping(struct device *dev, return ret; data = dev_iommu_priv_get(dev); - m4udev = data->dev; - mtk_mapping = m4udev->archdata.iommu; + mtk_mapping = data->mapping; if (!mtk_mapping) { /* MTK iommu support 4GB iova address space. */ mtk_mapping = arm_iommu_create_mapping(&platform_bus_type, @@ -410,7 +408,7 @@ static int mtk_iommu_create_mapping(struct device *dev, if (IS_ERR(mtk_mapping)) return PTR_ERR(mtk_mapping); - m4udev->archdata.iommu = mtk_mapping; + data->mapping = mtk_mapping; } return 0; @@ -459,7 +457,7 @@ static void mtk_iommu_probe_finalize(struct device *dev) int err; data = dev_iommu_priv_get(dev); - mtk_mapping = data->dev->archdata.iommu; + mtk_mapping = data->mapping; err = arm_iommu_attach_device(dev, mtk_mapping); if (err) From ad962d864c61a73757085847027a532cbe2c1353 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:32 +0200 Subject: [PATCH 18/76] x86: Remove dev->archdata.iommu pointer There are no users left, all drivers have been converted to use the per-device private pointer offered by IOMMU core. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Acked-by: Borislav Petkov Link: https://lore.kernel.org/r/20200625130836.1916-10-joro@8bytes.org --- arch/x86/include/asm/device.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h index 49bd6cf3eec9..7c0a52ca2f4d 100644 --- a/arch/x86/include/asm/device.h +++ b/arch/x86/include/asm/device.h @@ -3,9 +3,6 @@ #define _ASM_X86_DEVICE_H struct dev_archdata { -#ifdef CONFIG_IOMMU_API - void *iommu; /* hook for IOMMU specific extension */ -#endif }; struct pdev_archdata { From 0b660afe310855c6eef35f8082594f32e3b05511 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:33 +0200 Subject: [PATCH 19/76] ia64: Remove dev->archdata.iommu pointer There are no users left, all drivers have been converted to use the per-device private pointer offered by IOMMU core. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-11-joro@8bytes.org --- arch/ia64/include/asm/device.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/ia64/include/asm/device.h b/arch/ia64/include/asm/device.h index 3eb397415381..918b198cd5bb 100644 --- a/arch/ia64/include/asm/device.h +++ b/arch/ia64/include/asm/device.h @@ -6,9 +6,6 @@ #define _ASM_IA64_DEVICE_H struct dev_archdata { -#ifdef CONFIG_IOMMU_API - void *iommu; /* hook for IOMMU specific extension */ -#endif }; struct pdev_archdata { From fb0fd5f70c1bfc9cdf5406e82d5de0ecdbe1e80c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:34 +0200 Subject: [PATCH 20/76] arm: Remove dev->archdata.iommu pointer There are no users left, all drivers have been converted to use the per-device private pointer offered by IOMMU core. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-12-joro@8bytes.org --- arch/arm/include/asm/device.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index c675bc0d5aa8..be666f58bf7a 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -9,9 +9,6 @@ struct dev_archdata { #ifdef CONFIG_DMABOUNCE struct dmabounce_device_info *dmabounce; #endif -#ifdef CONFIG_IOMMU_API - void *iommu; /* private IOMMU data */ -#endif #ifdef CONFIG_ARM_DMA_USE_IOMMU struct dma_iommu_mapping *mapping; #endif From 5866a75b50b381fa05028e7ff3f2272ed006e321 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:35 +0200 Subject: [PATCH 21/76] arm64: Remove dev->archdata.iommu pointer There are no users left, all drivers have been converted to use the per-device private pointer offered by IOMMU core. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Acked-by: Will Deacon Link: https://lore.kernel.org/r/20200625130836.1916-13-joro@8bytes.org --- arch/arm64/include/asm/device.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/include/asm/device.h b/arch/arm64/include/asm/device.h index 12b778d55342..996498751318 100644 --- a/arch/arm64/include/asm/device.h +++ b/arch/arm64/include/asm/device.h @@ -6,9 +6,6 @@ #define __ASM_DEVICE_H struct dev_archdata { -#ifdef CONFIG_IOMMU_API - void *iommu; /* private IOMMU data */ -#endif }; struct pdev_archdata { From 6255c8c8d256b38b550b35c6362750f64ce46b8d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 25 Jun 2020 15:08:36 +0200 Subject: [PATCH 22/76] powerpc/dma: Remove dev->archdata.iommu_domain There are no users left, so remove the pointer and save some memory. Signed-off-by: Joerg Roedel Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20200625130836.1916-14-joro@8bytes.org --- arch/powerpc/include/asm/device.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index 266542769e4b..1bc595213338 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -34,9 +34,6 @@ struct dev_archdata { struct iommu_table *iommu_table_base; #endif -#ifdef CONFIG_IOMMU_API - void *iommu_domain; -#endif #ifdef CONFIG_PPC64 struct pci_dn *pci_data; #endif From ca37faf3d7005b5588f045edfac1d82799c408a7 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 30 Jun 2020 10:17:56 +0200 Subject: [PATCH 23/76] iommu: Move sg_table wrapper out of CONFIG_IOMMU_SUPPORT Move the recently added sg_table wrapper out of CONFIG_IOMMU_SUPPORT to let the client code copile also when IOMMU support is disabled. Fixes: 48530d9fab0d ("iommu: add generic helper for mapping sgtable objects") Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20200630081756.18526-1-m.szyprowski@samsung.com Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 5f0b7859d2eb..5657d4fef9f2 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -457,22 +457,6 @@ extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t io extern void iommu_set_fault_handler(struct iommu_domain *domain, iommu_fault_handler_t handler, void *token); -/** - * iommu_map_sgtable - Map the given buffer to the IOMMU domain - * @domain: The IOMMU domain to perform the mapping - * @iova: The start address to map the buffer - * @sgt: The sg_table object describing the buffer - * @prot: IOMMU protection bits - * - * Creates a mapping at @iova for the buffer described by a scatterlist - * stored in the given sg_table object in the provided IOMMU domain. - */ -static inline size_t iommu_map_sgtable(struct iommu_domain *domain, - unsigned long iova, struct sg_table *sgt, int prot) -{ - return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot); -} - extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); extern void generic_iommu_put_resv_regions(struct device *dev, @@ -1079,6 +1063,22 @@ static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) } #endif /* CONFIG_IOMMU_API */ +/** + * iommu_map_sgtable - Map the given buffer to the IOMMU domain + * @domain: The IOMMU domain to perform the mapping + * @iova: The start address to map the buffer + * @sgt: The sg_table object describing the buffer + * @prot: IOMMU protection bits + * + * Creates a mapping at @iova for the buffer described by a scatterlist + * stored in the given sg_table object in the provided IOMMU domain. + */ +static inline size_t iommu_map_sgtable(struct iommu_domain *domain, + unsigned long iova, struct sg_table *sgt, int prot) +{ + return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot); +} + #ifdef CONFIG_IOMMU_DEBUGFS extern struct dentry *iommu_debugfs_dir; void iommu_debugfs_setup(void); From 557d4bec009330567ce99692918abbe9c77528e0 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Fri, 5 Jun 2020 00:00:25 -0700 Subject: [PATCH 24/76] iommu: Add include/uapi/linux/iommu.h to MAINTAINERS file When include/uapi/linux/iommu.h was created it was never added to the file list in MAINTAINERS. Signed-off-by: Jerry Snitselaar Cc: Joerg Roedel Link: https://lore.kernel.org/r/20200605070025.216124-1-jsnitsel@redhat.com Signed-off-by: Joerg Roedel --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 496fd4eafb68..c23352059a6b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9014,6 +9014,7 @@ F: drivers/iommu/ F: include/linux/iommu.h F: include/linux/iova.h F: include/linux/of_iommu.h +F: include/uapi/linux/iommu.h IO_URING M: Jens Axboe From ecd7274fb4cd2d6c035a52d59ca3d6ec936a07be Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 4 Jun 2020 18:11:15 +0100 Subject: [PATCH 25/76] iommu: Remove unused IOMMU_SYS_CACHE_ONLY flag The IOMMU_SYS_CACHE_ONLY flag was never exposed via the DMA API and has no in-tree users. Remove it. Cc: Robin Murphy Cc: "Isaac J. Manjarres" Cc: Joerg Roedel Cc: Rob Clark Reviewed-by: Christoph Hellwig Reviewed-by: Sai Prakash Ranjan Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm.c | 3 --- include/linux/iommu.h | 6 ------ 2 files changed, 9 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 04fbd4bf0ff9..8f175c02f8e3 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -438,9 +438,6 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); - else if (prot & IOMMU_SYS_CACHE_ONLY) - pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE - << ARM_LPAE_PTE_ATTRINDX_SHIFT); } if (prot & IOMMU_CACHE) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 5f0b7859d2eb..bee1a8fa1fb1 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -31,12 +31,6 @@ * if the IOMMU page table format is equivalent. */ #define IOMMU_PRIV (1 << 5) -/* - * Non-coherent masters can use this page protection flag to set cacheable - * memory attributes for only a transparent outer level of cache, also known as - * the last-level or system cache. - */ -#define IOMMU_SYS_CACHE_ONLY (1 << 6) struct iommu_ops; struct iommu_group; From 976492922a6a9c06f6f61436848adf2f03702e5c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 24 Jun 2020 11:24:51 +0100 Subject: [PATCH 26/76] iommu/arm-smmu: Update impl quirks comment The comment about implementation and integration quirks being mutually-exclusive is out of date, and in fact the code is already structured for the case it anticipates, so document that properly. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/1e742177e084621f3454fbaf768325a6c215656a.1592994291.git.robin.murphy@arm.com Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-impl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index c75b9d957b70..1fb4bb994dbf 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -153,10 +153,9 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) const struct device_node *np = smmu->dev->of_node; /* - * We will inevitably have to combine model-specific implementation - * quirks with platform-specific integration quirks, but everything - * we currently support happens to work out as straightforward - * mutually-exclusive assignments. + * Set the impl for model-specific implementation quirks first, + * such that platform integration quirks can pick it up and + * inherit from it if necessary. */ switch (smmu->model) { case ARM_MMU500: @@ -168,6 +167,7 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) break; } + /* This is implicitly MMU-400 */ if (of_property_read_bool(np, "calxeda,smmu-secure-config-access")) smmu->impl = &calxeda_impl; From 7b6b70d88b10c61af7050820ca041ff9e73482ae Mon Sep 17 00:00:00 2001 From: Jonathan Marek Date: Tue, 9 Jun 2020 15:40:19 -0400 Subject: [PATCH 27/76] dt-bindings: arm-smmu: Add sm8150 and sm8250 compatible strings Add compatible strings for sm8150 and sm8250 iommus to documentation. Signed-off-by: Jonathan Marek Acked-by: Rob Herring Link: https://lore.kernel.org/r/20200609194030.17756-2-jonathan@marek.ca Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index d7ceb4c34423..3ac741236c8b 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -37,6 +37,8 @@ properties: - enum: - qcom,sc7180-smmu-500 - qcom,sdm845-smmu-500 + - qcom,sm8150-smmu-500 + - qcom,sm8250-smmu-500 - const: arm,mmu-500 - items: - const: arm,mmu-500 From 2c5c3cfb2da577bb23c3d9d771cc08300dec3559 Mon Sep 17 00:00:00 2001 From: Jonathan Marek Date: Tue, 9 Jun 2020 15:40:20 -0400 Subject: [PATCH 28/76] iommu: arm-smmu-impl: Use qcom impl for sm8150 and sm8250 compatibles Use the qcom implementation for IOMMU hardware on sm8150 and sm8250 SoCs. Signed-off-by: Jonathan Marek Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20200609194030.17756-3-jonathan@marek.ca Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-impl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index 1fb4bb994dbf..22a9acd76955 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -172,7 +172,9 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) smmu->impl = &calxeda_impl; if (of_device_is_compatible(np, "qcom,sdm845-smmu-500") || - of_device_is_compatible(np, "qcom,sc7180-smmu-500")) + of_device_is_compatible(np, "qcom,sc7180-smmu-500") || + of_device_is_compatible(np, "qcom,sm8150-smmu-500") || + of_device_is_compatible(np, "qcom,sm8250-smmu-500")) return qcom_smmu_impl_init(smmu); return smmu; From b4ceb4a5359ed1c9ba4a20acf3a70d4bbead3248 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 3 Jul 2020 17:03:19 +0100 Subject: [PATCH 29/76] iommu: Tidy up Kconfig for SoC IOMMUs Wacky COMPILE_TEST dependencies based on who used to define dev_archdata.iommu can go. Dependencies on ARM or ARM64 already implied by the ARCH_* platform selection can go. The entire IOMMU_SUPPORT menu already depends on MMU, so those can go. IOMMU_DMA is for the architecture's DMA API implementation to choose, and its interface to IOMMU drivers is properly stubbed out if disabled, so dependencies on or selections of that can go (AMD_IOMMU is the current exception since the x86 drivers have to provide their own entire dma_map_ops implementation). Since commit ed6ccf10f24b ("dma-mapping: properly stub out the DMA API for !CONFIG_HAS_DMA"), drivers which simply use the dma-mapping API should not need to depend on HAS_DMA, so those can go. And a long-dead option for code removed from the MSM driver 4 years ago can also go. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/7fb9c74dc6bd12a4619ca44c92408e91352f1be0.1593791968.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6dc49ed8377a..4dcd85bbff99 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -128,10 +128,6 @@ config MSM_IOMMU If unsure, say N here. -config IOMMU_PGTABLES_L2 - def_bool y - depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n - # AMD IOMMU support config AMD_IOMMU bool "AMD IOMMU support" @@ -274,7 +270,6 @@ config IRQ_REMAP # OMAP IOMMU support config OMAP_IOMMU bool "OMAP IOMMU Support" - depends on ARM && MMU || (COMPILE_TEST && (ARM || ARM64 || IA64 || SPARC)) depends on ARCH_OMAP2PLUS || COMPILE_TEST select IOMMU_API help @@ -292,7 +287,6 @@ config OMAP_IOMMU_DEBUG config ROCKCHIP_IOMMU bool "Rockchip IOMMU Support" - depends on ARM || ARM64 || (COMPILE_TEST && (ARM64 || IA64 || SPARC)) depends on ARCH_ROCKCHIP || COMPILE_TEST select IOMMU_API select ARM_DMA_USE_IOMMU @@ -308,7 +302,6 @@ config SUN50I_IOMMU depends on ARCH_SUNXI || COMPILE_TEST select ARM_DMA_USE_IOMMU select IOMMU_API - select IOMMU_DMA help Support for the IOMMU introduced in the Allwinner H6 SoCs. @@ -335,7 +328,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && MMU || (COMPILE_TEST && (ARM || ARM64 || IA64 || SPARC)) + depends on ARCH_EXYNOS || COMPILE_TEST depends on !CPU_BIG_ENDIAN # revisit driver if we can enable big-endian ptes select IOMMU_API select ARM_DMA_USE_IOMMU @@ -358,7 +351,7 @@ config EXYNOS_IOMMU_DEBUG config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" - depends on ARM || IOMMU_DMA + depends on ARM || ARM64 depends on ARCH_RENESAS || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API select IOMMU_IO_PGTABLE_LPAE @@ -380,7 +373,7 @@ config SPAPR_TCE_IOMMU # ARM IOMMU support config ARM_SMMU tristate "ARM Ltd. System MMU (SMMU) Support" - depends on (ARM64 || ARM || (COMPILE_TEST && !GENERIC_ATOMIC64)) && MMU + depends on ARM64 || ARM || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU if ARM @@ -466,11 +459,9 @@ config S390_AP_IOMMU config MTK_IOMMU bool "MTK IOMMU Support" - depends on HAS_DMA depends on ARCH_MEDIATEK || COMPILE_TEST select ARM_DMA_USE_IOMMU select IOMMU_API - select IOMMU_DMA select IOMMU_IO_PGTABLE_ARMV7S select MEMORY select MTK_SMI From 97215a7df4351fdd9141418568be872fb1032d6e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 3 Jul 2020 17:03:20 +0100 Subject: [PATCH 30/76] iommu/renesas: Expand COMPILE_TEST coverage This driver shouldn't need anything architecture-specific (that isn't under CONFIG_ARM protection already), and has already been accessible from certain x86 configurations by virtue of the previously-cleaned-up "ARM || IOMMU_DMA" dependency. Allow COMPILE_TEST for all architectures. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/1fe2006aa98f008a2e689adba6e8c96e9197f903.1593791968.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 4dcd85bbff99..af6c081d164f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -351,7 +351,6 @@ config EXYNOS_IOMMU_DEBUG config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" - depends on ARM || ARM64 depends on ARCH_RENESAS || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API select IOMMU_IO_PGTABLE_LPAE From 349b00c9c0a052af6720552919c9812dad9013e7 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:18 +0800 Subject: [PATCH 31/76] dt-bindings: mediatek: Add bindings for MT6779 This patch adds description for MT6779 IOMMU. MT6779 has two iommus, they are mm_iommu and apu_iommu which both use ARM Short-Descriptor translation format. In addition, mm_iommu and apu_iommu are two independent HW instance , we need to set them separately. The MT6779 IOMMU hardware diagram is as below, it is only a brief diagram about iommu, it don't focus on the part of smi_larb, so I don't describe the smi_larb detailedly. EMI | -------------------------------------- | | MM_IOMMU APU_IOMMU | | SMI_COMMOM----------- APU_BUS | | | SMI_LARB(0~11) | | | | | | | -------------- | | | | | Multimedia engine CCU VPU MDLA EMDA All the connections are hardware fixed, software can not adjust it. Signed-off-by: Chao Hao Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20200703044127.27438-2-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- .../bindings/iommu/mediatek,iommu.txt | 2 + include/dt-bindings/memory/mt6779-larb-port.h | 206 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 include/dt-bindings/memory/mt6779-larb-port.h diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt index ce59a505f5a4..c1ccd8582eb2 100644 --- a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt +++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt @@ -58,6 +58,7 @@ Required properties: - compatible : must be one of the following string: "mediatek,mt2701-m4u" for mt2701 which uses generation one m4u HW. "mediatek,mt2712-m4u" for mt2712 which uses generation two m4u HW. + "mediatek,mt6779-m4u" for mt6779 which uses generation two m4u HW. "mediatek,mt7623-m4u", "mediatek,mt2701-m4u" for mt7623 which uses generation one m4u HW. "mediatek,mt8173-m4u" for mt8173 which uses generation two m4u HW. @@ -78,6 +79,7 @@ Required properties: Specifies the mtk_m4u_id as defined in dt-binding/memory/mt2701-larb-port.h for mt2701, mt7623 dt-binding/memory/mt2712-larb-port.h for mt2712, + dt-binding/memory/mt6779-larb-port.h for mt6779, dt-binding/memory/mt8173-larb-port.h for mt8173, and dt-binding/memory/mt8183-larb-port.h for mt8183. diff --git a/include/dt-bindings/memory/mt6779-larb-port.h b/include/dt-bindings/memory/mt6779-larb-port.h new file mode 100644 index 000000000000..2ad0899fbf2f --- /dev/null +++ b/include/dt-bindings/memory/mt6779-larb-port.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 MediaTek Inc. + * Author: Chao Hao + */ + +#ifndef _DTS_IOMMU_PORT_MT6779_H_ +#define _DTS_IOMMU_PORT_MT6779_H_ + +#define MTK_M4U_ID(larb, port) (((larb) << 5) | (port)) + +#define M4U_LARB0_ID 0 +#define M4U_LARB1_ID 1 +#define M4U_LARB2_ID 2 +#define M4U_LARB3_ID 3 +#define M4U_LARB4_ID 4 +#define M4U_LARB5_ID 5 +#define M4U_LARB6_ID 6 +#define M4U_LARB7_ID 7 +#define M4U_LARB8_ID 8 +#define M4U_LARB9_ID 9 +#define M4U_LARB10_ID 10 +#define M4U_LARB11_ID 11 + +/* larb0 */ +#define M4U_PORT_DISP_POSTMASK0 MTK_M4U_ID(M4U_LARB0_ID, 0) +#define M4U_PORT_DISP_OVL0_HDR MTK_M4U_ID(M4U_LARB0_ID, 1) +#define M4U_PORT_DISP_OVL1_HDR MTK_M4U_ID(M4U_LARB0_ID, 2) +#define M4U_PORT_DISP_OVL0 MTK_M4U_ID(M4U_LARB0_ID, 3) +#define M4U_PORT_DISP_OVL1 MTK_M4U_ID(M4U_LARB0_ID, 4) +#define M4U_PORT_DISP_PVRIC0 MTK_M4U_ID(M4U_LARB0_ID, 5) +#define M4U_PORT_DISP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 6) +#define M4U_PORT_DISP_WDMA0 MTK_M4U_ID(M4U_LARB0_ID, 7) +#define M4U_PORT_DISP_FAKE0 MTK_M4U_ID(M4U_LARB0_ID, 8) + +/* larb1 */ +#define M4U_PORT_DISP_OVL0_2L_HDR MTK_M4U_ID(M4U_LARB1_ID, 0) +#define M4U_PORT_DISP_OVL1_2L_HDR MTK_M4U_ID(M4U_LARB1_ID, 1) +#define M4U_PORT_DISP_OVL0_2L MTK_M4U_ID(M4U_LARB1_ID, 2) +#define M4U_PORT_DISP_OVL1_2L MTK_M4U_ID(M4U_LARB1_ID, 3) +#define M4U_PORT_DISP_RDMA1 MTK_M4U_ID(M4U_LARB1_ID, 4) +#define M4U_PORT_MDP_PVRIC0 MTK_M4U_ID(M4U_LARB1_ID, 5) +#define M4U_PORT_MDP_PVRIC1 MTK_M4U_ID(M4U_LARB1_ID, 6) +#define M4U_PORT_MDP_RDMA0 MTK_M4U_ID(M4U_LARB1_ID, 7) +#define M4U_PORT_MDP_RDMA1 MTK_M4U_ID(M4U_LARB1_ID, 8) +#define M4U_PORT_MDP_WROT0_R MTK_M4U_ID(M4U_LARB1_ID, 9) +#define M4U_PORT_MDP_WROT0_W MTK_M4U_ID(M4U_LARB1_ID, 10) +#define M4U_PORT_MDP_WROT1_R MTK_M4U_ID(M4U_LARB1_ID, 11) +#define M4U_PORT_MDP_WROT1_W MTK_M4U_ID(M4U_LARB1_ID, 12) +#define M4U_PORT_DISP_FAKE1 MTK_M4U_ID(M4U_LARB1_ID, 13) + +/* larb2-VDEC */ +#define M4U_PORT_HW_VDEC_MC_EXT MTK_M4U_ID(M4U_LARB2_ID, 0) +#define M4U_PORT_HW_VDEC_UFO_EXT MTK_M4U_ID(M4U_LARB2_ID, 1) +#define M4U_PORT_HW_VDEC_PP_EXT MTK_M4U_ID(M4U_LARB2_ID, 2) +#define M4U_PORT_HW_VDEC_PRED_RD_EXT MTK_M4U_ID(M4U_LARB2_ID, 3) +#define M4U_PORT_HW_VDEC_PRED_WR_EXT MTK_M4U_ID(M4U_LARB2_ID, 4) +#define M4U_PORT_HW_VDEC_PPWRAP_EXT MTK_M4U_ID(M4U_LARB2_ID, 5) +#define M4U_PORT_HW_VDEC_TILE_EXT MTK_M4U_ID(M4U_LARB2_ID, 6) +#define M4U_PORT_HW_VDEC_VLD_EXT MTK_M4U_ID(M4U_LARB2_ID, 7) +#define M4U_PORT_HW_VDEC_VLD2_EXT MTK_M4U_ID(M4U_LARB2_ID, 8) +#define M4U_PORT_HW_VDEC_AVC_MV_EXT MTK_M4U_ID(M4U_LARB2_ID, 9) +#define M4U_PORT_HW_VDEC_UFO_ENC_EXT MTK_M4U_ID(M4U_LARB2_ID, 10) +#define M4U_PORT_HW_VDEC_RG_CTRL_DMA_EXT MTK_M4U_ID(M4U_LARB2_ID, 11) + +/* larb3-VENC */ +#define M4U_PORT_VENC_RCPU MTK_M4U_ID(M4U_LARB3_ID, 0) +#define M4U_PORT_VENC_REC MTK_M4U_ID(M4U_LARB3_ID, 1) +#define M4U_PORT_VENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 2) +#define M4U_PORT_VENC_SV_COMV MTK_M4U_ID(M4U_LARB3_ID, 3) +#define M4U_PORT_VENC_RD_COMV MTK_M4U_ID(M4U_LARB3_ID, 4) +#define M4U_PORT_VENC_NBM_RDMA MTK_M4U_ID(M4U_LARB3_ID, 5) +#define M4U_PORT_VENC_NBM_RDMA_LITE MTK_M4U_ID(M4U_LARB3_ID, 6) +#define M4U_PORT_JPGENC_Y_RDMA MTK_M4U_ID(M4U_LARB3_ID, 7) +#define M4U_PORT_JPGENC_C_RDMA MTK_M4U_ID(M4U_LARB3_ID, 8) +#define M4U_PORT_JPGENC_Q_TABLE MTK_M4U_ID(M4U_LARB3_ID, 9) +#define M4U_PORT_JPGENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 10) +#define M4U_PORT_JPGDEC_WDMA MTK_M4U_ID(M4U_LARB3_ID, 11) +#define M4U_PORT_JPGDEC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 12) +#define M4U_PORT_VENC_NBM_WDMA MTK_M4U_ID(M4U_LARB3_ID, 13) +#define M4U_PORT_VENC_NBM_WDMA_LITE MTK_M4U_ID(M4U_LARB3_ID, 14) +#define M4U_PORT_VENC_CUR_LUMA MTK_M4U_ID(M4U_LARB3_ID, 15) +#define M4U_PORT_VENC_CUR_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 16) +#define M4U_PORT_VENC_REF_LUMA MTK_M4U_ID(M4U_LARB3_ID, 17) +#define M4U_PORT_VENC_REF_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 18) + +/* larb4-dummy */ + +/* larb5-IMG */ +#define M4U_PORT_IMGI_D1 MTK_M4U_ID(M4U_LARB5_ID, 0) +#define M4U_PORT_IMGBI_D1 MTK_M4U_ID(M4U_LARB5_ID, 1) +#define M4U_PORT_DMGI_D1 MTK_M4U_ID(M4U_LARB5_ID, 2) +#define M4U_PORT_DEPI_D1 MTK_M4U_ID(M4U_LARB5_ID, 3) +#define M4U_PORT_LCEI_D1 MTK_M4U_ID(M4U_LARB5_ID, 4) +#define M4U_PORT_SMTI_D1 MTK_M4U_ID(M4U_LARB5_ID, 5) +#define M4U_PORT_SMTO_D2 MTK_M4U_ID(M4U_LARB5_ID, 6) +#define M4U_PORT_SMTO_D1 MTK_M4U_ID(M4U_LARB5_ID, 7) +#define M4U_PORT_CRZO_D1 MTK_M4U_ID(M4U_LARB5_ID, 8) +#define M4U_PORT_IMG3O_D1 MTK_M4U_ID(M4U_LARB5_ID, 9) +#define M4U_PORT_VIPI_D1 MTK_M4U_ID(M4U_LARB5_ID, 10) +#define M4U_PORT_WPE_RDMA1 MTK_M4U_ID(M4U_LARB5_ID, 11) +#define M4U_PORT_WPE_RDMA0 MTK_M4U_ID(M4U_LARB5_ID, 12) +#define M4U_PORT_WPE_WDMA MTK_M4U_ID(M4U_LARB5_ID, 13) +#define M4U_PORT_TIMGO_D1 MTK_M4U_ID(M4U_LARB5_ID, 14) +#define M4U_PORT_MFB_RDMA0 MTK_M4U_ID(M4U_LARB5_ID, 15) +#define M4U_PORT_MFB_RDMA1 MTK_M4U_ID(M4U_LARB5_ID, 16) +#define M4U_PORT_MFB_RDMA2 MTK_M4U_ID(M4U_LARB5_ID, 17) +#define M4U_PORT_MFB_RDMA3 MTK_M4U_ID(M4U_LARB5_ID, 18) +#define M4U_PORT_MFB_WDMA MTK_M4U_ID(M4U_LARB5_ID, 19) +#define M4U_PORT_RESERVE1 MTK_M4U_ID(M4U_LARB5_ID, 20) +#define M4U_PORT_RESERVE2 MTK_M4U_ID(M4U_LARB5_ID, 21) +#define M4U_PORT_RESERVE3 MTK_M4U_ID(M4U_LARB5_ID, 22) +#define M4U_PORT_RESERVE4 MTK_M4U_ID(M4U_LARB5_ID, 23) +#define M4U_PORT_RESERVE5 MTK_M4U_ID(M4U_LARB5_ID, 24) +#define M4U_PORT_RESERVE6 MTK_M4U_ID(M4U_LARB5_ID, 25) + +/* larb6-IMG-VPU */ +#define M4U_PORT_IMG_IPUO MTK_M4U_ID(M4U_LARB6_ID, 0) +#define M4U_PORT_IMG_IPU3O MTK_M4U_ID(M4U_LARB6_ID, 1) +#define M4U_PORT_IMG_IPUI MTK_M4U_ID(M4U_LARB6_ID, 2) + +/* larb7-DVS */ +#define M4U_PORT_DVS_RDMA MTK_M4U_ID(M4U_LARB7_ID, 0) +#define M4U_PORT_DVS_WDMA MTK_M4U_ID(M4U_LARB7_ID, 1) +#define M4U_PORT_DVP_RDMA MTK_M4U_ID(M4U_LARB7_ID, 2) +#define M4U_PORT_DVP_WDMA MTK_M4U_ID(M4U_LARB7_ID, 3) + +/* larb8-IPESYS */ +#define M4U_PORT_FDVT_RDA MTK_M4U_ID(M4U_LARB8_ID, 0) +#define M4U_PORT_FDVT_RDB MTK_M4U_ID(M4U_LARB8_ID, 1) +#define M4U_PORT_FDVT_WRA MTK_M4U_ID(M4U_LARB8_ID, 2) +#define M4U_PORT_FDVT_WRB MTK_M4U_ID(M4U_LARB8_ID, 3) +#define M4U_PORT_FE_RD0 MTK_M4U_ID(M4U_LARB8_ID, 4) +#define M4U_PORT_FE_RD1 MTK_M4U_ID(M4U_LARB8_ID, 5) +#define M4U_PORT_FE_WR0 MTK_M4U_ID(M4U_LARB8_ID, 6) +#define M4U_PORT_FE_WR1 MTK_M4U_ID(M4U_LARB8_ID, 7) +#define M4U_PORT_RSC_RDMA0 MTK_M4U_ID(M4U_LARB8_ID, 8) +#define M4U_PORT_RSC_WDMA MTK_M4U_ID(M4U_LARB8_ID, 9) + +/* larb9-CAM */ +#define M4U_PORT_CAM_IMGO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 0) +#define M4U_PORT_CAM_RRZO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 1) +#define M4U_PORT_CAM_LSCI_R1_C MTK_M4U_ID(M4U_LARB9_ID, 2) +#define M4U_PORT_CAM_BPCI_R1_C MTK_M4U_ID(M4U_LARB9_ID, 3) +#define M4U_PORT_CAM_YUVO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 4) +#define M4U_PORT_CAM_UFDI_R2_C MTK_M4U_ID(M4U_LARB9_ID, 5) +#define M4U_PORT_CAM_RAWI_R2_C MTK_M4U_ID(M4U_LARB9_ID, 6) +#define M4U_PORT_CAM_RAWI_R5_C MTK_M4U_ID(M4U_LARB9_ID, 7) +#define M4U_PORT_CAM_CAMSV_1 MTK_M4U_ID(M4U_LARB9_ID, 8) +#define M4U_PORT_CAM_CAMSV_2 MTK_M4U_ID(M4U_LARB9_ID, 9) +#define M4U_PORT_CAM_CAMSV_3 MTK_M4U_ID(M4U_LARB9_ID, 10) +#define M4U_PORT_CAM_CAMSV_4 MTK_M4U_ID(M4U_LARB9_ID, 11) +#define M4U_PORT_CAM_CAMSV_5 MTK_M4U_ID(M4U_LARB9_ID, 12) +#define M4U_PORT_CAM_CAMSV_6 MTK_M4U_ID(M4U_LARB9_ID, 13) +#define M4U_PORT_CAM_AAO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 14) +#define M4U_PORT_CAM_AFO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 15) +#define M4U_PORT_CAM_FLKO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 16) +#define M4U_PORT_CAM_LCESO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 17) +#define M4U_PORT_CAM_CRZO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 18) +#define M4U_PORT_CAM_LTMSO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 19) +#define M4U_PORT_CAM_RSSO_R1_C MTK_M4U_ID(M4U_LARB9_ID, 20) +#define M4U_PORT_CAM_CCUI MTK_M4U_ID(M4U_LARB9_ID, 21) +#define M4U_PORT_CAM_CCUO MTK_M4U_ID(M4U_LARB9_ID, 22) +#define M4U_PORT_CAM_FAKE MTK_M4U_ID(M4U_LARB9_ID, 23) + +/* larb10-CAM_A */ +#define M4U_PORT_CAM_IMGO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 0) +#define M4U_PORT_CAM_RRZO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 1) +#define M4U_PORT_CAM_LSCI_R1_A MTK_M4U_ID(M4U_LARB10_ID, 2) +#define M4U_PORT_CAM_BPCI_R1_A MTK_M4U_ID(M4U_LARB10_ID, 3) +#define M4U_PORT_CAM_YUVO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 4) +#define M4U_PORT_CAM_UFDI_R2_A MTK_M4U_ID(M4U_LARB10_ID, 5) +#define M4U_PORT_CAM_RAWI_R2_A MTK_M4U_ID(M4U_LARB10_ID, 6) +#define M4U_PORT_CAM_RAWI_R5_A MTK_M4U_ID(M4U_LARB10_ID, 7) +#define M4U_PORT_CAM_IMGO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 8) +#define M4U_PORT_CAM_RRZO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 9) +#define M4U_PORT_CAM_LSCI_R1_B MTK_M4U_ID(M4U_LARB10_ID, 10) +#define M4U_PORT_CAM_BPCI_R1_B MTK_M4U_ID(M4U_LARB10_ID, 11) +#define M4U_PORT_CAM_YUVO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 12) +#define M4U_PORT_CAM_UFDI_R2_B MTK_M4U_ID(M4U_LARB10_ID, 13) +#define M4U_PORT_CAM_RAWI_R2_B MTK_M4U_ID(M4U_LARB10_ID, 14) +#define M4U_PORT_CAM_RAWI_R5_B MTK_M4U_ID(M4U_LARB10_ID, 15) +#define M4U_PORT_CAM_CAMSV_0 MTK_M4U_ID(M4U_LARB10_ID, 16) +#define M4U_PORT_CAM_AAO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 17) +#define M4U_PORT_CAM_AFO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 18) +#define M4U_PORT_CAM_FLKO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 19) +#define M4U_PORT_CAM_LCESO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 20) +#define M4U_PORT_CAM_CRZO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 21) +#define M4U_PORT_CAM_AAO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 22) +#define M4U_PORT_CAM_AFO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 23) +#define M4U_PORT_CAM_FLKO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 24) +#define M4U_PORT_CAM_LCESO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 25) +#define M4U_PORT_CAM_CRZO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 26) +#define M4U_PORT_CAM_LTMSO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 27) +#define M4U_PORT_CAM_RSSO_R1_A MTK_M4U_ID(M4U_LARB10_ID, 28) +#define M4U_PORT_CAM_LTMSO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 29) +#define M4U_PORT_CAM_RSSO_R1_B MTK_M4U_ID(M4U_LARB10_ID, 30) + +/* larb11-CAM-VPU */ +#define M4U_PORT_CAM_IPUO MTK_M4U_ID(M4U_LARB11_ID, 0) +#define M4U_PORT_CAM_IPU2O MTK_M4U_ID(M4U_LARB11_ID, 1) +#define M4U_PORT_CAM_IPU3O MTK_M4U_ID(M4U_LARB11_ID, 2) +#define M4U_PORT_CAM_IPUI MTK_M4U_ID(M4U_LARB11_ID, 3) +#define M4U_PORT_CAM_IPU2I MTK_M4U_ID(M4U_LARB11_ID, 4) + +#endif From 75eed350877c4cdae2bd9e8410c66292143bd410 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:19 +0800 Subject: [PATCH 32/76] iommu/mediatek: Rename the register STANDARD_AXI_MODE(0x48) to MISC_CTRL For iommu offset=0x48 register, only the previous mt8173/mt8183 use the name STANDARD_AXI_MODE, all the latest SoC extend the register more feature by different bits, for example: axi_mode, in_order_en, coherent_en and so on. So rename REG_MMU_MISC_CTRL may be more proper. This patch only rename the register name, no functional change. Signed-off-by: Chao Hao Reviewed-by: Yong Wu Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20200703044127.27438-3-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 14 +++++++------- drivers/iommu/mtk_iommu.h | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 2be96f1cdbd2..88d3df5b91c2 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -41,7 +41,7 @@ #define F_INVLD_EN0 BIT(0) #define F_INVLD_EN1 BIT(1) -#define REG_MMU_STANDARD_AXI_MODE 0x048 +#define REG_MMU_MISC_CTRL 0x048 #define REG_MMU_DCM_DIS 0x050 #define REG_MMU_CTRL_REG 0x110 @@ -573,8 +573,10 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) } writel_relaxed(0, data->base + REG_MMU_DCM_DIS); - if (data->plat_data->reset_axi) - writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE); + if (data->plat_data->reset_axi) { + /* The register is called STANDARD_AXI_MODE in this case */ + writel_relaxed(0, data->base + REG_MMU_MISC_CTRL); + } if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, dev_name(data->dev), (void *)data)) { @@ -718,8 +720,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev) struct mtk_iommu_suspend_reg *reg = &data->reg; void __iomem *base = data->base; - reg->standard_axi_mode = readl_relaxed(base + - REG_MMU_STANDARD_AXI_MODE); + reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); @@ -743,8 +744,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); return ret; } - writel_relaxed(reg->standard_axi_mode, - base + REG_MMU_STANDARD_AXI_MODE); + writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index ea949a324e33..7212e6fcf982 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -18,7 +18,10 @@ #include struct mtk_iommu_suspend_reg { - u32 standard_axi_mode; + union { + u32 standard_axi_mode;/* v1 */ + u32 misc_ctrl;/* v2 */ + }; u32 dcm_dis; u32 ctrl_reg; u32 int_control0; From 6b717796227ec8d4303adcdc574165d06e499f0f Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:20 +0800 Subject: [PATCH 33/76] iommu/mediatek: Use a u32 flags to describe different HW features Given the fact that we are adding more and more plat_data bool values, it would make sense to use a u32 flags register and add the appropriate macro definitions to set and check for a flag present. No functional change. Suggested-by: Matthias Brugger Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Yong Wu Link: https://lore.kernel.org/r/20200703044127.27438-4-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 28 +++++++++++++++++----------- drivers/iommu/mtk_iommu.h | 7 +------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 88d3df5b91c2..40ca564d97af 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -100,6 +100,15 @@ #define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf) #define MTK_M4U_TO_PORT(id) ((id) & 0x1f) +#define HAS_4GB_MODE BIT(0) +/* HW will use the EMI clock if there isn't the "bclk". */ +#define HAS_BCLK BIT(1) +#define HAS_VLD_PA_RNG BIT(2) +#define RESET_AXI BIT(3) + +#define MTK_IOMMU_HAS_FLAG(pdata, _x) \ + ((((pdata)->flags) & (_x)) == (_x)) + struct mtk_iommu_domain { struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; @@ -563,7 +572,8 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) upper_32_bits(data->protect_base); writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); - if (data->enable_4GB && data->plat_data->has_vld_pa_rng) { + if (data->enable_4GB && + MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { /* * If 4GB mode is enabled, the validate PA range is from * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. @@ -573,7 +583,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) } writel_relaxed(0, data->base + REG_MMU_DCM_DIS); - if (data->plat_data->reset_axi) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { /* The register is called STANDARD_AXI_MODE in this case */ writel_relaxed(0, data->base + REG_MMU_MISC_CTRL); } @@ -618,7 +628,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) /* Whether the current dram is over 4GB */ data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT)); - if (!data->plat_data->has_4gb_mode) + if (!MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) data->enable_4GB = false; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -631,7 +641,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) if (data->irq < 0) return data->irq; - if (data->plat_data->has_bclk) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { data->bclk = devm_clk_get(dev, "bclk"); if (IS_ERR(data->bclk)) return PTR_ERR(data->bclk); @@ -763,23 +773,19 @@ static const struct dev_pm_ops mtk_iommu_pm_ops = { static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, - .has_4gb_mode = true, - .has_bclk = true, - .has_vld_pa_rng = true, + .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, }; static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, - .has_4gb_mode = true, - .has_bclk = true, - .reset_axi = true, + .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI, .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */ }; static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, - .reset_axi = true, + .flags = RESET_AXI, .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1}, }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 7212e6fcf982..5225a9170aaa 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -39,12 +39,7 @@ enum mtk_iommu_plat { struct mtk_iommu_plat_data { enum mtk_iommu_plat m4u_plat; - bool has_4gb_mode; - - /* HW will use the EMI clock if there isn't the "bclk". */ - bool has_bclk; - bool has_vld_pa_rng; - bool reset_axi; + u32 flags; unsigned char larbid_remap[MTK_LARB_NR_MAX]; }; From 4bb2bf4c6ad36d5aef9fc7ecd01e89ae4f8d7ec7 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:21 +0800 Subject: [PATCH 34/76] iommu/mediatek: Setting MISC_CTRL register Add F_MMU_IN_ORDER_WR_EN_MASK and F_MMU_STANDARD_AXI_MODE_EN_MASK definitions in MISC_CTRL register. F_MMU_STANDARD_AXI_MODE_EN_MASK: If we set F_MMU_STANDARD_AXI_MODE_EN_MASK (bit[3][19] = 0, not follow standard AXI protocol), the iommu will priorize sending of urgent read command over a normal read command. This improves the performance. F_MMU_IN_ORDER_WR_EN_MASK: If we set F_MMU_IN_ORDER_WR_EN_MASK (bit[1][17] = 0, out-of-order write), the iommu will re-order write commands and send the write commands with higher priority. Otherwise the sending of write commands will be done in order. The feature is controlled by OUT_ORDER_WR_EN platform data flag. Suggested-by: Yong Wu Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Matthias Brugger Link: https://lore.kernel.org/r/20200703044127.27438-5-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 40ca564d97af..219d7aa6f059 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -42,6 +42,9 @@ #define F_INVLD_EN1 BIT(1) #define REG_MMU_MISC_CTRL 0x048 +#define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) +#define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) + #define REG_MMU_DCM_DIS 0x050 #define REG_MMU_CTRL_REG 0x110 @@ -105,6 +108,7 @@ #define HAS_BCLK BIT(1) #define HAS_VLD_PA_RNG BIT(2) #define RESET_AXI BIT(3) +#define OUT_ORDER_WR_EN BIT(4) #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -585,8 +589,14 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { /* The register is called STANDARD_AXI_MODE in this case */ - writel_relaxed(0, data->base + REG_MMU_MISC_CTRL); + regval = 0; + } else { + regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); + regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; + if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) + regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; } + writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, dev_name(data->dev), (void *)data)) { From b053bc7183c86dc52e68e9891d269992e79a5251 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:22 +0800 Subject: [PATCH 35/76] iommu/mediatek: Move inv_sel_reg into the plat_data For mt6779, MMU_INV_SEL register's offset is changed from 0x38 to 0x2c, so we can put inv_sel_reg in the plat_data to use it. In addition, we renamed it to REG_MMU_INV_SEL_GEN1 and use it before mt6779. Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Yong Wu Link: https://lore.kernel.org/r/20200703044127.27438-6-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 9 ++++++--- drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 219d7aa6f059..533b8f76f592 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -37,7 +37,7 @@ #define REG_MMU_INVLD_START_A 0x024 #define REG_MMU_INVLD_END_A 0x028 -#define REG_MMU_INV_SEL 0x038 +#define REG_MMU_INV_SEL_GEN1 0x038 #define F_INVLD_EN0 BIT(0) #define F_INVLD_EN1 BIT(1) @@ -178,7 +178,7 @@ static void mtk_iommu_tlb_flush_all(void *cookie) for_each_m4u(data) { writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, - data->base + REG_MMU_INV_SEL); + data->base + data->plat_data->inv_sel_reg); writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); wmb(); /* Make sure the tlb flush all done */ } @@ -195,7 +195,7 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, for_each_m4u(data) { spin_lock_irqsave(&data->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, - data->base + REG_MMU_INV_SEL); + data->base + data->plat_data->inv_sel_reg); writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); writel_relaxed(iova + size - 1, @@ -784,18 +784,21 @@ static const struct dev_pm_ops mtk_iommu_pm_ops = { static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, + .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, }; static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI, + .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */ }; static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, .flags = RESET_AXI, + .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1}, }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 5225a9170aaa..cf53f5e80d22 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -40,6 +40,7 @@ enum mtk_iommu_plat { struct mtk_iommu_plat_data { enum mtk_iommu_plat m4u_plat; u32 flags; + u32 inv_sel_reg; unsigned char larbid_remap[MTK_LARB_NR_MAX]; }; From 37276e00da7d3a0d643d325aef81b7283fe23010 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:23 +0800 Subject: [PATCH 36/76] iommu/mediatek: Add sub_comm id in translation fault The max larb number that a iommu HW support is 8(larb0~larb7 in the below diagram). If the larb's number is over 8, we use a sub_common for merging several larbs into one larb. At this case, we will extend larb_id: bit[11:9] means common-id; bit[8:7] means subcommon-id; >From these two variables, we could get the real larb number when translation fault happen. The diagram is as below: EMI | IOMMU | ----------------- | | common1 common0 | | ----------------- | smi common | ------------------------------------ | | | | | | 3'd0 3'd1 3'd2 3'd3 ... 3'd7 <-common_id(max is 8) | | | | | | Larb0 Larb1 | Larb3 ... Larb7 | smi sub common | -------------------------- | | | | 2'd0 2'd1 2'd2 2'd3 <-sub_common_id(max is 4) | | | | Larb8 Larb9 Larb10 Larb11 In this patch we extend larb_remap[] to larb_remap[8][4] for this. larb_remap[x][y]: x means common-id above, y means subcommon_id above. We can also distinguish if the M4U HW has sub_common by HAS_SUB_COMM macro. Signed-off-by: Chao Hao Reviewed-by: Yong Wu Reviewed-by: Matthias Brugger Cc: Matthias Brugger Link: https://lore.kernel.org/r/20200703044127.27438-7-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 21 ++++++++++++++------- drivers/iommu/mtk_iommu.h | 5 ++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 533b8f76f592..0d96dcd8612b 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -91,6 +91,8 @@ #define REG_MMU1_INVLD_PA 0x148 #define REG_MMU0_INT_ID 0x150 #define REG_MMU1_INT_ID 0x154 +#define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) +#define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) @@ -109,6 +111,7 @@ #define HAS_VLD_PA_RNG BIT(2) #define RESET_AXI BIT(3) #define OUT_ORDER_WR_EN BIT(4) +#define HAS_SUB_COMM BIT(5) #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -239,7 +242,7 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) struct mtk_iommu_data *data = dev_id; struct mtk_iommu_domain *dom = data->m4u_dom; u32 int_state, regval, fault_iova, fault_pa; - unsigned int fault_larb, fault_port; + unsigned int fault_larb, fault_port, sub_comm = 0; bool layer, write; /* Read error info from registers */ @@ -255,10 +258,14 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) } layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; - fault_larb = F_MMU_INT_ID_LARB_ID(regval); fault_port = F_MMU_INT_ID_PORT_ID(regval); - - fault_larb = data->plat_data->larbid_remap[fault_larb]; + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { + fault_larb = F_MMU_INT_ID_COMM_ID(regval); + sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); + } else { + fault_larb = F_MMU_INT_ID_LARB_ID(regval); + } + fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; if (report_iommu_fault(&dom->domain, data->dev, fault_iova, write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { @@ -785,21 +792,21 @@ static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, - .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, }; static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, - .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */ + .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ }; static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, .flags = RESET_AXI, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, - .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1}, + .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; static const struct of_device_id mtk_iommu_of_ids[] = { diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index cf53f5e80d22..46d0d47b22e1 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -17,6 +17,9 @@ #include #include +#define MTK_LARB_COM_MAX 8 +#define MTK_LARB_SUBCOM_MAX 4 + struct mtk_iommu_suspend_reg { union { u32 standard_axi_mode;/* v1 */ @@ -41,7 +44,7 @@ struct mtk_iommu_plat_data { enum mtk_iommu_plat m4u_plat; u32 flags; u32 inv_sel_reg; - unsigned char larbid_remap[MTK_LARB_NR_MAX]; + unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; }; struct mtk_iommu_domain; From 35c1b48d27dac486835435e703749236b14dcd8f Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:24 +0800 Subject: [PATCH 37/76] iommu/mediatek: Add REG_MMU_WR_LEN_CTRL register definition Some platforms(ex: mt6779) need to improve performance by setting REG_MMU_WR_LEN_CTRL register. And we can use WR_THROT_EN macro to control whether we need to set the register. If the register uses default value, iommu will send command to EMI without restriction, when the number of commands become more and more, it will drop the EMI performance. So when more than ten_commands(default value) don't be handled for EMI, iommu will stop send command to EMI for keeping EMI's performace by enabling write throttling mechanism(bit[5][21]=0) in MMU_WR_LEN_CTRL register. Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Matthias Brugger Link: https://lore.kernel.org/r/20200703044127.27438-8-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 11 +++++++++++ drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 12 insertions(+) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0d96dcd8612b..5c8e141668fc 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -46,6 +46,8 @@ #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) #define REG_MMU_DCM_DIS 0x050 +#define REG_MMU_WR_LEN_CTRL 0x054 +#define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) #define REG_MMU_CTRL_REG 0x110 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) @@ -112,6 +114,7 @@ #define RESET_AXI BIT(3) #define OUT_ORDER_WR_EN BIT(4) #define HAS_SUB_COMM BIT(5) +#define WR_THROT_EN BIT(6) #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -593,6 +596,12 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); } writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { + /* write command throttling mode */ + regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); + regval &= ~F_MMU_WR_THROT_DIS_MASK; + writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); + } if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { /* The register is called STANDARD_AXI_MODE in this case */ @@ -747,6 +756,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev) struct mtk_iommu_suspend_reg *reg = &data->reg; void __iomem *base = data->base; + reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); @@ -771,6 +781,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); return ret; } + writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 46d0d47b22e1..31edd05e2eb1 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -31,6 +31,7 @@ struct mtk_iommu_suspend_reg { u32 int_main_control; u32 ivrp_paddr; u32 vld_pa_rng; + u32 wr_len_ctrl; }; enum mtk_iommu_plat { From 829316b3bc897a3275a0354505fde3ccd0053936 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:25 +0800 Subject: [PATCH 38/76] iommu/mediatek: Extend protect pa alignment value Starting with mt6779, iommu needs to extend to 256 bytes from 128 bytes which can send the max number of data for memory protection pa alignment. So we can use a separate patch to modify it. Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20200703044127.27438-9-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 5c8e141668fc..e71003037ffa 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -98,7 +98,7 @@ #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) -#define MTK_PROTECT_PA_ALIGN 128 +#define MTK_PROTECT_PA_ALIGN 256 /* * Get the local arbiter ID and the portid within the larb arbiter From 864444130eed304835b09c86a5bf2ff05bc2f4a2 Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:26 +0800 Subject: [PATCH 39/76] iommu/mediatek: Modify MMU_CTRL register setting The MMU_CTRL register of MT8173 is different from other SoCs. The in_order_wr_en is bit[9] which is zero by default. Other SoCs have the vitcim_tlb_en feature mapped to bit[12]. This bit is set to one by default. We need to preserve the bit when setting F_MMU_TF_PROT_TO_PROGRAM_ADDR as otherwise the bit will be cleared and IOMMU performance will drop. Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Matthias Brugger Cc: Yong Wu Link: https://lore.kernel.org/r/20200703044127.27438-10-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index e71003037ffa..a816030d00f1 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -555,11 +555,13 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return ret; } - if (data->plat_data->m4u_plat == M4U_MT8173) + if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; - else - regval = F_MMU_TF_PROT_TO_PROGRAM_ADDR; + } else { + regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); + regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; + } writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); regval = F_L2_MULIT_HIT_EN | From 068c86e92f3fc9df7d95a39710b40be811d861aa Mon Sep 17 00:00:00 2001 From: Chao Hao Date: Fri, 3 Jul 2020 12:41:27 +0800 Subject: [PATCH 40/76] iommu/mediatek: Add mt6779 basic support 1. Start from mt6779, INVLDT_SEL move to offset=0x2c, so we add REG_MMU_INV_SEL_GEN2 definition and mt6779 uses it. 2. Add mt6779_data to support mm_iommu HW init. Signed-off-by: Chao Hao Reviewed-by: Matthias Brugger Cc: Yong Wu Link: https://lore.kernel.org/r/20200703044127.27438-11-chao.hao@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 9 +++++++++ drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 10 insertions(+) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index a816030d00f1..59e5a62a34db 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -37,6 +37,7 @@ #define REG_MMU_INVLD_START_A 0x024 #define REG_MMU_INVLD_END_A 0x028 +#define REG_MMU_INV_SEL_GEN2 0x02c #define REG_MMU_INV_SEL_GEN1 0x038 #define F_INVLD_EN0 BIT(0) #define F_INVLD_EN1 BIT(1) @@ -808,6 +809,13 @@ static const struct mtk_iommu_plat_data mt2712_data = { .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, }; +static const struct mtk_iommu_plat_data mt6779_data = { + .m4u_plat = M4U_MT6779, + .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, +}; + static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI, @@ -824,6 +832,7 @@ static const struct mtk_iommu_plat_data mt8183_data = { static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, + { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, {} diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 31edd05e2eb1..214898578026 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -37,6 +37,7 @@ struct mtk_iommu_suspend_reg { enum mtk_iommu_plat { M4U_MT2701, M4U_MT2712, + M4U_MT6779, M4U_MT8173, M4U_MT8183, }; From f512eefc9bf94e4f943c95833518bdbc80c93f0e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Wed, 8 Jul 2020 23:04:34 +0200 Subject: [PATCH 41/76] iommu/omap: Replace HTTP links with HTTPS ones Rationale: Reduces attack surface on kernel devs opening the links for MITM as HTTPS traffic is much harder to manipulate. Deterministic algorithm: For each file: If not .svg: For each line: If doesn't contain `\bxmlns\b`: For each link, `\bhttp://[^# \t\r\n]*(?:\w|/)`: If neither `\bgnu\.org/license`, nor `\bmozilla\.org/MPL\b`: If both the HTTP and HTTPS versions return 200 OK and serve the same content: Replace HTTP with HTTPS. Signed-off-by: Alexander A. Klimov Link: https://lore.kernel.org/r/20200708210434.22518-1-grandmaster@al2klimov.de Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index c8282cc212cb..ec1c8cd0e419 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -3,7 +3,7 @@ * omap iommu: tlb and pagetable primitives * * Copyright (C) 2008-2010 Nokia Corporation - * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright (C) 2013-2017 Texas Instruments Incorporated - https://www.ti.com/ * * Written by Hiroshi DOYU , * Paul Mundt and Toshihiro Kobayashi From 397e18b4bb2d27389361ceaf154a7acced648a72 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 13 Jul 2020 12:16:48 +0200 Subject: [PATCH 42/76] iommu/mediatek: Include liunx/dma-mapping.h This fixes a compile error when cross-compiling the driver on x86-32. Signed-off-by: Joerg Roedel Link: https://lore.kernel.org/r/20200713101648.32056-1-joro@8bytes.org --- drivers/iommu/mtk_iommu.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 214898578026..6253e98d810c 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #define MTK_LARB_COM_MAX 8 From d724794b4534105e4169f8c9641235541bc7b74f Mon Sep 17 00:00:00 2001 From: Ming-Fan Chen Date: Wed, 8 Jan 2020 14:41:29 +0800 Subject: [PATCH 43/76] dt-bindings: mediatek: Add binding for MT6779 SMI This patch add description for MT6779 SMI. There are GALS in smi-larb but without clock of GALS alone. changelog since v2: Add GALS for mt6779 in smi-common.txt Signed-off-by: Ming-Fan Chen Acked-by: Rob Herring Link: https://lore.kernel.org/r/1578465691-30692-3-git-send-email-ming-fan.chen@mediatek.com Signed-off-by: Joerg Roedel --- .../bindings/memory-controllers/mediatek,smi-common.txt | 5 +++-- .../bindings/memory-controllers/mediatek,smi-larb.txt | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt index b478ade4da65..b64573680b42 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt @@ -5,7 +5,7 @@ The hardware block diagram please check bindings/iommu/mediatek,iommu.txt Mediatek SMI have two generations of HW architecture, here is the list which generation the SoCs use: generation 1: mt2701 and mt7623. -generation 2: mt2712, mt8173 and mt8183. +generation 2: mt2712, mt6779, mt8173 and mt8183. There's slight differences between the two SMI, for generation 2, the register which control the iommu port is at each larb's register base. But @@ -18,6 +18,7 @@ Required properties: - compatible : must be one of : "mediatek,mt2701-smi-common" "mediatek,mt2712-smi-common" + "mediatek,mt6779-smi-common" "mediatek,mt7623-smi-common", "mediatek,mt2701-smi-common" "mediatek,mt8173-smi-common" "mediatek,mt8183-smi-common" @@ -35,7 +36,7 @@ Required properties: and these 2 option clocks for generation 2 smi HW: - "gals0": the path0 clock of GALS(Global Async Local Sync). - "gals1": the path1 clock of GALS(Global Async Local Sync). - Here is the list which has this GALS: mt8183. + Here is the list which has this GALS: mt6779 and mt8183. Example: smi_common: smi@14022000 { diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt index 4b369b3e1a69..8f19dfe7d80e 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt @@ -6,6 +6,7 @@ Required properties: - compatible : must be one of : "mediatek,mt2701-smi-larb" "mediatek,mt2712-smi-larb" + "mediatek,mt6779-smi-larb" "mediatek,mt7623-smi-larb", "mediatek,mt2701-smi-larb" "mediatek,mt8173-smi-larb" "mediatek,mt8183-smi-larb" @@ -21,7 +22,7 @@ Required properties: - "gals": the clock for GALS(Global Async Local Sync). Here is the list which has this GALS: mt8183. -Required property for mt2701, mt2712 and mt7623: +Required property for mt2701, mt2712, mt6779 and mt7623: - mediatek,larb-id :the hardware id of this larb. Example: From fc492f339f766591d6399d380edb89b2ab8bc89c Mon Sep 17 00:00:00 2001 From: Ming-Fan Chen Date: Wed, 8 Jan 2020 14:41:30 +0800 Subject: [PATCH 44/76] memory: mtk-smi: Add basic support for MT6779 Add smi-larb and smi-common nodes and config_port for MT6779. changelog since v2: Split basic nodes and config_port support from initial golden setting patch Signed-off-by: Ming-Fan Chen Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/1578465691-30692-4-git-send-email-ming-fan.chen@mediatek.com Signed-off-by: Joerg Roedel --- drivers/memory/mtk-smi.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index a113e811faab..109c7e51d551 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -239,6 +239,13 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt2712 = { .larb_direct_to_common_mask = BIT(8) | BIT(9), /* bdpsys */ }; +static const struct mtk_smi_larb_gen mtk_smi_larb_mt6779 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .larb_direct_to_common_mask = + BIT(4) | BIT(6) | BIT(11) | BIT(12) | BIT(13), + /* DUMMY | IPU0 | IPU1 | CCU | MDLA */ +}; + static const struct mtk_smi_larb_gen mtk_smi_larb_mt8183 = { .has_gals = true, .config_port = mtk_smi_larb_config_port_gen2_general, @@ -259,6 +266,10 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = { .compatible = "mediatek,mt2712-smi-larb", .data = &mtk_smi_larb_mt2712 }, + { + .compatible = "mediatek,mt6779-smi-larb", + .data = &mtk_smi_larb_mt6779 + }, { .compatible = "mediatek,mt8183-smi-larb", .data = &mtk_smi_larb_mt8183 @@ -388,6 +399,13 @@ static const struct mtk_smi_common_plat mtk_smi_common_gen2 = { .gen = MTK_SMI_GEN2, }; +static const struct mtk_smi_common_plat mtk_smi_common_mt6779 = { + .gen = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(4) | + F_MMU1_LARB(5) | F_MMU1_LARB(6) | F_MMU1_LARB(7), +}; + static const struct mtk_smi_common_plat mtk_smi_common_mt8183 = { .gen = MTK_SMI_GEN2, .has_gals = true, @@ -408,6 +426,10 @@ static const struct of_device_id mtk_smi_common_of_ids[] = { .compatible = "mediatek,mt2712-smi-common", .data = &mtk_smi_common_gen2, }, + { + .compatible = "mediatek,mt6779-smi-common", + .data = &mtk_smi_common_mt6779, + }, { .compatible = "mediatek,mt8183-smi-common", .data = &mtk_smi_common_mt8183, From 6a79a5a3842b6a9f639fe2874dd6ae0bd4b24d1a Mon Sep 17 00:00:00 2001 From: Tomasz Nowicki Date: Wed, 15 Jul 2020 09:06:46 +0200 Subject: [PATCH 45/76] iommu/arm-smmu: Call configuration impl hook before consuming features 'cfg_probe' hook is called at the very end of configuration probing procedure and therefore features override and workaround may become complex like for ID register fixups. In preparation for adding Marvell errata move 'cfg_probe' a bit earlier to have chance to adjust the detected features before we start consuming them. Since the Cavium quirk (the only user) does not alter features it is safe to do so. Suggested-by: Robin Murphy Signed-off-by: Tomasz Nowicki Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200715070649.18733-2-tn@semihalf.com Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 243bc4cb2705..19f906de6420 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1728,7 +1728,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) unsigned int size; u32 id; bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; - int i; + int i, ret; dev_notice(smmu->dev, "probing hardware configuration...\n"); dev_notice(smmu->dev, "SMMUv%d with:\n", @@ -1891,6 +1891,12 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } + if (smmu->impl && smmu->impl->cfg_probe) { + ret = smmu->impl->cfg_probe(smmu); + if (ret) + return ret; + } + /* Now we've corralled the various formats, what'll it do? */ if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; @@ -1918,9 +1924,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", smmu->ipa_size, smmu->pa_size); - if (smmu->impl && smmu->impl->cfg_probe) - return smmu->impl->cfg_probe(smmu); - return 0; } From f2d9848aeb9fa71523bbfb226203ffb7d50877d2 Mon Sep 17 00:00:00 2001 From: Hanna Hawa Date: Wed, 15 Jul 2020 09:06:47 +0200 Subject: [PATCH 46/76] iommu/arm-smmu: Workaround for Marvell Armada-AP806 SoC erratum #582743 Due to erratum #582743, the Marvell Armada-AP806 can't access 64bit to ARM SMMUv2 registers. Provide implementation relevant hooks: - split the writeq/readq to two accesses of writel/readl. - mask the MMU_IDR2.PTFSv8 fields to not use AArch64 format (but only AARCH32_L) since with AArch64 format 32 bits access is not supported. Note that most 64-bit registers like TTBRn can be accessed as two 32-bit halves without issue, and AArch32 format ensures that the register writes which must be atomic (for TLBI etc.) need only be 32-bit. Signed-off-by: Hanna Hawa Signed-off-by: Gregory CLEMENT Signed-off-by: Tomasz Nowicki Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200715070649.18733-3-tn@semihalf.com Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.rst | 3 ++ drivers/iommu/arm-smmu-impl.c | 45 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index 936cf2a59ca4..157214d3abe1 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -125,6 +125,9 @@ stable kernels. | Cavium | ThunderX2 Core | #219 | CAVIUM_TX2_ERRATUM_219 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ +| Marvell | ARM-MMU-500 | #582743 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ ++----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index 22a9acd76955..c87d825f651e 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -147,6 +147,48 @@ static const struct arm_smmu_impl arm_mmu500_impl = { .reset = arm_mmu500_reset, }; +static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off) +{ + /* + * Marvell Armada-AP806 erratum #582743. + * Split all the readq to double readl + */ + return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off); +} + +static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off, + u64 val) +{ + /* + * Marvell Armada-AP806 erratum #582743. + * Split all the writeq to double writel + */ + hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off); +} + +static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu) +{ + + /* + * Armada-AP806 erratum #582743. + * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64 + * formats altogether and allow using 32 bits access on the + * interconnect. + */ + smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K | + ARM_SMMU_FEAT_FMT_AARCH64_16K | + ARM_SMMU_FEAT_FMT_AARCH64_64K); + + return 0; +} + +static const struct arm_smmu_impl mrvl_mmu500_impl = { + .read_reg64 = mrvl_mmu500_readq, + .write_reg64 = mrvl_mmu500_writeq, + .cfg_probe = mrvl_mmu500_cfg_probe, + .reset = arm_mmu500_reset, +}; + struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) { @@ -177,5 +219,8 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) of_device_is_compatible(np, "qcom,sm8250-smmu-500")) return qcom_smmu_impl_init(smmu); + if (of_device_is_compatible(np, "marvell,ap806-smmu-500")) + smmu->impl = &mrvl_mmu500_impl; + return smmu; } From e85e84d19b9dac3d3ac92c9f6a24d034691f1243 Mon Sep 17 00:00:00 2001 From: Tomasz Nowicki Date: Wed, 15 Jul 2020 09:06:48 +0200 Subject: [PATCH 47/76] dt-bindings: arm-smmu: add compatible string for Marvell Armada-AP806 SMMU-500 Add specific compatible string for Marvell usage due to errata of accessing 64bits registers of ARM SMMU, in AP806. AP806 SoC uses the generic ARM-MMU500, and there's no specific implementation of Marvell, this compatible is used for errata only. Signed-off-by: Hanna Hawa Signed-off-by: Gregory CLEMENT Signed-off-by: Tomasz Nowicki Reviewed-by: Rob Herring Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200715070649.18733-4-tn@semihalf.com Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 3ac741236c8b..93fb9fe068b9 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -40,6 +40,10 @@ properties: - qcom,sm8150-smmu-500 - qcom,sm8250-smmu-500 - const: arm,mmu-500 + - description: Marvell SoCs implementing "arm,mmu-500" + items: + - const: marvell,ap806-smmu-500 + - const: arm,mmu-500 - items: - const: arm,mmu-500 - const: arm,smmu-v2 From 49fbb25030265c660de732513f18275d88ff99d3 Mon Sep 17 00:00:00 2001 From: John Garry Date: Tue, 23 Jun 2020 01:28:37 +0800 Subject: [PATCH 48/76] iommu/arm-smmu-v3: Fix trivial typo Set "cmq" -> "cmdq". Signed-off-by: John Garry Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index f578677a5c41..a8e814c652fe 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1479,7 +1479,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, } /* - * Try to unlock the cmq lock. This will fail if we're the last + * Try to unlock the cmdq lock. This will fail if we're the last * reader, in which case we can safely update cmdq->q.llq.cons */ if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) { From cd8479cf0de9d1950b51032a216fbf630f8542a4 Mon Sep 17 00:00:00 2001 From: Krishna Reddy Date: Sat, 18 Jul 2020 12:34:53 -0700 Subject: [PATCH 49/76] iommu/arm-smmu: move TLB timeout and spin count macros Move TLB timeout and spin count macros to header file to allow using the same from vendor specific implementations. Signed-off-by: Krishna Reddy Reviewed-by: Jon Hunter Reviewed-by: Nicolin Chen Reviewed-by: Pritesh Raithatha Reviewed-by: Robin Murphy Reviewed-by: Thierry Reding Link: https://lore.kernel.org/r/20200718193457.30046-2-vdumpa@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 3 --- drivers/iommu/arm-smmu.h | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 19f906de6420..cdd15ead9bc4 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -52,9 +52,6 @@ */ #define QCOM_DUMMY_VAL -1 -#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ -#define TLB_SPIN_COUNT 10 - #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index d172c024be61..c7d0122a7c6c 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -236,6 +236,8 @@ enum arm_smmu_cbar_type { /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 +#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ +#define TLB_SPIN_COUNT 10 /* Shared driver definitions */ enum arm_smmu_arch_version { From 6c019f4e697ebed2225b20cc5d6276673834853d Mon Sep 17 00:00:00 2001 From: Krishna Reddy Date: Sat, 18 Jul 2020 12:34:54 -0700 Subject: [PATCH 50/76] iommu/arm-smmu: ioremap smmu mmio region before implementation init ioremap smmu mmio region before calling into implementation init. This is necessary to allow mapped address available during vendor specific implementation init. Signed-off-by: Krishna Reddy Reviewed-by: Jon Hunter Reviewed-by: Nicolin Chen Reviewed-by: Pritesh Raithatha Reviewed-by: Robin Murphy Reviewed-by: Thierry Reding Link: https://lore.kernel.org/r/20200718193457.30046-3-vdumpa@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index cdd15ead9bc4..de520115d3df 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -2123,10 +2123,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) if (err) return err; - smmu = arm_smmu_impl_init(smmu); - if (IS_ERR(smmu)) - return PTR_ERR(smmu); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ioaddr = res->start; smmu->base = devm_ioremap_resource(dev, res); @@ -2138,6 +2134,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) */ smmu->numpage = resource_size(res); + smmu = arm_smmu_impl_init(smmu); + if (IS_ERR(smmu)) + return PTR_ERR(smmu); + num_irqs = 0; while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { num_irqs++; From aab5a1c882760a5bc990b14e5c0c2ee4dab5f1ac Mon Sep 17 00:00:00 2001 From: Krishna Reddy Date: Sat, 18 Jul 2020 12:34:55 -0700 Subject: [PATCH 51/76] iommu/arm-smmu: add NVIDIA implementation for ARM MMU-500 usage NVIDIA's Tegra194 SoC has three ARM MMU-500 instances. It uses two of the ARM MMU-500s together to interleave IOVA accesses across them and must be programmed identically. This implementation supports programming the two ARM MMU-500s that must be programmed identically. The third ARM MMU-500 instance is supported by standard arm-smmu.c driver itself. Signed-off-by: Krishna Reddy Reviewed-by: Jon Hunter Reviewed-by: Nicolin Chen Reviewed-by: Pritesh Raithatha Reviewed-by: Robin Murphy Reviewed-by: Thierry Reding Link: https://lore.kernel.org/r/20200718193457.30046-4-vdumpa@nvidia.com Signed-off-by: Will Deacon --- MAINTAINERS | 2 + drivers/iommu/Makefile | 2 +- drivers/iommu/arm-smmu-impl.c | 3 + drivers/iommu/arm-smmu-nvidia.c | 179 ++++++++++++++++++++++++++++++++ drivers/iommu/arm-smmu.c | 1 + drivers/iommu/arm-smmu.h | 1 + 6 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 drivers/iommu/arm-smmu-nvidia.c diff --git a/MAINTAINERS b/MAINTAINERS index 496fd4eafb68..ee2c0ba13a0f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16810,8 +16810,10 @@ F: drivers/i2c/busses/i2c-tegra.c TEGRA IOMMU DRIVERS M: Thierry Reding +R: Krishna Reddy L: linux-tegra@vger.kernel.org S: Supported +F: drivers/iommu/arm-smmu-nvidia.c F: drivers/iommu/tegra* TEGRA KBC DRIVER diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 342190196dfb..2b8203db73ec 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -15,7 +15,7 @@ obj-$(CONFIG_AMD_IOMMU) += amd/iommu.o amd/init.o amd/quirks.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd/debugfs.o obj-$(CONFIG_AMD_IOMMU_V2) += amd/iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o -arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o +arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-nvidia.o arm-smmu-qcom.o obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o obj-$(CONFIG_DMAR_TABLE) += intel/dmar.o obj-$(CONFIG_INTEL_IOMMU) += intel/iommu.o intel/pasid.o diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index c87d825f651e..f4ff124a1967 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -213,6 +213,9 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) if (of_property_read_bool(np, "calxeda,smmu-secure-config-access")) smmu->impl = &calxeda_impl; + if (of_device_is_compatible(np, "nvidia,tegra194-smmu")) + return nvidia_smmu_impl_init(smmu); + if (of_device_is_compatible(np, "qcom,sdm845-smmu-500") || of_device_is_compatible(np, "qcom,sc7180-smmu-500") || of_device_is_compatible(np, "qcom,sm8150-smmu-500") || diff --git a/drivers/iommu/arm-smmu-nvidia.c b/drivers/iommu/arm-smmu-nvidia.c new file mode 100644 index 000000000000..2f55e5793d34 --- /dev/null +++ b/drivers/iommu/arm-smmu-nvidia.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019-2020 NVIDIA CORPORATION. All rights reserved. + +#include +#include +#include +#include +#include + +#include "arm-smmu.h" + +/* + * Tegra194 has three ARM MMU-500 Instances. + * Two of them are used together and must be programmed identically for + * interleaved IOVA accesses across them and translates accesses from + * non-isochronous HW devices. + * Third one is used for translating accesses from isochronous HW devices. + * This implementation supports programming of the two instances that must + * be programmed identically. + * The third instance usage is through standard arm-smmu driver itself and + * is out of scope of this implementation. + */ +#define NUM_SMMU_INSTANCES 2 + +struct nvidia_smmu { + struct arm_smmu_device smmu; + void __iomem *bases[NUM_SMMU_INSTANCES]; +}; + +static inline void __iomem *nvidia_smmu_page(struct arm_smmu_device *smmu, + unsigned int inst, int page) +{ + struct nvidia_smmu *nvidia_smmu; + + nvidia_smmu = container_of(smmu, struct nvidia_smmu, smmu); + return nvidia_smmu->bases[inst] + (page << smmu->pgshift); +} + +static u32 nvidia_smmu_read_reg(struct arm_smmu_device *smmu, + int page, int offset) +{ + void __iomem *reg = nvidia_smmu_page(smmu, 0, page) + offset; + + return readl_relaxed(reg); +} + +static void nvidia_smmu_write_reg(struct arm_smmu_device *smmu, + int page, int offset, u32 val) +{ + unsigned int i; + + for (i = 0; i < NUM_SMMU_INSTANCES; i++) { + void __iomem *reg = nvidia_smmu_page(smmu, i, page) + offset; + + writel_relaxed(val, reg); + } +} + +static u64 nvidia_smmu_read_reg64(struct arm_smmu_device *smmu, + int page, int offset) +{ + void __iomem *reg = nvidia_smmu_page(smmu, 0, page) + offset; + + return readq_relaxed(reg); +} + +static void nvidia_smmu_write_reg64(struct arm_smmu_device *smmu, + int page, int offset, u64 val) +{ + unsigned int i; + + for (i = 0; i < NUM_SMMU_INSTANCES; i++) { + void __iomem *reg = nvidia_smmu_page(smmu, i, page) + offset; + + writeq_relaxed(val, reg); + } +} + +static void nvidia_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, + int sync, int status) +{ + unsigned int delay; + + arm_smmu_writel(smmu, page, sync, 0); + + for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { + unsigned int spin_cnt; + + for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { + u32 val = 0; + unsigned int i; + + for (i = 0; i < NUM_SMMU_INSTANCES; i++) { + void __iomem *reg; + + reg = nvidia_smmu_page(smmu, i, page) + status; + val |= readl_relaxed(reg); + } + + if (!(val & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) + return; + + cpu_relax(); + } + + udelay(delay); + } + + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); +} + +static int nvidia_smmu_reset(struct arm_smmu_device *smmu) +{ + unsigned int i; + + for (i = 0; i < NUM_SMMU_INSTANCES; i++) { + u32 val; + void __iomem *reg = nvidia_smmu_page(smmu, i, ARM_SMMU_GR0) + + ARM_SMMU_GR0_sGFSR; + + /* clear global FSR */ + val = readl_relaxed(reg); + writel_relaxed(val, reg); + } + + return 0; +} + +static const struct arm_smmu_impl nvidia_smmu_impl = { + .read_reg = nvidia_smmu_read_reg, + .write_reg = nvidia_smmu_write_reg, + .read_reg64 = nvidia_smmu_read_reg64, + .write_reg64 = nvidia_smmu_write_reg64, + .reset = nvidia_smmu_reset, + .tlb_sync = nvidia_smmu_tlb_sync, +}; + +struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu) +{ + struct resource *res; + struct device *dev = smmu->dev; + struct nvidia_smmu *nvidia_smmu; + struct platform_device *pdev = to_platform_device(dev); + + nvidia_smmu = devm_kzalloc(dev, sizeof(*nvidia_smmu), GFP_KERNEL); + if (!nvidia_smmu) + return ERR_PTR(-ENOMEM); + + /* + * Copy the data from struct arm_smmu_device *smmu allocated in + * arm-smmu.c. The smmu from struct nvidia_smmu replaces the smmu + * pointer used in arm-smmu.c once this function returns. + * This is necessary to derive nvidia_smmu from smmu pointer passed + * through arm_smmu_impl function calls subsequently. + */ + nvidia_smmu->smmu = *smmu; + /* Instance 0 is ioremapped by arm-smmu.c. */ + nvidia_smmu->bases[0] = smmu->base; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) + return ERR_PTR(-ENODEV); + + nvidia_smmu->bases[1] = devm_ioremap_resource(dev, res); + if (IS_ERR(nvidia_smmu->bases[1])) + return ERR_CAST(nvidia_smmu->bases[1]); + + nvidia_smmu->smmu.impl = &nvidia_smmu_impl; + + /* + * Free the struct arm_smmu_device *smmu allocated in arm-smmu.c. + * Once this function returns, arm-smmu.c would use arm_smmu_device + * allocated as part of struct nvidia_smmu. + */ + devm_kfree(dev, smmu); + + return &nvidia_smmu->smmu; +} diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index de520115d3df..35422a17f610 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1946,6 +1946,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, + { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, { }, }; diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index c7d0122a7c6c..fad63efa1a72 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -452,6 +452,7 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); +struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); int arm_mmu500_reset(struct arm_smmu_device *smmu); From 3d2deb0cdb69e85d05760c6c72189d7653709ee1 Mon Sep 17 00:00:00 2001 From: Krishna Reddy Date: Sat, 18 Jul 2020 12:34:56 -0700 Subject: [PATCH 52/76] dt-bindings: arm-smmu: add binding for Tegra194 SMMU Add binding for NVIDIA's Tegra194 SoC SMMU. Signed-off-by: Krishna Reddy Reviewed-by: Jon Hunter Reviewed-by: Rob Herring Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20200718193457.30046-5-vdumpa@nvidia.com Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/arm,smmu.yaml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 93fb9fe068b9..503160a7b9a0 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -44,6 +44,11 @@ properties: items: - const: marvell,ap806-smmu-500 - const: arm,mmu-500 + - description: NVIDIA SoCs that program two ARM MMU-500s identically + items: + - enum: + - nvidia,tegra194-smmu + - const: nvidia,smmu-500 - items: - const: arm,mmu-500 - const: arm,smmu-v2 @@ -61,7 +66,8 @@ properties: - cavium,smmu-v2 reg: - maxItems: 1 + minItems: 1 + maxItems: 2 '#global-interrupts': description: The number of global interrupts exposed by the device. @@ -144,6 +150,23 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + enum: + - nvidia,tegra194-smmu + then: + properties: + reg: + minItems: 2 + maxItems: 2 + else: + properties: + reg: + maxItems: 1 + examples: - |+ /* SMMU with stream matching or stream indexing */ From aa7ec73297df57a86308fee78d2bf86e22ea0bae Mon Sep 17 00:00:00 2001 From: Krishna Reddy Date: Sat, 18 Jul 2020 12:34:57 -0700 Subject: [PATCH 53/76] iommu/arm-smmu: Add global/context fault implementation hooks Add global/context fault hooks to allow vendor specific implementations override default fault interrupt handlers. Update NVIDIA implementation to override the default global/context fault interrupt handlers and handle interrupts across the two ARM MMU-500s that are programmed identically. Signed-off-by: Krishna Reddy Reviewed-by: Jon Hunter Reviewed-by: Nicolin Chen Reviewed-by: Pritesh Raithatha Reviewed-by: Robin Murphy Reviewed-by: Thierry Reding Link: https://lore.kernel.org/r/20200718193457.30046-6-vdumpa@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-nvidia.c | 99 +++++++++++++++++++++++++++++++++ drivers/iommu/arm-smmu.c | 17 +++++- drivers/iommu/arm-smmu.h | 3 + 3 files changed, 117 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm-smmu-nvidia.c b/drivers/iommu/arm-smmu-nvidia.c index 2f55e5793d34..31368057e9be 100644 --- a/drivers/iommu/arm-smmu-nvidia.c +++ b/drivers/iommu/arm-smmu-nvidia.c @@ -127,6 +127,103 @@ static int nvidia_smmu_reset(struct arm_smmu_device *smmu) return 0; } +static irqreturn_t nvidia_smmu_global_fault_inst(int irq, + struct arm_smmu_device *smmu, + int inst) +{ + u32 gfsr, gfsynr0, gfsynr1, gfsynr2; + void __iomem *gr0_base = nvidia_smmu_page(smmu, inst, 0); + + gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); + if (!gfsr) + return IRQ_NONE; + + gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); + gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); + gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); + + dev_err_ratelimited(smmu->dev, + "Unexpected global fault, this could be serious\n"); + dev_err_ratelimited(smmu->dev, + "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", + gfsr, gfsynr0, gfsynr1, gfsynr2); + + writel_relaxed(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); + return IRQ_HANDLED; +} + +static irqreturn_t nvidia_smmu_global_fault(int irq, void *dev) +{ + unsigned int inst; + irqreturn_t ret = IRQ_NONE; + struct arm_smmu_device *smmu = dev; + + for (inst = 0; inst < NUM_SMMU_INSTANCES; inst++) { + irqreturn_t irq_ret; + + irq_ret = nvidia_smmu_global_fault_inst(irq, smmu, inst); + if (irq_ret == IRQ_HANDLED) + ret = IRQ_HANDLED; + } + + return ret; +} + +static irqreturn_t nvidia_smmu_context_fault_bank(int irq, + struct arm_smmu_device *smmu, + int idx, int inst) +{ + u32 fsr, fsynr, cbfrsynra; + unsigned long iova; + void __iomem *gr1_base = nvidia_smmu_page(smmu, inst, 1); + void __iomem *cb_base = nvidia_smmu_page(smmu, inst, smmu->numpage + idx); + + fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + if (!(fsr & ARM_SMMU_FSR_FAULT)) + return IRQ_NONE; + + fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); + iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); + cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(idx)); + + dev_err_ratelimited(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); + + writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR); + return IRQ_HANDLED; +} + +static irqreturn_t nvidia_smmu_context_fault(int irq, void *dev) +{ + int idx; + unsigned int inst; + irqreturn_t ret = IRQ_NONE; + struct arm_smmu_device *smmu; + struct iommu_domain *domain = dev; + struct arm_smmu_domain *smmu_domain; + + smmu_domain = container_of(domain, struct arm_smmu_domain, domain); + smmu = smmu_domain->smmu; + + for (inst = 0; inst < NUM_SMMU_INSTANCES; inst++) { + irqreturn_t irq_ret; + + /* + * Interrupt line is shared between all contexts. + * Check for faults across all contexts. + */ + for (idx = 0; idx < smmu->num_context_banks; idx++) { + irq_ret = nvidia_smmu_context_fault_bank(irq, smmu, + idx, inst); + if (irq_ret == IRQ_HANDLED) + ret = IRQ_HANDLED; + } + } + + return ret; +} + static const struct arm_smmu_impl nvidia_smmu_impl = { .read_reg = nvidia_smmu_read_reg, .write_reg = nvidia_smmu_write_reg, @@ -134,6 +231,8 @@ static const struct arm_smmu_impl nvidia_smmu_impl = { .write_reg64 = nvidia_smmu_write_reg64, .reset = nvidia_smmu_reset, .tlb_sync = nvidia_smmu_tlb_sync, + .global_fault = nvidia_smmu_global_fault, + .context_fault = nvidia_smmu_context_fault, }; struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 35422a17f610..56b7103e1368 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -670,6 +670,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + irqreturn_t (*context_fault)(int irq, void *dev); mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) @@ -832,7 +833,13 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, * handler seeing a half-initialised domain state. */ irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, + + if (smmu->impl && smmu->impl->context_fault) + context_fault = smmu->impl->context_fault; + else + context_fault = arm_smmu_context_fault; + + ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", @@ -2108,6 +2115,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; int num_irqs, i, err; + irqreturn_t (*global_fault)(int irq, void *dev); smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { @@ -2194,9 +2202,14 @@ static int arm_smmu_device_probe(struct platform_device *pdev) smmu->num_context_irqs = smmu->num_context_banks; } + if (smmu->impl && smmu->impl->global_fault) + global_fault = smmu->impl->global_fault; + else + global_fault = arm_smmu_global_fault; + for (i = 0; i < smmu->num_global_irqs; ++i) { err = devm_request_irq(smmu->dev, smmu->irqs[i], - arm_smmu_global_fault, + global_fault, IRQF_SHARED, "arm-smmu global fault", smmu); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index fad63efa1a72..d890a4a968e8 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -389,6 +390,8 @@ struct arm_smmu_impl { void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync, int status); int (*def_domain_type)(struct device *dev); + irqreturn_t (*global_fault)(int irq, void *dev); + irqreturn_t (*context_fault)(int irq, void *dev); }; static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) From 4b2aa7a6f9b793cadbda898476c8a16d374f1b3a Mon Sep 17 00:00:00 2001 From: Marian-Cristian Rotariu Date: Tue, 14 Jul 2020 11:20:54 +0100 Subject: [PATCH 54/76] iommu/ipmmu-vmsa: Hook up R8A774E1 DT matching code Add support for RZ/G2H (R8A774E1) SoC IPMMUs. Signed-off-by: Marian-Cristian Rotariu Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/1594722055-9298-2-git-send-email-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index b57b1f213a48..8072db8b379b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -740,6 +740,7 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { { .soc_id = "r8a774a1", }, { .soc_id = "r8a774b1", }, { .soc_id = "r8a774c0", }, + { .soc_id = "r8a774e1", }, { .soc_id = "r8a7795", }, { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, @@ -752,6 +753,7 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { { .soc_id = "r8a774b1", }, { .soc_id = "r8a774c0", }, + { .soc_id = "r8a774e1", }, { .soc_id = "r8a7795", .revision = "ES3.*" }, { .soc_id = "r8a77961", }, { .soc_id = "r8a77965", }, @@ -964,6 +966,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a774c0", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a774e1", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a7795", .data = &ipmmu_features_rcar_gen3, From f3e048b78ad37dff3ba81243f2533f336a240072 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 14 Jul 2020 11:20:55 +0100 Subject: [PATCH 55/76] iommu/ipmmu-vmsa: Add an entry for r8a77961 in soc_rcar_gen3[] Add an entry for r8a77961 in soc_rcar_gen3[] list so that we dont enable iommu unconditionally. Fixes: 17fe161816398 ("iommu/renesas: Add support for r8a77961") Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/1594722055-9298-3-git-send-email-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 8072db8b379b..d411dc50f719 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -742,6 +742,7 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { { .soc_id = "r8a774c0", }, { .soc_id = "r8a774e1", }, { .soc_id = "r8a7795", }, + { .soc_id = "r8a77961", }, { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, { .soc_id = "r8a77970", }, From 9314006dc3324cd449db90f23745c4f3e275d2d4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 14 Jul 2020 12:59:28 +0100 Subject: [PATCH 56/76] iommu/exynos: Rename update_pte() The name "update_pte" is a little too generic, and can end up clashing with architecture pagetable code leaked out of common mm headers. Rename it to something more appropriately namespaced. Reported-by: kernel test robot Acked-by: Marek Szyprowski Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/829bb5dc18e734870b75db673ddce86e7e37fc73.1594727968.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 60c8a56e4a3f..75cdd37fae38 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -721,7 +721,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { } }; -static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) +static inline void exynos_iommu_set_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); @@ -933,7 +933,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, if (!pent) return ERR_PTR(-ENOMEM); - update_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); + exynos_iommu_set_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); kmemleak_ignore(pent); *pgcounter = NUM_LV2ENTRIES; handle = dma_map_single(dma_dev, pent, LV2TABLE_SIZE, @@ -994,7 +994,7 @@ static int lv1set_section(struct exynos_iommu_domain *domain, *pgcnt = 0; } - update_pte(sent, mk_lv1ent_sect(paddr, prot)); + exynos_iommu_set_pte(sent, mk_lv1ent_sect(paddr, prot)); spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { @@ -1018,7 +1018,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, if (WARN_ON(!lv2ent_fault(pent))) return -EADDRINUSE; - update_pte(pent, mk_lv2ent_spage(paddr, prot)); + exynos_iommu_set_pte(pent, mk_lv2ent_spage(paddr, prot)); *pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; @@ -1150,7 +1150,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } /* workaround for h/w bug in System MMU v3.3 */ - update_pte(ent, ZERO_LV2LINK); + exynos_iommu_set_pte(ent, ZERO_LV2LINK); size = SECT_SIZE; goto done; } @@ -1171,7 +1171,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } if (lv2ent_small(ent)) { - update_pte(ent, 0); + exynos_iommu_set_pte(ent, 0); size = SPAGE_SIZE; domain->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; From dee9d154f40c58d02f69acdaa5cfd1eae6ebc28b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 14 Jul 2020 20:22:11 +0100 Subject: [PATCH 57/76] iommu/omap: Check for failure of a call to omap_iommu_dump_ctx It is possible for the call to omap_iommu_dump_ctx to return a negative error number, so check for the failure and return the error number rather than pass the negative value to simple_read_from_buffer. Fixes: 14e0e6796a0d ("OMAP: iommu: add initial debugfs support") Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20200714192211.744776-1-colin.king@canonical.com Addresses-Coverity: ("Improper use of negative value") Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu-debug.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index 8e19bfa94121..a99afb5d9011 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -98,8 +98,11 @@ static ssize_t debug_read_regs(struct file *file, char __user *userbuf, mutex_lock(&iommu_debug_lock); bytes = omap_iommu_dump_ctx(obj, p, count); + if (bytes < 0) + goto err; bytes = simple_read_from_buffer(userbuf, count, ppos, buf, bytes); +err: mutex_unlock(&iommu_debug_lock); kfree(buf); From 092550eacd040a349917a6bbe90ad543258e1695 Mon Sep 17 00:00:00 2001 From: Libing Zhou Date: Wed, 22 Jul 2020 14:44:50 +0800 Subject: [PATCH 58/76] iommu/amd: Remove double zero check The free_pages() does zero check, therefore remove double zero check here. Signed-off-by: Libing Zhou Link: https://lore.kernel.org/r/20200722064450.GA63618@hzling02.china.nsn-net.net Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 232683ea10e0..958050c213f9 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -720,21 +720,14 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) static void __init free_ppr_log(struct amd_iommu *iommu) { - if (iommu->ppr_log == NULL) - return; - free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); } static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP - if (iommu->ga_log) - free_pages((unsigned long)iommu->ga_log, - get_order(GA_LOG_SIZE)); - if (iommu->ga_log_tail) - free_pages((unsigned long)iommu->ga_log_tail, - get_order(8)); + free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE)); + free_pages((unsigned long)iommu->ga_log_tail, get_order(8)); #endif } From 1b0b2a84c979347f5385a8cb3901d3a46569d57c Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Mon, 13 Jul 2020 22:25:42 +0800 Subject: [PATCH 59/76] iommu: Make some functions static The sparse tool complains as follows: drivers/iommu/iommu.c:386:5: warning: symbol 'iommu_insert_resv_region' was not declared. Should it be static? drivers/iommu/iommu.c:2182:5: warning: symbol '__iommu_map' was not declared. Should it be static? Those functions are not used outside of iommu.c, so mark them static. Reported-by: Hulk Robot Signed-off-by: Wei Yongjun Link: https://lore.kernel.org/r/20200713142542.50294-1-weiyongjun1@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 1ed1e14a1f0c..40947f5bc6c5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -383,8 +383,8 @@ static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) * Elements are sorted by start address and overlapping segments * of the same type are merged. */ -int iommu_insert_resv_region(struct iommu_resv_region *new, - struct list_head *regions) +static int iommu_insert_resv_region(struct iommu_resv_region *new, + struct list_head *regions) { struct iommu_resv_region *iter, *tmp, *nr, *top; LIST_HEAD(stack); @@ -2179,8 +2179,8 @@ static size_t iommu_pgsize(struct iommu_domain *domain, return pgsize; } -int __iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +static int __iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { const struct iommu_ops *ops = domain->ops; unsigned long orig_iova = iova; From 5f77d6ca5ca74e4b4a5e2e010f7ff50c45dea326 Mon Sep 17 00:00:00 2001 From: Liu Yi L Date: Fri, 24 Jul 2020 09:49:14 +0800 Subject: [PATCH 60/76] iommu/vt-d: Enforce PASID devTLB field mask Set proper masks to avoid invalid input spillover to reserved bits. Signed-off-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200724014925.15523-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/intel-iommu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 3e8fa1c7a1e6..311117b50e93 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -381,8 +381,8 @@ enum { #define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK) #define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) -#define QI_DEV_EIOTLB_GLOB(g) ((u64)g) -#define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32) +#define QI_DEV_EIOTLB_GLOB(g) ((u64)(g) & 0x1) +#define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32) #define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) #define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ From 78df6c86f0691f5b6e325006aeb470de443351ea Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Fri, 24 Jul 2020 09:49:15 +0800 Subject: [PATCH 61/76] iommu/vt-d: Remove global page support in devTLB flush Global pages support is removed from VT-d spec 3.0 for dev TLB invalidation. This patch is to remove the bits for vSVA. Similar change already made for the native SVA. See the link below. Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/linux-iommu/20190830142919.GE11578@8bytes.org/T/ Link: https://lore.kernel.org/r/20200724014925.15523-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 4 +--- drivers/iommu/intel/iommu.c | 4 ++-- include/linux/intel-iommu.h | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 683b812c5c47..9be08b9400ee 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1438,8 +1438,7 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, /* PASID-based device IOTLB Invalidate */ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, - u32 pasid, u16 qdep, u64 addr, - unsigned int size_order, u64 granu) + u32 pasid, u16 qdep, u64 addr, unsigned int size_order) { unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1); struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0}; @@ -1447,7 +1446,6 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) | QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid); - desc.qw1 = QI_DEV_EIOTLB_GLOB(granu); /* * If S bit is 0, we only flush a single page. If S bit is set, diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index d759e7234e98..bdd1e7d81178 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5474,7 +5474,7 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, info->pfsid, pasid, info->ats_qdep, inv_info->addr_info.addr, - size, granu); + size); break; case IOMMU_CACHE_INV_TYPE_DEV_IOTLB: if (info->ats_enabled) @@ -5482,7 +5482,7 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, info->pfsid, pasid, info->ats_qdep, inv_info->addr_info.addr, - size, granu); + size); else pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n"); break; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 311117b50e93..c7a8aae36771 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -381,7 +381,6 @@ enum { #define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK) #define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) -#define QI_DEV_EIOTLB_GLOB(g) ((u64)(g) & 0x1) #define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32) #define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) #define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) @@ -705,7 +704,7 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, u32 pasid, u16 qdep, u64 addr, - unsigned int size_order, u64 granu); + unsigned int size_order); void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu, int pasid); From e7e69461a83264dbce2b4ff480f858f3f1454db7 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Fri, 24 Jul 2020 09:49:16 +0800 Subject: [PATCH 62/76] iommu/vt-d: Fix PASID devTLB invalidation DevTLB flush can be used for both DMA request with and without PASIDs. The former uses PASID#0 (RID2PASID), latter uses non-zero PASID for SVA usage. This patch adds a check for PASID value such that devTLB flush with PASID is used for SVA case. This is more efficient in that multiple PASIDs can be used by a single device, when tearing down a PASID entry we shall flush only the devTLB specific to a PASID. Fixes: 6f7db75e1c46 ("iommu/vt-d: Add second level page table") Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200724014925.15523-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index c81f0f17c6ba..fa0154cce537 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -486,7 +486,16 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, qdep = info->ats_qdep; pfsid = info->pfsid; - qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT); + /* + * When PASID 0 is used, it indicates RID2PASID(DMA request w/o PASID), + * devTLB flush w/o PASID should be used. For non-zero PASID under + * SVA usage, device could do DMA with multiple PASIDs. It is more + * efficient to flush devTLB specific to the PASID. + */ + if (pasid == PASID_RID2PASID) + qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT); + else + qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT); } void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, From 288d08e78008828416ffaa85ef274b4e29ef3dae Mon Sep 17 00:00:00 2001 From: Liu Yi L Date: Fri, 24 Jul 2020 09:49:17 +0800 Subject: [PATCH 63/76] iommu/vt-d: Handle non-page aligned address Address information for device TLB invalidation comes from userspace when device is directly assigned to a guest with vIOMMU support. VT-d requires page aligned address. This patch checks and enforce address to be page aligned, otherwise reserved bits can be set in the invalidation descriptor. Unrecoverable fault will be reported due to non-zero value in the reserved bits. Fixes: 61a06a16e36d8 ("iommu/vt-d: Support flushing more translation cache types") Signed-off-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200724014925.15523-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 9be08b9400ee..f6cbe3f95c8d 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1456,9 +1456,26 @@ void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, * Max Invs Pending (MIP) is set to 0 for now until we have DIT in * ECAP. */ - desc.qw1 |= addr & ~mask; - if (size_order) + if (addr & GENMASK_ULL(size_order + VTD_PAGE_SHIFT, 0)) + pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n", + addr, size_order); + + /* Take page address */ + desc.qw1 = QI_DEV_EIOTLB_ADDR(addr); + + if (size_order) { + /* + * Existing 0s in address below size_order may be the least + * significant bit, we must set them to 1s to avoid having + * smaller size than desired. + */ + desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1, + VTD_PAGE_SHIFT); + /* Clear size_order bit to indicate size */ + desc.qw1 &= ~mask; + /* Set the S bit to indicate flushing more than 1 page */ desc.qw1 |= QI_DEV_EIOTLB_SIZE; + } qi_submit_sync(iommu, &desc, 1, 0); } From 0fa1a15fa9b37be3899d7fec552c77b67baa8ac7 Mon Sep 17 00:00:00 2001 From: Liu Yi L Date: Fri, 24 Jul 2020 09:49:18 +0800 Subject: [PATCH 64/76] iommu/vt-d: Fix devTLB flush for vSVA For guest SVA usage, in order to optimize for less VMEXIT, guest request of IOTLB flush also includes device TLB. On the host side, IOMMU driver performs IOTLB and implicit devTLB invalidation. When PASID-selective granularity is requested by the guest we need to derive the equivalent address range for devTLB instead of using the address information in the UAPI data. The reason for that is, unlike IOTLB flush, devTLB flush does not support PASID-selective granularity. This is to say, we need to set the following in the PASID based devTLB invalidation descriptor: - entire 64 bit range in address ~(0x1 << 63) - S bit = 1 (VT-d CH 6.5.2.6). Without this fix, device TLB flush range is not set properly for PASID selective granularity. This patch also merged devTLB flush code for both implicit and explicit cases. Fixes: 6ee1b77ba3ac ("iommu/vt-d: Add svm/sva invalidate function") Signed-off-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200724014925.15523-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index bdd1e7d81178..34a0ebfc2fcf 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5416,7 +5416,7 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, sid = PCI_DEVID(bus, devfn); /* Size is only valid in address selective invalidation */ - if (inv_info->granularity != IOMMU_INV_GRANU_PASID) + if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) size = to_vtd_size(inv_info->addr_info.granule_size, inv_info->addr_info.nb_granules); @@ -5425,6 +5425,7 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, IOMMU_CACHE_INV_TYPE_NR) { int granu = 0; u64 pasid = 0; + u64 addr = 0; granu = to_vtd_granularity(cache_type, inv_info->granularity); if (granu == -EINVAL) { @@ -5464,24 +5465,34 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, (granu == QI_GRAN_NONG_PASID) ? -1 : 1 << size, inv_info->addr_info.flags & IOMMU_INV_ADDR_FLAGS_LEAF); + if (!info->ats_enabled) + break; /* * Always flush device IOTLB if ATS is enabled. vIOMMU * in the guest may assume IOTLB flush is inclusive, * which is more efficient. */ - if (info->ats_enabled) - qi_flush_dev_iotlb_pasid(iommu, sid, - info->pfsid, pasid, - info->ats_qdep, - inv_info->addr_info.addr, - size); - break; + fallthrough; case IOMMU_CACHE_INV_TYPE_DEV_IOTLB: + /* + * PASID based device TLB invalidation does not support + * IOMMU_INV_GRANU_PASID granularity but only supports + * IOMMU_INV_GRANU_ADDR. + * The equivalent of that is we set the size to be the + * entire range of 64 bit. User only provides PASID info + * without address info. So we set addr to 0. + */ + if (inv_info->granularity == IOMMU_INV_GRANU_PASID) { + size = 64 - VTD_PAGE_SHIFT; + addr = 0; + } else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) { + addr = inv_info->addr_info.addr; + } + if (info->ats_enabled) qi_flush_dev_iotlb_pasid(iommu, sid, info->pfsid, pasid, - info->ats_qdep, - inv_info->addr_info.addr, + info->ats_qdep, addr, size); else pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n"); From 1ff00279655d95ae9c285c39878aedf9ff008d25 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Fri, 24 Jul 2020 09:49:19 +0800 Subject: [PATCH 65/76] iommu/vt-d: Warn on out-of-range invalidation address For guest requested IOTLB invalidation, address and mask are provided as part of the invalidation data. VT-d HW silently ignores any address bits below the mask. SW shall also allow such case but give warning if address does not align with the mask. This patch relax the fault handling from error to warning and proceed with invalidation request with the given mask. Fixes: 6ee1b77ba3ac0 ("iommu/vt-d: Add svm/sva invalidate function") Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200724014925.15523-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 34a0ebfc2fcf..0a4831532442 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5447,13 +5447,12 @@ intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev, switch (BIT(cache_type)) { case IOMMU_CACHE_INV_TYPE_IOTLB: + /* HW will ignore LSB bits based on address mask */ if (inv_info->granularity == IOMMU_INV_GRANU_ADDR && size && (inv_info->addr_info.addr & ((BIT(VTD_PAGE_SHIFT + size)) - 1))) { - pr_err_ratelimited("Address out of range, 0x%llx, size order %llu\n", + pr_err_ratelimited("User address not aligned, 0x%llx, size order %llu\n", inv_info->addr_info.addr, size); - ret = -ERANGE; - goto out_unlock; } /* From d315e9e684d1efd4cb2e8cd70b8d71dec02fcf1f Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Fri, 24 Jul 2020 09:49:20 +0800 Subject: [PATCH 66/76] iommu/vt-d: Disable multiple GPASID-dev bind For the unlikely use case where multiple aux domains from the same pdev are attached to a single guest and then bound to a single process (thus same PASID) within that guest, we cannot easily support this case by refcounting the number of users. As there is only one SL page table per PASID while we have multiple aux domains thus multiple SL page tables for the same PASID. Extra unbinding guest PASID can happen due to race between normal and exception cases. Termination of one aux domain may affect others unless we actively track and switch aux domains to ensure the validity of SL page tables and TLB states in the shared PASID entry. Support for sharing second level PGDs across domains can reduce the complexity but this is not available due to the limitations on VFIO container architecture. We can revisit this decision once sharing PGDs are available. Overall, the complexity and potential glitch do not warrant this unlikely use case thereby removed by this patch. Fixes: 56722a4398a30 ("iommu/vt-d: Add bind guest PASID support") Signed-off-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Lu Baolu Reviewed-by: Eric Auger Cc: Kevin Tian Link: https://lore.kernel.org/r/20200724014925.15523-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 6c87c807a0ab..d386853121a2 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -277,20 +277,16 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, goto out; } + /* + * Do not allow multiple bindings of the same device-PASID since + * there is only one SL page tables per PASID. We may revisit + * once sharing PGD across domains are supported. + */ for_each_svm_dev(sdev, svm, dev) { - /* - * For devices with aux domains, we should allow - * multiple bind calls with the same PASID and pdev. - */ - if (iommu_dev_feature_enabled(dev, - IOMMU_DEV_FEAT_AUX)) { - sdev->users++; - } else { - dev_warn_ratelimited(dev, - "Already bound with PASID %u\n", - svm->pasid); - ret = -EBUSY; - } + dev_warn_ratelimited(dev, + "Already bound with PASID %u\n", + svm->pasid); + ret = -EBUSY; goto out; } } else { From dd6692f1b883bac46036000a1e3a0b3785f89e87 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 24 Jul 2020 09:49:21 +0800 Subject: [PATCH 67/76] iommu/vt-d: Refactor device_to_iommu() helper It is refactored in two ways: - Make it global so that it could be used in other files. - Make bus/devfn optional so that callers could ignore these two returned values when they only want to get the coresponding iommu pointer. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20200724014925.15523-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 55 +++++++++++-------------------------- drivers/iommu/intel/svm.c | 8 +++--- include/linux/intel-iommu.h | 3 +- 3 files changed, 21 insertions(+), 45 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 0a4831532442..ce8458e8119c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -778,16 +778,16 @@ is_downstream_to_pci_bridge(struct device *dev, struct device *bridge) return false; } -static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) +struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) { struct dmar_drhd_unit *drhd = NULL; + struct pci_dev *pdev = NULL; struct intel_iommu *iommu; struct device *tmp; - struct pci_dev *pdev = NULL; u16 segment = 0; int i; - if (iommu_dummy(dev)) + if (!dev || iommu_dummy(dev)) return NULL; if (dev_is_pci(dev)) { @@ -818,8 +818,10 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf if (pdev && pdev->is_virtfn) goto got_pdev; - *bus = drhd->devices[i].bus; - *devfn = drhd->devices[i].devfn; + if (bus && devfn) { + *bus = drhd->devices[i].bus; + *devfn = drhd->devices[i].devfn; + } goto out; } @@ -829,8 +831,10 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf if (pdev && drhd->include_all) { got_pdev: - *bus = pdev->bus->number; - *devfn = pdev->devfn; + if (bus && devfn) { + *bus = pdev->bus->number; + *devfn = pdev->devfn; + } goto out; } } @@ -5146,11 +5150,10 @@ static int aux_domain_add_dev(struct dmar_domain *domain, struct device *dev) { int ret; - u8 bus, devfn; unsigned long flags; struct intel_iommu *iommu; - iommu = device_to_iommu(dev, &bus, &devfn); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu) return -ENODEV; @@ -5236,9 +5239,8 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct intel_iommu *iommu; int addr_width; - u8 bus, devfn; - iommu = device_to_iommu(dev, &bus, &devfn); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu) return -ENODEV; @@ -5668,9 +5670,8 @@ static bool intel_iommu_capable(enum iommu_cap cap) static struct iommu_device *intel_iommu_probe_device(struct device *dev) { struct intel_iommu *iommu; - u8 bus, devfn; - iommu = device_to_iommu(dev, &bus, &devfn); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu) return ERR_PTR(-ENODEV); @@ -5683,9 +5684,8 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) static void intel_iommu_release_device(struct device *dev) { struct intel_iommu *iommu; - u8 bus, devfn; - iommu = device_to_iommu(dev, &bus, &devfn); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu) return; @@ -5835,37 +5835,14 @@ static struct iommu_group *intel_iommu_device_group(struct device *dev) return generic_device_group(dev); } -#ifdef CONFIG_INTEL_IOMMU_SVM -struct intel_iommu *intel_svm_device_to_iommu(struct device *dev) -{ - struct intel_iommu *iommu; - u8 bus, devfn; - - if (iommu_dummy(dev)) { - dev_warn(dev, - "No IOMMU translation for device; cannot enable SVM\n"); - return NULL; - } - - iommu = device_to_iommu(dev, &bus, &devfn); - if ((!iommu)) { - dev_err(dev, "No IOMMU for device; cannot enable SVM\n"); - return NULL; - } - - return iommu; -} -#endif /* CONFIG_INTEL_IOMMU_SVM */ - static int intel_iommu_enable_auxd(struct device *dev) { struct device_domain_info *info; struct intel_iommu *iommu; unsigned long flags; - u8 bus, devfn; int ret; - iommu = device_to_iommu(dev, &bus, &devfn); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu || dmar_disabled) return -EINVAL; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index d386853121a2..65d2327dcd0d 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -231,7 +231,7 @@ static LIST_HEAD(global_svm_list); int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, struct iommu_gpasid_bind_data *data) { - struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); + struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); struct dmar_domain *dmar_domain; struct intel_svm_dev *sdev; struct intel_svm *svm; @@ -369,7 +369,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, int intel_svm_unbind_gpasid(struct device *dev, int pasid) { - struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); + struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); struct intel_svm_dev *sdev; struct intel_svm *svm; int ret = -EINVAL; @@ -426,7 +426,7 @@ static int intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops, struct mm_struct *mm, struct intel_svm_dev **sd) { - struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); + struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); struct device_domain_info *info; struct intel_svm_dev *sdev; struct intel_svm *svm = NULL; @@ -604,7 +604,7 @@ static int intel_svm_unbind_mm(struct device *dev, int pasid) struct intel_svm *svm; int ret = -EINVAL; - iommu = intel_svm_device_to_iommu(dev); + iommu = device_to_iommu(dev, NULL, NULL); if (!iommu) goto out; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index c7a8aae36771..a57ffbcc84c7 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -727,6 +727,7 @@ void iommu_flush_write_buffer(struct intel_iommu *iommu); int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev); struct dmar_domain *find_domain(struct device *dev); struct device_domain_info *get_domain_info(struct device *dev); +struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); #ifdef CONFIG_INTEL_IOMMU_SVM extern void intel_svm_check(struct intel_iommu *iommu); @@ -765,8 +766,6 @@ struct intel_svm { struct list_head devs; struct list_head list; }; - -extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev); #else static inline void intel_svm_check(struct intel_iommu *iommu) {} #endif From 19abcf70c2b16834a607db515b26a32b233c79eb Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 24 Jul 2020 09:49:22 +0800 Subject: [PATCH 68/76] iommu/vt-d: Add a helper to get svm and sdev for pasid There are several places in the code that need to get the pointers of svm and sdev according to a pasid and device. Add a helper to achieve this for code consolidation and readability. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20200724014925.15523-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 115 +++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 65d2327dcd0d..c104a50a625c 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -228,13 +228,57 @@ static LIST_HEAD(global_svm_list); list_for_each_entry((sdev), &(svm)->devs, list) \ if ((d) != (sdev)->dev) {} else +static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, + struct intel_svm **rsvm, + struct intel_svm_dev **rsdev) +{ + struct intel_svm_dev *d, *sdev = NULL; + struct intel_svm *svm; + + /* The caller should hold the pasid_mutex lock */ + if (WARN_ON(!mutex_is_locked(&pasid_mutex))) + return -EINVAL; + + if (pasid == INVALID_IOASID || pasid >= PASID_MAX) + return -EINVAL; + + svm = ioasid_find(NULL, pasid, NULL); + if (IS_ERR(svm)) + return PTR_ERR(svm); + + if (!svm) + goto out; + + /* + * If we found svm for the PASID, there must be at least one device + * bond. + */ + if (WARN_ON(list_empty(&svm->devs))) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(d, &svm->devs, list) { + if (d->dev == dev) { + sdev = d; + break; + } + } + rcu_read_unlock(); + +out: + *rsvm = svm; + *rsdev = sdev; + + return 0; +} + int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, struct iommu_gpasid_bind_data *data) { struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); + struct intel_svm_dev *sdev = NULL; struct dmar_domain *dmar_domain; - struct intel_svm_dev *sdev; - struct intel_svm *svm; + struct intel_svm *svm = NULL; int ret = 0; if (WARN_ON(!iommu) || !data) @@ -261,35 +305,23 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, dmar_domain = to_dmar_domain(domain); mutex_lock(&pasid_mutex); - svm = ioasid_find(NULL, data->hpasid, NULL); - if (IS_ERR(svm)) { - ret = PTR_ERR(svm); + ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev); + if (ret) goto out; - } - - if (svm) { - /* - * If we found svm for the PASID, there must be at - * least one device bond, otherwise svm should be freed. - */ - if (WARN_ON(list_empty(&svm->devs))) { - ret = -EINVAL; - goto out; - } + if (sdev) { /* * Do not allow multiple bindings of the same device-PASID since * there is only one SL page tables per PASID. We may revisit * once sharing PGD across domains are supported. */ - for_each_svm_dev(sdev, svm, dev) { - dev_warn_ratelimited(dev, - "Already bound with PASID %u\n", - svm->pasid); - ret = -EBUSY; - goto out; - } - } else { + dev_warn_ratelimited(dev, "Already bound with PASID %u\n", + svm->pasid); + ret = -EBUSY; + goto out; + } + + if (!svm) { /* We come here when PASID has never been bond to a device. */ svm = kzalloc(sizeof(*svm), GFP_KERNEL); if (!svm) { @@ -372,25 +404,17 @@ int intel_svm_unbind_gpasid(struct device *dev, int pasid) struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); struct intel_svm_dev *sdev; struct intel_svm *svm; - int ret = -EINVAL; + int ret; if (WARN_ON(!iommu)) return -EINVAL; mutex_lock(&pasid_mutex); - svm = ioasid_find(NULL, pasid, NULL); - if (!svm) { - ret = -EINVAL; + ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); + if (ret) goto out; - } - if (IS_ERR(svm)) { - ret = PTR_ERR(svm); - goto out; - } - - for_each_svm_dev(sdev, svm, dev) { - ret = 0; + if (sdev) { if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) sdev->users--; if (!sdev->users) { @@ -414,7 +438,6 @@ int intel_svm_unbind_gpasid(struct device *dev, int pasid) kfree(svm); } } - break; } out: mutex_unlock(&pasid_mutex); @@ -592,7 +615,7 @@ success: if (sd) *sd = sdev; ret = 0; - out: +out: return ret; } @@ -608,17 +631,11 @@ static int intel_svm_unbind_mm(struct device *dev, int pasid) if (!iommu) goto out; - svm = ioasid_find(NULL, pasid, NULL); - if (!svm) + ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); + if (ret) goto out; - if (IS_ERR(svm)) { - ret = PTR_ERR(svm); - goto out; - } - - for_each_svm_dev(sdev, svm, dev) { - ret = 0; + if (sdev) { sdev->users--; if (!sdev->users) { list_del_rcu(&sdev->list); @@ -647,10 +664,8 @@ static int intel_svm_unbind_mm(struct device *dev, int pasid) kfree(svm); } } - break; } - out: - +out: return ret; } From eb8d93ea3c1d35b0baf693dd0b7c87ec62358fc9 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 24 Jul 2020 09:49:23 +0800 Subject: [PATCH 69/76] iommu/vt-d: Report page request faults for guest SVA A pasid might be bound to a page table from a VM guest via the iommu ops.sva_bind_gpasid. In this case, when a DMA page fault is detected on the physical IOMMU, we need to inject the page fault request into the guest. After the guest completes handling the page fault, a page response need to be sent back via the iommu ops.page_response(). This adds support to report a page request fault. Any external module which is interested in handling this fault should regiester a notifier with iommu_register_device_fault_handler(). Co-developed-by: Jacob Pan Co-developed-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20200724014925.15523-11-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 103 +++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index c104a50a625c..140114ab1375 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -811,8 +811,63 @@ qi_retry: } } +static int prq_to_iommu_prot(struct page_req_dsc *req) +{ + int prot = 0; + + if (req->rd_req) + prot |= IOMMU_FAULT_PERM_READ; + if (req->wr_req) + prot |= IOMMU_FAULT_PERM_WRITE; + if (req->exe_req) + prot |= IOMMU_FAULT_PERM_EXEC; + if (req->pm_req) + prot |= IOMMU_FAULT_PERM_PRIV; + + return prot; +} + +static int +intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) +{ + struct iommu_fault_event event; + + if (!dev || !dev_is_pci(dev)) + return -ENODEV; + + /* Fill in event data for device specific processing */ + memset(&event, 0, sizeof(struct iommu_fault_event)); + event.fault.type = IOMMU_FAULT_PAGE_REQ; + event.fault.prm.addr = desc->addr; + event.fault.prm.pasid = desc->pasid; + event.fault.prm.grpid = desc->prg_index; + event.fault.prm.perm = prq_to_iommu_prot(desc); + + if (desc->lpig) + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; + if (desc->pasid_present) { + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + } + if (desc->priv_data_present) { + /* + * Set last page in group bit if private data is present, + * page response is required as it does for LPIG. + * iommu_report_device_fault() doesn't understand this vendor + * specific requirement thus we set last_page as a workaround. + */ + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; + memcpy(event.fault.prm.private_data, desc->priv_data, + sizeof(desc->priv_data)); + } + + return iommu_report_device_fault(dev, &event); +} + static irqreturn_t prq_event_thread(int irq, void *d) { + struct intel_svm_dev *sdev = NULL; struct intel_iommu *iommu = d; struct intel_svm *svm = NULL; int head, tail, handled = 0; @@ -824,7 +879,6 @@ static irqreturn_t prq_event_thread(int irq, void *d) tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; while (head != tail) { - struct intel_svm_dev *sdev; struct vm_area_struct *vma; struct page_req_dsc *req; struct qi_desc resp; @@ -860,6 +914,20 @@ static irqreturn_t prq_event_thread(int irq, void *d) } } + if (!sdev || sdev->sid != req->rid) { + struct intel_svm_dev *t; + + sdev = NULL; + rcu_read_lock(); + list_for_each_entry_rcu(t, &svm->devs, list) { + if (t->sid == req->rid) { + sdev = t; + break; + } + } + rcu_read_unlock(); + } + result = QI_RESP_INVALID; /* Since we're using init_mm.pgd directly, we should never take * any faults on kernel addresses. */ @@ -870,6 +938,17 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (!is_canonical_address(address)) goto bad_req; + /* + * If prq is to be handled outside iommu driver via receiver of + * the fault notifiers, we skip the page response here. + */ + if (svm->flags & SVM_FLAG_GUEST_MODE) { + if (sdev && !intel_svm_prq_report(sdev->dev, req)) + goto prq_advance; + else + goto bad_req; + } + /* If the mm is already defunct, don't handle faults. */ if (!mmget_not_zero(svm->mm)) goto bad_req; @@ -888,24 +967,11 @@ static irqreturn_t prq_event_thread(int irq, void *d) goto invalid; result = QI_RESP_SUCCESS; - invalid: +invalid: mmap_read_unlock(svm->mm); mmput(svm->mm); - bad_req: - /* Accounting for major/minor faults? */ - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) { - if (sdev->sid == req->rid) - break; - } - /* Other devices can go away, but the drivers are not permitted - * to unbind while any page faults might be in flight. So it's - * OK to drop the 'lock' here now we have it. */ - rcu_read_unlock(); - - if (WARN_ON(&sdev->list == &svm->devs)) - sdev = NULL; - +bad_req: + WARN_ON(!sdev); if (sdev && sdev->ops && sdev->ops->fault_cb) { int rwxp = (req->rd_req << 3) | (req->wr_req << 2) | (req->exe_req << 1) | (req->pm_req); @@ -916,7 +982,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) and these can be NULL. Do not use them below this point! */ sdev = NULL; svm = NULL; - no_pasid: +no_pasid: if (req->lpig || req->priv_data_present) { /* * Per VT-d spec. v3.0 ch7.7, system software must @@ -941,6 +1007,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) resp.qw3 = 0; qi_submit_sync(iommu, &resp, 1, 0); } +prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; } From 8b73712115ebd603b75876f7d96d59e41d9107ad Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 24 Jul 2020 09:49:24 +0800 Subject: [PATCH 70/76] iommu/vt-d: Add page response ops support After page requests are handled, software must respond to the device which raised the page request with the result. This is done through the iommu ops.page_response if the request was reported to outside of vendor iommu driver through iommu_report_device_fault(). This adds the VT-d implementation of page_response ops. Co-developed-by: Jacob Pan Co-developed-by: Liu Yi L Signed-off-by: Jacob Pan Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20200724014925.15523-12-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 1 + drivers/iommu/intel/svm.c | 99 +++++++++++++++++++++++++++++++++++++ include/linux/intel-iommu.h | 3 ++ 3 files changed, 103 insertions(+) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ce8458e8119c..1cb3a2a050c3 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -6067,6 +6067,7 @@ const struct iommu_ops intel_iommu_ops = { .sva_bind = intel_svm_bind, .sva_unbind = intel_svm_unbind, .sva_get_pasid = intel_svm_get_pasid, + .page_response = intel_svm_page_response, #endif }; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 140114ab1375..85ce8daa3177 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -1078,3 +1078,102 @@ int intel_svm_get_pasid(struct iommu_sva *sva) return pasid; } + +int intel_svm_page_response(struct device *dev, + struct iommu_fault_event *evt, + struct iommu_page_response *msg) +{ + struct iommu_fault_page_request *prm; + struct intel_svm_dev *sdev = NULL; + struct intel_svm *svm = NULL; + struct intel_iommu *iommu; + bool private_present; + bool pasid_present; + bool last_page; + u8 bus, devfn; + int ret = 0; + u16 sid; + + if (!dev || !dev_is_pci(dev)) + return -ENODEV; + + iommu = device_to_iommu(dev, &bus, &devfn); + if (!iommu) + return -ENODEV; + + if (!msg || !evt) + return -EINVAL; + + mutex_lock(&pasid_mutex); + + prm = &evt->fault.prm; + sid = PCI_DEVID(bus, devfn); + pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; + last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; + + if (!pasid_present) { + ret = -EINVAL; + goto out; + } + + if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { + ret = -EINVAL; + goto out; + } + + ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); + if (ret || !sdev) { + ret = -ENODEV; + goto out; + } + + /* + * For responses from userspace, need to make sure that the + * pasid has been bound to its mm. + */ + if (svm->flags & SVM_FLAG_GUEST_MODE) { + struct mm_struct *mm; + + mm = get_task_mm(current); + if (!mm) { + ret = -EINVAL; + goto out; + } + + if (mm != svm->mm) { + ret = -ENODEV; + mmput(mm); + goto out; + } + + mmput(mm); + } + + /* + * Per VT-d spec. v3.0 ch7.7, system software must respond + * with page group response if private data is present (PDP) + * or last page in group (LPIG) bit is set. This is an + * additional VT-d requirement beyond PCI ATS spec. + */ + if (last_page || private_present) { + struct qi_desc desc; + + desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | + QI_PGRP_PASID_P(pasid_present) | + QI_PGRP_PDP(private_present) | + QI_PGRP_RESP_CODE(msg->code) | + QI_PGRP_RESP_TYPE; + desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); + desc.qw2 = 0; + desc.qw3 = 0; + if (private_present) + memcpy(&desc.qw2, prm->private_data, + sizeof(prm->private_data)); + + qi_submit_sync(iommu, &desc, 1, 0); + } +out: + mutex_unlock(&pasid_mutex); + return ret; +} diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index a57ffbcc84c7..9ff5e340948b 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -740,6 +740,9 @@ struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata); void intel_svm_unbind(struct iommu_sva *handle); int intel_svm_get_pasid(struct iommu_sva *handle); +int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, + struct iommu_page_response *msg); + struct svm_dev_ops; struct intel_svm_dev { From 02f3effddfd04f3f08a24d23a82d1c1c6d89b777 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 24 Jul 2020 09:49:25 +0800 Subject: [PATCH 71/76] iommu/vt-d: Rename intel-pasid.h to pasid.h As Intel VT-d files have been moved to its own subdirectory, the prefix makes no sense. No functional changes. Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20200724014925.15523-13-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/debugfs.c | 2 +- drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/pasid.c | 2 +- drivers/iommu/intel/{intel-pasid.h => pasid.h} | 2 +- drivers/iommu/intel/svm.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename drivers/iommu/intel/{intel-pasid.h => pasid.h} (98%) diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index cf1ebb98e418..efea7f02abd9 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -15,7 +15,7 @@ #include -#include "intel-pasid.h" +#include "pasid.h" struct tbl_walk { u16 bus; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 1cb3a2a050c3..b3faf99457a6 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -48,7 +48,7 @@ #include #include "../irq_remapping.h" -#include "intel-pasid.h" +#include "pasid.h" #define ROOT_SIZE VTD_PAGE_SIZE #define CONTEXT_SIZE VTD_PAGE_SIZE diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index fa0154cce537..e6faedf42fd4 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -19,7 +19,7 @@ #include #include -#include "intel-pasid.h" +#include "pasid.h" /* * Intel IOMMU system wide PASID name space: diff --git a/drivers/iommu/intel/intel-pasid.h b/drivers/iommu/intel/pasid.h similarity index 98% rename from drivers/iommu/intel/intel-pasid.h rename to drivers/iommu/intel/pasid.h index c5318d40e0fa..c9850766c3a9 100644 --- a/drivers/iommu/intel/intel-pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * intel-pasid.h - PASID idr, table and entry header + * pasid.h - PASID idr, table and entry header * * Copyright (C) 2018 Intel Corporation * diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 85ce8daa3177..442623ac4b47 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -20,7 +20,7 @@ #include #include -#include "intel-pasid.h" +#include "pasid.h" static irqreturn_t prq_event_thread(int irq, void *d); static void intel_svm_drain_prq(struct device *dev, int pasid); From 9930264fd997dad4c2cd8914cec7acab7c0ed984 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 12 Jun 2020 11:39:54 +0800 Subject: [PATCH 72/76] iommu: Mark __iommu_map_sg() as static Now __iommu_map_sg() is used only in iommu.c file, so mark it static. Signed-off-by: Baolin Wang Acked-by: Will Deacon Link: https://lore.kernel.org/r/ab722e9970739929738066777b8ee7930e32abd5.1591930156.git.baolin.wang@linux.alibaba.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 40947f5bc6c5..15ce5ec3edb2 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2330,9 +2330,9 @@ size_t iommu_unmap_fast(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_unmap_fast); -size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, - struct scatterlist *sg, unsigned int nents, int prot, - gfp_t gfp) +static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot, + gfp_t gfp) { size_t len = 0, mapped = 0; phys_addr_t start; From f34ce7a7018c2f71d78fc7f512f6daf01e487114 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 12 Jun 2020 11:39:55 +0800 Subject: [PATCH 73/76] iommu: Add gfp parameter to io_pgtable_ops->map() Now the ARM page tables are always allocated by GFP_ATOMIC parameter, but the iommu_ops->map() function has been added a gfp_t parameter by commit 781ca2de89ba ("iommu: Add gfp parameter to iommu_ops::map"), thus io_pgtable_ops->map() should use the gfp parameter passed from iommu_ops->map() to allocate page pages, which can avoid wasting the memory allocators atomic pools for some non-atomic contexts. Signed-off-by: Baolin Wang Acked-by: Will Deacon Link: https://lore.kernel.org/r/3093df4cb95497aaf713fca623ce4ecebb197c2e.1591930156.git.baolin.wang@linux.alibaba.com Signed-off-by: Joerg Roedel --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 2 +- drivers/iommu/arm-smmu-v3.c | 2 +- drivers/iommu/arm-smmu.c | 2 +- drivers/iommu/io-pgtable-arm-v7s.c | 18 +++++++++--------- drivers/iommu/io-pgtable-arm.c | 18 +++++++++--------- drivers/iommu/ipmmu-vmsa.c | 2 +- drivers/iommu/msm_iommu.c | 2 +- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/qcom_iommu.c | 2 +- include/linux/io-pgtable.h | 2 +- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index ed28aeba6d59..5a39eee8cf83 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -262,7 +262,7 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, while (len) { size_t pgsize = get_pgsize(iova | paddr, len); - ops->map(ops, iova, paddr, pgsize, prot); + ops->map(ops, iova, paddr, pgsize, prot, GFP_KERNEL); iova += pgsize; paddr += pgsize; len -= pgsize; diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index f578677a5c41..7b59f06b3913 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2850,7 +2850,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map(ops, iova, paddr, size, prot); + return ops->map(ops, iova, paddr, size, prot, gfp); } static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 243bc4cb2705..dc1d2535798a 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1227,7 +1227,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, return -ENODEV; arm_smmu_rpm_get(smmu); - ret = ops->map(ops, iova, paddr, size, prot); + ret = ops->map(ops, iova, paddr, size, prot, gfp); arm_smmu_rpm_put(smmu); return ret; diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 4272fe4e17f4..a688f22cbe3b 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -470,7 +470,7 @@ static arm_v7s_iopte arm_v7s_install_table(arm_v7s_iopte *table, static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, int prot, - int lvl, arm_v7s_iopte *ptep) + int lvl, arm_v7s_iopte *ptep, gfp_t gfp) { struct io_pgtable_cfg *cfg = &data->iop.cfg; arm_v7s_iopte pte, *cptep; @@ -491,7 +491,7 @@ static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = READ_ONCE(*ptep); if (!pte) { - cptep = __arm_v7s_alloc_table(lvl + 1, GFP_ATOMIC, data); + cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data); if (!cptep) return -ENOMEM; @@ -512,11 +512,11 @@ static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, } /* Rinse, repeat */ - return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep); + return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); } static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int prot) + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable *iop = &data->iop; @@ -530,7 +530,7 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, paddr >= (1ULL << data->iop.cfg.oas))) return -ERANGE; - ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd); + ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd, gfp); /* * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. @@ -922,12 +922,12 @@ static int __init arm_v7s_do_selftests(void) if (ops->map(ops, iova, iova, size, IOMMU_READ | IOMMU_WRITE | IOMMU_NOEXEC | - IOMMU_CACHE)) + IOMMU_CACHE, GFP_KERNEL)) return __FAIL(ops); /* Overlapping mappings */ if (!ops->map(ops, iova, iova + size, size, - IOMMU_READ | IOMMU_NOEXEC)) + IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL)) return __FAIL(ops); if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) @@ -946,7 +946,7 @@ static int __init arm_v7s_do_selftests(void) return __FAIL(ops); /* Remap of partial unmap */ - if (ops->map(ops, iova_start + size, size, size, IOMMU_READ)) + if (ops->map(ops, iova_start + size, size, size, IOMMU_READ, GFP_KERNEL)) return __FAIL(ops); if (ops->iova_to_phys(ops, iova_start + size + 42) @@ -967,7 +967,7 @@ static int __init arm_v7s_do_selftests(void) return __FAIL(ops); /* Remap full block */ - if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) + if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL)) return __FAIL(ops); if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 04fbd4bf0ff9..4a5a7b083a9b 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -355,7 +355,7 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, arm_lpae_iopte prot, - int lvl, arm_lpae_iopte *ptep) + int lvl, arm_lpae_iopte *ptep, gfp_t gfp) { arm_lpae_iopte *cptep, pte; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); @@ -376,7 +376,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = READ_ONCE(*ptep); if (!pte) { - cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); + cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg); if (!cptep) return -ENOMEM; @@ -396,7 +396,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, } /* Rinse, repeat */ - return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); + return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); } static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, @@ -461,7 +461,7 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, } static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int iommu_prot) + phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; @@ -483,7 +483,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return -ERANGE; prot = arm_lpae_prot_to_pte(data, iommu_prot); - ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); + ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, gfp); /* * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. @@ -1178,12 +1178,12 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) if (ops->map(ops, iova, iova, size, IOMMU_READ | IOMMU_WRITE | IOMMU_NOEXEC | - IOMMU_CACHE)) + IOMMU_CACHE, GFP_KERNEL)) return __FAIL(ops, i); /* Overlapping mappings */ if (!ops->map(ops, iova, iova + size, size, - IOMMU_READ | IOMMU_NOEXEC)) + IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL)) return __FAIL(ops, i); if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) @@ -1198,7 +1198,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) return __FAIL(ops, i); /* Remap of partial unmap */ - if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) + if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL)) return __FAIL(ops, i); if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) @@ -1216,7 +1216,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) return __FAIL(ops, i); /* Remap full block */ - if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) + if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL)) return __FAIL(ops, i); if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 4c2972f3153b..87475b2f7ef1 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -687,7 +687,7 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, if (!domain) return -ENODEV; - return domain->iop->map(domain->iop, iova, paddr, size, prot); + return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp); } static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index f773cc85f311..3615cd6241c4 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -491,7 +491,7 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova, int ret; spin_lock_irqsave(&priv->pgtlock, flags); - ret = priv->iop->map(priv->iop, iova, pa, len, prot); + ret = priv->iop->map(priv->iop, iova, pa, len, prot, GFP_ATOMIC); spin_unlock_irqrestore(&priv->pgtlock, flags); return ret; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 2be96f1cdbd2..b7b16414a217 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -397,7 +397,7 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, paddr |= BIT_ULL(32); /* Synchronize with the tlb_lock */ - return dom->iop->map(dom->iop, iova, paddr, size, prot); + return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); } static size_t mtk_iommu_unmap(struct iommu_domain *domain, diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index c3e1fbd1988c..cfcfd7553b30 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -441,7 +441,7 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova, return -ENODEV; spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags); - ret = ops->map(ops, iova, paddr, size, prot); + ret = ops->map(ops, iova, paddr, size, prot, GFP_ATOMIC); spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags); return ret; } diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 53d53c6c2be9..23285ba645db 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -155,7 +155,7 @@ struct io_pgtable_cfg { */ struct io_pgtable_ops { int (*map)(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int prot); + phys_addr_t paddr, size_t size, int prot, gfp_t gfp); size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather); phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, From b1012ca8dc4f9b1a1fe8e2cb1590dd6d43ea3849 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 23 Jul 2020 09:34:37 +0800 Subject: [PATCH 74/76] iommu/vt-d: Skip TE disabling on quirky gfx dedicated iommu The VT-d spec requires (10.4.4 Global Command Register, TE field) that: Hardware implementations supporting DMA draining must drain any in-flight DMA read/write requests queued within the Root-Complex before completing the translation enable command and reflecting the status of the command through the TES field in the Global Status register. Unfortunately, some integrated graphic devices fail to do so after some kind of power state transition. As the result, the system might stuck in iommu_disable_translation(), waiting for the completion of TE transition. This provides a quirk list for those devices and skips TE disabling if the qurik hits. Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=208363 Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=206571 Signed-off-by: Lu Baolu Tested-by: Koba Ko Tested-by: Jun Miao Cc: Ashok Raj Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200723013437.2268-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 1 + drivers/iommu/intel/iommu.c | 27 +++++++++++++++++++++++++++ include/linux/dmar.h | 1 + include/linux/intel-iommu.h | 2 ++ 4 files changed, 31 insertions(+) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index f6cbe3f95c8d..93e6345f3414 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1102,6 +1102,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) } drhd->iommu = iommu; + iommu->drhd = drhd; return 0; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b3faf99457a6..0c2d582ff8cd 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -356,6 +356,7 @@ static int intel_iommu_strict; static int intel_iommu_superpage = 1; static int iommu_identity_mapping; static int intel_no_bounce; +static int iommu_skip_te_disable; #define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 @@ -1633,6 +1634,10 @@ static void iommu_disable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flag; + if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated && + (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap))) + return; + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->gcmd &= ~DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -4043,6 +4048,7 @@ static void __init init_no_remapping_devices(void) /* This IOMMU has *only* gfx devices. Either bypass it or set the gfx_mapped flag, as appropriate */ + drhd->gfx_dedicated = 1; if (!dmar_map_gfx) { drhd->ignored = 1; for_each_active_dev_scope(drhd->devices, @@ -6170,6 +6176,27 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_g DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt); +static void quirk_igfx_skip_te_disable(struct pci_dev *dev) +{ + unsigned short ver; + + if (!IS_GFX_DEVICE(dev)) + return; + + ver = (dev->device >> 8) & 0xff; + if (ver != 0x45 && ver != 0x46 && ver != 0x4c && + ver != 0x4e && ver != 0x8a && ver != 0x98 && + ver != 0x9a) + return; + + if (risky_device(dev)) + return; + + pci_info(dev, "Skip IOMMU disabling for graphics\n"); + iommu_skip_te_disable = 1; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable); + /* On Tylersburg chipsets, some BIOSes have been known to enable the ISOCH DMAR unit for the Azalia sound device, but not give it any TLB entries, which causes it to deadlock. Check for that. We do diff --git a/include/linux/dmar.h b/include/linux/dmar.h index d7bf029df737..65565820328a 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -48,6 +48,7 @@ struct dmar_drhd_unit { u16 segment; /* PCI domain */ u8 ignored:1; /* ignore drhd */ u8 include_all:1; + u8 gfx_dedicated:1; /* graphic dedicated */ struct intel_iommu *iommu; }; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 9ff5e340948b..b1ed2f25f7c0 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -599,6 +599,8 @@ struct intel_iommu { struct iommu_device iommu; /* IOMMU core code handle */ int node; u32 flags; /* Software defined flags */ + + struct dmar_drhd_unit *drhd; }; /* PCI domain-device relationship */ From ab65ba57e3acb55920999f96a6152228b52a2f49 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 30 Jun 2020 13:06:35 -0700 Subject: [PATCH 75/76] iommu/vt-d: Move Kconfig and Makefile bits down into intel directory Move Intel Kconfig and Makefile bits down into intel directory with the rest of the Intel specific files. Signed-off-by: Jerry Snitselaar Reviewed-by: Lu Baolu Cc: Joerg Roedel Cc: Lu Baolu Link: https://lore.kernel.org/r/20200630200636.48600-2-jsnitsel@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 86 +----------------------------------- drivers/iommu/Makefile | 8 +--- drivers/iommu/intel/Kconfig | 86 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/Makefile | 7 +++ 4 files changed, 96 insertions(+), 91 deletions(-) create mode 100644 drivers/iommu/intel/Kconfig create mode 100644 drivers/iommu/intel/Makefile diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index af6c081d164f..e9e1238aac21 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -172,91 +172,7 @@ config AMD_IOMMU_DEBUGFS This option is -NOT- intended for production environments, and should not generally be enabled. -# Intel IOMMU support -config DMAR_TABLE - bool - -config INTEL_IOMMU - bool "Support for Intel IOMMU using DMA Remapping Devices" - depends on PCI_MSI && ACPI && (X86 || IA64) - select IOMMU_API - select IOMMU_IOVA - select NEED_DMA_MAP_STATE - select DMAR_TABLE - select SWIOTLB - select IOASID - help - DMA remapping (DMAR) devices support enables independent address - translations for Direct Memory Access (DMA) from devices. - These DMA remapping devices are reported via ACPI tables - and include PCI device scope covered by these DMA - remapping devices. - -config INTEL_IOMMU_DEBUGFS - bool "Export Intel IOMMU internals in Debugfs" - depends on INTEL_IOMMU && IOMMU_DEBUGFS - help - !!!WARNING!!! - - DO NOT ENABLE THIS OPTION UNLESS YOU REALLY KNOW WHAT YOU ARE DOING!!! - - Expose Intel IOMMU internals in Debugfs. - - This option is -NOT- intended for production environments, and should - only be enabled for debugging Intel IOMMU. - -config INTEL_IOMMU_SVM - bool "Support for Shared Virtual Memory with Intel IOMMU" - depends on INTEL_IOMMU && X86_64 - select PCI_PASID - select PCI_PRI - select MMU_NOTIFIER - select IOASID - help - Shared Virtual Memory (SVM) provides a facility for devices - to access DMA resources through process address space by - means of a Process Address Space ID (PASID). - -config INTEL_IOMMU_DEFAULT_ON - def_bool y - prompt "Enable Intel DMA Remapping Devices by default" - depends on INTEL_IOMMU - help - Selecting this option will enable a DMAR device at boot time if - one is found. If this option is not selected, DMAR support can - be enabled by passing intel_iommu=on to the kernel. - -config INTEL_IOMMU_BROKEN_GFX_WA - bool "Workaround broken graphics drivers (going away soon)" - depends on INTEL_IOMMU && BROKEN && X86 - help - Current Graphics drivers tend to use physical address - for DMA and avoid using DMA APIs. Setting this config - option permits the IOMMU driver to set a unity map for - all the OS-visible memory. Hence the driver can continue - to use physical addresses for DMA, at least until this - option is removed in the 2.6.32 kernel. - -config INTEL_IOMMU_FLOPPY_WA - def_bool y - depends on INTEL_IOMMU && X86 - help - Floppy disk drivers are known to bypass DMA API calls - thereby failing to work when IOMMU is enabled. This - workaround will setup a 1:1 mapping for the first - 16MiB to make floppy (an ISA device) work. - -config INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON - bool "Enable Intel IOMMU scalable mode by default" - depends on INTEL_IOMMU - help - Selecting this option will enable by default the scalable mode if - hardware presents the capability. The scalable mode is defined in - VT-d 3.0. The scalable mode capability could be checked by reading - /sys/devices/virtual/iommu/dmar*/intel-iommu/ecap. If this option - is not selected, scalable mode support could also be enabled by - passing intel_iommu=sm_on to the kernel. If not sure, please use - the default value. +source "drivers/iommu/intel/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 342190196dfb..71dd2f382e78 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +obj-y += intel/ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o @@ -17,13 +18,8 @@ obj-$(CONFIG_AMD_IOMMU_V2) += amd/iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o -obj-$(CONFIG_DMAR_TABLE) += intel/dmar.o -obj-$(CONFIG_INTEL_IOMMU) += intel/iommu.o intel/pasid.o -obj-$(CONFIG_INTEL_IOMMU) += intel/trace.o -obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel/debugfs.o -obj-$(CONFIG_INTEL_IOMMU_SVM) += intel/svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o -obj-$(CONFIG_IRQ_REMAP) += intel/irq_remapping.o irq_remapping.o +obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig new file mode 100644 index 000000000000..877beec9d987 --- /dev/null +++ b/drivers/iommu/intel/Kconfig @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Intel IOMMU support +config DMAR_TABLE + bool + +config INTEL_IOMMU + bool "Support for Intel IOMMU using DMA Remapping Devices" + depends on PCI_MSI && ACPI && (X86 || IA64) + select IOMMU_API + select IOMMU_IOVA + select NEED_DMA_MAP_STATE + select DMAR_TABLE + select SWIOTLB + select IOASID + help + DMA remapping (DMAR) devices support enables independent address + translations for Direct Memory Access (DMA) from devices. + These DMA remapping devices are reported via ACPI tables + and include PCI device scope covered by these DMA + remapping devices. + +config INTEL_IOMMU_DEBUGFS + bool "Export Intel IOMMU internals in Debugfs" + depends on INTEL_IOMMU && IOMMU_DEBUGFS + help + !!!WARNING!!! + + DO NOT ENABLE THIS OPTION UNLESS YOU REALLY KNOW WHAT YOU ARE DOING!!! + + Expose Intel IOMMU internals in Debugfs. + + This option is -NOT- intended for production environments, and should + only be enabled for debugging Intel IOMMU. + +config INTEL_IOMMU_SVM + bool "Support for Shared Virtual Memory with Intel IOMMU" + depends on INTEL_IOMMU && X86_64 + select PCI_PASID + select PCI_PRI + select MMU_NOTIFIER + select IOASID + help + Shared Virtual Memory (SVM) provides a facility for devices + to access DMA resources through process address space by + means of a Process Address Space ID (PASID). + +config INTEL_IOMMU_DEFAULT_ON + def_bool y + prompt "Enable Intel DMA Remapping Devices by default" + depends on INTEL_IOMMU + help + Selecting this option will enable a DMAR device at boot time if + one is found. If this option is not selected, DMAR support can + be enabled by passing intel_iommu=on to the kernel. + +config INTEL_IOMMU_BROKEN_GFX_WA + bool "Workaround broken graphics drivers (going away soon)" + depends on INTEL_IOMMU && BROKEN && X86 + help + Current Graphics drivers tend to use physical address + for DMA and avoid using DMA APIs. Setting this config + option permits the IOMMU driver to set a unity map for + all the OS-visible memory. Hence the driver can continue + to use physical addresses for DMA, at least until this + option is removed in the 2.6.32 kernel. + +config INTEL_IOMMU_FLOPPY_WA + def_bool y + depends on INTEL_IOMMU && X86 + help + Floppy disk drivers are known to bypass DMA API calls + thereby failing to work when IOMMU is enabled. This + workaround will setup a 1:1 mapping for the first + 16MiB to make floppy (an ISA device) work. + +config INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON + bool "Enable Intel IOMMU scalable mode by default" + depends on INTEL_IOMMU + help + Selecting this option will enable by default the scalable mode if + hardware presents the capability. The scalable mode is defined in + VT-d 3.0. The scalable mode capability could be checked by reading + /sys/devices/virtual/iommu/dmar*/intel-iommu/ecap. If this option + is not selected, scalable mode support could also be enabled by + passing intel_iommu=sm_on to the kernel. If not sure, please use + the default value. diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile new file mode 100644 index 000000000000..fb8e1e8c8029 --- /dev/null +++ b/drivers/iommu/intel/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_DMAR_TABLE) += dmar.o +obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o +obj-$(CONFIG_INTEL_IOMMU) += trace.o +obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o +obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o +obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o From cbe94c6e1a7d11050050c4d5b89bb278c163e8d6 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 30 Jun 2020 13:06:36 -0700 Subject: [PATCH 76/76] iommu/amd: Move Kconfig and Makefile bits down into amd directory Move AMD Kconfig and Makefile bits down into the amd directory with the rest of the AMD specific files. Signed-off-by: Jerry Snitselaar Cc: Joerg Roedel Cc: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20200630200636.48600-3-jsnitsel@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 45 +------------------------------------- drivers/iommu/Makefile | 5 +---- drivers/iommu/amd/Kconfig | 44 +++++++++++++++++++++++++++++++++++++ drivers/iommu/amd/Makefile | 4 ++++ 4 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 drivers/iommu/amd/Kconfig create mode 100644 drivers/iommu/amd/Makefile diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index e9e1238aac21..8ef9860b8d7f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -128,50 +128,7 @@ config MSM_IOMMU If unsure, say N here. -# AMD IOMMU support -config AMD_IOMMU - bool "AMD IOMMU support" - select SWIOTLB - select PCI_MSI - select PCI_ATS - select PCI_PRI - select PCI_PASID - select IOMMU_API - select IOMMU_IOVA - select IOMMU_DMA - depends on X86_64 && PCI && ACPI - help - With this option you can enable support for AMD IOMMU hardware in - your system. An IOMMU is a hardware component which provides - remapping of DMA memory accesses from devices. With an AMD IOMMU you - can isolate the DMA memory of different devices and protect the - system from misbehaving device drivers or hardware. - - You can find out if your system has an AMD IOMMU if you look into - your BIOS for an option to enable it or if you have an IVRS ACPI - table. - -config AMD_IOMMU_V2 - tristate "AMD IOMMU Version 2 driver" - depends on AMD_IOMMU - select MMU_NOTIFIER - help - This option enables support for the AMD IOMMUv2 features of the IOMMU - hardware. Select this option if you want to use devices that support - the PCI PRI and PASID interface. - -config AMD_IOMMU_DEBUGFS - bool "Enable AMD IOMMU internals in DebugFS" - depends on AMD_IOMMU && IOMMU_DEBUGFS - help - !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! - - DO NOT ENABLE THIS OPTION UNLESS YOU REALLY, -REALLY- KNOW WHAT YOU ARE DOING!!! - Exposes AMD IOMMU device internals in DebugFS. - - This option is -NOT- intended for production environments, and should - not generally be enabled. - +source "drivers/iommu/amd/Kconfig" source "drivers/iommu/intel/Kconfig" config IRQ_REMAP diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 71dd2f382e78..f356bc12b1c7 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += intel/ +obj-y += amd/ intel/ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o @@ -12,9 +12,6 @@ obj-$(CONFIG_IOASID) += ioasid.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o -obj-$(CONFIG_AMD_IOMMU) += amd/iommu.o amd/init.o amd/quirks.o -obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd/debugfs.o -obj-$(CONFIG_AMD_IOMMU_V2) += amd/iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig new file mode 100644 index 000000000000..1f061d91e0b8 --- /dev/null +++ b/drivers/iommu/amd/Kconfig @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +# AMD IOMMU support +config AMD_IOMMU + bool "AMD IOMMU support" + select SWIOTLB + select PCI_MSI + select PCI_ATS + select PCI_PRI + select PCI_PASID + select IOMMU_API + select IOMMU_IOVA + select IOMMU_DMA + depends on X86_64 && PCI && ACPI + help + With this option you can enable support for AMD IOMMU hardware in + your system. An IOMMU is a hardware component which provides + remapping of DMA memory accesses from devices. With an AMD IOMMU you + can isolate the DMA memory of different devices and protect the + system from misbehaving device drivers or hardware. + + You can find out if your system has an AMD IOMMU if you look into + your BIOS for an option to enable it or if you have an IVRS ACPI + table. + +config AMD_IOMMU_V2 + tristate "AMD IOMMU Version 2 driver" + depends on AMD_IOMMU + select MMU_NOTIFIER + help + This option enables support for the AMD IOMMUv2 features of the IOMMU + hardware. Select this option if you want to use devices that support + the PCI PRI and PASID interface. + +config AMD_IOMMU_DEBUGFS + bool "Enable AMD IOMMU internals in DebugFS" + depends on AMD_IOMMU && IOMMU_DEBUGFS + help + !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! + + DO NOT ENABLE THIS OPTION UNLESS YOU REALLY, -REALLY- KNOW WHAT YOU ARE DOING!!! + Exposes AMD IOMMU device internals in DebugFS. + + This option is -NOT- intended for production environments, and should + not generally be enabled. diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile new file mode 100644 index 000000000000..dc5a2fa4fd37 --- /dev/null +++ b/drivers/iommu/amd/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o +obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o +obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o