From 54c6d242fa32cba8313936e3a35f27dc2c7c3e04 Mon Sep 17 00:00:00 2001 From: Cosmin-Gabriel Samoila Date: Sun, 13 Mar 2016 02:17:27 +0200 Subject: iommu/io-pgtable: Fix a brace coding style issue. Fixed a coding style issue. Signed-off-by: Cosmin-Gabriel Samoila Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 876f6a76d288..127558d83667 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -25,8 +25,7 @@ #include "io-pgtable.h" static const struct io_pgtable_init_fns * -io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = -{ +io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE [ARM_32_LPAE_S1] = &io_pgtable_arm_32_lpae_s1_init_fns, [ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns, -- cgit v1.2.3 From 1afe23194d0580bc332fe27c4e8717f6562348c5 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 14 Mar 2016 06:01:10 +0800 Subject: iommu/io-pgtable: Add MTK 4GB mode in Short-descriptor In MT8173, Normally the first 1GB PA is for the HW SRAM and Regs, so the PA will be 33bits if the dram size is 4GB. We have a "DRAM 4GB mode" toggle bit for this. If it's enabled, from CPU's point of view, the dram PA will be from 0x1_00000000~0x1_ffffffff. In short descriptor, the pagetable descriptor is always 32bit. Mediatek extend bit9 in the lvl1 and lvl2 pgtable descriptor as the 4GB mode. In the 4GB mode, the bit9 must be set, then M4U help add 0x1_00000000 based on the PA in pagetable. Thus the M4U output address to EMI is always 33bits(the input address is still 32bits). We add a special quirk for this MTK-4GB mode. And in the standard spec, Bit9 in the lvl1 is "IMPLEMENTATION DEFINED", while it's AP[2] in the lvl2, therefore if this quirk is enabled, NO_PERMS is also expected. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 13 ++++++++++++- drivers/iommu/io-pgtable.h | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 9488e3c97bcb..0f93dc2f98a3 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -121,6 +121,8 @@ #define ARM_V7S_TEX_MASK 0x7 #define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT) +#define ARM_V7S_ATTR_MTK_4GB BIT(9) /* MTK extend it for 4GB mode */ + /* *well, except for TEX on level 2 large pages, of course :( */ #define ARM_V7S_CONT_PAGE_TEX_SHIFT 6 #define ARM_V7S_CONT_PAGE_TEX_MASK (ARM_V7S_TEX_MASK << ARM_V7S_CONT_PAGE_TEX_SHIFT) @@ -364,6 +366,9 @@ static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) pte |= ARM_V7S_ATTR_NS_SECTION; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_4GB) + pte |= ARM_V7S_ATTR_MTK_4GB; + if (num_entries > 1) pte = arm_v7s_pte_to_cont(pte, lvl); @@ -625,9 +630,15 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | - IO_PGTABLE_QUIRK_TLBI_ON_MAP)) + IO_PGTABLE_QUIRK_TLBI_ON_MAP | + IO_PGTABLE_QUIRK_ARM_MTK_4GB)) return NULL; + /* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_4GB && + !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS)) + return NULL; + data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index d4f502742e3b..969d82cc92ca 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -60,10 +60,16 @@ struct io_pgtable_cfg { * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid * (unmapped) entries but the hardware might do so anyway, perform * TLB maintenance when mapping as well as when unmapping. + * + * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all + * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit + * when the SoC is in "4GB mode" and they can only access the high + * remap of DRAM (0x1_00000000 to 0x1_ffffffff). */ #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) + #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; -- cgit v1.2.3 From 01e23c93868884327828a01e864135f05e515ae5 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 14 Mar 2016 06:01:11 +0800 Subject: iommu/mediatek: Add 4GB mode support This patch add 4GB mode support for m4u. Signed-off-by: Yong Wu Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 929a66a81b2b..db745533ea92 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -11,6 +11,7 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ +#include #include #include #include @@ -56,7 +57,7 @@ #define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) #define REG_MMU_IVRP_PADDR 0x114 -#define F_MMU_IVRP_PA_SET(pa) ((pa) >> 1) +#define F_MMU_IVRP_PA_SET(pa, ext) (((pa) >> 1) | ((!!(ext)) << 31)) #define REG_MMU_INT_CONTROL0 0x120 #define F_L2_MULIT_HIT_EN BIT(0) @@ -125,6 +126,7 @@ struct mtk_iommu_data { struct mtk_iommu_domain *m4u_dom; struct iommu_group *m4u_group; struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ + bool enable_4GB; }; static struct iommu_ops mtk_iommu_ops; @@ -257,6 +259,9 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) .iommu_dev = data->dev, }; + if (data->enable_4GB) + dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_4GB; + dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); if (!dom->iop) { dev_err(data->dev, "Failed to alloc io pgtable\n"); @@ -530,7 +535,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) F_INT_PRETETCH_TRANSATION_FIFO_FAULT; writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); - writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), data->base + REG_MMU_IVRP_PADDR); writel_relaxed(0, data->base + REG_MMU_DCM_DIS); @@ -591,6 +596,9 @@ static int mtk_iommu_probe(struct platform_device *pdev) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); + /* Whether the current dram is over 4GB */ + data->enable_4GB = !!(max_pfn > (0xffffffffUL >> PAGE_SHIFT)); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); data->base = devm_ioremap_resource(dev, res); if (IS_ERR(data->base)) @@ -690,7 +698,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); - writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), base + REG_MMU_IVRP_PADDR); return 0; } -- cgit v1.2.3 From c43fce4eebae257ca413733690e2076757282093 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Thu, 17 Mar 2016 14:12:25 -0600 Subject: iommu/vt-d: Ratelimit fault handler Fault rates can easily overwhelm the console and make the system unresponsive. Ratelimit to allow an opportunity for maintenance. Signed-off-by: Alex Williamson Fixes: 0ac2491f57af ('x86, dmar: move page fault handling code to dmar.c') Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 8ffd7568fc91..8f8bfffbf1e6 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1602,10 +1602,17 @@ irqreturn_t dmar_fault(int irq, void *dev_id) int reg, fault_index; u32 fault_status; unsigned long flag; + bool ratelimited; + static DEFINE_RATELIMIT_STATE(rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + /* Disable printing, simply clear the fault when ratelimited */ + ratelimited = !__ratelimit(&rs); raw_spin_lock_irqsave(&iommu->register_lock, flag); fault_status = readl(iommu->reg + DMAR_FSTS_REG); - if (fault_status) + if (fault_status && !ratelimited) pr_err("DRHD: handling fault status reg %x\n", fault_status); /* TBD: ignore advanced fault log currently */ @@ -1627,24 +1634,28 @@ irqreturn_t dmar_fault(int irq, void *dev_id) if (!(data & DMA_FRCD_F)) break; - fault_reason = dma_frcd_fault_reason(data); - type = dma_frcd_type(data); + if (!ratelimited) { + fault_reason = dma_frcd_fault_reason(data); + type = dma_frcd_type(data); - data = readl(iommu->reg + reg + - fault_index * PRIMARY_FAULT_REG_LEN + 8); - source_id = dma_frcd_source_id(data); + data = readl(iommu->reg + reg + + fault_index * PRIMARY_FAULT_REG_LEN + 8); + source_id = dma_frcd_source_id(data); + + guest_addr = dmar_readq(iommu->reg + reg + + fault_index * PRIMARY_FAULT_REG_LEN); + guest_addr = dma_frcd_page_addr(guest_addr); + } - guest_addr = dmar_readq(iommu->reg + reg + - fault_index * PRIMARY_FAULT_REG_LEN); - guest_addr = dma_frcd_page_addr(guest_addr); /* clear the fault */ writel(DMA_FRCD_F, iommu->reg + reg + fault_index * PRIMARY_FAULT_REG_LEN + 12); raw_spin_unlock_irqrestore(&iommu->register_lock, flag); - dmar_fault_do_one(iommu, type, fault_reason, - source_id, guest_addr); + if (!ratelimited) + dmar_fault_do_one(iommu, type, fault_reason, + source_id, guest_addr); fault_index++; if (fault_index >= cap_num_fault_regs(iommu->cap)) -- cgit v1.2.3 From a0fe14d7dcf5db2f101b4fe8689ecabb255ab7d3 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Thu, 17 Mar 2016 14:12:31 -0600 Subject: iommu/vt-d: Improve fault handler error messages Remove new line in error logs, avoid duplicate and explicit pr_fmt. Suggested-by: Joe Perches Signed-off-by: Alex Williamson Fixes: 0ac2491f57af ('x86, dmar: move page fault handling code to dmar.c') Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 8f8bfffbf1e6..6a86b5d1defa 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1579,18 +1579,14 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, reason = dmar_get_fault_reason(fault_reason, &fault_type); if (fault_type == INTR_REMAP) - pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] " - "fault index %llx\n" - "INTR-REMAP:[fault reason %02d] %s\n", - (source_id >> 8), PCI_SLOT(source_id & 0xFF), + pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n", + source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr >> 48, fault_reason, reason); else - pr_err("DMAR:[%s] Request device [%02x:%02x.%d] " - "fault addr %llx \n" - "DMAR:[fault reason %02d] %s\n", - (type ? "DMA Read" : "DMA Write"), - (source_id >> 8), PCI_SLOT(source_id & 0xFF), + pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n", + type ? "DMA Read" : "DMA Write", + source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason); return 0; } -- cgit v1.2.3 From 8d7f2d84ed2d44b05e1ce88fa4b74886af46a139 Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Mon, 21 Mar 2016 12:00:23 +0100 Subject: iommu/rockchip: Don't feed NULL res pointers to devres If we do, devres prints a "invalid resource" string in the error loglevel. Signed-off-by: Tomeu Vizoso Reviewed-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index a6f593a0a29e..0253ab35c33b 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1049,6 +1049,8 @@ static int rk_iommu_probe(struct platform_device *pdev) for (i = 0; i < pdev->num_resources; i++) { res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) + continue; iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(iommu->bases[i])) continue; -- cgit v1.2.3 From 3d1a2442d2c08c872150db7a554647c06f6e864f Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Wed, 23 Mar 2016 20:34:19 +0200 Subject: x86/vt-d: Fix comment for dma_pte_free_pagetable() dma_pte_free_pagetable no longer depends on last level ptes being clear, it clears them itself. Fix up the comment to match. Cc: Jiang Liu Suggested-by: Alex Williamson Signed-off-by: Michael S. Tsirkin Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a2e1b7f14df2..01005531891b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1143,7 +1143,7 @@ next: } while (!first_pte_in_page(++pte) && pfn <= last_pfn); } -/* free page table pages. last level pte should already be cleared */ +/* clear last level (leaf) ptes and free page table pages. */ static void dma_pte_free_pagetable(struct dmar_domain *domain, unsigned long start_pfn, unsigned long last_pfn) -- cgit v1.2.3 From 521f40823ebf2818045546d45aa378eba1c41717 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 4 Apr 2016 17:46:18 -0500 Subject: iommu/omap: Remove iopgtable_clear_entry_all() from driver remove The function iopgtable_clear_entry_all() is used for clearing all the page table entries. These entries are neither created nor initialized during the OMAP IOMMU driver probe, and are managed only when a client device attaches to the IOMMU. So, there is no need to invoke this function on a driver remove. Removing this fixes a NULL pointer dereference crash if the IOMMU device is unbound from the driver with no client device attached to the IOMMU device. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 3dc5b65f3990..c05d48f88596 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -987,7 +987,6 @@ static int omap_iommu_remove(struct platform_device *pdev) { struct omap_iommu *obj = platform_get_drvdata(pdev); - iopgtable_clear_entry_all(obj); omap_iommu_debugfs_remove(obj); pm_runtime_disable(obj->dev); -- cgit v1.2.3 From 7c1ab60008755fc229b623a875f1e42de31cdb64 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 4 Apr 2016 17:46:19 -0500 Subject: iommu/omap: Replace BUG() in iopgtable_store_entry_core() The iopgtable_store_entry_core() function uses a BUG() statement for an unsupported page size entry programming. Replace this with a less severe WARN_ON() and perform a graceful bailout on error. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index c05d48f88596..f6cf728ee32a 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -628,10 +628,12 @@ iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) break; default: fn = NULL; - BUG(); break; } + if (WARN_ON(!fn)) + return -EINVAL; + prot = get_iopte_attr(e); spin_lock(&obj->page_table_lock); -- cgit v1.2.3 From 433c434a269e6a39753f0a97b4be903f96d030a9 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 4 Apr 2016 17:46:20 -0500 Subject: iommu/omap: Use WARN_ON for page table alignment check The OMAP IOMMU page table needs to be aligned on a 16K boundary, and the current code uses a BUG_ON on the alignment sanity check in the .domain_alloc() ops implementation. Replace this with a less severe WARN_ON and bail out gracefully. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index f6cf728ee32a..e2583cce2cc1 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1162,7 +1162,8 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) * should never fail, but please keep this around to ensure * we keep the hardware happy */ - BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)); + if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE))) + goto fail_align; clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE); spin_lock_init(&omap_domain->lock); @@ -1173,6 +1174,8 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) return &omap_domain->domain; +fail_align: + kfree(omap_domain->pgtable); fail_nomem: kfree(omap_domain); out: -- cgit v1.2.3 From a5c0e0b4ac07434056c5c4ecafeb9a5cff3d1a9d Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 4 Apr 2016 17:46:21 -0500 Subject: iommu/omap: Align code with open parenthesis This patch fixes one existing alignment checkpatch check warning of the type "Alignment should match open parenthesis" in the OMAP IOMMU debug source file. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu-debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index 9bc20e2119a3..505548aafeff 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -136,7 +136,7 @@ static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, struct seq_file *s) { seq_printf(s, "%08x %08x %01x\n", cr->cam, cr->ram, - (cr->cam & MMU_CAM_P) ? 1 : 0); + (cr->cam & MMU_CAM_P) ? 1 : 0); return 0; } -- cgit v1.2.3 From 7d7d38afb3e8fdfebfd867cc0ff4b5c45c14053c Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Fri, 1 Apr 2016 09:05:57 -0400 Subject: iommu/amd: Adding Extended Feature Register check for PC support The IVHD header type 11h and 40h introduce the PCSup bit in the EFR Register Image bit fileds. This should be used to determine the IOMMU performance support instead of relying on the PNCounters and PNBanks. Note also that the PNCouters and PNBanks bits in the IOMMU attributes field of IVHD headers type 11h are incorrectly programmed on some systems. So, we should not rely on it to determine the performance counter/banks size. Instead, these values should be read from the MMIO Offset 0030h IOMMU Extended Feature Register. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index bf4959f4225b..dff1e01174d1 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -99,7 +99,11 @@ struct ivhd_header { u64 mmio_phys; u16 pci_seg; u16 info; - u32 efr; + u32 efr_attr; + + /* Following only valid on IVHD type 11h and 40h */ + u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ + u64 res; } __attribute__((packed)); /* @@ -1078,13 +1082,25 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->pci_seg = h->pci_seg; iommu->mmio_phys = h->mmio_phys; - /* Check if IVHD EFR contains proper max banks/counters */ - if ((h->efr != 0) && - ((h->efr & (0xF << 13)) != 0) && - ((h->efr & (0x3F << 17)) != 0)) { - iommu->mmio_phys_end = MMIO_REG_END_OFFSET; - } else { - iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + switch (h->type) { + case 0x10: + /* Check if IVHD EFR contains proper max banks/counters */ + if ((h->efr_attr != 0) && + ((h->efr_attr & (0xF << 13)) != 0) && + ((h->efr_attr & (0x3F << 17)) != 0)) + iommu->mmio_phys_end = MMIO_REG_END_OFFSET; + else + iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + break; + case 0x11: + case 0x40: + if (h->efr_reg & (1 << 9)) + iommu->mmio_phys_end = MMIO_REG_END_OFFSET; + else + iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + break; + default: + return -EINVAL; } iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys, -- cgit v1.2.3 From ac7ccf6765af5a255cec82fa95ec11f6c769022c Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Fri, 1 Apr 2016 09:05:58 -0400 Subject: iommu/amd: Modify ivhd_header structure to support type 11h and 40h This patch modifies the existing struct ivhd_header, which currently only support IVHD type 0x10, to add new fields from IVHD type 11h and 40h. It also modifies the pointer calculation to allow support for IVHD type 11h and 40h Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index dff1e01174d1..22e078bfe35b 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -398,6 +398,22 @@ static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); } +static inline u32 get_ivhd_header_size(struct ivhd_header *h) +{ + u32 size = 0; + + switch (h->type) { + case 0x10: + size = 24; + break; + case 0x11: + case 0x40: + size = 40; + break; + } + return size; +} + /**************************************************************************** * * The functions below belong to the first pass of AMD IOMMU ACPI table @@ -424,7 +440,14 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) u8 *p = (void *)h, *end = (void *)h; struct ivhd_entry *dev; - p += sizeof(*h); + u32 ivhd_size = get_ivhd_header_size(h); + + if (!ivhd_size) { + pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type); + return -EINVAL; + } + + p += ivhd_size; end += h->length; while (p < end) { @@ -789,6 +812,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, u32 dev_i, ext_flags = 0; bool alias = false; struct ivhd_entry *e; + u32 ivhd_size; int ret; @@ -804,7 +828,14 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, /* * Done. Now parse the device entries */ - p += sizeof(struct ivhd_header); + ivhd_size = get_ivhd_header_size(h); + if (!ivhd_size) { + pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type); + return -EINVAL; + } + + p += ivhd_size; + end += h->length; -- cgit v1.2.3 From 8c7142f56fedfc6824b5bca56fee1f443e01746b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Fri, 1 Apr 2016 09:05:59 -0400 Subject: iommu/amd: Use the most comprehensive IVHD type that the driver can support The IVRS in more recent AMD system usually contains multiple IVHD block types (e.g. 0x10, 0x11, and 0x40) for each IOMMU. The newer IVHD types provide more information (e.g. new features specified in the IOMMU spec), while maintain compatibility with the older IVHD type. Having multiple IVHD type allows older IOMMU drivers to still function (e.g. using the older IVHD type 0x10) while the newer IOMMU driver can use the newer IVHD types (e.g. 0x11 and 0x40). Therefore, the IOMMU driver should only make use of the newest IVHD type that it can support. This patch adds new logic to determine the highest level of IVHD type it can support, and use it throughout the to initialize the driver. This requires adding another pass to the IVRS parsing to determine appropriate IVHD type (see function get_highest_supported_ivhd_type()) before parsing the contents. [Vincent: fix the build error of IVHD_DEV_ACPI_HID flag not found] Signed-off-by: Wan Zongshun Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 107 ++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 29 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 22e078bfe35b..8f4961225724 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -44,7 +44,7 @@ */ #define IVRS_HEADER_LENGTH 48 -#define ACPI_IVHD_TYPE 0x10 +#define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 #define ACPI_IVMD_TYPE_ALL 0x20 #define ACPI_IVMD_TYPE 0x21 #define ACPI_IVMD_TYPE_RANGE 0x22 @@ -58,6 +58,7 @@ #define IVHD_DEV_EXT_SELECT 0x46 #define IVHD_DEV_EXT_SELECT_RANGE 0x47 #define IVHD_DEV_SPECIAL 0x48 +#define IVHD_DEV_ACPI_HID 0xf0 #define IVHD_SPECIAL_IOAPIC 1 #define IVHD_SPECIAL_HPET 2 @@ -137,6 +138,7 @@ bool amd_iommu_irq_remap __read_mostly; static bool amd_iommu_detected; static bool __initdata amd_iommu_disabled; +static int amd_iommu_target_ivhd_type; u16 amd_iommu_last_bdf; /* largest PCI device id we have to handle */ @@ -428,7 +430,15 @@ static inline u32 get_ivhd_header_size(struct ivhd_header *h) */ static inline int ivhd_entry_length(u8 *ivhd) { - return 0x04 << (*ivhd >> 6); + u32 type = ((struct ivhd_entry *)ivhd)->type; + + if (type < 0x80) { + return 0x04 << (*ivhd >> 6); + } else if (type == IVHD_DEV_ACPI_HID) { + /* For ACPI_HID, offset 21 is uid len */ + return *((u8 *)ivhd + 21) + 22; + } + return 0; } /* @@ -475,6 +485,22 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) return 0; } +static int __init check_ivrs_checksum(struct acpi_table_header *table) +{ + int i; + u8 checksum = 0, *p = (u8 *)table; + + for (i = 0; i < table->length; ++i) + checksum += p[i]; + if (checksum != 0) { + /* ACPI table corrupt */ + pr_err(FW_BUG "AMD-Vi: IVRS invalid checksum\n"); + return -ENODEV; + } + + return 0; +} + /* * Iterate over all IVHD entries in the ACPI table and find the highest device * id which we need to handle. This is the first of three functions which parse @@ -482,31 +508,19 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) */ static int __init find_last_devid_acpi(struct acpi_table_header *table) { - int i; - u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table; + u8 *p = (u8 *)table, *end = (u8 *)table; struct ivhd_header *h; - /* - * Validate checksum here so we don't need to do it when - * we actually parse the table - */ - for (i = 0; i < table->length; ++i) - checksum += p[i]; - if (checksum != 0) - /* ACPI table corrupt */ - return -ENODEV; - p += IVRS_HEADER_LENGTH; end += table->length; while (p < end) { h = (struct ivhd_header *)p; - switch (h->type) { - case ACPI_IVHD_TYPE: - find_last_devid_from_ivhd(h); - break; - default: - break; + if (h->type == amd_iommu_target_ivhd_type) { + int ret = find_last_devid_from_ivhd(h); + + if (ret) + return ret; } p += h->length; } @@ -1164,6 +1178,32 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) return 0; } +/** + * get_highest_supported_ivhd_type - Look up the appropriate IVHD type + * @ivrs Pointer to the IVRS header + * + * This function search through all IVDB of the maximum supported IVHD + */ +static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) +{ + u8 *base = (u8 *)ivrs; + struct ivhd_header *ivhd = (struct ivhd_header *) + (base + IVRS_HEADER_LENGTH); + u8 last_type = ivhd->type; + u16 devid = ivhd->devid; + + while (((u8 *)ivhd - base < ivrs->length) && + (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) { + u8 *p = (u8 *) ivhd; + + if (ivhd->devid == devid) + last_type = ivhd->type; + ivhd = (struct ivhd_header *)(p + ivhd->length); + } + + return last_type; +} + /* * Iterates over all IOMMU entries in the ACPI table, allocates the * IOMMU structure and initializes it with init_iommu_one() @@ -1180,8 +1220,7 @@ static int __init init_iommu_all(struct acpi_table_header *table) while (p < end) { h = (struct ivhd_header *)p; - switch (*p) { - case ACPI_IVHD_TYPE: + if (*p == amd_iommu_target_ivhd_type) { DUMP_printk("device: %02x:%02x.%01x cap: %04x " "seg: %d flags: %01x info %04x\n", @@ -1198,9 +1237,6 @@ static int __init init_iommu_all(struct acpi_table_header *table) ret = init_iommu_one(iommu, h); if (ret) return ret; - break; - default: - break; } p += h->length; @@ -1865,18 +1901,20 @@ static void __init free_dma_resources(void) * remapping setup code. * * This function basically parses the ACPI table for AMD IOMMU (IVRS) - * three times: + * four times: * - * 1 pass) Find the highest PCI device id the driver has to handle. + * 1 pass) Discover the most comprehensive IVHD type to use. + * + * 2 pass) Find the highest PCI device id the driver has to handle. * Upon this information the size of the data structures is * determined that needs to be allocated. * - * 2 pass) Initialize the data structures just allocated with the + * 3 pass) Initialize the data structures just allocated with the * information in the ACPI table about available AMD IOMMUs * in the system. It also maps the PCI devices in the * system to specific IOMMUs * - * 3 pass) After the basic data structures are allocated and + * 4 pass) After the basic data structures are allocated and * initialized we update them with information about memory * remapping requirements parsed out of the ACPI table in * this last pass. @@ -1903,6 +1941,17 @@ static int __init early_amd_iommu_init(void) return -EINVAL; } + /* + * Validate checksum here so we don't need to do it when + * we actually parse the table + */ + ret = check_ivrs_checksum(ivrs_base); + if (ret) + return ret; + + amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); + DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); + /* * First parse ACPI tables to find the largest Bus/Dev/Func * we need to handle. Upon this information the shared data -- cgit v1.2.3 From 2a0cb4e2d423c8aeafa79945279246f6b35ea8cf Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Fri, 1 Apr 2016 09:06:00 -0400 Subject: iommu/amd: Add new map for storing IVHD dev entry type HID This patch introduces acpihid_map, which is used to store the new IVHD device entry extracted from BIOS IVRS table. It also provides a utility function add_acpi_hid_device(), to add this types of devices to the map. Signed-off-by: Wan Zongshun Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 1 + drivers/iommu/amd_iommu_init.c | 122 ++++++++++++++++++++++++++++++++++++++++ drivers/iommu/amd_iommu_types.h | 14 +++++ 3 files changed, 137 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 374c129219ef..d8e59a84f7c0 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -72,6 +72,7 @@ static DEFINE_SPINLOCK(dev_data_list_lock); LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); +LIST_HEAD(acpihid_map); /* * Domain for untranslated devices - only allocated diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 8f4961225724..e7ebfa2a6c2c 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -60,6 +60,10 @@ #define IVHD_DEV_SPECIAL 0x48 #define IVHD_DEV_ACPI_HID 0xf0 +#define UID_NOT_PRESENT 0 +#define UID_IS_INTEGER 1 +#define UID_IS_CHARACTER 2 + #define IVHD_SPECIAL_IOAPIC 1 #define IVHD_SPECIAL_HPET 2 @@ -116,6 +120,11 @@ struct ivhd_entry { u16 devid; u8 flags; u32 ext; + u32 hidh; + u64 cid; + u8 uidf; + u8 uidl; + u8 uid; } __attribute__((packed)); /* @@ -224,8 +233,12 @@ enum iommu_init_state { #define EARLY_MAP_SIZE 4 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; +static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; + static int __initdata early_ioapic_map_size; static int __initdata early_hpet_map_size; +static int __initdata early_acpihid_map_size; + static bool __initdata cmdline_maps; static enum iommu_init_state init_state = IOMMU_START_STATE; @@ -765,6 +778,42 @@ static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) return 0; } +static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, + bool cmd_line) +{ + struct acpihid_map_entry *entry; + struct list_head *list = &acpihid_map; + + list_for_each_entry(entry, list, list) { + if (strcmp(entry->hid, hid) || + (*uid && *entry->uid && strcmp(entry->uid, uid)) || + !entry->cmd_line) + continue; + + pr_info("AMD-Vi: Command-line override for hid:%s uid:%s\n", + hid, uid); + *devid = entry->devid; + return 0; + } + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + memcpy(entry->uid, uid, strlen(uid)); + memcpy(entry->hid, hid, strlen(hid)); + entry->devid = *devid; + entry->cmd_line = cmd_line; + entry->root_devid = (entry->devid & (~0x7)); + + pr_info("AMD-Vi:%s, add hid:%s, uid:%s, rdevid:%d\n", + entry->cmd_line ? "cmd" : "ivrs", + entry->hid, entry->uid, entry->root_devid); + + list_add_tail(&entry->list, list); + return 0; +} + static int __init add_early_maps(void) { int i, ret; @@ -787,6 +836,15 @@ static int __init add_early_maps(void) return ret; } + for (i = 0; i < early_acpihid_map_size; ++i) { + ret = add_acpi_hid_device(early_acpihid_map[i].hid, + early_acpihid_map[i].uid, + &early_acpihid_map[i].devid, + early_acpihid_map[i].cmd_line); + if (ret) + return ret; + } + return 0; } @@ -1007,6 +1065,70 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } + case IVHD_DEV_ACPI_HID: { + u16 devid; + u8 hid[ACPIHID_HID_LEN] = {0}; + u8 uid[ACPIHID_UID_LEN] = {0}; + int ret; + + if (h->type != 0x40) { + pr_err(FW_BUG "Invalid IVHD device type %#x\n", + e->type); + break; + } + + memcpy(hid, (u8 *)(&e->ext), ACPIHID_HID_LEN - 1); + hid[ACPIHID_HID_LEN - 1] = '\0'; + + if (!(*hid)) { + pr_err(FW_BUG "Invalid HID.\n"); + break; + } + + switch (e->uidf) { + case UID_NOT_PRESENT: + + if (e->uidl != 0) + pr_warn(FW_BUG "Invalid UID length.\n"); + + break; + case UID_IS_INTEGER: + + sprintf(uid, "%d", e->uid); + + break; + case UID_IS_CHARACTER: + + memcpy(uid, (u8 *)(&e->uid), ACPIHID_UID_LEN - 1); + uid[ACPIHID_UID_LEN - 1] = '\0'; + + break; + default: + break; + } + + DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", + hid, uid, + PCI_BUS_NUM(devid), + PCI_SLOT(devid), + PCI_FUNC(devid)); + + devid = e->devid; + flags = e->flags; + + ret = add_acpi_hid_device(hid, uid, &devid, false); + if (ret) + return ret; + + /* + * add_special_device might update the devid in case a + * command-line override is present. So call + * set_dev_entry_from_acpi after add_special_device. + */ + set_dev_entry_from_acpi(iommu, devid, e->flags, 0); + + break; + } default: break; } diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 9d32b20a5e9a..b6b14d288b0b 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -527,6 +527,19 @@ struct amd_iommu { #endif }; +#define ACPIHID_UID_LEN 256 +#define ACPIHID_HID_LEN 9 + +struct acpihid_map_entry { + struct list_head list; + u8 uid[ACPIHID_UID_LEN]; + u8 hid[ACPIHID_HID_LEN]; + u16 devid; + u16 root_devid; + bool cmd_line; + struct iommu_group *group; +}; + struct devid_map { struct list_head list; u8 id; @@ -537,6 +550,7 @@ struct devid_map { /* Map HPET and IOAPIC ids to the devid used by the IOMMU */ extern struct list_head ioapic_map; extern struct list_head hpet_map; +extern struct list_head acpihid_map; /* * List with all IOMMUs in the system. This list is not locked because it is -- cgit v1.2.3 From ca3bf5d47cec8b7614bcb2e9132c40081d6d81db Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Fri, 1 Apr 2016 09:06:01 -0400 Subject: iommu/amd: Introduces ivrs_acpihid kernel parameter This patch introduces a new kernel parameter, ivrs_acpihid. This is used to override existing ACPI-HID IVHD device entry, or add an entry in case it is missing in the IVHD. Signed-off-by: Wan Zongshun Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- Documentation/kernel-parameters.txt | 7 +++++++ drivers/iommu/amd_iommu_init.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) (limited to 'drivers/iommu') diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index ecc74fa4bfde..8c881a5f7af0 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1767,6 +1767,13 @@ bytes respectively. Such letter suffixes can also be entirely omitted. PCI device 00:14.0 write the parameter as: ivrs_hpet[0]=00:14.0 + ivrs_acpihid [HW,X86_64] + Provide an override to the ACPI-HID:UID<->DEVICE-ID + mapping provided in the IVRS ACPI table. For + example, to map UART-HID:UID AMD0020:0 to + PCI device 00:14.5 write the parameter as: + ivrs_acpihid[00:14.5]=AMD0020:0 + js= [HW,JOY] Analog joystick See Documentation/input/joystick.txt. diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index e7ebfa2a6c2c..9e0034196e10 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -2477,10 +2477,43 @@ static int __init parse_ivrs_hpet(char *str) return 1; } +static int __init parse_ivrs_acpihid(char *str) +{ + u32 bus, dev, fn; + char *hid, *uid, *p; + char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0}; + int ret, i; + + ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid); + if (ret != 4) { + pr_err("AMD-Vi: Invalid command line: ivrs_acpihid(%s)\n", str); + return 1; + } + + p = acpiid; + hid = strsep(&p, ":"); + uid = p; + + if (!hid || !(*hid) || !uid) { + pr_err("AMD-Vi: Invalid command line: hid or uid\n"); + return 1; + } + + i = early_acpihid_map_size++; + memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); + memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); + early_acpihid_map[i].devid = + ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); + early_acpihid_map[i].cmd_line = true; + + return 1; +} + __setup("amd_iommu_dump", parse_amd_iommu_dump); __setup("amd_iommu=", parse_amd_iommu_options); __setup("ivrs_ioapic", parse_ivrs_ioapic); __setup("ivrs_hpet", parse_ivrs_hpet); +__setup("ivrs_acpihid", parse_ivrs_acpihid); IOMMU_INIT_FINISH(amd_iommu_detect, gart_iommu_hole_init, -- cgit v1.2.3 From 7aba6cb9ee9db7849d0bf57891d9c7feb4e89457 Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Fri, 1 Apr 2016 09:06:02 -0400 Subject: iommu/amd: Make call-sites of get_device_id aware of its return value This patch is to make the call-sites of get_device_id aware of its return value. Signed-off-by: Wan Zongshun Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 51 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 10 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d8e59a84f7c0..400867f01d35 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -279,9 +279,11 @@ static void init_unity_mappings_for_device(struct device *dev, struct dma_ops_domain *dma_dom) { struct unity_map_entry *e; - u16 devid; + int devid; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; list_for_each_entry(e, &amd_iommu_unity_map, list) { if (!(devid >= e->devid_start && devid <= e->devid_end)) @@ -296,7 +298,7 @@ static void init_unity_mappings_for_device(struct device *dev, */ static bool check_device(struct device *dev) { - u16 devid; + int devid; if (!dev || !dev->dma_mask) return false; @@ -306,6 +308,8 @@ static bool check_device(struct device *dev) return false; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return false; /* Out of our scope? */ if (devid > amd_iommu_last_bdf) @@ -342,11 +346,16 @@ static int iommu_init_device(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct iommu_dev_data *dev_data; + int devid; if (dev->archdata.iommu) return 0; - dev_data = find_dev_data(get_device_id(dev)); + devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return devid; + + dev_data = find_dev_data(devid); if (!dev_data) return -ENOMEM; @@ -367,9 +376,13 @@ static int iommu_init_device(struct device *dev) static void iommu_ignore_device(struct device *dev) { - u16 devid, alias; + u16 alias; + int devid; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; + alias = amd_iommu_alias_table[devid]; memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); @@ -381,8 +394,14 @@ static void iommu_ignore_device(struct device *dev) static void iommu_uninit_device(struct device *dev) { - struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev)); + int devid; + struct iommu_dev_data *dev_data; + + devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; + dev_data = search_dev_data(devid); if (!dev_data) return; @@ -2314,13 +2333,15 @@ static int amd_iommu_add_device(struct device *dev) struct iommu_dev_data *dev_data; struct iommu_domain *domain; struct amd_iommu *iommu; - u16 devid; - int ret; + int ret, devid; if (!check_device(dev) || get_dev_data(dev)) return 0; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return devid; + iommu = amd_iommu_rlookup_table[devid]; ret = iommu_init_device(dev); @@ -2358,12 +2379,15 @@ out: static void amd_iommu_remove_device(struct device *dev) { struct amd_iommu *iommu; - u16 devid; + int devid; if (!check_device(dev)) return; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; + iommu = amd_iommu_rlookup_table[devid]; iommu_uninit_device(dev); @@ -3035,12 +3059,14 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, { struct iommu_dev_data *dev_data = dev->archdata.iommu; struct amd_iommu *iommu; - u16 devid; + int devid; if (!check_device(dev)) return; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; if (dev_data->domain != NULL) detach_device(dev); @@ -3158,9 +3184,11 @@ static void amd_iommu_get_dm_regions(struct device *dev, struct list_head *head) { struct unity_map_entry *entry; - u16 devid; + int devid; devid = get_device_id(dev); + if (IS_ERR_VALUE(devid)) + return; list_for_each_entry(entry, &amd_iommu_unity_map, list) { struct iommu_dm_region *region; @@ -3862,6 +3890,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info) case X86_IRQ_ALLOC_TYPE_MSI: case X86_IRQ_ALLOC_TYPE_MSIX: devid = get_device_id(&info->msi_dev->dev); + if (IS_ERR_VALUE(devid)) + return NULL; + iommu = amd_iommu_rlookup_table[devid]; if (iommu) return iommu->msi_domain; -- cgit v1.2.3 From 2bf9a0a12749b2c45964020795859d4a1f228a1d Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Fri, 1 Apr 2016 09:06:03 -0400 Subject: iommu/amd: Add iommu support for ACPI HID devices Current IOMMU driver make assumption that the downstream devices are PCI. With the newly added ACPI-HID IVHD device entry support, this is no longer true. This patch is to add dev type check and to distinguish the pci and acpihid device code path. Signed-off-by: Wan Zongshun Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 69 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 400867f01d35..0df651a379df 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -216,13 +217,60 @@ static struct iommu_dev_data *find_dev_data(u16 devid) return dev_data; } -static inline u16 get_device_id(struct device *dev) +static inline int match_hid_uid(struct device *dev, + struct acpihid_map_entry *entry) +{ + const char *hid, *uid; + + hid = acpi_device_hid(ACPI_COMPANION(dev)); + uid = acpi_device_uid(ACPI_COMPANION(dev)); + + if (!hid || !(*hid)) + return -ENODEV; + + if (!uid || !(*uid)) + return strcmp(hid, entry->hid); + + if (!(*entry->uid)) + return strcmp(hid, entry->hid); + + return (strcmp(hid, entry->hid) || strcmp(uid, entry->uid)); +} + +static inline u16 get_pci_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); return PCI_DEVID(pdev->bus->number, pdev->devfn); } +static inline int get_acpihid_device_id(struct device *dev, + struct acpihid_map_entry **entry) +{ + struct acpihid_map_entry *p; + + list_for_each_entry(p, &acpihid_map, list) { + if (!match_hid_uid(dev, p)) { + if (entry) + *entry = p; + return p->devid; + } + } + return -EINVAL; +} + +static inline int get_device_id(struct device *dev) +{ + int devid; + + if (dev_is_pci(dev)) + devid = get_pci_device_id(dev); + else + devid = get_acpihid_device_id(dev, NULL); + + return devid; +} + static struct iommu_dev_data *get_dev_data(struct device *dev) { return dev->archdata.iommu; @@ -303,10 +351,6 @@ static bool check_device(struct device *dev) if (!dev || !dev->dma_mask) return false; - /* No PCI device */ - if (!dev_is_pci(dev)) - return false; - devid = get_device_id(dev); if (IS_ERR_VALUE(devid)) return false; @@ -344,7 +388,6 @@ out: static int iommu_init_device(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); struct iommu_dev_data *dev_data; int devid; @@ -359,10 +402,10 @@ static int iommu_init_device(struct device *dev) if (!dev_data) return -ENOMEM; - if (pci_iommuv2_capable(pdev)) { + if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) { struct amd_iommu *iommu; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; dev_data->iommu_v2 = iommu->is_iommu_v2; } @@ -2239,13 +2282,17 @@ static bool pci_pri_tlp_required(struct pci_dev *pdev) static int attach_device(struct device *dev, struct protection_domain *domain) { - struct pci_dev *pdev = to_pci_dev(dev); + struct pci_dev *pdev; struct iommu_dev_data *dev_data; unsigned long flags; int ret; dev_data = get_dev_data(dev); + if (!dev_is_pci(dev)) + goto skip_ats_check; + + pdev = to_pci_dev(dev); if (domain->flags & PD_IOMMUV2_MASK) { if (!dev_data->passthrough) return -EINVAL; @@ -2264,6 +2311,7 @@ static int attach_device(struct device *dev, dev_data->ats.qdep = pci_ats_queue_depth(pdev); } +skip_ats_check: write_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev_data, domain); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); @@ -2320,6 +2368,9 @@ static void detach_device(struct device *dev) __detach_device(dev_data); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + if (!dev_is_pci(dev)) + return; + if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2) pdev_iommuv2_disable(to_pci_dev(dev)); else if (dev_data->ats.enabled) -- cgit v1.2.3 From b097d11a0fa3f97be88774d09ee9ed1d8532a7b0 Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Fri, 1 Apr 2016 09:06:04 -0400 Subject: iommu/amd: Manage iommu_group for ACPI HID devices This patch creates a new function for finding or creating an IOMMU group for acpihid(ACPI Hardware ID) device. The acpihid devices with the same devid will be put into same group and there will have the same domain id and share the same page table. Signed-off-by: Wan Zongshun Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0df651a379df..713e7ea06210 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -276,6 +276,29 @@ static struct iommu_dev_data *get_dev_data(struct device *dev) return dev->archdata.iommu; } +/* +* Find or create an IOMMU group for a acpihid device. +*/ +static struct iommu_group *acpihid_device_group(struct device *dev) +{ + struct acpihid_map_entry *p, *entry = NULL; + u16 devid; + + devid = get_acpihid_device_id(dev, &entry); + if (devid < 0) + return ERR_PTR(devid); + + list_for_each_entry(p, &acpihid_map, list) { + if ((devid == p->devid) && p->group) + entry->group = p->group; + } + + if (!entry->group) + entry->group = generic_device_group(dev); + + return entry->group; +} + static bool pci_iommuv2_capable(struct pci_dev *pdev) { static const int caps[] = { @@ -2445,6 +2468,14 @@ static void amd_iommu_remove_device(struct device *dev) iommu_completion_wait(iommu); } +static struct iommu_group *amd_iommu_device_group(struct device *dev) +{ + if (dev_is_pci(dev)) + return pci_device_group(dev); + + return acpihid_device_group(dev); +} + /***************************************************************************** * * The next functions belong to the dma_ops mapping/unmapping code. @@ -3286,7 +3317,7 @@ static const struct iommu_ops amd_iommu_ops = { .iova_to_phys = amd_iommu_iova_to_phys, .add_device = amd_iommu_add_device, .remove_device = amd_iommu_remove_device, - .device_group = pci_device_group, + .device_group = amd_iommu_device_group, .get_dm_regions = amd_iommu_get_dm_regions, .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, -- cgit v1.2.3 From 9a4d3bf56c87be9ad8916e2013af04787d6bac9b Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Fri, 1 Apr 2016 09:06:05 -0400 Subject: iommu/amd: Set AMD iommu callbacks for amba bus AMD Uart DMA belongs to ACPI HID type device, and its driver is basing on AMBA Bus, need also IOMMU support. This patch is just to set the AMD iommu callbacks for amba bus. Signed-off-by: Wan Zongshun Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 713e7ea06210..c430c10fb645 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -2969,7 +2970,17 @@ static struct dma_map_ops amd_iommu_dma_ops = { int __init amd_iommu_init_api(void) { - return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + int err = 0; + + err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + if (err) + return err; +#ifdef CONFIG_ARM_AMBA + err = bus_set_iommu(&amba_bustype, &amd_iommu_ops); + if (err) + return err; +#endif + return 0; } int __init amd_iommu_init_dma_ops(void) -- cgit v1.2.3 From a0d284d2b1d9f7453ff1c5cc7854219daf4f7590 Mon Sep 17 00:00:00 2001 From: Andy Fleming Date: Wed, 16 Mar 2016 23:15:44 -0500 Subject: powerpc: Fix incorrect PPC32 PAMU dependency The Freescale PAMU can be enabled on both 32 and 64-bit Power chips. Commit 477ab7a19ce restricted PAMU to PPC32. PPC covers both. Signed-off-by: Andy Fleming Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index dd1dc39f84ff..e0dfb8676d61 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -76,8 +76,7 @@ config IOMMU_DMA config FSL_PAMU bool "Freescale IOMMU support" - depends on PPC32 - depends on PPC_E500MC || COMPILE_TEST + depends on PPC_E500MC || (COMPILE_TEST && PPC) select IOMMU_API select GENERIC_ALLOCATOR help -- cgit v1.2.3 From fb948251e4be1f4aedbe17ec96890ceee147c6d6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 5 Apr 2016 12:39:31 +0100 Subject: iommu/io-pgtable-arm: Support IOMMU_MMIO flag Teach the LPAE format to create Device mappings when asked. Signed-off-by: Robin Murphy Tested-by: Eric Auger Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index f433b516098a..a1ed1b73fed4 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -355,7 +355,10 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) pte |= ARM_LPAE_PTE_AP_RDONLY; - if (prot & IOMMU_CACHE) + if (prot & IOMMU_MMIO) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); } else { @@ -364,7 +367,9 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_HAP_READ; if (prot & IOMMU_WRITE) pte |= ARM_LPAE_PTE_HAP_WRITE; - if (prot & IOMMU_CACHE) + if (prot & IOMMU_MMIO) + pte |= ARM_LPAE_PTE_MEMATTR_DEV; + else if (prot & IOMMU_CACHE) pte |= ARM_LPAE_PTE_MEMATTR_OIWB; else pte |= ARM_LPAE_PTE_MEMATTR_NC; -- cgit v1.2.3 From e88ccab12a9e85c536544f1464bd75610b1c46d6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 5 Apr 2016 12:39:32 +0100 Subject: iommu/io-pgtable-arm-v7s: Support IOMMU_MMIO flag Teach the short-descriptor format to create Device mappings when asked. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 0f93dc2f98a3..8c6139986d7d 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -260,9 +260,10 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, struct io_pgtable_cfg *cfg) { bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS); - arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S | - ARM_V7S_ATTR_TEX(1); + arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S; + if (!(prot & IOMMU_MMIO)) + pte |= ARM_V7S_ATTR_TEX(1); if (ap) { pte |= ARM_V7S_PTE_AF | ARM_V7S_PTE_AP_UNPRIV; if (!(prot & IOMMU_WRITE)) @@ -272,7 +273,9 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, if ((prot & IOMMU_NOEXEC) && ap) pte |= ARM_V7S_ATTR_XN(lvl); - if (prot & IOMMU_CACHE) + if (prot & IOMMU_MMIO) + pte |= ARM_V7S_ATTR_B; + else if (prot & IOMMU_CACHE) pte |= ARM_V7S_ATTR_B | ARM_V7S_ATTR_C; return pte; @@ -281,10 +284,13 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) { int prot = IOMMU_READ; + arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl); - if (pte & (ARM_V7S_PTE_AP_RDONLY << ARM_V7S_ATTR_SHIFT(lvl))) + if (attr & ARM_V7S_PTE_AP_RDONLY) prot |= IOMMU_WRITE; - if (pte & ARM_V7S_ATTR_C) + if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0) + prot |= IOMMU_MMIO; + else if (pte & ARM_V7S_ATTR_C) prot |= IOMMU_CACHE; return prot; -- cgit v1.2.3 From 338c3149a221527e202ee26b1e35f76c965bb6c0 Mon Sep 17 00:00:00 2001 From: Jacek Lawrynowicz Date: Thu, 3 Mar 2016 15:38:02 +0100 Subject: PCI: Add support for multiple DMA aliases Solve IOMMU support issues with PCIe non-transparent bridges that use Requester ID look-up tables (RID-LUT), e.g., the PEX8733. The NTB connects devices in two independent PCI domains. Devices separated by the NTB are not able to discover each other. A PCI packet being forwared from one domain to another has to have its RID modified so it appears on correct bus and completions are forwarded back to the original domain through the NTB. The RID is translated using a preprogrammed table (LUT) and the PCI packet propagates upstream away from the NTB. If the destination system has IOMMU enabled, the packet will be discarded because the new RID is unknown to the IOMMU. Adding a DMA alias for the new RID allows IOMMU to properly recognize the packet. Each device behind the NTB has a unique RID assigned in the RID-LUT. The current DMA alias implementation supports only a single alias, so it's not possible to support mutiple devices behind the NTB when IOMMU is enabled. Enable all possible aliases on a given bus (256) that are stored in a bitset. Alias devfn is directly translated to a bit number. The bitset is not allocated for devices that have no need for DMA aliases. More details can be found in the following article: http://www.plxtech.com/files/pdf/technical/expresslane/RTC_Enabling%20MulitHostSystemDesigns.pdf Signed-off-by: Jacek Lawrynowicz Signed-off-by: Bjorn Helgaas Reviewed-by: Alex Williamson Acked-by: David Woodhouse Acked-by: Joerg Roedel --- drivers/iommu/iommu.c | 10 +++------- drivers/pci/pci.c | 19 +++++++++++++++++-- drivers/pci/probe.c | 1 + drivers/pci/search.c | 14 +++++++++----- include/linux/pci.h | 5 ++--- 5 files changed, 32 insertions(+), 17 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index bfd4f7c3b1d8..1b49e940a318 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -660,8 +660,8 @@ static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, } /* - * Look for aliases to or from the given device for exisiting groups. The - * dma_alias_devfn only supports aliases on the same bus, therefore the search + * Look for aliases to or from the given device for existing groups. DMA + * aliases are only supported on the same bus, therefore the search * space is quite small (especially since we're really only looking at pcie * device, and therefore only expect multiple slots on the root complex or * downstream switch ports). It's conceivable though that a pair of @@ -686,11 +686,7 @@ static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, continue; /* We alias them or they alias us */ - if (((pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && - pdev->dma_alias_devfn == tmp->devfn) || - ((tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && - tmp->dma_alias_devfn == pdev->devfn)) { - + if (pci_devs_are_dma_aliases(pdev, tmp)) { group = get_pci_alias_group(tmp, devfns); if (group) { pci_dev_put(tmp); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index c82ebd0f6982..0b90c2186f1c 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4588,12 +4588,27 @@ int pci_set_vga_state(struct pci_dev *dev, bool decode, */ void pci_add_dma_alias(struct pci_dev *dev, u8 devfn) { - dev->dma_alias_devfn = devfn; - dev->dev_flags |= PCI_DEV_FLAGS_DMA_ALIAS_DEVFN; + if (!dev->dma_alias_mask) + dev->dma_alias_mask = kcalloc(BITS_TO_LONGS(U8_MAX), + sizeof(long), GFP_KERNEL); + if (!dev->dma_alias_mask) { + dev_warn(&dev->dev, "Unable to allocate DMA alias mask\n"); + return; + } + + set_bit(devfn, dev->dma_alias_mask); dev_info(&dev->dev, "Enabling fixed DMA alias to %02x.%d\n", PCI_SLOT(devfn), PCI_FUNC(devfn)); } +bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) +{ + return (dev1->dma_alias_mask && + test_bit(dev2->devfn, dev1->dma_alias_mask)) || + (dev2->dma_alias_mask && + test_bit(dev1->devfn, dev2->dma_alias_mask)); +} + bool pci_device_is_present(struct pci_dev *pdev) { u32 v; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 8004f67c57ec..ae7daeb83e21 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1537,6 +1537,7 @@ static void pci_release_dev(struct device *dev) pcibios_release_device(pci_dev); pci_bus_put(pci_dev->bus); kfree(pci_dev->driver_override); + kfree(pci_dev->dma_alias_mask); kfree(pci_dev); } diff --git a/drivers/pci/search.c b/drivers/pci/search.c index a20ce7d5e2a7..33e0f033a48e 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -40,11 +40,15 @@ int pci_for_each_dma_alias(struct pci_dev *pdev, * If the device is broken and uses an alias requester ID for * DMA, iterate over that too. */ - if (unlikely(pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN)) { - ret = fn(pdev, PCI_DEVID(pdev->bus->number, - pdev->dma_alias_devfn), data); - if (ret) - return ret; + if (unlikely(pdev->dma_alias_mask)) { + u8 devfn; + + for_each_set_bit(devfn, pdev->dma_alias_mask, U8_MAX) { + ret = fn(pdev, PCI_DEVID(pdev->bus->number, devfn), + data); + if (ret) + return ret; + } } for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 7e7019064437..5581d05f7833 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -166,8 +166,6 @@ enum pci_dev_flags { PCI_DEV_FLAGS_ASSIGNED = (__force pci_dev_flags_t) (1 << 2), /* Flag for quirk use to store if quirk-specific ACS is enabled */ PCI_DEV_FLAGS_ACS_ENABLED_QUIRK = (__force pci_dev_flags_t) (1 << 3), - /* Flag to indicate the device uses dma_alias_devfn */ - PCI_DEV_FLAGS_DMA_ALIAS_DEVFN = (__force pci_dev_flags_t) (1 << 4), /* Use a PCIe-to-PCI bridge alias even if !pci_is_pcie */ PCI_DEV_FLAG_PCIE_BRIDGE_ALIAS = (__force pci_dev_flags_t) (1 << 5), /* Do not use bus resets for device */ @@ -273,7 +271,7 @@ struct pci_dev { u8 rom_base_reg; /* which config register controls the ROM */ u8 pin; /* which interrupt pin this device uses */ u16 pcie_flags_reg; /* cached PCIe Capabilities Register */ - u8 dma_alias_devfn;/* devfn of DMA alias, if any */ + unsigned long *dma_alias_mask;/* mask of enabled devfn aliases */ struct pci_driver *driver; /* which driver has allocated this device */ u64 dma_mask; /* Mask of the bits of bus address this @@ -1989,6 +1987,7 @@ static inline struct eeh_dev *pci_dev_to_eeh_dev(struct pci_dev *pdev) #endif void pci_add_dma_alias(struct pci_dev *dev, u8 devfn); +bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2); int pci_for_each_dma_alias(struct pci_dev *pdev, int (*fn)(struct pci_dev *pdev, u16 alias, void *data), void *data); -- cgit v1.2.3 From 93984fbd4e33cc861d5b49caed02a02cbfb01340 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 4 Apr 2016 22:25:00 +0200 Subject: x86/cpufeature: Replace cpu_has_apic with boot_cpu_has() usage Signed-off-by: Borislav Petkov Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: iommu@lists.linux-foundation.org Cc: linux-pm@vger.kernel.org Cc: oprofile-list@lists.sf.net Link: http://lkml.kernel.org/r/1459801503-15600-8-git-send-email-bp@alien8.de Signed-off-by: Ingo Molnar --- arch/x86/events/core.c | 2 +- arch/x86/include/asm/cpufeature.h | 1 - arch/x86/include/asm/irq_work.h | 2 +- arch/x86/kernel/acpi/boot.c | 8 ++++---- arch/x86/kernel/apic/apic.c | 20 ++++++++++---------- arch/x86/kernel/apic/apic_noop.c | 4 ++-- arch/x86/kernel/apic/io_apic.c | 2 +- arch/x86/kernel/apic/ipi.c | 2 +- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/cpu/amd.c | 4 ++-- arch/x86/kernel/cpu/intel.c | 2 +- arch/x86/kernel/cpu/mcheck/mce_intel.c | 2 +- arch/x86/kernel/cpu/mcheck/therm_throt.c | 2 +- arch/x86/kernel/devicetree.c | 2 +- arch/x86/kernel/smpboot.c | 2 +- arch/x86/oprofile/nmi_int.c | 2 +- arch/x86/pci/xen.c | 2 +- drivers/cpufreq/longhaul.c | 2 +- drivers/iommu/irq_remapping.c | 2 +- 19 files changed, 32 insertions(+), 33 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 041e442a3e28..54c17455600e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1518,7 +1518,7 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) static void __init pmu_check_apic(void) { - if (cpu_has_apic) + if (boot_cpu_has(X86_FEATURE_APIC)) return; x86_pmu.apic = 0; diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 8f58cd215f6d..c532961c7439 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -118,7 +118,6 @@ extern const char * const x86_bug_flags[NBUGINTS*32]; set_bit(bit, (unsigned long *)cpu_caps_set); \ } while (0) -#define cpu_has_apic boot_cpu_has(X86_FEATURE_APIC) #define cpu_has_fxsr boot_cpu_has(X86_FEATURE_FXSR) #define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) #define cpu_has_xsaves boot_cpu_has(X86_FEATURE_XSAVES) diff --git a/arch/x86/include/asm/irq_work.h b/arch/x86/include/asm/irq_work.h index d0afb05c84fc..f70604125286 100644 --- a/arch/x86/include/asm/irq_work.h +++ b/arch/x86/include/asm/irq_work.h @@ -5,7 +5,7 @@ static inline bool arch_irq_work_has_interrupt(void) { - return cpu_has_apic; + return boot_cpu_has(X86_FEATURE_APIC); } #endif /* _ASM_IRQ_WORK_H */ diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 8c2f1ef6ca23..2522e564269e 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -136,7 +136,7 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) { struct acpi_table_madt *madt = NULL; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return -EINVAL; madt = (struct acpi_table_madt *)table; @@ -951,7 +951,7 @@ static int __init early_acpi_parse_madt_lapic_addr_ovr(void) { int count; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return -ENODEV; /* @@ -979,7 +979,7 @@ static int __init acpi_parse_madt_lapic_entries(void) int ret; struct acpi_subtable_proc madt_proc[2]; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return -ENODEV; /* @@ -1125,7 +1125,7 @@ static int __init acpi_parse_madt_ioapic_entries(void) if (acpi_disabled || acpi_noirq) return -ENODEV; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return -ENODEV; /* diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 0b6509f1a4fe..60078a67d7e3 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1085,7 +1085,7 @@ void lapic_shutdown(void) { unsigned long flags; - if (!cpu_has_apic && !apic_from_smp_config()) + if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) return; local_irq_save(flags); @@ -1134,7 +1134,7 @@ void __init init_bsp_APIC(void) * Don't do the setup now if we have a SMP BIOS as the * through-I/O-APIC virtual wire mode might be active. */ - if (smp_found_config || !cpu_has_apic) + if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC)) return; /* @@ -1445,7 +1445,7 @@ static void __x2apic_disable(void) { u64 msr; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return; rdmsrl(MSR_IA32_APICBASE, msr); @@ -1632,7 +1632,7 @@ void __init enable_IR_x2apic(void) */ static int __init detect_init_APIC(void) { - if (!cpu_has_apic) { + if (!boot_cpu_has(X86_FEATURE_APIC)) { pr_info("No local APIC present\n"); return -1; } @@ -1711,14 +1711,14 @@ static int __init detect_init_APIC(void) goto no_apic; case X86_VENDOR_INTEL: if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 || - (boot_cpu_data.x86 == 5 && cpu_has_apic)) + (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC))) break; goto no_apic; default: goto no_apic; } - if (!cpu_has_apic) { + if (!boot_cpu_has(X86_FEATURE_APIC)) { /* * Over-ride BIOS and try to enable the local APIC only if * "lapic" specified. @@ -2233,19 +2233,19 @@ int __init APIC_init_uniprocessor(void) return -1; } #ifdef CONFIG_X86_64 - if (!cpu_has_apic) { + if (!boot_cpu_has(X86_FEATURE_APIC)) { disable_apic = 1; pr_info("Apic disabled by BIOS\n"); return -1; } #else - if (!smp_found_config && !cpu_has_apic) + if (!smp_found_config && !boot_cpu_has(X86_FEATURE_APIC)) return -1; /* * Complain if the BIOS pretends there is one. */ - if (!cpu_has_apic && + if (!boot_cpu_has(X86_FEATURE_APIC) && APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) { pr_err("BIOS bug, local APIC 0x%x not detected!...\n", boot_cpu_physical_apicid); @@ -2426,7 +2426,7 @@ static void apic_pm_activate(void) static int __init init_lapic_sysfs(void) { /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ - if (cpu_has_apic) + if (boot_cpu_has(X86_FEATURE_APIC)) register_syscore_ops(&lapic_syscore_ops); return 0; diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 331a7a07c48f..13d19ed58514 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -100,13 +100,13 @@ static void noop_vector_allocation_domain(int cpu, struct cpumask *retmask, static u32 noop_apic_read(u32 reg) { - WARN_ON_ONCE((cpu_has_apic && !disable_apic)); + WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); return 0; } static void noop_apic_write(u32 reg, u32 v) { - WARN_ON_ONCE(cpu_has_apic && !disable_apic); + WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); } struct apic apic_noop = { diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index fdb0fbfb1197..84e33ff5a6d5 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1454,7 +1454,7 @@ void native_disable_io_apic(void) ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry); } - if (cpu_has_apic || apic_from_smp_config()) + if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config()) disconnect_bsp_APIC(ioapic_i8259.pin != -1); } diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 28bde88b0085..2a0f225afebd 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -230,7 +230,7 @@ int safe_smp_processor_id(void) { int apicid, cpuid; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return 0; apicid = hard_smp_processor_id(); diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index ad59d70bcb1a..26d3ccc63e40 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -943,7 +943,7 @@ static int __init print_ICs(void) print_PIC(); /* don't print out if apic is not there */ - if (!cpu_has_apic && !apic_from_smp_config()) + if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) return 0; print_local_APICs(show_lapic); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 19d7dcfc8b3e..54f7b44dcf01 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -565,9 +565,9 @@ static void early_init_amd(struct cpuinfo_x86 *c) * can safely set X86_FEATURE_EXTD_APICID unconditionally for families * after 16h. */ - if (cpu_has_apic && c->x86 > 0x16) { + if (boot_cpu_has(X86_FEATURE_APIC) && c->x86 > 0x16) { set_cpu_cap(c, X86_FEATURE_EXTD_APICID); - } else if (cpu_has_apic && c->x86 >= 0xf) { + } else if (boot_cpu_has(X86_FEATURE_APIC) && c->x86 >= 0xf) { /* check CPU config space for extended APIC ID */ unsigned int val; val = read_pci_config(0, 24, 0, 0x68); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index f71a34944b56..1d5582259b20 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -281,7 +281,7 @@ static void intel_workarounds(struct cpuinfo_x86 *c) * integrated APIC (see 11AP erratum in "Pentium Processor * Specification Update"). */ - if (cpu_has_apic && (c->x86<<8 | c->x86_model<<4) == 0x520 && + if (boot_cpu_has(X86_FEATURE_APIC) && (c->x86<<8 | c->x86_model<<4) == 0x520 && (c->x86_mask < 0x6 || c->x86_mask == 0xb)) set_cpu_bug(c, X86_BUG_11AP); diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c index 1e8bb6c94f14..1defb8ea882c 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_intel.c +++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c @@ -84,7 +84,7 @@ static int cmci_supported(int *banks) */ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return 0; - if (!cpu_has_apic || lapic_get_maxlvt() < 6) + if (!boot_cpu_has(X86_FEATURE_APIC) || lapic_get_maxlvt() < 6) return 0; rdmsrl(MSR_IA32_MCG_CAP, cap); *banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff); diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 0b445c2ff735..615793321c49 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -447,7 +447,7 @@ asmlinkage __visible void smp_trace_thermal_interrupt(struct pt_regs *regs) /* Thermal monitoring depends on APIC, ACPI and clock modulation */ static int intel_thermal_supported(struct cpuinfo_x86 *c) { - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return 0; if (!cpu_has(c, X86_FEATURE_ACPI) || !cpu_has(c, X86_FEATURE_ACC)) return 0; diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 1f4acd68b98b..3fe45f84ced4 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -151,7 +151,7 @@ static void __init dtb_lapic_setup(void) return; /* Did the boot loader setup the local APIC ? */ - if (!cpu_has_apic) { + if (!boot_cpu_has(X86_FEATURE_APIC)) { if (apic_force_enable(r.start)) return; } diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index a2065d3b3b39..1fe4130b14d9 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1231,7 +1231,7 @@ static int __init smp_sanity_check(unsigned max_cpus) * If we couldn't find a local APIC, then get out of here now! */ if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && - !cpu_has_apic) { + !boot_cpu_has(X86_FEATURE_APIC)) { if (!disable_apic) { pr_err("BIOS bug, local APIC #%d not detected!...\n", boot_cpu_physical_apicid); diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index 25171e9595f7..28c04123b6dd 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -700,7 +700,7 @@ int __init op_nmi_init(struct oprofile_operations *ops) char *cpu_type = NULL; int ret = 0; - if (!cpu_has_apic) + if (!boot_cpu_has(X86_FEATURE_APIC)) return -ENODEV; if (force_cpu_type == timer) diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index beac4dfdade6..4bd08b0fc8ea 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -445,7 +445,7 @@ void __init xen_msi_init(void) uint32_t eax = cpuid_eax(xen_cpuid_base() + 4); if (((eax & XEN_HVM_CPUID_X2APIC_VIRT) && x2apic_mode) || - ((eax & XEN_HVM_CPUID_APIC_ACCESS_VIRT) && cpu_has_apic)) + ((eax & XEN_HVM_CPUID_APIC_ACCESS_VIRT) && boot_cpu_has(X86_FEATURE_APIC))) return; } diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c index 0f6b229afcb9..247bfa8eaddb 100644 --- a/drivers/cpufreq/longhaul.c +++ b/drivers/cpufreq/longhaul.c @@ -945,7 +945,7 @@ static int __init longhaul_init(void) } #endif #ifdef CONFIG_X86_IO_APIC - if (cpu_has_apic) { + if (boot_cpu_has(X86_FEATURE_APIC)) { printk(KERN_ERR PFX "APIC detected. Longhaul is currently " "broken in this configuration.\n"); return -ENODEV; diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 8adaaeae3268..49721b4e1975 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -36,7 +36,7 @@ static void irq_remapping_disable_io_apic(void) * As this gets called during crash dump, keep this simple for * now. */ - if (cpu_has_apic || apic_from_smp_config()) + if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config()) disconnect_bsp_APIC(0); } -- cgit v1.2.3 From 2d8e1f039de842701d35a857a7c5d9d47bc1c639 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 11 Apr 2016 10:14:46 +0300 Subject: iommu/amd: Signedness bug in acpihid_device_group() "devid" needs to be signed for the error handling to work. Fixes: b097d11a0fa3f ('iommu/amd: Manage iommu_group for ACPI HID devices') Signed-off-by: Dan Carpenter Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c430c10fb645..12f77798a0a4 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -283,7 +283,7 @@ static struct iommu_dev_data *get_dev_data(struct device *dev) static struct iommu_group *acpihid_device_group(struct device *dev) { struct acpihid_map_entry *p, *entry = NULL; - u16 devid; + int devid; devid = get_acpihid_device_id(dev, &entry); if (devid < 0) -- cgit v1.2.3 From cb6c27bb09122ceb0effda0b15885123870a6af7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 4 Apr 2016 17:49:22 +0200 Subject: iommu/arm-smmu: Make use of phandle iterators in device-tree parsing Remove the usage of of_parse_phandle_with_args() and replace it by the phandle-iterator implementation so that we can parse out all of the potentially present 128 stream-ids. Signed-off-by: Joerg Roedel Acked-by: Will Deacon Signed-off-by: Rob Herring --- drivers/iommu/arm-smmu.c | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 2409e3bd3df2..a1c965cdcfad 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -49,7 +49,7 @@ #include "io-pgtable.h" /* Maximum number of stream IDs assigned to a single device */ -#define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS +#define MAX_MASTER_STREAMIDS 128 /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 @@ -357,6 +357,12 @@ struct arm_smmu_domain { struct iommu_domain domain; }; +struct arm_smmu_phandle_args { + struct device_node *np; + int args_count; + uint32_t args[MAX_MASTER_STREAMIDS]; +}; + static struct iommu_ops arm_smmu_ops; static DEFINE_SPINLOCK(arm_smmu_devices_lock); @@ -466,7 +472,7 @@ static int insert_smmu_master(struct arm_smmu_device *smmu, static int register_smmu_master(struct arm_smmu_device *smmu, struct device *dev, - struct of_phandle_args *masterspec) + struct arm_smmu_phandle_args *masterspec) { int i; struct arm_smmu_master *master; @@ -1737,7 +1743,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; struct rb_node *node; - struct of_phandle_args masterspec; + struct of_phandle_iterator it; + struct arm_smmu_phandle_args *masterspec; int num_irqs, i, err; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); @@ -1798,20 +1805,35 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) i = 0; smmu->masters = RB_ROOT; - while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters", - "#stream-id-cells", i, - &masterspec)) { - err = register_smmu_master(smmu, dev, &masterspec); + + err = -ENOMEM; + /* No need to zero the memory for masterspec */ + masterspec = kmalloc(sizeof(*masterspec), GFP_KERNEL); + if (!masterspec) + goto out_put_masters; + + of_for_each_phandle(&it, err, dev->of_node, + "mmu-masters", "#stream-id-cells", 0) { + int count = of_phandle_iterator_args(&it, masterspec->args, + MAX_MASTER_STREAMIDS); + masterspec->np = of_node_get(it.node); + masterspec->args_count = count; + + err = register_smmu_master(smmu, dev, masterspec); if (err) { dev_err(dev, "failed to add master %s\n", - masterspec.np->name); + masterspec->np->name); + kfree(masterspec); goto out_put_masters; } i++; } + dev_notice(dev, "registered %d master devices\n", i); + kfree(masterspec); + parse_driver_options(smmu); if (smmu->version > ARM_SMMU_V1 && -- cgit v1.2.3 From 314f1dc140844a73034e285ddac92b9e1ffecb5b Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:32:45 +0300 Subject: iommu/vt-d: refactoring of deferred flush entries Currently, deferred flushes' info is striped between several lists in the flush tables. Instead, move all information about a specific flush to a single entry in this table. This patch does not introduce any functional change. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 48 +++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e1852e845d21..ec562078247f 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -458,15 +458,19 @@ static void flush_unmaps_timeout(unsigned long data); static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0); +struct deferred_flush_entry { + struct iova *iova; + struct dmar_domain *domain; + struct page *freelist; +}; + #define HIGH_WATER_MARK 250 -struct deferred_flush_tables { +struct deferred_flush_table { int next; - struct iova *iova[HIGH_WATER_MARK]; - struct dmar_domain *domain[HIGH_WATER_MARK]; - struct page *freelist[HIGH_WATER_MARK]; + struct deferred_flush_entry entries[HIGH_WATER_MARK]; }; -static struct deferred_flush_tables *deferred_flush; +static struct deferred_flush_table *deferred_flush; /* bitmap for indexing intel_iommus */ static int g_num_of_iommus; @@ -3111,7 +3115,7 @@ static int __init init_dmars(void) } deferred_flush = kzalloc(g_num_of_iommus * - sizeof(struct deferred_flush_tables), GFP_KERNEL); + sizeof(struct deferred_flush_table), GFP_KERNEL); if (!deferred_flush) { ret = -ENOMEM; goto free_g_iommus; @@ -3518,22 +3522,25 @@ static void flush_unmaps(void) DMA_TLB_GLOBAL_FLUSH); for (j = 0; j < deferred_flush[i].next; j++) { unsigned long mask; - struct iova *iova = deferred_flush[i].iova[j]; - struct dmar_domain *domain = deferred_flush[i].domain[j]; + struct deferred_flush_entry *entry = + &deferred_flush->entries[j]; + struct iova *iova = entry->iova; + struct dmar_domain *domain = entry->domain; + struct page *freelist = entry->freelist; /* On real hardware multiple invalidations are expensive */ if (cap_caching_mode(iommu->cap)) iommu_flush_iotlb_psi(iommu, domain, iova->pfn_lo, iova_size(iova), - !deferred_flush[i].freelist[j], 0); + !freelist, 0); else { mask = ilog2(mm_to_dma_pfn(iova_size(iova))); - iommu_flush_dev_iotlb(deferred_flush[i].domain[j], + iommu_flush_dev_iotlb(domain, (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask); } - __free_iova(&deferred_flush[i].domain[j]->iovad, iova); - if (deferred_flush[i].freelist[j]) - dma_free_pagelist(deferred_flush[i].freelist[j]); + __free_iova(&domain->iovad, iova); + if (freelist) + dma_free_pagelist(freelist); } deferred_flush[i].next = 0; } @@ -3553,8 +3560,9 @@ static void flush_unmaps_timeout(unsigned long data) static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist) { unsigned long flags; - int next, iommu_id; + int entry_id, iommu_id; struct intel_iommu *iommu; + struct deferred_flush_entry *entry; spin_lock_irqsave(&async_umap_flush_lock, flags); if (list_size == HIGH_WATER_MARK) @@ -3563,11 +3571,13 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *f iommu = domain_get_iommu(dom); iommu_id = iommu->seq_id; - next = deferred_flush[iommu_id].next; - deferred_flush[iommu_id].domain[next] = dom; - deferred_flush[iommu_id].iova[next] = iova; - deferred_flush[iommu_id].freelist[next] = freelist; - deferred_flush[iommu_id].next++; + entry_id = deferred_flush[iommu_id].next; + ++(deferred_flush[iommu_id].next); + + entry = &deferred_flush[iommu_id].entries[entry_id]; + entry->domain = dom; + entry->iova = iova; + entry->freelist = freelist; if (!timer_on) { mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10)); -- cgit v1.2.3 From aa4732406e1290dd18a8d2078977996c152a4be7 Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:33:02 +0300 Subject: iommu/vt-d: per-cpu deferred invalidation queues The IOMMU's IOTLB invalidation is a costly process. When iommu mode is not set to "strict", it is done asynchronously. Current code amortizes the cost of invalidating IOTLB entries by batching all the invalidations in the system and performing a single global invalidation instead. The code queues pending invalidations in a global queue that is accessed under the global "async_umap_flush_lock" spinlock, which can result is significant spinlock contention. This patch splits this deferred queue into multiple per-cpu deferred queues, and thus gets rid of the "async_umap_flush_lock" and its contention. To keep existing deferred invalidation behavior, it still invalidates the pending invalidations of all CPUs whenever a CPU reaches its watermark or a timeout occurs. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 133 ++++++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 41 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ec562078247f..a64b6f3b9a66 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -456,8 +457,6 @@ static LIST_HEAD(dmar_rmrr_units); static void flush_unmaps_timeout(unsigned long data); -static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0); - struct deferred_flush_entry { struct iova *iova; struct dmar_domain *domain; @@ -470,17 +469,19 @@ struct deferred_flush_table { struct deferred_flush_entry entries[HIGH_WATER_MARK]; }; -static struct deferred_flush_table *deferred_flush; +struct deferred_flush_data { + spinlock_t lock; + int timer_on; + struct timer_list timer; + long size; + struct deferred_flush_table *tables; +}; + +DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush); /* bitmap for indexing intel_iommus */ static int g_num_of_iommus; -static DEFINE_SPINLOCK(async_umap_flush_lock); -static LIST_HEAD(unmaps_to_do); - -static int timer_on; -static long list_size; - static void domain_exit(struct dmar_domain *domain); static void domain_remove_dev_info(struct dmar_domain *domain); static void dmar_remove_one_dev_info(struct dmar_domain *domain, @@ -1922,8 +1923,12 @@ static void domain_exit(struct dmar_domain *domain) return; /* Flush any lazy unmaps that may reference this domain */ - if (!intel_iommu_strict) - flush_unmaps_timeout(0); + if (!intel_iommu_strict) { + int cpu; + + for_each_possible_cpu(cpu) + flush_unmaps_timeout(cpu); + } /* Remove associated devices and clear attached or cached domains */ rcu_read_lock(); @@ -3081,7 +3086,7 @@ static int __init init_dmars(void) bool copied_tables = false; struct device *dev; struct intel_iommu *iommu; - int i, ret; + int i, ret, cpu; /* * for each drhd @@ -3114,11 +3119,20 @@ static int __init init_dmars(void) goto error; } - deferred_flush = kzalloc(g_num_of_iommus * - sizeof(struct deferred_flush_table), GFP_KERNEL); - if (!deferred_flush) { - ret = -ENOMEM; - goto free_g_iommus; + for_each_possible_cpu(cpu) { + struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush, + cpu); + + dfd->tables = kzalloc(g_num_of_iommus * + sizeof(struct deferred_flush_table), + GFP_KERNEL); + if (!dfd->tables) { + ret = -ENOMEM; + goto free_g_iommus; + } + + spin_lock_init(&dfd->lock); + setup_timer(&dfd->timer, flush_unmaps_timeout, cpu); } for_each_active_iommu(iommu, drhd) { @@ -3295,8 +3309,9 @@ free_iommu: disable_dmar_iommu(iommu); free_dmar_iommu(iommu); } - kfree(deferred_flush); free_g_iommus: + for_each_possible_cpu(cpu) + kfree(per_cpu_ptr(&deferred_flush, cpu)->tables); kfree(g_iommus); error: return ret; @@ -3501,29 +3516,31 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page, dir, *dev->dma_mask); } -static void flush_unmaps(void) +static void flush_unmaps(struct deferred_flush_data *flush_data) { int i, j; - timer_on = 0; + flush_data->timer_on = 0; /* just flush them all */ for (i = 0; i < g_num_of_iommus; i++) { struct intel_iommu *iommu = g_iommus[i]; + struct deferred_flush_table *flush_table = + &flush_data->tables[i]; if (!iommu) continue; - if (!deferred_flush[i].next) + if (!flush_table->next) continue; /* In caching mode, global flushes turn emulation expensive */ if (!cap_caching_mode(iommu->cap)) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); - for (j = 0; j < deferred_flush[i].next; j++) { + for (j = 0; j < flush_table->next; j++) { unsigned long mask; struct deferred_flush_entry *entry = - &deferred_flush->entries[j]; + &flush_table->entries[j]; struct iova *iova = entry->iova; struct dmar_domain *domain = entry->domain; struct page *freelist = entry->freelist; @@ -3542,19 +3559,20 @@ static void flush_unmaps(void) if (freelist) dma_free_pagelist(freelist); } - deferred_flush[i].next = 0; + flush_table->next = 0; } - list_size = 0; + flush_data->size = 0; } -static void flush_unmaps_timeout(unsigned long data) +static void flush_unmaps_timeout(unsigned long cpuid) { + struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid); unsigned long flags; - spin_lock_irqsave(&async_umap_flush_lock, flags); - flush_unmaps(); - spin_unlock_irqrestore(&async_umap_flush_lock, flags); + spin_lock_irqsave(&flush_data->lock, flags); + flush_unmaps(flush_data); + spin_unlock_irqrestore(&flush_data->lock, flags); } static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist) @@ -3563,28 +3581,44 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *f int entry_id, iommu_id; struct intel_iommu *iommu; struct deferred_flush_entry *entry; + struct deferred_flush_data *flush_data; + unsigned int cpuid; - spin_lock_irqsave(&async_umap_flush_lock, flags); - if (list_size == HIGH_WATER_MARK) - flush_unmaps(); + cpuid = get_cpu(); + flush_data = per_cpu_ptr(&deferred_flush, cpuid); + + /* Flush all CPUs' entries to avoid deferring too much. If + * this becomes a bottleneck, can just flush us, and rely on + * flush timer for the rest. + */ + if (flush_data->size == HIGH_WATER_MARK) { + int cpu; + + for_each_online_cpu(cpu) + flush_unmaps_timeout(cpu); + } + + spin_lock_irqsave(&flush_data->lock, flags); iommu = domain_get_iommu(dom); iommu_id = iommu->seq_id; - entry_id = deferred_flush[iommu_id].next; - ++(deferred_flush[iommu_id].next); + entry_id = flush_data->tables[iommu_id].next; + ++(flush_data->tables[iommu_id].next); - entry = &deferred_flush[iommu_id].entries[entry_id]; + entry = &flush_data->tables[iommu_id].entries[entry_id]; entry->domain = dom; entry->iova = iova; entry->freelist = freelist; - if (!timer_on) { - mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10)); - timer_on = 1; + if (!flush_data->timer_on) { + mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10)); + flush_data->timer_on = 1; } - list_size++; - spin_unlock_irqrestore(&async_umap_flush_lock, flags); + flush_data->size++; + spin_unlock_irqrestore(&flush_data->lock, flags); + + put_cpu(); } static void intel_unmap(struct device *dev, dma_addr_t dev_addr) @@ -4508,6 +4542,23 @@ static struct notifier_block intel_iommu_memory_nb = { .priority = 0 }; +static int intel_iommu_cpu_notifier(struct notifier_block *nfb, + unsigned long action, void *v) +{ + unsigned int cpu = (unsigned long)v; + + switch (action) { + case CPU_DEAD: + case CPU_DEAD_FROZEN: + flush_unmaps_timeout(cpu); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block intel_iommu_cpu_nb = { + .notifier_call = intel_iommu_cpu_notifier, +}; static ssize_t intel_iommu_show_version(struct device *dev, struct device_attribute *attr, @@ -4641,7 +4692,6 @@ int __init intel_iommu_init(void) up_write(&dmar_global_lock); pr_info("Intel(R) Virtualization Technology for Directed I/O\n"); - init_timer(&unmap_timer); #ifdef CONFIG_SWIOTLB swiotlb = 0; #endif @@ -4658,6 +4708,7 @@ int __init intel_iommu_init(void) bus_register_notifier(&pci_bus_type, &device_nb); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); + register_hotcpu_notifier(&intel_iommu_cpu_nb); intel_iommu_enabled = 1; -- cgit v1.2.3 From f5c0c08b1e0976cb493d503108f1b897ce58bc5d Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:33:11 +0300 Subject: iommu/vt-d: correct flush_unmaps pfn usage Change flush_unmaps() to correctly pass iommu_flush_iotlb_psi() dma addresses. (x86_64 mm and dma have the same size for pages at the moment, but this usage improves consistency.) Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a64b6f3b9a66..6aacfa4eee2a 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3548,7 +3548,8 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) /* On real hardware multiple invalidations are expensive */ if (cap_caching_mode(iommu->cap)) iommu_flush_iotlb_psi(iommu, domain, - iova->pfn_lo, iova_size(iova), + mm_to_dma_pfn(iova->pfn_lo), + mm_to_dma_pfn(iova_size(iova)), !freelist, 0); else { mask = ilog2(mm_to_dma_pfn(iova_size(iova))); -- cgit v1.2.3 From 769530e4ba4898748a293592f0920275b40cbb93 Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:33:25 +0300 Subject: iommu/vt-d: only unmap mapped entries Current unmap implementation unmaps the entire area covered by the IOVA range, which is a power-of-2 aligned region. The corresponding map, however, only maps those pages originally mapped by the user. This discrepancy can lead to unmapping of already unmapped entries, which is unneeded work. With this patch, only mapped pages are unmapped. This is also a baseline for a map/unmap implementation based on IOVAs and not iova structures, which will allow caching. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 6aacfa4eee2a..63b8c026ef9c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -459,6 +459,7 @@ static void flush_unmaps_timeout(unsigned long data); struct deferred_flush_entry { struct iova *iova; + unsigned long nrpages; struct dmar_domain *domain; struct page *freelist; }; @@ -3542,6 +3543,7 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) struct deferred_flush_entry *entry = &flush_table->entries[j]; struct iova *iova = entry->iova; + unsigned long nrpages = entry->nrpages; struct dmar_domain *domain = entry->domain; struct page *freelist = entry->freelist; @@ -3549,10 +3551,9 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) if (cap_caching_mode(iommu->cap)) iommu_flush_iotlb_psi(iommu, domain, mm_to_dma_pfn(iova->pfn_lo), - mm_to_dma_pfn(iova_size(iova)), - !freelist, 0); + nrpages, !freelist, 0); else { - mask = ilog2(mm_to_dma_pfn(iova_size(iova))); + mask = ilog2(nrpages); iommu_flush_dev_iotlb(domain, (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask); } @@ -3576,7 +3577,8 @@ static void flush_unmaps_timeout(unsigned long cpuid) spin_unlock_irqrestore(&flush_data->lock, flags); } -static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist) +static void add_unmap(struct dmar_domain *dom, struct iova *iova, + unsigned long nrpages, struct page *freelist) { unsigned long flags; int entry_id, iommu_id; @@ -3610,6 +3612,7 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *f entry = &flush_data->tables[iommu_id].entries[entry_id]; entry->domain = dom; entry->iova = iova; + entry->nrpages = nrpages; entry->freelist = freelist; if (!flush_data->timer_on) { @@ -3622,10 +3625,11 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *f put_cpu(); } -static void intel_unmap(struct device *dev, dma_addr_t dev_addr) +static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) { struct dmar_domain *domain; unsigned long start_pfn, last_pfn; + unsigned long nrpages; struct iova *iova; struct intel_iommu *iommu; struct page *freelist; @@ -3643,8 +3647,9 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr) (unsigned long long)dev_addr)) return; + nrpages = aligned_nrpages(dev_addr, size); start_pfn = mm_to_dma_pfn(iova->pfn_lo); - last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1; + last_pfn = start_pfn + nrpages - 1; pr_debug("Device %s unmapping: pfn %lx-%lx\n", dev_name(dev), start_pfn, last_pfn); @@ -3653,12 +3658,12 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr) if (intel_iommu_strict) { iommu_flush_iotlb_psi(iommu, domain, start_pfn, - last_pfn - start_pfn + 1, !freelist, 0); + nrpages, !freelist, 0); /* free iova */ __free_iova(&domain->iovad, iova); dma_free_pagelist(freelist); } else { - add_unmap(domain, iova, freelist); + add_unmap(domain, iova, nrpages, freelist); /* * queue up the release of the unmap to save the 1/6th of the * cpu used up by the iotlb flush operation... @@ -3670,7 +3675,7 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - intel_unmap(dev, dev_addr); + intel_unmap(dev, dev_addr, size); } static void *intel_alloc_coherent(struct device *dev, size_t size, @@ -3729,7 +3734,7 @@ static void intel_free_coherent(struct device *dev, size_t size, void *vaddr, size = PAGE_ALIGN(size); order = get_order(size); - intel_unmap(dev, dma_handle); + intel_unmap(dev, dma_handle, size); if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) __free_pages(page, order); } @@ -3738,7 +3743,16 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) { - intel_unmap(dev, sglist[0].dma_address); + dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK; + unsigned long nrpages = 0; + struct scatterlist *sg; + int i; + + for_each_sg(sglist, sg, nelems, i) { + nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg)); + } + + intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT); } static int intel_nontranslate_map_sg(struct device *hddev, -- cgit v1.2.3 From 0824c5920b16fe11034f3b5d2d48456d282d83f9 Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 19:03:35 +0300 Subject: iommu/vt-d: avoid dev iotlb logic for domains with no dev iotlbs This patch avoids taking the device_domain_lock in iommu_flush_dev_iotlb() for domains with no dev iotlb devices. Signed-off-by: Omer Peleg [gvdl@google.com: fixed locking issues] Signed-off-by: Godfrey van der Linden [mad@cs.technion.ac.il: rebased and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 63b8c026ef9c..d100583e15e3 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -391,6 +391,7 @@ struct dmar_domain { * domain ids are 16 bit wide according * to VT-d spec, section 9.3 */ + bool has_iotlb_device; struct list_head devices; /* all devices' list */ struct iova_domain iovad; /* iova's that belong to this domain */ @@ -1464,10 +1465,35 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu, return NULL; } +static void domain_update_iotlb(struct dmar_domain *domain) +{ + struct device_domain_info *info; + bool has_iotlb_device = false; + + assert_spin_locked(&device_domain_lock); + + list_for_each_entry(info, &domain->devices, link) { + struct pci_dev *pdev; + + if (!info->dev || !dev_is_pci(info->dev)) + continue; + + pdev = to_pci_dev(info->dev); + if (pdev->ats_enabled) { + has_iotlb_device = true; + break; + } + } + + domain->has_iotlb_device = has_iotlb_device; +} + static void iommu_enable_dev_iotlb(struct device_domain_info *info) { struct pci_dev *pdev; + assert_spin_locked(&device_domain_lock); + if (!info || !dev_is_pci(info->dev)) return; @@ -1487,6 +1513,7 @@ static void iommu_enable_dev_iotlb(struct device_domain_info *info) #endif if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) { info->ats_enabled = 1; + domain_update_iotlb(info->domain); info->ats_qdep = pci_ats_queue_depth(pdev); } } @@ -1495,6 +1522,8 @@ static void iommu_disable_dev_iotlb(struct device_domain_info *info) { struct pci_dev *pdev; + assert_spin_locked(&device_domain_lock); + if (!dev_is_pci(info->dev)) return; @@ -1503,6 +1532,7 @@ static void iommu_disable_dev_iotlb(struct device_domain_info *info) if (info->ats_enabled) { pci_disable_ats(pdev); info->ats_enabled = 0; + domain_update_iotlb(info->domain); } #ifdef CONFIG_INTEL_IOMMU_SVM if (info->pri_enabled) { @@ -1523,6 +1553,9 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, unsigned long flags; struct device_domain_info *info; + if (!domain->has_iotlb_device) + return; + spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &domain->devices, link) { if (!info->ats_enabled) @@ -1740,6 +1773,7 @@ static struct dmar_domain *alloc_domain(int flags) memset(domain, 0, sizeof(*domain)); domain->nid = -1; domain->flags = flags; + domain->has_iotlb_device = false; INIT_LIST_HEAD(&domain->devices); return domain; -- cgit v1.2.3 From 2aac630429d986a43ac59525a4cff47a624dc58e Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:33:57 +0300 Subject: iommu/vt-d: change intel-iommu to use IOVA frame numbers Make intel-iommu map/unmap/invalidate work with IOVA pfns instead of pointers to "struct iova". This avoids using the iova struct from the IOVA red-black tree and the resulting explicit find_iova() on unmap. This patch will allow us to cache IOVAs in the next patch, in order to avoid rbtree operations for the majority of map/unmap operations. Note: In eliminating the find_iova() operation, we have also eliminated the sanity check previously done in the unmap flow. Arguably, this was overhead that is better avoided in production code, but it could be brought back as a debug option for driver development. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased, fixed to not break iova api, and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 61 +++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d100583e15e3..a8babc43e6d4 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -459,7 +459,7 @@ static LIST_HEAD(dmar_rmrr_units); static void flush_unmaps_timeout(unsigned long data); struct deferred_flush_entry { - struct iova *iova; + unsigned long iova_pfn; unsigned long nrpages; struct dmar_domain *domain; struct page *freelist; @@ -3353,7 +3353,7 @@ error: } /* This takes a number of _MM_ pages, not VTD pages */ -static struct iova *intel_alloc_iova(struct device *dev, +static unsigned long intel_alloc_iova(struct device *dev, struct dmar_domain *domain, unsigned long nrpages, uint64_t dma_mask) { @@ -3373,16 +3373,16 @@ static struct iova *intel_alloc_iova(struct device *dev, iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(DMA_BIT_MASK(32)), 1); if (iova) - return iova; + return iova->pfn_lo; } iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); if (unlikely(!iova)) { pr_err("Allocating %ld-page iova for %s failed", nrpages, dev_name(dev)); - return NULL; + return 0; } - return iova; + return iova->pfn_lo; } static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) @@ -3480,7 +3480,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, { struct dmar_domain *domain; phys_addr_t start_paddr; - struct iova *iova; + unsigned long iova_pfn; int prot = 0; int ret; struct intel_iommu *iommu; @@ -3498,8 +3498,8 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, iommu = domain_get_iommu(domain); size = aligned_nrpages(paddr, size); - iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask); - if (!iova) + iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask); + if (!iova_pfn) goto error; /* @@ -3517,7 +3517,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, * might have two guest_addr mapping to the same host paddr, but this * is not a big problem */ - ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo), + ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn), mm_to_dma_pfn(paddr_pfn), size, prot); if (ret) goto error; @@ -3525,18 +3525,18 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, /* it's a non-present to present mapping. Only flush if caching mode */ if (cap_caching_mode(iommu->cap)) iommu_flush_iotlb_psi(iommu, domain, - mm_to_dma_pfn(iova->pfn_lo), + mm_to_dma_pfn(iova_pfn), size, 0, 1); else iommu_flush_write_buffer(iommu); - start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT; + start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT; start_paddr += paddr & ~PAGE_MASK; return start_paddr; error: - if (iova) - __free_iova(&domain->iovad, iova); + if (iova_pfn) + free_iova(&domain->iovad, iova_pfn); pr_err("Device %s request: %zx@%llx dir %d --- failed\n", dev_name(dev), size, (unsigned long long)paddr, dir); return 0; @@ -3576,7 +3576,7 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) unsigned long mask; struct deferred_flush_entry *entry = &flush_table->entries[j]; - struct iova *iova = entry->iova; + unsigned long iova_pfn = entry->iova_pfn; unsigned long nrpages = entry->nrpages; struct dmar_domain *domain = entry->domain; struct page *freelist = entry->freelist; @@ -3584,14 +3584,14 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) /* On real hardware multiple invalidations are expensive */ if (cap_caching_mode(iommu->cap)) iommu_flush_iotlb_psi(iommu, domain, - mm_to_dma_pfn(iova->pfn_lo), + mm_to_dma_pfn(iova_pfn), nrpages, !freelist, 0); else { mask = ilog2(nrpages); iommu_flush_dev_iotlb(domain, - (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask); + (uint64_t)iova_pfn << PAGE_SHIFT, mask); } - __free_iova(&domain->iovad, iova); + free_iova(&domain->iovad, iova_pfn); if (freelist) dma_free_pagelist(freelist); } @@ -3611,7 +3611,7 @@ static void flush_unmaps_timeout(unsigned long cpuid) spin_unlock_irqrestore(&flush_data->lock, flags); } -static void add_unmap(struct dmar_domain *dom, struct iova *iova, +static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn, unsigned long nrpages, struct page *freelist) { unsigned long flags; @@ -3645,7 +3645,7 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, entry = &flush_data->tables[iommu_id].entries[entry_id]; entry->domain = dom; - entry->iova = iova; + entry->iova_pfn = iova_pfn; entry->nrpages = nrpages; entry->freelist = freelist; @@ -3664,7 +3664,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) struct dmar_domain *domain; unsigned long start_pfn, last_pfn; unsigned long nrpages; - struct iova *iova; + unsigned long iova_pfn; struct intel_iommu *iommu; struct page *freelist; @@ -3676,13 +3676,10 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) iommu = domain_get_iommu(domain); - iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr)); - if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n", - (unsigned long long)dev_addr)) - return; + iova_pfn = IOVA_PFN(dev_addr); nrpages = aligned_nrpages(dev_addr, size); - start_pfn = mm_to_dma_pfn(iova->pfn_lo); + start_pfn = mm_to_dma_pfn(iova_pfn); last_pfn = start_pfn + nrpages - 1; pr_debug("Device %s unmapping: pfn %lx-%lx\n", @@ -3694,10 +3691,10 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) iommu_flush_iotlb_psi(iommu, domain, start_pfn, nrpages, !freelist, 0); /* free iova */ - __free_iova(&domain->iovad, iova); + free_iova(&domain->iovad, iova_pfn); dma_free_pagelist(freelist); } else { - add_unmap(domain, iova, nrpages, freelist); + add_unmap(domain, iova_pfn, nrpages, freelist); /* * queue up the release of the unmap to save the 1/6th of the * cpu used up by the iotlb flush operation... @@ -3810,7 +3807,7 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele struct dmar_domain *domain; size_t size = 0; int prot = 0; - struct iova *iova = NULL; + unsigned long iova_pfn; int ret; struct scatterlist *sg; unsigned long start_vpfn; @@ -3829,9 +3826,9 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele for_each_sg(sglist, sg, nelems, i) size += aligned_nrpages(sg->offset, sg->length); - iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), + iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), *dev->dma_mask); - if (!iova) { + if (!iova_pfn) { sglist->dma_length = 0; return 0; } @@ -3846,13 +3843,13 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) prot |= DMA_PTE_WRITE; - start_vpfn = mm_to_dma_pfn(iova->pfn_lo); + start_vpfn = mm_to_dma_pfn(iova_pfn); ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot); if (unlikely(ret)) { dma_pte_free_pagetable(domain, start_vpfn, start_vpfn + size - 1); - __free_iova(&domain->iovad, iova); + free_iova(&domain->iovad, iova_pfn); return 0; } -- cgit v1.2.3 From 9257b4a206fc0229dd5f84b78e4d1ebf3f91d270 Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:34:11 +0300 Subject: iommu/iova: introduce per-cpu caching to iova allocation IOVA allocation has two problems that impede high-throughput I/O. First, it can do a linear search over the allocated IOVA ranges. Second, the rbtree spinlock that serializes IOVA allocations becomes contended. Address these problems by creating an API for caching allocated IOVA ranges, so that the IOVA allocator isn't accessed frequently. This patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs without taking the rbtree spinlock. The per-CPU caches are backed by a global cache, to avoid invoking the (linear-time) IOVA allocator without needing to make the per-CPU cache size excessive. This design is based on magazines, as described in "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources" (currently available at https://www.usenix.org/legacy/event/usenix01/bonwick.html) Adding caching on top of the existing rbtree allocator maintains the property that IOVAs are densely packed in the IO virtual address space, which is important for keeping IOMMU page table usage low. To keep the cache size reasonable, we bound the IOVA space a CPU can cache by 32 MiB (we cache a bounded number of IOVA ranges, and only ranges of size <= 128 KiB). The shared global cache is bounded at 4 MiB of IOVA space. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin [dwmw2: split out VT-d part into a separate patch] Signed-off-by: David Woodhouse --- drivers/iommu/iova.c | 417 ++++++++++++++++++++++++++++++++++++++++++++++++--- include/linux/iova.h | 23 ++- 2 files changed, 414 insertions(+), 26 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index fa0adef32bd6..ba764a0835d3 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -20,6 +20,17 @@ #include #include #include +#include +#include + +static bool iova_rcache_insert(struct iova_domain *iovad, + unsigned long pfn, + unsigned long size); +static unsigned long iova_rcache_get(struct iova_domain *iovad, + unsigned long size, + unsigned long limit_pfn); +static void init_iova_rcaches(struct iova_domain *iovad); +static void free_iova_rcaches(struct iova_domain *iovad); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, @@ -38,6 +49,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->granule = granule; iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = pfn_32bit; + init_iova_rcaches(iovad); } EXPORT_SYMBOL_GPL(init_iova_domain); @@ -291,33 +303,18 @@ alloc_iova(struct iova_domain *iovad, unsigned long size, } EXPORT_SYMBOL_GPL(alloc_iova); -/** - * find_iova - find's an iova for a given pfn - * @iovad: - iova domain in question. - * @pfn: - page frame number - * This function finds and returns an iova belonging to the - * given doamin which matches the given pfn. - */ -struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) +static struct iova * +private_find_iova(struct iova_domain *iovad, unsigned long pfn) { - unsigned long flags; - struct rb_node *node; + struct rb_node *node = iovad->rbroot.rb_node; + + assert_spin_locked(&iovad->iova_rbtree_lock); - /* Take the lock so that no other thread is manipulating the rbtree */ - spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - node = iovad->rbroot.rb_node; while (node) { struct iova *iova = container_of(node, struct iova, node); /* If pfn falls within iova's range, return iova */ if ((pfn >= iova->pfn_lo) && (pfn <= iova->pfn_hi)) { - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); - /* We are not holding the lock while this iova - * is referenced by the caller as the same thread - * which called this function also calls __free_iova() - * and it is by design that only one thread can possibly - * reference a particular iova and hence no conflict. - */ return iova; } @@ -327,9 +324,35 @@ struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) node = node->rb_right; } - spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); return NULL; } + +static void private_free_iova(struct iova_domain *iovad, struct iova *iova) +{ + assert_spin_locked(&iovad->iova_rbtree_lock); + __cached_rbnode_delete_update(iovad, iova); + rb_erase(&iova->node, &iovad->rbroot); + free_iova_mem(iova); +} + +/** + * find_iova - finds an iova for a given pfn + * @iovad: - iova domain in question. + * @pfn: - page frame number + * This function finds and returns an iova belonging to the + * given doamin which matches the given pfn. + */ +struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) +{ + unsigned long flags; + struct iova *iova; + + /* Take the lock so that no other thread is manipulating the rbtree */ + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); + iova = private_find_iova(iovad, pfn); + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + return iova; +} EXPORT_SYMBOL_GPL(find_iova); /** @@ -344,10 +367,8 @@ __free_iova(struct iova_domain *iovad, struct iova *iova) unsigned long flags; spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); - __cached_rbnode_delete_update(iovad, iova); - rb_erase(&iova->node, &iovad->rbroot); + private_free_iova(iovad, iova); spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); - free_iova_mem(iova); } EXPORT_SYMBOL_GPL(__free_iova); @@ -369,6 +390,63 @@ free_iova(struct iova_domain *iovad, unsigned long pfn) } EXPORT_SYMBOL_GPL(free_iova); +/** + * alloc_iova_fast - allocates an iova from rcache + * @iovad: - iova domain in question + * @size: - size of page frames to allocate + * @limit_pfn: - max limit address + * This function tries to satisfy an iova allocation from the rcache, + * and falls back to regular allocation on failure. +*/ +unsigned long +alloc_iova_fast(struct iova_domain *iovad, unsigned long size, + unsigned long limit_pfn) +{ + bool flushed_rcache = false; + unsigned long iova_pfn; + struct iova *new_iova; + + iova_pfn = iova_rcache_get(iovad, size, limit_pfn); + if (iova_pfn) + return iova_pfn; + +retry: + new_iova = alloc_iova(iovad, size, limit_pfn, true); + if (!new_iova) { + unsigned int cpu; + + if (flushed_rcache) + return 0; + + /* Try replenishing IOVAs by flushing rcache. */ + flushed_rcache = true; + for_each_online_cpu(cpu) + free_cpu_cached_iovas(cpu, iovad); + goto retry; + } + + return new_iova->pfn_lo; +} +EXPORT_SYMBOL_GPL(alloc_iova_fast); + +/** + * free_iova_fast - free iova pfn range into rcache + * @iovad: - iova domain in question. + * @pfn: - pfn that is allocated previously + * @size: - # of pages in range + * This functions frees an iova range by trying to put it into the rcache, + * falling back to regular iova deallocation via free_iova() if this fails. + */ +void +free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) +{ + if (iova_rcache_insert(iovad, pfn, size)) + return; + + free_iova(iovad, pfn); +} +EXPORT_SYMBOL_GPL(free_iova_fast); + /** * put_iova_domain - destroys the iova doamin * @iovad: - iova domain in question. @@ -379,6 +457,7 @@ void put_iova_domain(struct iova_domain *iovad) struct rb_node *node; unsigned long flags; + free_iova_rcaches(iovad); spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); node = rb_first(&iovad->rbroot); while (node) { @@ -550,5 +629,295 @@ error: return NULL; } +/* + * Magazine caches for IOVA ranges. For an introduction to magazines, + * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab + * Allocator to Many CPUs and Arbitrary Resources" by Bonwick and Adams. + * For simplicity, we use a static magazine size and don't implement the + * dynamic size tuning described in the paper. + */ + +#define IOVA_MAG_SIZE 128 + +struct iova_magazine { + unsigned long size; + unsigned long pfns[IOVA_MAG_SIZE]; +}; + +struct iova_cpu_rcache { + spinlock_t lock; + struct iova_magazine *loaded; + struct iova_magazine *prev; +}; + +static struct iova_magazine *iova_magazine_alloc(gfp_t flags) +{ + return kzalloc(sizeof(struct iova_magazine), flags); +} + +static void iova_magazine_free(struct iova_magazine *mag) +{ + kfree(mag); +} + +static void +iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) +{ + unsigned long flags; + int i; + + if (!mag) + return; + + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); + + for (i = 0 ; i < mag->size; ++i) { + struct iova *iova = private_find_iova(iovad, mag->pfns[i]); + + BUG_ON(!iova); + private_free_iova(iovad, iova); + } + + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + + mag->size = 0; +} + +static bool iova_magazine_full(struct iova_magazine *mag) +{ + return (mag && mag->size == IOVA_MAG_SIZE); +} + +static bool iova_magazine_empty(struct iova_magazine *mag) +{ + return (!mag || mag->size == 0); +} + +static unsigned long iova_magazine_pop(struct iova_magazine *mag, + unsigned long limit_pfn) +{ + BUG_ON(iova_magazine_empty(mag)); + + if (mag->pfns[mag->size - 1] >= limit_pfn) + return 0; + + return mag->pfns[--mag->size]; +} + +static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) +{ + BUG_ON(iova_magazine_full(mag)); + + mag->pfns[mag->size++] = pfn; +} + +static void init_iova_rcaches(struct iova_domain *iovad) +{ + struct iova_cpu_rcache *cpu_rcache; + struct iova_rcache *rcache; + unsigned int cpu; + int i; + + for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { + rcache = &iovad->rcaches[i]; + spin_lock_init(&rcache->lock); + rcache->depot_size = 0; + rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size()); + if (WARN_ON(!rcache->cpu_rcaches)) + continue; + for_each_possible_cpu(cpu) { + cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu); + spin_lock_init(&cpu_rcache->lock); + cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL); + cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL); + } + } +} + +/* + * Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and + * return true on success. Can fail if rcache is full and we can't free + * space, and free_iova() (our only caller) will then return the IOVA + * range to the rbtree instead. + */ +static bool __iova_rcache_insert(struct iova_domain *iovad, + struct iova_rcache *rcache, + unsigned long iova_pfn) +{ + struct iova_magazine *mag_to_free = NULL; + struct iova_cpu_rcache *cpu_rcache; + bool can_insert = false; + unsigned long flags; + + cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + spin_lock_irqsave(&cpu_rcache->lock, flags); + + if (!iova_magazine_full(cpu_rcache->loaded)) { + can_insert = true; + } else if (!iova_magazine_full(cpu_rcache->prev)) { + swap(cpu_rcache->prev, cpu_rcache->loaded); + can_insert = true; + } else { + struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC); + + if (new_mag) { + spin_lock(&rcache->lock); + if (rcache->depot_size < MAX_GLOBAL_MAGS) { + rcache->depot[rcache->depot_size++] = + cpu_rcache->loaded; + } else { + mag_to_free = cpu_rcache->loaded; + } + spin_unlock(&rcache->lock); + + cpu_rcache->loaded = new_mag; + can_insert = true; + } + } + + if (can_insert) + iova_magazine_push(cpu_rcache->loaded, iova_pfn); + + spin_unlock_irqrestore(&cpu_rcache->lock, flags); + + if (mag_to_free) { + iova_magazine_free_pfns(mag_to_free, iovad); + iova_magazine_free(mag_to_free); + } + + return can_insert; +} + +static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, + unsigned long size) +{ + unsigned int log_size = order_base_2(size); + + if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) + return false; + + return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn); +} + +/* + * Caller wants to allocate a new IOVA range from 'rcache'. If we can + * satisfy the request, return a matching non-NULL range and remove + * it from the 'rcache'. + */ +static unsigned long __iova_rcache_get(struct iova_rcache *rcache, + unsigned long limit_pfn) +{ + struct iova_cpu_rcache *cpu_rcache; + unsigned long iova_pfn = 0; + bool has_pfn = false; + unsigned long flags; + + cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + spin_lock_irqsave(&cpu_rcache->lock, flags); + + if (!iova_magazine_empty(cpu_rcache->loaded)) { + has_pfn = true; + } else if (!iova_magazine_empty(cpu_rcache->prev)) { + swap(cpu_rcache->prev, cpu_rcache->loaded); + has_pfn = true; + } else { + spin_lock(&rcache->lock); + if (rcache->depot_size > 0) { + iova_magazine_free(cpu_rcache->loaded); + cpu_rcache->loaded = rcache->depot[--rcache->depot_size]; + has_pfn = true; + } + spin_unlock(&rcache->lock); + } + + if (has_pfn) + iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn); + + spin_unlock_irqrestore(&cpu_rcache->lock, flags); + + return iova_pfn; +} + +/* + * Try to satisfy IOVA allocation range from rcache. Fail if requested + * size is too big or the DMA limit we are given isn't satisfied by the + * top element in the magazine. + */ +static unsigned long iova_rcache_get(struct iova_domain *iovad, + unsigned long size, + unsigned long limit_pfn) +{ + unsigned int log_size = order_base_2(size); + + if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) + return 0; + + return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn); +} + +/* + * Free a cpu's rcache. + */ +static void free_cpu_iova_rcache(unsigned int cpu, struct iova_domain *iovad, + struct iova_rcache *rcache) +{ + struct iova_cpu_rcache *cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu); + unsigned long flags; + + spin_lock_irqsave(&cpu_rcache->lock, flags); + + iova_magazine_free_pfns(cpu_rcache->loaded, iovad); + iova_magazine_free(cpu_rcache->loaded); + + iova_magazine_free_pfns(cpu_rcache->prev, iovad); + iova_magazine_free(cpu_rcache->prev); + + spin_unlock_irqrestore(&cpu_rcache->lock, flags); +} + +/* + * free rcache data structures. + */ +static void free_iova_rcaches(struct iova_domain *iovad) +{ + struct iova_rcache *rcache; + unsigned long flags; + unsigned int cpu; + int i, j; + + for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { + rcache = &iovad->rcaches[i]; + for_each_possible_cpu(cpu) + free_cpu_iova_rcache(cpu, iovad, rcache); + spin_lock_irqsave(&rcache->lock, flags); + free_percpu(rcache->cpu_rcaches); + for (j = 0; j < rcache->depot_size; ++j) { + iova_magazine_free_pfns(rcache->depot[j], iovad); + iova_magazine_free(rcache->depot[j]); + } + spin_unlock_irqrestore(&rcache->lock, flags); + } +} + +/* + * free all the IOVA ranges cached by a cpu (used when cpu is unplugged) + */ +void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) +{ + struct iova_cpu_rcache *cpu_rcache; + struct iova_rcache *rcache; + unsigned long flags; + int i; + + for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { + rcache = &iovad->rcaches[i]; + cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu); + spin_lock_irqsave(&cpu_rcache->lock, flags); + iova_magazine_free_pfns(cpu_rcache->loaded, iovad); + iova_magazine_free_pfns(cpu_rcache->prev, iovad); + spin_unlock_irqrestore(&cpu_rcache->lock, flags); + } +} + MODULE_AUTHOR("Anil S Keshavamurthy "); MODULE_LICENSE("GPL"); diff --git a/include/linux/iova.h b/include/linux/iova.h index 92f7177db2ce..f27bb2c62fca 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -19,8 +19,21 @@ /* iova structure */ struct iova { struct rb_node node; - unsigned long pfn_hi; /* IOMMU dish out addr hi */ - unsigned long pfn_lo; /* IOMMU dish out addr lo */ + unsigned long pfn_hi; /* Highest allocated pfn */ + unsigned long pfn_lo; /* Lowest allocated pfn */ +}; + +struct iova_magazine; +struct iova_cpu_rcache; + +#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */ +#define MAX_GLOBAL_MAGS 32 /* magazines per bin */ + +struct iova_rcache { + spinlock_t lock; + unsigned long depot_size; + struct iova_magazine *depot[MAX_GLOBAL_MAGS]; + struct iova_cpu_rcache __percpu *cpu_rcaches; }; /* holds all the iova translations for a domain */ @@ -31,6 +44,7 @@ struct iova_domain { unsigned long granule; /* pfn granularity for this domain */ unsigned long start_pfn; /* Lower limit for this domain */ unsigned long dma_32bit_pfn; + struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ }; static inline unsigned long iova_size(struct iova *iova) @@ -78,6 +92,10 @@ void __free_iova(struct iova_domain *iovad, struct iova *iova); struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned); +void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, + unsigned long size); +unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, + unsigned long limit_pfn); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); @@ -87,5 +105,6 @@ struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); struct iova *split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi); +void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); #endif -- cgit v1.2.3 From 22e2f9fa63b092923873fc8a52955151f4d83274 Mon Sep 17 00:00:00 2001 From: Omer Peleg Date: Wed, 20 Apr 2016 11:34:11 +0300 Subject: iommu/vt-d: Use per-cpu IOVA caching Commit 9257b4a2 ('iommu/iova: introduce per-cpu caching to iova allocation') introduced per-CPU IOVA caches to massively improve scalability. Use them. Signed-off-by: Omer Peleg [mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message] Signed-off-by: Adam Morrison Reviewed-by: Shaohua Li Reviewed-by: Ben Serebrin [dwmw2: split out VT-d part into a separate patch] Signed-off-by: David Woodhouse --- drivers/iommu/intel-iommu.c | 47 +++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a8babc43e6d4..76e833278db0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3357,7 +3357,7 @@ static unsigned long intel_alloc_iova(struct device *dev, struct dmar_domain *domain, unsigned long nrpages, uint64_t dma_mask) { - struct iova *iova = NULL; + unsigned long iova_pfn = 0; /* Restrict dma_mask to the width that the iommu can handle */ dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask); @@ -3370,19 +3370,19 @@ static unsigned long intel_alloc_iova(struct device *dev, * DMA_BIT_MASK(32) and if that fails then try allocating * from higher range */ - iova = alloc_iova(&domain->iovad, nrpages, - IOVA_PFN(DMA_BIT_MASK(32)), 1); - if (iova) - return iova->pfn_lo; + iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, + IOVA_PFN(DMA_BIT_MASK(32))); + if (iova_pfn) + return iova_pfn; } - iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); - if (unlikely(!iova)) { + iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask)); + if (unlikely(!iova_pfn)) { pr_err("Allocating %ld-page iova for %s failed", nrpages, dev_name(dev)); return 0; } - return iova->pfn_lo; + return iova_pfn; } static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) @@ -3536,7 +3536,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, error: if (iova_pfn) - free_iova(&domain->iovad, iova_pfn); + free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size)); pr_err("Device %s request: %zx@%llx dir %d --- failed\n", dev_name(dev), size, (unsigned long long)paddr, dir); return 0; @@ -3591,7 +3591,7 @@ static void flush_unmaps(struct deferred_flush_data *flush_data) iommu_flush_dev_iotlb(domain, (uint64_t)iova_pfn << PAGE_SHIFT, mask); } - free_iova(&domain->iovad, iova_pfn); + free_iova_fast(&domain->iovad, iova_pfn, nrpages); if (freelist) dma_free_pagelist(freelist); } @@ -3691,7 +3691,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) iommu_flush_iotlb_psi(iommu, domain, start_pfn, nrpages, !freelist, 0); /* free iova */ - free_iova(&domain->iovad, iova_pfn); + free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages)); dma_free_pagelist(freelist); } else { add_unmap(domain, iova_pfn, nrpages, freelist); @@ -3849,7 +3849,7 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele if (unlikely(ret)) { dma_pte_free_pagetable(domain, start_vpfn, start_vpfn + size - 1); - free_iova(&domain->iovad, iova_pfn); + free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size)); return 0; } @@ -4588,6 +4588,28 @@ static struct notifier_block intel_iommu_memory_nb = { .priority = 0 }; +static void free_all_cpu_cached_iovas(unsigned int cpu) +{ + int i; + + for (i = 0; i < g_num_of_iommus; i++) { + struct intel_iommu *iommu = g_iommus[i]; + struct dmar_domain *domain; + u16 did; + + if (!iommu) + continue; + + for (did = 0; did < 0xffff; did++) { + domain = get_iommu_domain(iommu, did); + + if (!domain) + continue; + free_cpu_cached_iovas(cpu, &domain->iovad); + } + } +} + static int intel_iommu_cpu_notifier(struct notifier_block *nfb, unsigned long action, void *v) { @@ -4596,6 +4618,7 @@ static int intel_iommu_cpu_notifier(struct notifier_block *nfb, switch (action) { case CPU_DEAD: case CPU_DEAD_FROZEN: + free_all_cpu_cached_iovas(cpu); flush_unmaps_timeout(cpu); break; } -- cgit v1.2.3 From 9ee35e4c6f42e792974872ee1ec4115718ce05bc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 21 Apr 2016 18:21:31 +0200 Subject: iommu/amd: Don't use IS_ERR_VALUE to check integer values Use the better 'var < 0' check. Fixes: 7aba6cb9ee9d ('iommu/amd: Make call-sites of get_device_id aware of its return value') Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 12f77798a0a4..a2a51c164c7f 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -354,7 +354,7 @@ static void init_unity_mappings_for_device(struct device *dev, int devid; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; list_for_each_entry(e, &amd_iommu_unity_map, list) { @@ -376,7 +376,7 @@ static bool check_device(struct device *dev) return false; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return false; /* Out of our scope? */ @@ -419,7 +419,7 @@ static int iommu_init_device(struct device *dev) return 0; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return devid; dev_data = find_dev_data(devid); @@ -447,7 +447,7 @@ static void iommu_ignore_device(struct device *dev) int devid; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; alias = amd_iommu_alias_table[devid]; @@ -465,7 +465,7 @@ static void iommu_uninit_device(struct device *dev) struct iommu_dev_data *dev_data; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; dev_data = search_dev_data(devid); @@ -2414,7 +2414,7 @@ static int amd_iommu_add_device(struct device *dev) return 0; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return devid; iommu = amd_iommu_rlookup_table[devid]; @@ -2460,7 +2460,7 @@ static void amd_iommu_remove_device(struct device *dev) return; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; iommu = amd_iommu_rlookup_table[devid]; @@ -3158,7 +3158,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, return; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; if (dev_data->domain != NULL) @@ -3280,7 +3280,7 @@ static void amd_iommu_get_dm_regions(struct device *dev, int devid; devid = get_device_id(dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return; list_for_each_entry(entry, &amd_iommu_unity_map, list) { @@ -3983,7 +3983,7 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info) case X86_IRQ_ALLOC_TYPE_MSI: case X86_IRQ_ALLOC_TYPE_MSIX: devid = get_device_id(&info->msi_dev->dev); - if (IS_ERR_VALUE(devid)) + if (devid < 0) return NULL; iommu = amd_iommu_rlookup_table[devid]; -- cgit v1.2.3 From fd6c50ee3af3935d8dfa38b0a81b2386fd915cfe Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 21 Apr 2016 18:24:02 +0200 Subject: iommu/amd: Move get_device_id() and friends to beginning of file They will be needed there later. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 108 +++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 54 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index a2a51c164c7f..119148552aa0 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -164,60 +164,6 @@ struct dma_ops_domain { * ****************************************************************************/ -static struct protection_domain *to_pdomain(struct iommu_domain *dom) -{ - return container_of(dom, struct protection_domain, domain); -} - -static struct iommu_dev_data *alloc_dev_data(u16 devid) -{ - struct iommu_dev_data *dev_data; - unsigned long flags; - - dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); - if (!dev_data) - return NULL; - - dev_data->devid = devid; - - spin_lock_irqsave(&dev_data_list_lock, flags); - list_add_tail(&dev_data->dev_data_list, &dev_data_list); - spin_unlock_irqrestore(&dev_data_list_lock, flags); - - return dev_data; -} - -static struct iommu_dev_data *search_dev_data(u16 devid) -{ - struct iommu_dev_data *dev_data; - unsigned long flags; - - spin_lock_irqsave(&dev_data_list_lock, flags); - list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { - if (dev_data->devid == devid) - goto out_unlock; - } - - dev_data = NULL; - -out_unlock: - spin_unlock_irqrestore(&dev_data_list_lock, flags); - - return dev_data; -} - -static struct iommu_dev_data *find_dev_data(u16 devid) -{ - struct iommu_dev_data *dev_data; - - dev_data = search_dev_data(devid); - - if (dev_data == NULL) - dev_data = alloc_dev_data(devid); - - return dev_data; -} - static inline int match_hid_uid(struct device *dev, struct acpihid_map_entry *entry) { @@ -272,6 +218,60 @@ static inline int get_device_id(struct device *dev) return devid; } +static struct protection_domain *to_pdomain(struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} + +static struct iommu_dev_data *alloc_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return NULL; + + dev_data->devid = devid; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_add_tail(&dev_data->dev_data_list, &dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static struct iommu_dev_data *search_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { + if (dev_data->devid == devid) + goto out_unlock; + } + + dev_data = NULL; + +out_unlock: + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static struct iommu_dev_data *find_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + + dev_data = search_dev_data(devid); + + if (dev_data == NULL) + dev_data = alloc_dev_data(devid); + + return dev_data; +} + static struct iommu_dev_data *get_dev_data(struct device *dev) { return dev->archdata.iommu; -- cgit v1.2.3 From 4e3e9b6997b24383264031198bf8905b3746221e Mon Sep 17 00:00:00 2001 From: Tirumalesh Chalamarla Date: Tue, 23 Feb 2016 10:19:00 -0800 Subject: iommu/arm-smmu: Add support for 16 bit VMID This patch adds support for 16-bit VMIDs on implementations of SMMUv2 that support it. Signed-off-by: Tirumalesh Chalamarla [will: commit messsage and comments] Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 2409e3bd3df2..25e884a75f6b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -94,6 +94,7 @@ #define sCR0_VMIDPNE (1 << 11) #define sCR0_PTM (1 << 12) #define sCR0_FB (1 << 13) +#define sCR0_VMID16EN (1 << 31) #define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 @@ -141,6 +142,7 @@ #define ID2_PTFS_4K (1 << 12) #define ID2_PTFS_16K (1 << 13) #define ID2_PTFS_64K (1 << 14) +#define ID2_VMID16 (1 << 15) /* Global TLB invalidation */ #define ARM_SMMU_GR0_TLBIVMID 0x64 @@ -193,6 +195,8 @@ #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define CBA2R_RW64_32BIT (0 << 0) #define CBA2R_RW64_64BIT (1 << 0) +#define CBA2R_VMID_SHIFT 16 +#define CBA2R_VMID_MASK 0xffff /* Translation context bank */ #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) @@ -305,6 +309,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) +#define ARM_SMMU_FEAT_VMID16 (1 << 6) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -734,16 +739,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); if (smmu->version > ARM_SMMU_V1) { - /* - * CBA2R. - * *Must* be initialised before CBAR thanks to VMID16 - * architectural oversight affected some implementations. - */ #ifdef CONFIG_64BIT reg = CBA2R_RW64_64BIT; #else reg = CBA2R_RW64_32BIT; #endif + /* 16-bit VMIDs live in CBA2R */ + if (smmu->features & ARM_SMMU_FEAT_VMID16) + reg |= ARM_SMMU_CB_VMID(cfg) << CBA2R_VMID_SHIFT; + writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } @@ -759,7 +763,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, if (stage1) { reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); - } else { + } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { + /* 8-bit VMIDs live in CBAR */ reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT; } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); @@ -1529,6 +1534,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Don't upgrade barriers */ reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); + if (smmu->features & ARM_SMMU_FEAT_VMID16) + reg |= sCR0_VMID16EN; + /* Push the button */ __arm_smmu_tlb_sync(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); @@ -1679,6 +1687,9 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); smmu->pa_size = size; + if (id & ID2_VMID16) + smmu->features |= ARM_SMMU_FEAT_VMID16; + /* * What the page table walker can address actually depends on which * descriptor format is in use, but since a) we don't know that yet, -- cgit v1.2.3 From 1bd37a6835bef0ecd2138cb42f9088fd890f9939 Mon Sep 17 00:00:00 2001 From: Tirumalesh Chalamarla Date: Fri, 4 Mar 2016 13:56:09 -0800 Subject: iommu/arm-smmu: Workaround for ThunderX erratum #27704 Due to erratum #27704, the CN88xx SMMUv2 implementation supports only shared ASID and VMID numberspaces. This patch ensures that ASID and VMIDs are unique across all SMMU instances on affected Cavium systems. Signed-off-by: Tirumalesh Chalamarla Signed-off-by: Akula Geethasowjanya [will: commit message, comments and formatting] Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.txt | 1 + .../devicetree/bindings/iommu/arm,smmu.txt | 1 + drivers/iommu/arm-smmu.c | 39 ++++++++++++++++------ 3 files changed, 30 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index ba4b6acfc545..806f91cdd45d 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -57,3 +57,4 @@ stable kernels. | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | | Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 | | Cavium | ThunderX Core | #27456 | CAVIUM_ERRATUM_27456 | +| Cavium | ThunderX SMMUv2 | #27704 | N/A | diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index 718074501fcb..19fe6f2c83f6 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -16,6 +16,7 @@ conditions. "arm,mmu-400" "arm,mmu-401" "arm,mmu-500" + "cavium,smmu-v2" depending on the particular implementation and/or the version of the architecture implemented. diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 25e884a75f6b..e933679a3266 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -334,6 +334,8 @@ struct arm_smmu_device { struct list_head list; struct rb_root masters; + + u32 cavium_id_base; /* Specific to Cavium */ }; struct arm_smmu_cfg { @@ -343,8 +345,8 @@ struct arm_smmu_cfg { }; #define INVALID_IRPTNDX 0xff -#define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx) -#define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1) +#define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx) +#define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1) enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, @@ -372,6 +374,8 @@ struct arm_smmu_option_prop { const char *prop; }; +static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0); + static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, { 0, NULL}, @@ -583,11 +587,11 @@ static void arm_smmu_tlb_inv_context(void *cookie) if (stage1) { base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); - writel_relaxed(ARM_SMMU_CB_ASID(cfg), + writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg), base + ARM_SMMU_CB_S1_TLBIASID); } else { base = ARM_SMMU_GR0(smmu); - writel_relaxed(ARM_SMMU_CB_VMID(cfg), + writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), base + ARM_SMMU_GR0_TLBIVMID); } @@ -609,7 +613,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { iova &= ~12UL; - iova |= ARM_SMMU_CB_ASID(cfg); + iova |= ARM_SMMU_CB_ASID(smmu, cfg); do { writel_relaxed(iova, reg); iova += granule; @@ -617,7 +621,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, #ifdef CONFIG_64BIT } else { iova >>= 12; - iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48; + iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48; do { writeq_relaxed(iova, reg); iova += granule >> 12; @@ -637,7 +641,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, #endif } else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; - writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg); + writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg); } } @@ -746,7 +750,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, #endif /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) - reg |= ARM_SMMU_CB_VMID(cfg) << CBA2R_VMID_SHIFT; + reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT; writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } @@ -765,7 +769,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { /* 8-bit VMIDs live in CBAR */ - reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT; + reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT; } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); @@ -773,11 +777,11 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, if (stage1) { reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; - reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT; + reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; - reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT; + reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1); } else { reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; @@ -1737,6 +1741,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 }, { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 }, { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 }, + { .compatible = "cavium,smmu-v2", .data = (void *)ARM_SMMU_V2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1847,6 +1852,18 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } + /* + * Cavium CN88xx erratum #27704. + * Ensure ASID and VMID allocation is unique across all SMMUs in + * the system. + */ + if (of_device_is_compatible(dev->of_node, "cavium,smmu-v2")) { + smmu->cavium_id_base = + atomic_add_return(smmu->num_context_banks, + &cavium_smmu_context_count); + smmu->cavium_id_base -= smmu->num_context_banks; + } + INIT_LIST_HEAD(&smmu->list); spin_lock(&arm_smmu_devices_lock); list_add(&smmu->list, &arm_smmu_devices); -- cgit v1.2.3 From 67b65a3fb8e658d00ad1bb06e341f09b1f93a25c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:57 +0100 Subject: iommu/arm-smmu: Differentiate specific implementations As the inevitable reality of implementation-specific errata workarounds begin to accrue alongside our integration quirk handling, it's about time the driver had a decent way of keeping track. Extend the per-SMMU data so we can identify specific implementations in an efficient and firmware-agnostic manner. Acked-by: Tirumalesh Chalamarla Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index e933679a3266..2d5f357de69c 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -278,6 +278,10 @@ enum arm_smmu_arch_version { ARM_SMMU_V2, }; +enum arm_smmu_implementation { + GENERIC_SMMU, +}; + struct arm_smmu_smr { u8 idx; u16 mask; @@ -315,6 +319,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) u32 options; enum arm_smmu_arch_version version; + enum arm_smmu_implementation model; u32 num_context_banks; u32 num_s2_context_banks; @@ -1735,13 +1740,24 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return 0; } +struct arm_smmu_match_data { + enum arm_smmu_arch_version version; + enum arm_smmu_implementation model; +}; + +#define ARM_SMMU_MATCH_DATA(name, ver, imp) \ +static struct arm_smmu_match_data name = { .version = ver, .model = imp } + +ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); + static const struct of_device_id arm_smmu_of_match[] = { - { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 }, - { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 }, - { .compatible = "cavium,smmu-v2", .data = (void *)ARM_SMMU_V2 }, + { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, + { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, + { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, + { .compatible = "cavium,smmu-v2", .data = &smmu_generic_v2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1749,6 +1765,7 @@ MODULE_DEVICE_TABLE(of, arm_smmu_of_match); static int arm_smmu_device_dt_probe(struct platform_device *pdev) { const struct of_device_id *of_id; + const struct arm_smmu_match_data *data; struct resource *res; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; @@ -1764,7 +1781,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) smmu->dev = dev; of_id = of_match_node(arm_smmu_of_match, dev->of_node); - smmu->version = (enum arm_smmu_arch_version)of_id->data; + data = of_id->data; + smmu->version = data->version; + smmu->model = data->model; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); smmu->base = devm_ioremap_resource(dev, res); -- cgit v1.2.3 From e086d912d4d78781652669618e7fb01a4d466703 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:58 +0100 Subject: iommu/arm-smmu: Convert ThunderX workaround to new method With a framework for implementation-specific funtionality in place, the currently-FDT-dependent ThunderX workaround gets to be the first user. Acked-by: Tirumalesh Chalamarla Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 2d5f357de69c..d8bc20a0efb9 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -280,6 +280,7 @@ enum arm_smmu_arch_version { enum arm_smmu_implementation { GENERIC_SMMU, + CAVIUM_SMMUV2, }; struct arm_smmu_smr { @@ -1686,6 +1687,17 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) } dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", smmu->num_context_banks, smmu->num_s2_context_banks); + /* + * Cavium CN88xx erratum #27704. + * Ensure ASID and VMID allocation is unique across all SMMUs in + * the system. + */ + if (smmu->model == CAVIUM_SMMUV2) { + smmu->cavium_id_base = + atomic_add_return(smmu->num_context_banks, + &cavium_smmu_context_count); + smmu->cavium_id_base -= smmu->num_context_banks; + } /* ID2 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); @@ -1750,6 +1762,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, @@ -1757,7 +1770,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, - { .compatible = "cavium,smmu-v2", .data = &smmu_generic_v2 }, + { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1871,18 +1884,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } - /* - * Cavium CN88xx erratum #27704. - * Ensure ASID and VMID allocation is unique across all SMMUs in - * the system. - */ - if (of_device_is_compatible(dev->of_node, "cavium,smmu-v2")) { - smmu->cavium_id_base = - atomic_add_return(smmu->num_context_banks, - &cavium_smmu_context_count); - smmu->cavium_id_base -= smmu->num_context_banks; - } - INIT_LIST_HEAD(&smmu->list); spin_lock(&arm_smmu_devices_lock); list_add(&smmu->list, &arm_smmu_devices); -- cgit v1.2.3 From f0cfffc48cac516e37711786227f6808491913a5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:59 +0100 Subject: iommu/arm-smmu: Work around MMU-500 prefetch errata MMU-500 erratum #841119 is tickled by a particular set of circumstances interacting with the next-page prefetcher. Since said prefetcher is quite dumb and actually detrimental to performance in some cases (by causing unwanted TLB evictions for non-sequential access patterns), we lose very little by turning it off, and what we gain is a guarantee that the erratum is never hit. As a bonus, the same workaround will also prevent erratum #826419 once v7 short descriptor support is implemented. CC: Catalin Marinas CC: Will Deacon Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.txt | 1 + drivers/iommu/arm-smmu.c | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index 806f91cdd45d..c6938e50e71f 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -53,6 +53,7 @@ stable kernels. | ARM | Cortex-A57 | #832075 | ARM64_ERRATUM_832075 | | ARM | Cortex-A57 | #852523 | N/A | | ARM | Cortex-A57 | #834220 | ARM64_ERRATUM_834220 | +| ARM | MMU-500 | #841119,#826419 | N/A | | | | | | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | | Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 | diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index d8bc20a0efb9..085fc8d808a5 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -203,6 +203,7 @@ #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift)) #define ARM_SMMU_CB_SCTLR 0x0 +#define ARM_SMMU_CB_ACTLR 0x4 #define ARM_SMMU_CB_RESUME 0x8 #define ARM_SMMU_CB_TTBCR2 0x10 #define ARM_SMMU_CB_TTBR0 0x20 @@ -234,6 +235,8 @@ #define SCTLR_M (1 << 0) #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) +#define ARM_MMU500_ACTLR_CPRE (1 << 1) + #define CB_PAR_F (1 << 0) #define ATSR_ACTIVE (1 << 0) @@ -280,6 +283,7 @@ enum arm_smmu_arch_version { enum arm_smmu_implementation { GENERIC_SMMU, + ARM_MMU500, CAVIUM_SMMUV2, }; @@ -1517,6 +1521,15 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); + /* + * Disable MMU-500's not-particularly-beneficial next-page + * prefetcher for the sake of errata #841119 and #826419. + */ + if (smmu->model == ARM_MMU500) { + reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); + reg &= ~ARM_MMU500_ACTLR_CPRE; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); + } } /* Invalidate the TLB, just in case */ @@ -1762,6 +1775,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { @@ -1769,7 +1783,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, - { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, + { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; -- cgit v1.2.3 From f9a05f05b12a42f2c52f3d3b8cc71fe2a6e60bce Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:13:01 +0100 Subject: iommu/arm-smmu: Tidy up 64-bit/atomic I/O accesses With {read,write}q_relaxed now able to fall back to the common nonatomic-hi-lo helper, make use of that so that we don't have to open-code our own. In the process, also convert the other remaining split accesses, and repurpose the custom accessor to smooth out the couple of troublesome instances where we really want to avoid nonatomic writes (and a 64-bit access is unnecessary in the 32-bit context formats we would use on a 32-bit CPU). This paves the way for getting rid of some of the assumptions currently baked into the driver which make it really awkward to use 32-bit context formats with SMMUv2 under a 64-bit kernel. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 48 +++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 085fc8d808a5..acff3326f818 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -71,16 +72,15 @@ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ ? 0x400 : 0)) +/* + * Some 64-bit registers only make sense to write atomically, but in such + * cases all the data relevant to AArch32 formats lies within the lower word, + * therefore this actually makes more sense than it might first appear. + */ #ifdef CONFIG_64BIT -#define smmu_writeq writeq_relaxed +#define smmu_write_atomic_lq writeq_relaxed #else -#define smmu_writeq(reg64, addr) \ - do { \ - u64 __val = (reg64); \ - void __iomem *__addr = (addr); \ - writel_relaxed(__val >> 32, __addr + 4); \ - writel_relaxed(__val, __addr); \ - } while (0) +#define smmu_write_atomic_lq writel_relaxed #endif /* Configuration registers */ @@ -211,11 +211,9 @@ #define ARM_SMMU_CB_TTBCR 0x30 #define ARM_SMMU_CB_S1_MAIR0 0x38 #define ARM_SMMU_CB_S1_MAIR1 0x3c -#define ARM_SMMU_CB_PAR_LO 0x50 -#define ARM_SMMU_CB_PAR_HI 0x54 +#define ARM_SMMU_CB_PAR 0x50 #define ARM_SMMU_CB_FSR 0x58 -#define ARM_SMMU_CB_FAR_LO 0x60 -#define ARM_SMMU_CB_FAR_HI 0x64 +#define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 #define ARM_SMMU_CB_S1_TLBIVA 0x600 #define ARM_SMMU_CB_S1_TLBIASID 0x610 @@ -645,7 +643,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, ARM_SMMU_CB_S2_TLBIIPAS2; iova >>= 12; do { - writeq_relaxed(iova, reg); + smmu_write_atomic_lq(iova, reg); iova += granule >> 12; } while (size -= granule); #endif @@ -664,7 +662,7 @@ static struct iommu_gather_ops arm_smmu_gather_ops = { static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { int flags, ret; - u32 fsr, far, fsynr, resume; + u32 fsr, fsynr, resume; unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -686,13 +684,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; - far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO); - iova = far; -#ifdef CONFIG_64BIT - far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI); - iova |= ((unsigned long)far << 32); -#endif - + iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); if (!report_iommu_fault(domain, smmu->dev, iova, flags)) { ret = IRQ_HANDLED; resume = RESUME_RETRY; @@ -788,14 +780,14 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1); } else { reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); } /* TTBCR */ @@ -1263,8 +1255,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, /* ATS1 registers can only be written atomically */ va = iova & ~0xfffUL; if (smmu->version == ARM_SMMU_V2) - smmu_writeq(va, cb_base + ARM_SMMU_CB_ATS1PR); - else + smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR); + else /* Register is only 32-bit in v1 */ writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, @@ -1275,9 +1267,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, return ops->iova_to_phys(ops, iova); } - phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO); - phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32; - + phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); if (phys & CB_PAR_F) { dev_err(dev, "translation fault!\n"); dev_err(dev, "PAR = 0x%llx\n", phys); -- cgit v1.2.3 From 7602b8710645da48b2937f05fa41d627a0e73dad Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 28 Apr 2016 17:12:09 +0100 Subject: iommu/arm-smmu: Decouple context format from kernel config The way the driver currently forces an AArch32 or AArch64 context format based on the kernel config and SMMU architecture version is suboptimal, in that it makes it very hard to support oddball mix-and-match cases like the SMMUv1 64KB supplement, or situations where the reduced table depth of an AArch32 short descriptor context may be desirable under an AArch64 kernel. It also only happens to work on current implementations which do support all the relevant formats. Introduce an explicit notion of context format, so we can manage that independently and get rid of the inflexible #ifdeffery. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 94 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 22 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index acff3326f818..f2ded69feba7 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -117,6 +117,8 @@ #define ID0_NTS (1 << 28) #define ID0_SMS (1 << 27) #define ID0_ATOSNS (1 << 26) +#define ID0_PTFS_NO_AARCH32 (1 << 25) +#define ID0_PTFS_NO_AARCH32S (1 << 24) #define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff @@ -317,6 +319,11 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) #define ARM_SMMU_FEAT_VMID16 (1 << 6) +#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) +#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) +#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) +#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) +#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -346,10 +353,18 @@ struct arm_smmu_device { u32 cavium_id_base; /* Specific to Cavium */ }; +enum arm_smmu_context_fmt { + ARM_SMMU_CTX_FMT_NONE, + ARM_SMMU_CTX_FMT_AARCH64, + ARM_SMMU_CTX_FMT_AARCH32_L, + ARM_SMMU_CTX_FMT_AARCH32_S, +}; + struct arm_smmu_cfg { u8 cbndx; u8 irptndx; u32 cbar; + enum arm_smmu_context_fmt fmt; }; #define INVALID_IRPTNDX 0xff @@ -619,14 +634,13 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; - if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { + if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { iova &= ~12UL; iova |= ARM_SMMU_CB_ASID(smmu, cfg); do { writel_relaxed(iova, reg); iova += granule; } while (size -= granule); -#ifdef CONFIG_64BIT } else { iova >>= 12; iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48; @@ -634,9 +648,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, writeq_relaxed(iova, reg); iova += granule >> 12; } while (size -= granule); -#endif } -#ifdef CONFIG_64BIT } else if (smmu->version == ARM_SMMU_V2) { reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : @@ -646,7 +658,6 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, smmu_write_atomic_lq(iova, reg); iova += granule >> 12; } while (size -= granule); -#endif } else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg); @@ -745,11 +756,10 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); if (smmu->version > ARM_SMMU_V1) { -#ifdef CONFIG_64BIT - reg = CBA2R_RW64_64BIT; -#else - reg = CBA2R_RW64_32BIT; -#endif + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) + reg = CBA2R_RW64_64BIT; + else + reg = CBA2R_RW64_32BIT; /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT; @@ -860,16 +870,40 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; + /* + * Choosing a suitable context format is even more fiddly. Until we + * grow some way for the caller to express a preference, and/or move + * the decision into the io-pgtable code where it arguably belongs, + * just aim for the closest thing to the rest of the system, and hope + * that the hardware isn't esoteric enough that we can't assume AArch64 + * support to be a superset of AArch32 support... + */ + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) + cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; + if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && + (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | + ARM_SMMU_FEAT_FMT_AARCH64_16K | + ARM_SMMU_FEAT_FMT_AARCH64_4K))) + cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; + + if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { + ret = -EINVAL; + goto out_unlock; + } + switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; start = smmu->num_s2_context_banks; ias = smmu->va_size; oas = smmu->ipa_size; - if (IS_ENABLED(CONFIG_64BIT)) + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S1; - else + } else { fmt = ARM_32_LPAE_S1; + ias = min(ias, 32UL); + oas = min(oas, 40UL); + } break; case ARM_SMMU_DOMAIN_NESTED: /* @@ -881,10 +915,13 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, start = 0; ias = smmu->ipa_size; oas = smmu->pa_size; - if (IS_ENABLED(CONFIG_64BIT)) + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S2; - else + } else { fmt = ARM_32_LPAE_S2; + ias = min(ias, 40UL); + oas = min(oas, 40UL); + } break; default: ret = -EINVAL; @@ -1670,6 +1707,12 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) ID0_NUMSIDB_MASK; } + if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; + if (!(id & ID0_PTFS_NO_AARCH32S)) + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; + } + /* ID1 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; @@ -1725,22 +1768,29 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->version == ARM_SMMU_V1) { smmu->va_size = smmu->ipa_size; - size = SZ_4K | SZ_2M | SZ_1G; } else { size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; smmu->va_size = arm_smmu_id_size_to_bits(size); -#ifndef CONFIG_64BIT - smmu->va_size = min(32UL, smmu->va_size); -#endif - size = 0; if (id & ID2_PTFS_4K) - size |= SZ_4K | SZ_2M | SZ_1G; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; if (id & ID2_PTFS_16K) - size |= SZ_16K | SZ_32M; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; if (id & ID2_PTFS_64K) - size |= SZ_64K | SZ_512M; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } + /* Now we've corralled the various formats, what'll it do? */ + size = 0; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) + size |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; + if (smmu->features & + (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) + size |= SZ_4K | SZ_2M | SZ_1G; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) + size |= SZ_16K | SZ_32M; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) + size |= SZ_64K | SZ_512M; + arm_smmu_ops.pgsize_bitmap &= size; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size); -- cgit v1.2.3 From b7862e3559f9ab4aaa258dcb846986601a7ca0b8 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:13:03 +0100 Subject: iommu/arm-smmu: Support SMMUv1 64KB supplement The 64KB Translation Granule Supplement to the SMMUv1 architecture allows an SMMUv1 implementation to support 64KB pages for stage 2 translations, using a constrained VMSAv8 descriptor format limited to 40-bit addresses. Now that we can freely mix and match context formats, we can actually handle having 4KB pages via an AArch32 context but 64KB pages via an AArch64 context, so plumb it in. It is assumed that any implementations will have hardware capabilities matching the format constraints, thus obviating the need for excessive sanity-checking; this is the case for MMU-401, the only ARM Ltd. implementation. CC: Eric Auger Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index f2ded69feba7..72dea314f9a1 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -277,7 +277,8 @@ MODULE_PARM_DESC(disable_bypass, "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); enum arm_smmu_arch_version { - ARM_SMMU_V1 = 1, + ARM_SMMU_V1, + ARM_SMMU_V1_64K, ARM_SMMU_V2, }; @@ -769,7 +770,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, /* CBAR */ reg = cfg->cbar; - if (smmu->version == ARM_SMMU_V1) + if (smmu->version < ARM_SMMU_V2) reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; /* @@ -934,7 +935,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_unlock; cfg->cbndx = ret; - if (smmu->version == ARM_SMMU_V1) { + if (smmu->version < ARM_SMMU_V2) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; } else { @@ -1619,7 +1620,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) bool cttw_dt, cttw_reg; dev_notice(smmu->dev, "probing hardware configuration...\n"); - dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); + dev_notice(smmu->dev, "SMMUv%d with:\n", + smmu->version == ARM_SMMU_V2 ? 2 : 1); /* ID0 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); @@ -1651,7 +1653,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENODEV; } - if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) { + if ((id & ID0_S1TS) && + ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; dev_notice(smmu->dev, "\taddress translation ops\n"); } @@ -1766,8 +1769,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "failed to set DMA mask for table walker\n"); - if (smmu->version == ARM_SMMU_V1) { + if (smmu->version < ARM_SMMU_V2) { smmu->va_size = smmu->ipa_size; + if (smmu->version == ARM_SMMU_V1_64K) + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } else { size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; smmu->va_size = arm_smmu_id_size_to_bits(size); @@ -1815,6 +1820,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); @@ -1822,7 +1828,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, - { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, @@ -1916,7 +1922,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) parse_driver_options(smmu); - if (smmu->version > ARM_SMMU_V1 && + if (smmu->version == ARM_SMMU_V2 && smmu->num_context_banks != smmu->num_context_irqs) { dev_err(dev, "found only %d context interrupt(s) but %d required\n", -- cgit v1.2.3 From 3ca3712a42f9e632eb41da94ca4eab4f1fb06fcb Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Tue, 3 May 2016 21:50:30 +0800 Subject: iommu/arm-smmu: Clear cache lock bit of ACR According MMU-500r2 TRM, section 3.7.1 Auxiliary Control registers, You can modify ACTLR only when the ACR.CACHE_LOCK bit is 0. So before clearing ARM_MMU500_ACTLR_CPRE of each context bank, need clear CACHE_LOCK bit of ACR register first. Since CACHE_LOCK bit is only present in MMU-500r2 onwards, need to check the major number of IDR7. Reviewed-by: Robin Murphy Signed-off-by: Peng Fan Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 72dea314f9a1..2fb9c33a4a08 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -98,6 +98,9 @@ #define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 +/* Auxiliary Configuration register */ +#define ARM_SMMU_GR0_sACR 0x10 + /* Identification registers */ #define ARM_SMMU_GR0_ID0 0x20 #define ARM_SMMU_GR0_ID1 0x24 @@ -146,6 +149,9 @@ #define ID2_PTFS_64K (1 << 14) #define ID2_VMID16 (1 << 15) +#define ID7_MAJOR_SHIFT 4 +#define ID7_MAJOR_MASK 0xf + /* Global TLB invalidation */ #define ARM_SMMU_GR0_TLBIVMID 0x64 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 @@ -237,6 +243,8 @@ #define ARM_MMU500_ACTLR_CPRE (1 << 1) +#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) + #define CB_PAR_F (1 << 0) #define ATSR_ACTIVE (1 << 0) @@ -1531,7 +1539,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) void __iomem *gr0_base = ARM_SMMU_GR0(smmu); void __iomem *cb_base; int i = 0; - u32 reg; + u32 reg, major; /* clear global FSR */ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); @@ -1544,6 +1552,19 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i)); } + /* + * Before clearing ARM_MMU500_ACTLR_CPRE, need to + * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK + * bit is only present in MMU-500r2 onwards. + */ + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); + major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; + if ((smmu->model == ARM_MMU500) && (major >= 2)) { + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); + reg &= ~ARM_MMU500_ACR_CACHE_LOCK; + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); + } + /* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); -- cgit v1.2.3 From 809eac54cdd62c67afea1e17080e681dfa33dc09 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 11 Apr 2016 12:32:31 +0100 Subject: iommu/dma: Implement scatterlist segment merging Stop wasting IOVA space by over-aligning scatterlist segments for a theoretical worst-case segment boundary mask, and instead take the real limits into account to merge consecutive segments wherever appropriate, so our callers can benefit from getting back nicely simplified lists. This also represents the last piece of functionality wanted by users of the current arch/arm implementation, thus brings us a small step closer to converting that over to the common code. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 84 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 23 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 58f2fe687a24..886cb3a78326 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -389,26 +389,58 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, /* * Prepare a successfully-mapped scatterlist to give back to the caller. - * Handling IOVA concatenation can come later, if needed + * + * At this point the segments are already laid out by iommu_dma_map_sg() to + * avoid individually crossing any boundaries, so we merely need to check a + * segment's start address to avoid concatenating across one. */ static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, dma_addr_t dma_addr) { - struct scatterlist *s; - int i; + struct scatterlist *s, *cur = sg; + unsigned long seg_mask = dma_get_seg_boundary(dev); + unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); + int i, count = 0; for_each_sg(sg, s, nents, i) { - /* Un-swizzling the fields here, hence the naming mismatch */ - unsigned int s_offset = sg_dma_address(s); + /* Restore this segment's original unaligned fields first */ + unsigned int s_iova_off = sg_dma_address(s); unsigned int s_length = sg_dma_len(s); - unsigned int s_dma_len = s->length; + unsigned int s_iova_len = s->length; - s->offset += s_offset; + s->offset += s_iova_off; s->length = s_length; - sg_dma_address(s) = dma_addr + s_offset; - dma_addr += s_dma_len; + sg_dma_address(s) = DMA_ERROR_CODE; + sg_dma_len(s) = 0; + + /* + * Now fill in the real DMA data. If... + * - there is a valid output segment to append to + * - and this segment starts on an IOVA page boundary + * - but doesn't fall at a segment boundary + * - and wouldn't make the resulting output segment too long + */ + if (cur_len && !s_iova_off && (dma_addr & seg_mask) && + (cur_len + s_length <= max_len)) { + /* ...then concatenate it with the previous one */ + cur_len += s_length; + } else { + /* Otherwise start the next output segment */ + if (i > 0) + cur = sg_next(cur); + cur_len = s_length; + count++; + + sg_dma_address(cur) = dma_addr + s_iova_off; + } + + sg_dma_len(cur) = cur_len; + dma_addr += s_iova_len; + + if (s_length + s_iova_off < s_iova_len) + cur_len = 0; } - return i; + return count; } /* @@ -446,34 +478,40 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, struct scatterlist *s, *prev = NULL; dma_addr_t dma_addr; size_t iova_len = 0; + unsigned long mask = dma_get_seg_boundary(dev); int i; /* * Work out how much IOVA space we need, and align the segments to * IOVA granules for the IOMMU driver to handle. With some clever * trickery we can modify the list in-place, but reversibly, by - * hiding the original data in the as-yet-unused DMA fields. + * stashing the unaligned parts in the as-yet-unused DMA fields. */ for_each_sg(sg, s, nents, i) { - size_t s_offset = iova_offset(iovad, s->offset); + size_t s_iova_off = iova_offset(iovad, s->offset); size_t s_length = s->length; + size_t pad_len = (mask - iova_len + 1) & mask; - sg_dma_address(s) = s_offset; + sg_dma_address(s) = s_iova_off; sg_dma_len(s) = s_length; - s->offset -= s_offset; - s_length = iova_align(iovad, s_length + s_offset); + s->offset -= s_iova_off; + s_length = iova_align(iovad, s_length + s_iova_off); s->length = s_length; /* - * The simple way to avoid the rare case of a segment - * crossing the boundary mask is to pad the previous one - * to end at a naturally-aligned IOVA for this one's size, - * at the cost of potentially over-allocating a little. + * Due to the alignment of our single IOVA allocation, we can + * depend on these assumptions about the segment boundary mask: + * - If mask size >= IOVA size, then the IOVA range cannot + * possibly fall across a boundary, so we don't care. + * - If mask size < IOVA size, then the IOVA range must start + * exactly on a boundary, therefore we can lay things out + * based purely on segment lengths without needing to know + * the actual addresses beforehand. + * - The mask must be a power of 2, so pad_len == 0 if + * iova_len == 0, thus we cannot dereference prev the first + * time through here (i.e. before it has a meaningful value). */ - if (prev) { - size_t pad_len = roundup_pow_of_two(s_length); - - pad_len = (pad_len - iova_len) & (pad_len - 1); + if (pad_len && pad_len < s_length - 1) { prev->length += pad_len; iova_len += pad_len; } -- cgit v1.2.3 From 53c92d793395fdab9edbd2f79b084bb6b2e6ae79 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 7 Apr 2016 18:42:05 +0100 Subject: iommu: of: enforce const-ness of struct iommu_ops As a set of driver-provided callbacks and static data, there is no compelling reason for struct iommu_ops to be mutable in core code, so enforce const-ness throughout. Acked-by: Thierry Reding Signed-off-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- arch/arm/include/asm/dma-mapping.h | 2 +- arch/arm/mm/dma-mapping.c | 6 +++--- arch/arm64/include/asm/dma-mapping.h | 2 +- arch/arm64/mm/dma-mapping.c | 4 ++-- drivers/iommu/of_iommu.c | 14 +++++++------- drivers/of/device.c | 2 +- include/linux/dma-mapping.h | 2 +- include/linux/of_iommu.h | 8 ++++---- 8 files changed, 20 insertions(+), 20 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 6ad1ceda62a5..02283eb2f5b2 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -118,7 +118,7 @@ static inline unsigned long dma_max_pfn(struct device *dev) #define arch_setup_dma_ops arch_setup_dma_ops extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent); + const struct iommu_ops *iommu, bool coherent); #define arch_teardown_dma_ops arch_teardown_dma_ops extern void arch_teardown_dma_ops(struct device *dev); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index c941e93048ad..5c2ca062c3fa 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2215,7 +2215,7 @@ static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) } static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { struct dma_iommu_mapping *mapping; @@ -2253,7 +2253,7 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) #else static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { return false; } @@ -2270,7 +2270,7 @@ static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent) + const struct iommu_ops *iommu, bool coherent) { struct dma_map_ops *dma_ops; diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h index ba437f090a74..7dbea6c070ec 100644 --- a/arch/arm64/include/asm/dma-mapping.h +++ b/arch/arm64/include/asm/dma-mapping.h @@ -48,7 +48,7 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent); + const struct iommu_ops *iommu, bool coherent); #define arch_setup_dma_ops arch_setup_dma_ops #ifdef CONFIG_IOMMU_DMA diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index a6e757cbab77..5d36907f9b12 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -979,13 +979,13 @@ void arch_teardown_dma_ops(struct device *dev) #else static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu) + const struct iommu_ops *iommu) { } #endif /* CONFIG_IOMMU_DMA */ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - struct iommu_ops *iommu, bool coherent) + const struct iommu_ops *iommu, bool coherent) { if (!dev->archdata.dma_ops) dev->archdata.dma_ops = &swiotlb_dma_ops; diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 5fea665af99d..af499aea0a1a 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -98,12 +98,12 @@ EXPORT_SYMBOL_GPL(of_get_dma_window); struct of_iommu_node { struct list_head list; struct device_node *np; - struct iommu_ops *ops; + const struct iommu_ops *ops; }; static LIST_HEAD(of_iommu_list); static DEFINE_SPINLOCK(of_iommu_lock); -void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops) { struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); @@ -119,10 +119,10 @@ void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) spin_unlock(&of_iommu_lock); } -struct iommu_ops *of_iommu_get_ops(struct device_node *np) +const struct iommu_ops *of_iommu_get_ops(struct device_node *np) { struct of_iommu_node *node; - struct iommu_ops *ops = NULL; + const struct iommu_ops *ops = NULL; spin_lock(&of_iommu_lock); list_for_each_entry(node, &of_iommu_list, list) @@ -134,12 +134,12 @@ struct iommu_ops *of_iommu_get_ops(struct device_node *np) return ops; } -struct iommu_ops *of_iommu_configure(struct device *dev, - struct device_node *master_np) +const struct iommu_ops *of_iommu_configure(struct device *dev, + struct device_node *master_np) { struct of_phandle_args iommu_spec; struct device_node *np; - struct iommu_ops *ops = NULL; + const struct iommu_ops *ops = NULL; int idx = 0; /* diff --git a/drivers/of/device.c b/drivers/of/device.c index e5f47cec75f3..fd5cfad7c403 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -88,7 +88,7 @@ void of_dma_configure(struct device *dev, struct device_node *np) int ret; bool coherent; unsigned long offset; - struct iommu_ops *iommu; + const struct iommu_ops *iommu; /* * Set default coherent_dma_mask to 32 bit. Drivers are expected to diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 9ea9aba28049..71c1b215ef66 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -514,7 +514,7 @@ extern u64 dma_get_required_mask(struct device *dev); #ifndef arch_setup_dma_ops static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, - u64 size, struct iommu_ops *iommu, + u64 size, const struct iommu_ops *iommu, bool coherent) { } #endif diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h index ffbe4707d4aa..bd02b44902d0 100644 --- a/include/linux/of_iommu.h +++ b/include/linux/of_iommu.h @@ -12,7 +12,7 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix, size_t *size); extern void of_iommu_init(void); -extern struct iommu_ops *of_iommu_configure(struct device *dev, +extern const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np); #else @@ -25,7 +25,7 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix, } static inline void of_iommu_init(void) { } -static inline struct iommu_ops *of_iommu_configure(struct device *dev, +static inline const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np) { return NULL; @@ -33,8 +33,8 @@ static inline struct iommu_ops *of_iommu_configure(struct device *dev, #endif /* CONFIG_OF_IOMMU */ -void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops); -struct iommu_ops *of_iommu_get_ops(struct device_node *np); +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops); +const struct iommu_ops *of_iommu_get_ops(struct device_node *np); extern struct of_device_id __iommu_of_table; -- cgit v1.2.3 From d16e0faab911cc0e100a1e8e93635b432566608e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 7 Apr 2016 18:42:06 +0100 Subject: iommu: Allow selecting page sizes per domain Many IOMMUs support multiple page table formats, meaning that any given domain may only support a subset of the hardware page sizes presented in iommu_ops->pgsize_bitmap. There are also certain use-cases where the creator of a domain may want to control which page sizes are used, for example to force the use of hugepage mappings to reduce pagetable walk depth. To this end, add a per-domain pgsize_bitmap to represent the subset of page sizes actually in use, to make it possible for domains with different requirements to coexist. Signed-off-by: Will Deacon [rm: hijacked and rebased original patch with new commit message] Signed-off-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/iommu.c | 22 ++++++++++++---------- drivers/iommu/mtk_iommu.c | 2 +- drivers/vfio/vfio_iommu_type1.c | 2 +- include/linux/iommu.h | 3 ++- 5 files changed, 17 insertions(+), 14 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 886cb3a78326..99432999b52f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -94,7 +94,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size return -ENODEV; /* Use the smallest supported page size for IOVA granularity */ - order = __ffs(domain->ops->pgsize_bitmap); + order = __ffs(domain->pgsize_bitmap); base_pfn = max_t(unsigned long, 1, base >> order); end_pfn = (base + size - 1) >> order; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b9df1411c894..ab4d014e3687 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -337,9 +337,9 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group, if (!domain || domain->type != IOMMU_DOMAIN_DMA) return 0; - BUG_ON(!domain->ops->pgsize_bitmap); + BUG_ON(!domain->pgsize_bitmap); - pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap); + pg_size = 1UL << __ffs(domain->pgsize_bitmap); INIT_LIST_HEAD(&mappings); iommu_get_dm_regions(dev, &mappings); @@ -1073,6 +1073,8 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, domain->ops = bus->iommu_ops; domain->type = type; + /* Assume all sizes by default; the driver may override this later */ + domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; return domain; } @@ -1297,7 +1299,7 @@ static size_t iommu_pgsize(struct iommu_domain *domain, pgsize = (1UL << (pgsize_idx + 1)) - 1; /* throw away page sizes not supported by the hardware */ - pgsize &= domain->ops->pgsize_bitmap; + pgsize &= domain->pgsize_bitmap; /* make sure we're still sane */ BUG_ON(!pgsize); @@ -1319,14 +1321,14 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, int ret = 0; if (unlikely(domain->ops->map == NULL || - domain->ops->pgsize_bitmap == 0UL)) + domain->pgsize_bitmap == 0UL)) return -ENODEV; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) return -EINVAL; /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * both the virtual address and the physical one, as well as @@ -1373,14 +1375,14 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) unsigned long orig_iova = iova; if (unlikely(domain->ops->unmap == NULL || - domain->ops->pgsize_bitmap == 0UL)) + domain->pgsize_bitmap == 0UL)) return -ENODEV; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) return -EINVAL; /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * The virtual address, as well as the size of the mapping, must be @@ -1426,10 +1428,10 @@ size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, unsigned int i, min_pagesz; int ret; - if (unlikely(domain->ops->pgsize_bitmap == 0UL)) + if (unlikely(domain->pgsize_bitmap == 0UL)) return 0; - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); for_each_sg(sg, s, nents, i) { phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; @@ -1510,7 +1512,7 @@ int iommu_domain_get_attr(struct iommu_domain *domain, break; case DOMAIN_ATTR_PAGING: paging = data; - *paging = (domain->ops->pgsize_bitmap != 0UL); + *paging = (domain->pgsize_bitmap != 0UL); break; case DOMAIN_ATTR_WINDOWS: count = data; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 929a66a81b2b..e6b25276cfec 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -264,7 +264,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) } /* Update our support page sizes bitmap */ - mtk_iommu_ops.pgsize_bitmap = dom->cfg.pgsize_bitmap; + dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], data->base + REG_MMU_PT_BASE_ADDR); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 75b24e93cedb..15a65823aad9 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -407,7 +407,7 @@ static unsigned long vfio_pgsize_bitmap(struct vfio_iommu *iommu) mutex_lock(&iommu->lock); list_for_each_entry(domain, &iommu->domain_list, next) - bitmap &= domain->domain->ops->pgsize_bitmap; + bitmap &= domain->domain->pgsize_bitmap; mutex_unlock(&iommu->lock); /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8a2570443b80..7811294bc0f7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -78,6 +78,7 @@ struct iommu_domain_geometry { struct iommu_domain { unsigned type; const struct iommu_ops *ops; + unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ iommu_fault_handler_t handler; void *handler_token; struct iommu_domain_geometry geometry; @@ -155,7 +156,7 @@ struct iommu_dm_region { * @domain_set_windows: Set the number of windows for a domain * @domain_get_windows: Return the number of windows for a domain * @of_xlate: add OF master IDs to iommu grouping - * @pgsize_bitmap: bitmap of supported page sizes + * @pgsize_bitmap: bitmap of all possible supported page sizes */ struct iommu_ops { bool (*capable)(enum iommu_cap); -- cgit v1.2.3 From 3b6b7e19e31a816ee02a8d4372cbea9ad7db3784 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 17:29:10 +0100 Subject: iommu/dma: Finish optimising higher-order allocations Now that we know exactly which page sizes our caller wants to use in the given domain, we can restrict higher-order allocation attempts to just those sizes, if any, and avoid wasting any time or effort on other sizes which offer no benefit. In the same vein, this also lets us accommodate a minimum order greater than 0 for special cases. Signed-off-by: Robin Murphy Acked-by: Will Deacon Tested-by: Yong Wu Signed-off-by: Joerg Roedel --- arch/arm64/mm/dma-mapping.c | 4 +-- drivers/iommu/dma-iommu.c | 60 +++++++++++++++++++++++++++++---------------- include/linux/dma-iommu.h | 4 +-- 3 files changed, 43 insertions(+), 25 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 5d36907f9b12..41d19a0fc9c0 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -562,8 +562,8 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, struct page **pages; pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent); - pages = iommu_dma_alloc(dev, iosize, gfp, ioprot, handle, - flush_page); + pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot, + handle, flush_page); if (!pages) return NULL; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 99432999b52f..ea5a9ebf0f78 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -190,11 +190,15 @@ static void __iommu_dma_free_pages(struct page **pages, int count) kvfree(pages); } -static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp) +static struct page **__iommu_dma_alloc_pages(unsigned int count, + unsigned long order_mask, gfp_t gfp) { struct page **pages; unsigned int i = 0, array_size = count * sizeof(*pages); - unsigned int order = MAX_ORDER; + + order_mask &= (2U << MAX_ORDER) - 1; + if (!order_mask) + return NULL; if (array_size <= PAGE_SIZE) pages = kzalloc(array_size, GFP_KERNEL); @@ -208,36 +212,38 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp) while (count) { struct page *page = NULL; - int j; + unsigned int order_size; /* * Higher-order allocations are a convenience rather * than a necessity, hence using __GFP_NORETRY until - * falling back to single-page allocations. + * falling back to minimum-order allocations. */ - for (order = min_t(unsigned int, order, __fls(count)); - order > 0; order--) { - page = alloc_pages(gfp | __GFP_NORETRY, order); + for (order_mask &= (2U << __fls(count)) - 1; + order_mask; order_mask &= ~order_size) { + unsigned int order = __fls(order_mask); + + order_size = 1U << order; + page = alloc_pages((order_mask - order_size) ? + gfp | __GFP_NORETRY : gfp, order); if (!page) continue; - if (PageCompound(page)) { - if (!split_huge_page(page)) - break; - __free_pages(page, order); - } else { + if (!order) + break; + if (!PageCompound(page)) { split_page(page, order); break; + } else if (!split_huge_page(page)) { + break; } + __free_pages(page, order); } - if (!page) - page = alloc_page(gfp); if (!page) { __iommu_dma_free_pages(pages, i); return NULL; } - j = 1 << order; - count -= j; - while (j--) + count -= order_size; + while (order_size--) pages[i++] = page++; } return pages; @@ -267,6 +273,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * attached to an iommu_dma_domain * @size: Size of buffer in bytes * @gfp: Allocation flags + * @attrs: DMA attributes for this allocation * @prot: IOMMU mapping flags * @handle: Out argument for allocated DMA handle * @flush_page: Arch callback which must ensure PAGE_SIZE bytes from the @@ -278,8 +285,8 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * Return: Array of struct page pointers describing the buffer, * or NULL on failure. */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, - gfp_t gfp, int prot, dma_addr_t *handle, +struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, + struct dma_attrs *attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); @@ -288,11 +295,22 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, struct page **pages; struct sg_table sgt; dma_addr_t dma_addr; - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; *handle = DMA_ERROR_CODE; - pages = __iommu_dma_alloc_pages(count, gfp); + min_size = alloc_sizes & -alloc_sizes; + if (min_size < PAGE_SIZE) { + min_size = PAGE_SIZE; + alloc_sizes |= PAGE_SIZE; + } else { + size = ALIGN(size, min_size); + } + if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) + alloc_sizes = min_size; + + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + pages = __iommu_dma_alloc_pages(count, alloc_sizes >> PAGE_SHIFT, gfp); if (!pages) return NULL; diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index fc481037478a..8443bbb5c071 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -38,8 +38,8 @@ int dma_direction_to_prot(enum dma_data_direction dir, bool coherent); * These implement the bulk of the relevant DMA mapping callbacks, but require * the arch code to take care of attributes and cache maintenance */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, - gfp_t gfp, int prot, dma_addr_t *handle, +struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, + struct dma_attrs *attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)); void iommu_dma_free(struct device *dev, struct page **pages, size_t size, dma_addr_t *handle); -- cgit v1.2.3 From e85e8f69cedb5fbc7cd16f56dd97220e61ed616e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 9 May 2016 16:58:37 +0200 Subject: iommu/amd: Remove statistics code The statistics are not really used for anything and should be replaced by generic and per-device statistic counters. Remove the code for now. Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 10 ----- drivers/iommu/amd_iommu.c | 95 ----------------------------------------- drivers/iommu/amd_iommu_types.h | 26 ----------- 3 files changed, 131 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index dd1dc39f84ff..3bd2548c22c3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -124,16 +124,6 @@ config AMD_IOMMU your BIOS for an option to enable it or if you have an IVRS ACPI table. -config AMD_IOMMU_STATS - bool "Export AMD IOMMU statistics to debugfs" - depends on AMD_IOMMU - select DEBUG_FS - ---help--- - This option enables code in the AMD IOMMU driver to collect various - statistics about whats happening in the driver and exports that - information to userspace via debugfs. - If unsure, say N. - config AMD_IOMMU_V2 tristate "AMD IOMMU Version 2 driver" depends on AMD_IOMMU diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 119148552aa0..de9f028be79e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -489,70 +489,6 @@ static void iommu_uninit_device(struct device *dev) */ } -#ifdef CONFIG_AMD_IOMMU_STATS - -/* - * Initialization code for statistics collection - */ - -DECLARE_STATS_COUNTER(compl_wait); -DECLARE_STATS_COUNTER(cnt_map_single); -DECLARE_STATS_COUNTER(cnt_unmap_single); -DECLARE_STATS_COUNTER(cnt_map_sg); -DECLARE_STATS_COUNTER(cnt_unmap_sg); -DECLARE_STATS_COUNTER(cnt_alloc_coherent); -DECLARE_STATS_COUNTER(cnt_free_coherent); -DECLARE_STATS_COUNTER(cross_page); -DECLARE_STATS_COUNTER(domain_flush_single); -DECLARE_STATS_COUNTER(domain_flush_all); -DECLARE_STATS_COUNTER(alloced_io_mem); -DECLARE_STATS_COUNTER(total_map_requests); -DECLARE_STATS_COUNTER(complete_ppr); -DECLARE_STATS_COUNTER(invalidate_iotlb); -DECLARE_STATS_COUNTER(invalidate_iotlb_all); -DECLARE_STATS_COUNTER(pri_requests); - -static struct dentry *stats_dir; -static struct dentry *de_fflush; - -static void amd_iommu_stats_add(struct __iommu_counter *cnt) -{ - if (stats_dir == NULL) - return; - - cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir, - &cnt->value); -} - -static void amd_iommu_stats_init(void) -{ - stats_dir = debugfs_create_dir("amd-iommu", NULL); - if (stats_dir == NULL) - return; - - de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir, - &amd_iommu_unmap_flush); - - amd_iommu_stats_add(&compl_wait); - amd_iommu_stats_add(&cnt_map_single); - amd_iommu_stats_add(&cnt_unmap_single); - amd_iommu_stats_add(&cnt_map_sg); - amd_iommu_stats_add(&cnt_unmap_sg); - amd_iommu_stats_add(&cnt_alloc_coherent); - amd_iommu_stats_add(&cnt_free_coherent); - amd_iommu_stats_add(&cross_page); - amd_iommu_stats_add(&domain_flush_single); - amd_iommu_stats_add(&domain_flush_all); - amd_iommu_stats_add(&alloced_io_mem); - amd_iommu_stats_add(&total_map_requests); - amd_iommu_stats_add(&complete_ppr); - amd_iommu_stats_add(&invalidate_iotlb); - amd_iommu_stats_add(&invalidate_iotlb_all); - amd_iommu_stats_add(&pri_requests); -} - -#endif - /**************************************************************************** * * Interrupt handling functions @@ -675,8 +611,6 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) { struct amd_iommu_fault fault; - INC_STATS_COUNTER(pri_requests); - if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n"); return; @@ -2641,11 +2575,6 @@ static dma_addr_t __map_single(struct device *dev, pages = iommu_num_pages(paddr, size, PAGE_SIZE); paddr &= PAGE_MASK; - INC_STATS_COUNTER(total_map_requests); - - if (pages > 1) - INC_STATS_COUNTER(cross_page); - if (align) align_mask = (1UL << get_order(size)) - 1; @@ -2666,8 +2595,6 @@ static dma_addr_t __map_single(struct device *dev, } address += offset; - ADD_STATS_COUNTER(alloced_io_mem, size); - if (unlikely(amd_iommu_np_cache)) { domain_flush_pages(&dma_dom->domain, address, size); domain_flush_complete(&dma_dom->domain); @@ -2715,8 +2642,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, start += PAGE_SIZE; } - SUB_STATS_COUNTER(alloced_io_mem, size); - dma_ops_free_addresses(dma_dom, dma_addr, pages); } @@ -2732,8 +2657,6 @@ static dma_addr_t map_page(struct device *dev, struct page *page, struct protection_domain *domain; u64 dma_mask; - INC_STATS_COUNTER(cnt_map_single); - domain = get_domain(dev); if (PTR_ERR(domain) == -EINVAL) return (dma_addr_t)paddr; @@ -2754,8 +2677,6 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, { struct protection_domain *domain; - INC_STATS_COUNTER(cnt_unmap_single); - domain = get_domain(dev); if (IS_ERR(domain)) return; @@ -2778,8 +2699,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, int mapped_elems = 0; u64 dma_mask; - INC_STATS_COUNTER(cnt_map_sg); - domain = get_domain(dev); if (IS_ERR(domain)) return 0; @@ -2825,8 +2744,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, struct scatterlist *s; int i; - INC_STATS_COUNTER(cnt_unmap_sg); - domain = get_domain(dev); if (IS_ERR(domain)) return; @@ -2849,8 +2766,6 @@ static void *alloc_coherent(struct device *dev, size_t size, struct protection_domain *domain; struct page *page; - INC_STATS_COUNTER(cnt_alloc_coherent); - domain = get_domain(dev); if (PTR_ERR(domain) == -EINVAL) { page = alloc_pages(flag, get_order(size)); @@ -2904,8 +2819,6 @@ static void free_coherent(struct device *dev, size_t size, struct protection_domain *domain; struct page *page; - INC_STATS_COUNTER(cnt_free_coherent); - page = virt_to_page(virt_addr); size = PAGE_ALIGN(size); @@ -2997,8 +2910,6 @@ int __init amd_iommu_init_dma_ops(void) if (!swiotlb) dma_ops = &nommu_dma_ops; - amd_iommu_stats_init(); - if (amd_iommu_unmap_flush) pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n"); else @@ -3489,8 +3400,6 @@ out: static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid, u64 address) { - INC_STATS_COUNTER(invalidate_iotlb); - return __flush_pasid(domain, pasid, address, false); } @@ -3511,8 +3420,6 @@ EXPORT_SYMBOL(amd_iommu_flush_page); static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid) { - INC_STATS_COUNTER(invalidate_iotlb_all); - return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, true); } @@ -3632,8 +3539,6 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid, struct amd_iommu *iommu; struct iommu_cmd cmd; - INC_STATS_COUNTER(complete_ppr); - dev_data = get_dev_data(&pdev->dev); iommu = amd_iommu_rlookup_table[dev_data->devid]; diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index b6b14d288b0b..590956ac704e 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -682,30 +682,4 @@ static inline int get_hpet_devid(int id) return -EINVAL; } -#ifdef CONFIG_AMD_IOMMU_STATS - -struct __iommu_counter { - char *name; - struct dentry *dent; - u64 value; -}; - -#define DECLARE_STATS_COUNTER(nm) \ - static struct __iommu_counter nm = { \ - .name = #nm, \ - } - -#define INC_STATS_COUNTER(name) name.value += 1 -#define ADD_STATS_COUNTER(name, x) name.value += (x) -#define SUB_STATS_COUNTER(name, x) name.value -= (x) - -#else /* CONFIG_AMD_IOMMU_STATS */ - -#define DECLARE_STATS_COUNTER(name) -#define INC_STATS_COUNTER(name) -#define ADD_STATS_COUNTER(name, x) -#define SUB_STATS_COUNTER(name, x) - -#endif /* CONFIG_AMD_IOMMU_STATS */ - #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */ -- cgit v1.2.3 From d546635731317a5f8923b1045d0f4403e8024a7d Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 9 May 2016 17:20:09 +0100 Subject: iommu/arm-smmu: Use per-domain page sizes. Now that we can accurately reflect the context format we choose for each domain, do that instead of imposing the global lowest-common-denominator restriction and potentially ending up with nothing. We currently have a strict 1:1 correspondence between domains and context banks, so we don't need to entertain the possibility of multiple formats _within_ a domain. Signed-off-by: Will Deacon [rm: split from original patch, added SMMUv3] Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 19 ++++++++++--------- drivers/iommu/arm-smmu.c | 27 +++++++++++++++------------ 2 files changed, 25 insertions(+), 21 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4ff73ff64e49..ebab33e77d67 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -590,6 +590,7 @@ struct arm_smmu_device { unsigned long ias; /* IPA */ unsigned long oas; /* PA */ + unsigned long pgsize_bitmap; #define ARM_SMMU_MAX_ASIDS (1 << 16) unsigned int asid_bits; @@ -1516,8 +1517,6 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, return 0; } -static struct iommu_ops arm_smmu_ops; - static int arm_smmu_domain_finalise(struct iommu_domain *domain) { int ret; @@ -1555,7 +1554,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) } pgtbl_cfg = (struct io_pgtable_cfg) { - .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, + .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, @@ -1566,7 +1565,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!pgtbl_ops) return -ENOMEM; - arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; smmu_domain->pgtbl_ops = pgtbl_ops; ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); @@ -2410,7 +2409,6 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) { u32 reg; bool coherent; - unsigned long pgsize_bitmap = 0; /* IDR0 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); @@ -2541,13 +2539,16 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) /* Page sizes */ if (reg & IDR5_GRAN64K) - pgsize_bitmap |= SZ_64K | SZ_512M; + smmu->pgsize_bitmap |= SZ_64K | SZ_512M; if (reg & IDR5_GRAN16K) - pgsize_bitmap |= SZ_16K | SZ_32M; + smmu->pgsize_bitmap |= SZ_16K | SZ_32M; if (reg & IDR5_GRAN4K) - pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; + smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; - arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap; + if (arm_smmu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + else + arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; /* Output address size */ switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) { diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 7cd4ad98904a..0360919a5737 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -351,6 +351,7 @@ struct arm_smmu_device { unsigned long va_size; unsigned long ipa_size; unsigned long pa_size; + unsigned long pgsize_bitmap; u32 num_global_irqs; u32 num_context_irqs; @@ -396,8 +397,6 @@ struct arm_smmu_domain { struct iommu_domain domain; }; -static struct iommu_ops arm_smmu_ops; - static DEFINE_SPINLOCK(arm_smmu_devices_lock); static LIST_HEAD(arm_smmu_devices); @@ -957,7 +956,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, } pgtbl_cfg = (struct io_pgtable_cfg) { - .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, + .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, @@ -971,8 +970,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_clear_smmu; } - /* Update our support page sizes to reflect the page table format */ - arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + /* Update the domain's page sizes to reflect the page table format */ + domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; /* Initialise the context bank with our page table cfg */ arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); @@ -1814,19 +1813,23 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) } /* Now we've corralled the various formats, what'll it do? */ - size = 0; if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) - size |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; + smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; if (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) - size |= SZ_4K | SZ_2M | SZ_1G; + smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) - size |= SZ_16K | SZ_32M; + smmu->pgsize_bitmap |= SZ_16K | SZ_32M; if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) - size |= SZ_64K | SZ_512M; + smmu->pgsize_bitmap |= SZ_64K | SZ_512M; + + if (arm_smmu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + else + arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; + dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", + smmu->pgsize_bitmap); - arm_smmu_ops.pgsize_bitmap &= size; - dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size); if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", -- cgit v1.2.3 From 287980e49ffc0f6d911601e7e352a812ed27768e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 27 May 2016 23:23:25 +0200 Subject: remove lots of IS_ERR_VALUE abuses Most users of IS_ERR_VALUE() in the kernel are wrong, as they pass an 'int' into a function that takes an 'unsigned long' argument. This happens to work because the type is sign-extended on 64-bit architectures before it gets converted into an unsigned type. However, anything that passes an 'unsigned short' or 'unsigned int' argument into IS_ERR_VALUE() is guaranteed to be broken, as are 8-bit integers and types that are wider than 'unsigned long'. Andrzej Hajda has already fixed a lot of the worst abusers that were causing actual bugs, but it would be nice to prevent any users that are not passing 'unsigned long' arguments. This patch changes all users of IS_ERR_VALUE() that I could find on 32-bit ARM randconfig builds and x86 allmodconfig. For the moment, this doesn't change the definition of IS_ERR_VALUE() because there are probably still architecture specific users elsewhere. Almost all the warnings I got are for files that are better off using 'if (err)' or 'if (err < 0)'. The only legitimate user I could find that we get a warning for is the (32-bit only) freescale fman driver, so I did not remove the IS_ERR_VALUE() there but changed the type to 'unsigned long'. For 9pfs, I just worked around one user whose calling conventions are so obscure that I did not dare change the behavior. I was using this definition for testing: #define IS_ERR_VALUE(x) ((unsigned long*)NULL == (typeof (x)*)NULL && \ unlikely((unsigned long long)(x) >= (unsigned long long)(typeof(x))-MAX_ERRNO)) which ends up making all 16-bit or wider types work correctly with the most plausible interpretation of what IS_ERR_VALUE() was supposed to return according to its users, but also causes a compile-time warning for any users that do not pass an 'unsigned long' argument. I suggested this approach earlier this year, but back then we ended up deciding to just fix the users that are obviously broken. After the initial warning that caused me to get involved in the discussion (fs/gfs2/dir.c) showed up again in the mainline kernel, Linus asked me to send the whole thing again. [ Updated the 9p parts as per Al Viro - Linus ] Signed-off-by: Arnd Bergmann Cc: Andrzej Hajda Cc: Andrew Morton Link: https://lkml.org/lkml/2016/1/7/363 Link: https://lkml.org/lkml/2016/5/27/486 Acked-by: Srinivas Kandagatla # For nvmem part Signed-off-by: Linus Torvalds --- drivers/acpi/acpi_dbg.c | 22 +++++++++++----------- drivers/ata/sata_highbank.c | 2 +- drivers/clk/tegra/clk-tegra210.c | 2 +- drivers/cpufreq/omap-cpufreq.c | 2 +- drivers/crypto/caam/ctrl.c | 2 +- drivers/dma/sun4i-dma.c | 16 ++++++++-------- drivers/gpio/gpio-xlp.c | 2 +- drivers/gpu/drm/sti/sti_vtg.c | 4 ++-- drivers/gpu/drm/tilcdc/tilcdc_tfp410.c | 2 +- drivers/gpu/host1x/hw/intr_hw.c | 2 +- drivers/iommu/arm-smmu-v3.c | 18 +++++++++--------- drivers/iommu/arm-smmu.c | 8 ++++---- drivers/irqchip/irq-clps711x.c | 2 +- drivers/irqchip/irq-gic.c | 2 +- drivers/irqchip/irq-hip04.c | 2 +- drivers/irqchip/spear-shirq.c | 2 +- drivers/media/i2c/adp1653.c | 10 +++++----- drivers/media/platform/s5p-tv/mixer_drv.c | 2 +- drivers/mfd/twl4030-irq.c | 2 +- drivers/mmc/core/mmc.c | 4 ++-- drivers/mmc/host/dw_mmc.c | 6 +++--- drivers/mmc/host/sdhci-esdhc-imx.c | 2 +- drivers/mmc/host/sdhci-of-at91.c | 2 +- drivers/mmc/host/sdhci.c | 4 ++-- drivers/net/ethernet/freescale/fman/fman.c | 2 +- drivers/net/ethernet/freescale/fman/fman_muram.c | 4 ++-- drivers/net/ethernet/freescale/fman/fman_muram.h | 4 ++-- drivers/net/wireless/ti/wlcore/spi.c | 4 ++-- drivers/nvmem/core.c | 22 +++++++++++----------- drivers/tty/serial/amba-pl011.c | 2 +- drivers/tty/serial/sprd_serial.c | 2 +- drivers/video/fbdev/da8xx-fb.c | 4 ++-- fs/afs/write.c | 4 ---- fs/binfmt_flat.c | 6 +++--- fs/gfs2/dir.c | 15 +++++++++------ kernel/pid.c | 2 +- net/9p/client.c | 8 ++++---- sound/soc/qcom/lpass-platform.c | 4 ++-- 38 files changed, 102 insertions(+), 103 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c index 15e4604efba7..1f4128487dd4 100644 --- a/drivers/acpi/acpi_dbg.c +++ b/drivers/acpi/acpi_dbg.c @@ -265,7 +265,7 @@ static int acpi_aml_write_kern(const char *buf, int len) char *p; ret = acpi_aml_lock_write(crc, ACPI_AML_OUT_KERN); - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; /* sync tail before inserting logs */ smp_mb(); @@ -286,7 +286,7 @@ static int acpi_aml_readb_kern(void) char *p; ret = acpi_aml_lock_read(crc, ACPI_AML_IN_KERN); - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; /* sync head before removing cmds */ smp_rmb(); @@ -330,7 +330,7 @@ again: goto again; break; } - if (IS_ERR_VALUE(ret)) + if (ret < 0) break; size += ret; count -= ret; @@ -373,7 +373,7 @@ again: if (ret == 0) goto again; } - if (IS_ERR_VALUE(ret)) + if (ret < 0) break; *(msg + size) = (char)ret; size++; @@ -526,7 +526,7 @@ static int acpi_aml_open(struct inode *inode, struct file *file) } acpi_aml_io.users++; err_lock: - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { if (acpi_aml_active_reader == file) acpi_aml_active_reader = NULL; } @@ -587,7 +587,7 @@ static int acpi_aml_read_user(char __user *buf, int len) char *p; ret = acpi_aml_lock_read(crc, ACPI_AML_OUT_USER); - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; /* sync head before removing logs */ smp_rmb(); @@ -602,7 +602,7 @@ static int acpi_aml_read_user(char __user *buf, int len) crc->tail = (crc->tail + n) & (ACPI_AML_BUF_SIZE - 1); ret = n; out: - acpi_aml_unlock_fifo(ACPI_AML_OUT_USER, !IS_ERR_VALUE(ret)); + acpi_aml_unlock_fifo(ACPI_AML_OUT_USER, !ret); return ret; } @@ -634,7 +634,7 @@ again: goto again; } } - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { if (!acpi_aml_running()) ret = 0; break; @@ -657,7 +657,7 @@ static int acpi_aml_write_user(const char __user *buf, int len) char *p; ret = acpi_aml_lock_write(crc, ACPI_AML_IN_USER); - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; /* sync tail before inserting cmds */ smp_mb(); @@ -672,7 +672,7 @@ static int acpi_aml_write_user(const char __user *buf, int len) crc->head = (crc->head + n) & (ACPI_AML_BUF_SIZE - 1); ret = n; out: - acpi_aml_unlock_fifo(ACPI_AML_IN_USER, !IS_ERR_VALUE(ret)); + acpi_aml_unlock_fifo(ACPI_AML_IN_USER, !ret); return n; } @@ -704,7 +704,7 @@ again: goto again; } } - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { if (!acpi_aml_running()) ret = 0; break; diff --git a/drivers/ata/sata_highbank.c b/drivers/ata/sata_highbank.c index 8638d575b2b9..aafb8cc03523 100644 --- a/drivers/ata/sata_highbank.c +++ b/drivers/ata/sata_highbank.c @@ -197,7 +197,7 @@ static void highbank_set_em_messages(struct device *dev, for (i = 0; i < SGPIO_PINS; i++) { err = of_get_named_gpio(np, "calxeda,sgpio-gpio", i); - if (IS_ERR_VALUE(err)) + if (err < 0) return; pdata->sgpio_gpio[i] = err; diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c index b8551813ec43..456cf586d2c2 100644 --- a/drivers/clk/tegra/clk-tegra210.c +++ b/drivers/clk/tegra/clk-tegra210.c @@ -1221,7 +1221,7 @@ static int tegra210_pll_fixed_mdiv_cfg(struct clk_hw *hw, p = rate >= params->vco_min ? 1 : -EINVAL; } - if (IS_ERR_VALUE(p)) + if (p < 0) return -EINVAL; cfg->m = tegra_pll_get_fixed_mdiv(hw, input_rate); diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index cead9bec4843..376e63ca94e8 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -54,7 +54,7 @@ static int omap_target(struct cpufreq_policy *policy, unsigned int index) freq = new_freq * 1000; ret = clk_round_rate(policy->clk, freq); - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { dev_warn(mpu_dev, "CPUfreq: Cannot find matching frequency for %lu\n", freq); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 44d30b45f3cc..5ad5f3009ae0 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -402,7 +402,7 @@ int caam_get_era(void) ret = of_property_read_u32(caam_node, "fsl,sec-era", &prop); of_node_put(caam_node); - return IS_ERR_VALUE(ret) ? -ENOTSUPP : prop; + return ret ? -ENOTSUPP : prop; } EXPORT_SYMBOL(caam_get_era); diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index e0df233dde92..57aa227bfadb 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -461,25 +461,25 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, /* Source burst */ ret = convert_burst(sconfig->src_maxburst); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ ret = convert_burst(sconfig->dst_maxburst); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); /* Source bus width */ ret = convert_buswidth(sconfig->src_addr_width); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); @@ -518,25 +518,25 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, /* Source burst */ ret = convert_burst(sconfig->src_maxburst); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ ret = convert_burst(sconfig->dst_maxburst); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); /* Source bus width */ ret = convert_buswidth(sconfig->src_addr_width); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c index 08897dc11915..1a33a19d95b9 100644 --- a/drivers/gpio/gpio-xlp.c +++ b/drivers/gpio/gpio-xlp.c @@ -393,7 +393,7 @@ static int xlp_gpio_probe(struct platform_device *pdev) irq_base = irq_alloc_descs(-1, 0, gc->ngpio, 0); else irq_base = irq_alloc_descs(-1, XLP_GPIO_IRQ_BASE, gc->ngpio, 0); - if (IS_ERR_VALUE(irq_base)) { + if (irq_base < 0) { dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); return irq_base; } diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c index 32c7986b63ab..6bf4ce466d20 100644 --- a/drivers/gpu/drm/sti/sti_vtg.c +++ b/drivers/gpu/drm/sti/sti_vtg.c @@ -437,7 +437,7 @@ static int vtg_probe(struct platform_device *pdev) return -EPROBE_DEFER; } else { vtg->irq = platform_get_irq(pdev, 0); - if (IS_ERR_VALUE(vtg->irq)) { + if (vtg->irq < 0) { DRM_ERROR("Failed to get VTG interrupt\n"); return vtg->irq; } @@ -447,7 +447,7 @@ static int vtg_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(dev, vtg->irq, vtg_irq, vtg_irq_thread, IRQF_ONESHOT, dev_name(dev), vtg); - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { DRM_ERROR("Failed to register VTG interrupt\n"); return ret; } diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c index 7716f42f8aab..6b8c5b3bf588 100644 --- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c +++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c @@ -342,7 +342,7 @@ static int tfp410_probe(struct platform_device *pdev) tfp410_mod->gpio = of_get_named_gpio_flags(node, "powerdn-gpio", 0, NULL); - if (IS_ERR_VALUE(tfp410_mod->gpio)) { + if (tfp410_mod->gpio < 0) { dev_warn(&pdev->dev, "No power down GPIO\n"); } else { ret = gpio_request(tfp410_mod->gpio, "DVI_PDn"); diff --git a/drivers/gpu/host1x/hw/intr_hw.c b/drivers/gpu/host1x/hw/intr_hw.c index 498b37e39058..e1e31e9e67cd 100644 --- a/drivers/gpu/host1x/hw/intr_hw.c +++ b/drivers/gpu/host1x/hw/intr_hw.c @@ -85,7 +85,7 @@ static int _host1x_intr_init_host_sync(struct host1x *host, u32 cpm, err = devm_request_irq(host->dev, host->intr_syncpt_irq, syncpt_thresh_isr, IRQF_SHARED, "host1x_syncpt", host); - if (IS_ERR_VALUE(err)) { + if (err < 0) { WARN_ON(1); return err; } diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index ebab33e77d67..94b68213c50d 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1477,7 +1477,7 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits); - if (IS_ERR_VALUE(asid)) + if (asid < 0) return asid; cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, @@ -1508,7 +1508,7 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); - if (IS_ERR_VALUE(vmid)) + if (vmid < 0) return vmid; cfg->vmid = (u16)vmid; @@ -1569,7 +1569,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) smmu_domain->pgtbl_ops = pgtbl_ops; ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); - if (IS_ERR_VALUE(ret)) + if (ret < 0) free_io_pgtable_ops(pgtbl_ops); return ret; @@ -1642,7 +1642,7 @@ static void arm_smmu_detach_dev(struct device *dev) struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); smmu_group->ste.bypass = true; - if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) + if (arm_smmu_install_ste_for_group(smmu_group) < 0) dev_warn(dev, "failed to install bypass STE\n"); smmu_group->domain = NULL; @@ -1694,7 +1694,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) smmu_group->ste.bypass = domain->type == IOMMU_DOMAIN_DMA; ret = arm_smmu_install_ste_for_group(smmu_group); - if (IS_ERR_VALUE(ret)) + if (ret < 0) smmu_group->domain = NULL; out_unlock: @@ -2235,7 +2235,7 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) arm_smmu_evtq_handler, arm_smmu_evtq_thread, 0, "arm-smmu-v3-evtq", smmu); - if (IS_ERR_VALUE(ret)) + if (ret < 0) dev_warn(smmu->dev, "failed to enable evtq irq\n"); } @@ -2244,7 +2244,7 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) ret = devm_request_irq(smmu->dev, irq, arm_smmu_cmdq_sync_handler, 0, "arm-smmu-v3-cmdq-sync", smmu); - if (IS_ERR_VALUE(ret)) + if (ret < 0) dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n"); } @@ -2252,7 +2252,7 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) if (irq) { ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, 0, "arm-smmu-v3-gerror", smmu); - if (IS_ERR_VALUE(ret)) + if (ret < 0) dev_warn(smmu->dev, "failed to enable gerror irq\n"); } @@ -2264,7 +2264,7 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) arm_smmu_priq_thread, 0, "arm-smmu-v3-priq", smmu); - if (IS_ERR_VALUE(ret)) + if (ret < 0) dev_warn(smmu->dev, "failed to enable priq irq\n"); else diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index e206ce7a4e4b..9345a3fcb706 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -950,7 +950,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); - if (IS_ERR_VALUE(ret)) + if (ret < 0) goto out_unlock; cfg->cbndx = ret; @@ -989,7 +989,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); cfg->irptndx = INVALID_IRPTNDX; @@ -1099,7 +1099,7 @@ static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu, for (i = 0; i < cfg->num_streamids; ++i) { int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0, smmu->num_mapping_groups); - if (IS_ERR_VALUE(idx)) { + if (idx < 0) { dev_err(smmu->dev, "failed to allocate free SMR\n"); goto err_free_smrs; } @@ -1233,7 +1233,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; /* diff --git a/drivers/irqchip/irq-clps711x.c b/drivers/irqchip/irq-clps711x.c index eb5eb0cd414d..2223b3f15d68 100644 --- a/drivers/irqchip/irq-clps711x.c +++ b/drivers/irqchip/irq-clps711x.c @@ -182,7 +182,7 @@ static int __init _clps711x_intc_init(struct device_node *np, writel_relaxed(0, clps711x_intc->intmr[2]); err = irq_alloc_descs(-1, 0, ARRAY_SIZE(clps711x_irqs), numa_node_id()); - if (IS_ERR_VALUE(err)) + if (err < 0) goto out_iounmap; clps711x_intc->ops.map = clps711x_intc_irq_map; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index b4e647179346..fbc4ae2afd29 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -1123,7 +1123,7 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, int irq_start, irq_base = irq_alloc_descs(irq_start, 16, gic_irqs, numa_node_id()); - if (IS_ERR_VALUE(irq_base)) { + if (irq_base < 0) { WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", irq_start); irq_base = irq_start; diff --git a/drivers/irqchip/irq-hip04.c b/drivers/irqchip/irq-hip04.c index 9688d2e2a636..9e25d8ce08e5 100644 --- a/drivers/irqchip/irq-hip04.c +++ b/drivers/irqchip/irq-hip04.c @@ -402,7 +402,7 @@ hip04_of_init(struct device_node *node, struct device_node *parent) nr_irqs -= hwirq_base; /* calculate # of irqs to allocate */ irq_base = irq_alloc_descs(-1, hwirq_base, nr_irqs, numa_node_id()); - if (IS_ERR_VALUE(irq_base)) { + if (irq_base < 0) { pr_err("failed to allocate IRQ numbers\n"); return -EINVAL; } diff --git a/drivers/irqchip/spear-shirq.c b/drivers/irqchip/spear-shirq.c index 1ccd2abed65f..1518ba31a80c 100644 --- a/drivers/irqchip/spear-shirq.c +++ b/drivers/irqchip/spear-shirq.c @@ -232,7 +232,7 @@ static int __init shirq_init(struct spear_shirq **shirq_blocks, int block_nr, nr_irqs += shirq_blocks[i]->nr_irqs; virq_base = irq_alloc_descs(-1, 0, nr_irqs, 0); - if (IS_ERR_VALUE(virq_base)) { + if (virq_base < 0) { pr_err("%s: irq desc alloc failed\n", __func__); goto err_unmap; } diff --git a/drivers/media/i2c/adp1653.c b/drivers/media/i2c/adp1653.c index 9e1731c565e7..e191e295c951 100644 --- a/drivers/media/i2c/adp1653.c +++ b/drivers/media/i2c/adp1653.c @@ -95,7 +95,7 @@ static int adp1653_get_fault(struct adp1653_flash *flash) int rval; fault = i2c_smbus_read_byte_data(client, ADP1653_REG_FAULT); - if (IS_ERR_VALUE(fault)) + if (fault < 0) return fault; flash->fault |= fault; @@ -105,13 +105,13 @@ static int adp1653_get_fault(struct adp1653_flash *flash) /* Clear faults. */ rval = i2c_smbus_write_byte_data(client, ADP1653_REG_OUT_SEL, 0); - if (IS_ERR_VALUE(rval)) + if (rval < 0) return rval; flash->led_mode->val = V4L2_FLASH_LED_MODE_NONE; rval = adp1653_update_hw(flash); - if (IS_ERR_VALUE(rval)) + if (rval) return rval; return flash->fault; @@ -158,7 +158,7 @@ static int adp1653_get_ctrl(struct v4l2_ctrl *ctrl) int rval; rval = adp1653_get_fault(flash); - if (IS_ERR_VALUE(rval)) + if (rval) return rval; ctrl->cur.val = 0; @@ -184,7 +184,7 @@ static int adp1653_set_ctrl(struct v4l2_ctrl *ctrl) int rval; rval = adp1653_get_fault(flash); - if (IS_ERR_VALUE(rval)) + if (rval) return rval; if ((rval & (ADP1653_REG_FAULT_FLT_SCP | ADP1653_REG_FAULT_FLT_OT | diff --git a/drivers/media/platform/s5p-tv/mixer_drv.c b/drivers/media/platform/s5p-tv/mixer_drv.c index 5ef67774971d..8a5d19469ddc 100644 --- a/drivers/media/platform/s5p-tv/mixer_drv.c +++ b/drivers/media/platform/s5p-tv/mixer_drv.c @@ -146,7 +146,7 @@ int mxr_power_get(struct mxr_device *mdev) /* returning 1 means that power is already enabled, * so zero success be returned */ - if (IS_ERR_VALUE(ret)) + if (ret < 0) return ret; return 0; } diff --git a/drivers/mfd/twl4030-irq.c b/drivers/mfd/twl4030-irq.c index 40e51b0baa46..b46c0cfc27d9 100644 --- a/drivers/mfd/twl4030-irq.c +++ b/drivers/mfd/twl4030-irq.c @@ -696,7 +696,7 @@ int twl4030_init_irq(struct device *dev, int irq_num) nr_irqs = TWL4030_PWR_NR_IRQS + TWL4030_CORE_NR_IRQS; irq_base = irq_alloc_descs(-1, 0, nr_irqs, 0); - if (IS_ERR_VALUE(irq_base)) { + if (irq_base < 0) { dev_err(dev, "Fail to allocate IRQ descs\n"); return irq_base; } diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index b81b08f81325..c984321d1881 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -1276,7 +1276,7 @@ static int mmc_select_hs200(struct mmc_card *card) * switch to HS200 mode if bus width is set successfully. */ err = mmc_select_bus_width(card); - if (!IS_ERR_VALUE(err)) { + if (!err) { val = EXT_CSD_TIMING_HS200 | card->drive_strength << EXT_CSD_DRV_STR_SHIFT; err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, @@ -1583,7 +1583,7 @@ static int mmc_init_card(struct mmc_host *host, u32 ocr, } else if (mmc_card_hs(card)) { /* Select the desired bus width optionally */ err = mmc_select_bus_width(card); - if (!IS_ERR_VALUE(err)) { + if (!err) { err = mmc_select_hs_ddr(card); if (err) goto free_card; diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c index 829a6eebcdce..2cc6123b1df9 100644 --- a/drivers/mmc/host/dw_mmc.c +++ b/drivers/mmc/host/dw_mmc.c @@ -1431,7 +1431,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc) int gpio_ro = mmc_gpio_get_ro(mmc); /* Use platform get_ro function, else try on board write protect */ - if (!IS_ERR_VALUE(gpio_ro)) + if (gpio_ro >= 0) read_only = gpio_ro; else read_only = @@ -1454,7 +1454,7 @@ static int dw_mci_get_cd(struct mmc_host *mmc) if ((mmc->caps & MMC_CAP_NEEDS_POLL) || (mmc->caps & MMC_CAP_NONREMOVABLE)) present = 1; - else if (!IS_ERR_VALUE(gpio_cd)) + else if (gpio_cd >= 0) present = gpio_cd; else present = (mci_readl(slot->host, CDETECT) & (1 << slot->id)) @@ -2927,7 +2927,7 @@ static void dw_mci_enable_cd(struct dw_mci *host) if (slot->mmc->caps & MMC_CAP_NEEDS_POLL) return; - if (IS_ERR_VALUE(mmc_gpio_get_cd(slot->mmc))) + if (mmc_gpio_get_cd(slot->mmc) < 0) break; } if (i == host->num_slots) diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 2d300d87cda8..9d3ae1f4bd3c 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1011,7 +1011,7 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev, if (ret) return ret; - if (!IS_ERR_VALUE(mmc_gpio_get_cd(host->mmc))) + if (mmc_gpio_get_cd(host->mmc) >= 0) host->quirks &= ~SDHCI_QUIRK_BROKEN_CARD_DETECTION; return 0; diff --git a/drivers/mmc/host/sdhci-of-at91.c b/drivers/mmc/host/sdhci-of-at91.c index 25f779e09d8e..d4cef713d246 100644 --- a/drivers/mmc/host/sdhci-of-at91.c +++ b/drivers/mmc/host/sdhci-of-at91.c @@ -289,7 +289,7 @@ static int sdhci_at91_probe(struct platform_device *pdev) * to enable polling via device tree with broken-cd property. */ if (!(host->mmc->caps & MMC_CAP_NONREMOVABLE) && - IS_ERR_VALUE(mmc_gpio_get_cd(host->mmc))) { + mmc_gpio_get_cd(host->mmc) < 0) { host->mmc->caps |= MMC_CAP_NEEDS_POLL; host->quirks &= ~SDHCI_QUIRK_BROKEN_CARD_DETECTION; } diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index e010ea4eb6f5..0e3d7c056cb1 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -1624,7 +1624,7 @@ static int sdhci_get_cd(struct mmc_host *mmc) * Try slot gpio detect, if defined it take precedence * over build in controller functionality */ - if (!IS_ERR_VALUE(gpio_cd)) + if (gpio_cd >= 0) return !!gpio_cd; /* If polling, assume that the card is always present. */ @@ -3077,7 +3077,7 @@ int sdhci_add_host(struct sdhci_host *host) if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) && !(mmc->caps & MMC_CAP_NONREMOVABLE) && - IS_ERR_VALUE(mmc_gpio_get_cd(host->mmc))) + mmc_gpio_get_cd(host->mmc) < 0) mmc->caps |= MMC_CAP_NEEDS_POLL; /* If there are external regulators, get them */ diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index bcb9dccada4d..1de2e1e51c2b 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -615,7 +615,7 @@ struct fman { struct fman_cfg *cfg; struct muram_info *muram; /* cam section in muram */ - int cam_offset; + unsigned long cam_offset; size_t cam_size; /* Fifo in MURAM */ int fifo_offset; diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.c b/drivers/net/ethernet/freescale/fman/fman_muram.c index 4eb0e9ac7182..47394c45b6e8 100644 --- a/drivers/net/ethernet/freescale/fman/fman_muram.c +++ b/drivers/net/ethernet/freescale/fman/fman_muram.c @@ -129,7 +129,7 @@ unsigned long fman_muram_offset_to_vbase(struct muram_info *muram, * * Return: address of the allocated memory; NULL otherwise. */ -int fman_muram_alloc(struct muram_info *muram, size_t size) +unsigned long fman_muram_alloc(struct muram_info *muram, size_t size) { unsigned long vaddr; @@ -150,7 +150,7 @@ int fman_muram_alloc(struct muram_info *muram, size_t size) * * Free an allocated memory from FM-MURAM partition. */ -void fman_muram_free_mem(struct muram_info *muram, u32 offset, size_t size) +void fman_muram_free_mem(struct muram_info *muram, unsigned long offset, size_t size) { unsigned long addr = fman_muram_offset_to_vbase(muram, offset); diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.h b/drivers/net/ethernet/freescale/fman/fman_muram.h index dbf0af9e5bb5..889649ad8931 100644 --- a/drivers/net/ethernet/freescale/fman/fman_muram.h +++ b/drivers/net/ethernet/freescale/fman/fman_muram.h @@ -44,8 +44,8 @@ struct muram_info *fman_muram_init(phys_addr_t base, size_t size); unsigned long fman_muram_offset_to_vbase(struct muram_info *muram, unsigned long offset); -int fman_muram_alloc(struct muram_info *muram, size_t size); +unsigned long fman_muram_alloc(struct muram_info *muram, size_t size); -void fman_muram_free_mem(struct muram_info *muram, u32 offset, size_t size); +void fman_muram_free_mem(struct muram_info *muram, unsigned long offset, size_t size); #endif /* __FM_MURAM_EXT */ diff --git a/drivers/net/wireless/ti/wlcore/spi.c b/drivers/net/wireless/ti/wlcore/spi.c index 020ac1a4b408..cea9443c22a6 100644 --- a/drivers/net/wireless/ti/wlcore/spi.c +++ b/drivers/net/wireless/ti/wlcore/spi.c @@ -382,7 +382,7 @@ static int wlcore_probe_of(struct spi_device *spi, struct wl12xx_spi_glue *glue, ret = of_property_read_u32(dt_node, "ref-clock-frequency", &pdev_data->ref_clock_freq); - if (IS_ERR_VALUE(ret)) { + if (ret) { dev_err(glue->dev, "can't get reference clock frequency (%d)\n", ret); return ret; @@ -425,7 +425,7 @@ static int wl1271_probe(struct spi_device *spi) } ret = wlcore_probe_of(spi, glue, &pdev_data); - if (IS_ERR_VALUE(ret)) { + if (ret) { dev_err(glue->dev, "can't get device tree parameters (%d)\n", ret); return ret; diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index bb4ea123547f..965911d9b36a 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -113,7 +113,7 @@ static ssize_t bin_attr_nvmem_read(struct file *filp, struct kobject *kobj, rc = nvmem_reg_read(nvmem, pos, buf, count); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return count; @@ -147,7 +147,7 @@ static ssize_t bin_attr_nvmem_write(struct file *filp, struct kobject *kobj, rc = nvmem_reg_write(nvmem, pos, buf, count); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return count; @@ -366,7 +366,7 @@ static int nvmem_add_cells(struct nvmem_device *nvmem, } rval = nvmem_cell_info_to_nvmem_cell(nvmem, &info[i], cells[i]); - if (IS_ERR_VALUE(rval)) { + if (rval) { kfree(cells[i]); goto err; } @@ -963,7 +963,7 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, rc = nvmem_reg_read(nvmem, cell->offset, buf, cell->bytes); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; /* shift bits in-place */ @@ -998,7 +998,7 @@ void *nvmem_cell_read(struct nvmem_cell *cell, size_t *len) return ERR_PTR(-ENOMEM); rc = __nvmem_cell_read(nvmem, cell, buf, len); - if (IS_ERR_VALUE(rc)) { + if (rc) { kfree(buf); return ERR_PTR(rc); } @@ -1083,7 +1083,7 @@ int nvmem_cell_write(struct nvmem_cell *cell, void *buf, size_t len) if (cell->bit_offset || cell->nbits) kfree(buf); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return len; @@ -1111,11 +1111,11 @@ ssize_t nvmem_device_cell_read(struct nvmem_device *nvmem, return -EINVAL; rc = nvmem_cell_info_to_nvmem_cell(nvmem, info, &cell); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; rc = __nvmem_cell_read(nvmem, &cell, buf, &len); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return len; @@ -1141,7 +1141,7 @@ int nvmem_device_cell_write(struct nvmem_device *nvmem, return -EINVAL; rc = nvmem_cell_info_to_nvmem_cell(nvmem, info, &cell); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return nvmem_cell_write(&cell, buf, cell.bytes); @@ -1170,7 +1170,7 @@ int nvmem_device_read(struct nvmem_device *nvmem, rc = nvmem_reg_read(nvmem, offset, buf, bytes); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; return bytes; @@ -1198,7 +1198,7 @@ int nvmem_device_write(struct nvmem_device *nvmem, rc = nvmem_reg_write(nvmem, offset, buf, bytes); - if (IS_ERR_VALUE(rc)) + if (rc) return rc; diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index a2aa655f56c4..1b7331e40d79 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2360,7 +2360,7 @@ static int pl011_probe_dt_alias(int index, struct device *dev) return ret; ret = of_alias_get_id(np, "serial"); - if (IS_ERR_VALUE(ret)) { + if (ret < 0) { seen_dev_without_alias = true; ret = index; } else { diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c index 18971063f95f..699447aa8b43 100644 --- a/drivers/tty/serial/sprd_serial.c +++ b/drivers/tty/serial/sprd_serial.c @@ -654,7 +654,7 @@ static int sprd_probe_dt_alias(int index, struct device *dev) return ret; ret = of_alias_get_id(np, "serial"); - if (IS_ERR_VALUE(ret)) + if (ret < 0) ret = index; else if (ret >= ARRAY_SIZE(sprd_port) || sprd_port[ret] != NULL) { dev_warn(dev, "requested serial port %d not available.\n", ret); diff --git a/drivers/video/fbdev/da8xx-fb.c b/drivers/video/fbdev/da8xx-fb.c index d8d583d32a37..c229b1a0d13b 100644 --- a/drivers/video/fbdev/da8xx-fb.c +++ b/drivers/video/fbdev/da8xx-fb.c @@ -713,7 +713,7 @@ static int da8xx_fb_config_clk_divider(struct da8xx_fb_par *par, if (par->lcdc_clk_rate != lcdc_clk_rate) { ret = clk_set_rate(par->lcdc_clk, lcdc_clk_rate); - if (IS_ERR_VALUE(ret)) { + if (ret) { dev_err(par->dev, "unable to set clock rate at %u\n", lcdc_clk_rate); @@ -784,7 +784,7 @@ static int lcd_init(struct da8xx_fb_par *par, const struct lcd_ctrl_config *cfg, int ret = 0; ret = da8xx_fb_calc_config_clk_divider(par, panel); - if (IS_ERR_VALUE(ret)) { + if (ret) { dev_err(par->dev, "unable to configure clock\n"); return ret; } diff --git a/fs/afs/write.c b/fs/afs/write.c index 65de439bdc4f..14d506efd1aa 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -643,10 +643,6 @@ ssize_t afs_file_write(struct kiocb *iocb, struct iov_iter *from) return 0; result = generic_file_write_iter(iocb, from); - if (IS_ERR_VALUE(result)) { - _leave(" = %zd", result); - return result; - } _leave(" = %zd", result); return result; diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c index f723cd3a455c..caf9e39bb82b 100644 --- a/fs/binfmt_flat.c +++ b/fs/binfmt_flat.c @@ -337,7 +337,7 @@ calc_reloc(unsigned long r, struct lib_info *p, int curid, int internalp) "(%d != %d)", (unsigned) r, curid, id); goto failed; } else if ( ! p->lib_list[id].loaded && - IS_ERR_VALUE(load_flat_shared_library(id, p))) { + load_flat_shared_library(id, p) < 0) { printk("BINFMT_FLAT: failed to load library %d", id); goto failed; } @@ -837,7 +837,7 @@ static int load_flat_shared_library(int id, struct lib_info *libs) res = prepare_binprm(&bprm); - if (!IS_ERR_VALUE(res)) + if (!res) res = load_flat_file(&bprm, libs, id, NULL); abort_creds(bprm.cred); @@ -883,7 +883,7 @@ static int load_flat_binary(struct linux_binprm * bprm) stack_len += FLAT_STACK_ALIGN - 1; /* reserve for upcoming alignment */ res = load_flat_file(bprm, &libinfo, 0, &stack_len); - if (IS_ERR_VALUE(res)) + if (res < 0) return res; /* Update data segment pointers for all libraries */ diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c index 4a01f30e9995..271d93905bac 100644 --- a/fs/gfs2/dir.c +++ b/fs/gfs2/dir.c @@ -783,12 +783,15 @@ static int get_leaf_nr(struct gfs2_inode *dip, u32 index, u64 *leaf_out) { __be64 *hash; + int error; hash = gfs2_dir_get_hash_table(dip); - if (IS_ERR(hash)) - return PTR_ERR(hash); - *leaf_out = be64_to_cpu(*(hash + index)); - return 0; + error = PTR_ERR_OR_ZERO(hash); + + if (!error) + *leaf_out = be64_to_cpu(*(hash + index)); + + return error; } static int get_first_leaf(struct gfs2_inode *dip, u32 index, @@ -798,7 +801,7 @@ static int get_first_leaf(struct gfs2_inode *dip, u32 index, int error; error = get_leaf_nr(dip, index, &leaf_no); - if (!IS_ERR_VALUE(error)) + if (!error) error = get_leaf(dip, leaf_no, bh_out); return error; @@ -1014,7 +1017,7 @@ static int dir_split_leaf(struct inode *inode, const struct qstr *name) index = name->hash >> (32 - dip->i_depth); error = get_leaf_nr(dip, index, &leaf_no); - if (IS_ERR_VALUE(error)) + if (error) return error; /* Get the old leaf block */ diff --git a/kernel/pid.c b/kernel/pid.c index 4d73a834c7e6..f66162f2359b 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -311,7 +311,7 @@ struct pid *alloc_pid(struct pid_namespace *ns) pid->level = ns->level; for (i = ns->level; i >= 0; i--) { nr = alloc_pidmap(tmp); - if (IS_ERR_VALUE(nr)) { + if (nr < 0) { retval = nr; goto out_free; } diff --git a/net/9p/client.c b/net/9p/client.c index ea79ee9a7348..3fc94a49ccd5 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -518,10 +518,10 @@ static int p9_check_errors(struct p9_client *c, struct p9_req_t *req) if (err) goto out_err; - if (p9_is_proto_dotu(c)) + if (p9_is_proto_dotu(c) && ecode < 512) err = -ecode; - if (!err || !IS_ERR_VALUE(err)) { + if (!err) { err = p9_errstr2errno(ename, strlen(ename)); p9_debug(P9_DEBUG_9P, "<<< RERROR (%d) %s\n", @@ -605,10 +605,10 @@ static int p9_check_zc_errors(struct p9_client *c, struct p9_req_t *req, if (err) goto out_err; - if (p9_is_proto_dotu(c)) + if (p9_is_proto_dotu(c) && ecode < 512) err = -ecode; - if (!err || !IS_ERR_VALUE(err)) { + if (!err) { err = p9_errstr2errno(ename, strlen(ename)); p9_debug(P9_DEBUG_9P, "<<< RERROR (%d) %s\n", diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c index 6e8665430bd5..ddfe34434765 100644 --- a/sound/soc/qcom/lpass-platform.c +++ b/sound/soc/qcom/lpass-platform.c @@ -491,7 +491,7 @@ static int lpass_platform_pcm_new(struct snd_soc_pcm_runtime *soc_runtime) data->rdma_ch = v->alloc_dma_channel(drvdata, SNDRV_PCM_STREAM_PLAYBACK); - if (IS_ERR_VALUE(data->rdma_ch)) + if (data->rdma_ch < 0) return data->rdma_ch; drvdata->substream[data->rdma_ch] = psubstream; @@ -518,7 +518,7 @@ static int lpass_platform_pcm_new(struct snd_soc_pcm_runtime *soc_runtime) data->wrdma_ch = v->alloc_dma_channel(drvdata, SNDRV_PCM_STREAM_CAPTURE); - if (IS_ERR_VALUE(data->wrdma_ch)) + if (data->wrdma_ch < 0) goto capture_alloc_err; drvdata->substream[data->wrdma_ch] = csubstream; -- cgit v1.2.3 From 9aeb26cfc2abc96be42b9df2d0f2dc5d805084ff Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 3 Jun 2016 11:50:30 +0100 Subject: iommu/arm-smmu: Wire up map_sg for arm-smmu-v3 The map_sg callback is missing from arm_smmu_ops, but is required by iommu.h. Similarly to most other IOMMU drivers, connect it to default_iommu_map_sg. Cc: Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 94b68213c50d..5f6b3bcab078 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1941,6 +1941,7 @@ static struct iommu_ops arm_smmu_ops = { .attach_dev = arm_smmu_attach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, + .map_sg = default_iommu_map_sg, .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, -- cgit v1.2.3 From 0076cd3d063a43f69eba2d0a1d13927897c8ed35 Mon Sep 17 00:00:00 2001 From: Wan Zongshun Date: Tue, 10 May 2016 09:21:01 -0400 Subject: iommu/amd: Set AMD iommu callbacks for platform bus driver AMD has more drivers will use ACPI to platform bus driver later, all those devices need iommu support, for example: eMMC driver. For latest AMD eMMC controller, it will utilize sdhci-acpi.c driver, which will rely on platform bus to match device and driver, where we will set 'dev' of struct platform_device as map_sg parameter passing to iommu driver for DMA request, so the iommu-ops are needed on the platform bus. Signed-off-by: Wan Zongshun Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 634f636393d5..921111e65c8c 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -2958,6 +2959,9 @@ int __init amd_iommu_init_api(void) if (err) return err; #endif + err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops); + if (err) + return err; return 0; } -- cgit v1.2.3 From ae8a7910fb0568531033bd6ebe65590f7a4fa6e2 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Wed, 1 Jun 2016 16:46:10 +0100 Subject: iommu/rockchip: Fix zap cache during device attach rk_iommu_command() takes a struct rk_iommu and iterates over the slave MMUs, so this is doubly wrong in that we're passing in the wrong pointer and talking to MMUs that we shouldn't be. Fixes: cd6438c5f844 ("iommu/rockchip: Reconstruct to support multi slaves") Cc: stable@vger.kernel.org Signed-off-by: John Keeping Tested-by: Heiko Stuebner Reviewed-by: Heiko Stuebner Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index c7d6156ff536..25b4627cb57f 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -815,7 +815,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, dte_addr = virt_to_phys(rk_domain->dt); for (i = 0; i < iommu->num_mmu; i++) { rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr); - rk_iommu_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); + rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); } -- cgit v1.2.3 From b54b874fbaf5e024723e50dfb035a9916d6752b4 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Fri, 20 May 2016 15:48:21 +0200 Subject: iommu/exynos: Suppress unbinding to prevent system failure Removal of IOMMU driver cannot be done reliably, so Exynos IOMMU driver doesn't support this operation. It is essential for system operation, so it makes sense to prevent unbinding by disabling bind/unbind sysfs feature for SYSMMU controller driver to avoid kernel ops or trashing memory caused by such operation. Signed-off-by: Marek Szyprowski CC: stable@vger.kernel.org # v4.2+ Reviewed-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 5ecc86cb74c8..e27e3b7df4e7 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -709,6 +709,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { .name = "exynos-sysmmu", .of_match_table = sysmmu_of_match, .pm = &sysmmu_pm_ops, + .suppress_bind_attrs = true, } }; -- cgit v1.2.3 From 86f004c77c5aba6761d2f6b308a79b1913bea50a Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 21 May 2016 02:41:51 +0000 Subject: iommu/vt-d: Reduce extra first level entry in iommu->domains In commit <8bf478163e69> ("iommu/vt-d: Split up iommu->domains array"), it it splits iommu->domains in two levels. Each first level contains 256 entries of second level. In case of the ndomains is exact a multiple of 256, it would have one more extra first level entry for current implementation. This patch refines this calculation to reduce the extra first level entry. Signed-off-by: Wei Yang Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a644d0cec2d8..748e5e4b3144 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1672,7 +1672,7 @@ static int iommu_init_domains(struct intel_iommu *iommu) return -ENOMEM; } - size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **); + size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **); iommu->domains = kzalloc(size, GFP_KERNEL); if (iommu->domains) { @@ -1737,7 +1737,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) static void free_dmar_iommu(struct intel_iommu *iommu) { if ((iommu->domains) && (iommu->domain_ids)) { - int elems = (cap_ndoms(iommu->cap) >> 8) + 1; + int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8; int i; for (i = 0; i < elems; i++) -- cgit v1.2.3 From 0c2b063f1813ac99238b9c61edb58752eb7762cf Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 23 May 2016 11:30:07 +0200 Subject: iommu/exynos: Return proper errors from getting clocks This patch reworks driver probe code to propagate error codes from clk_get() operation. This will allow to properly handle deferred probe in the future. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 56 +++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index e27e3b7df4e7..989365682d14 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -602,37 +602,31 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk = devm_clk_get(dev, "sysmmu"); - if (!IS_ERR(data->clk)) { - ret = clk_prepare(data->clk); - if (ret) { - dev_err(dev, "Failed to prepare clk\n"); - return ret; - } - } else { + if (PTR_ERR(data->clk) == -ENOENT) data->clk = NULL; - } + else if (IS_ERR(data->clk)) + return PTR_ERR(data->clk); + ret = clk_prepare(data->clk); + if (ret) + return ret; data->aclk = devm_clk_get(dev, "aclk"); - if (!IS_ERR(data->aclk)) { - ret = clk_prepare(data->aclk); - if (ret) { - dev_err(dev, "Failed to prepare aclk\n"); - return ret; - } - } else { + if (PTR_ERR(data->aclk) == -ENOENT) data->aclk = NULL; - } + else if (IS_ERR(data->aclk)) + return PTR_ERR(data->aclk); + ret = clk_prepare(data->aclk); + if (ret) + return ret; data->pclk = devm_clk_get(dev, "pclk"); - if (!IS_ERR(data->pclk)) { - ret = clk_prepare(data->pclk); - if (ret) { - dev_err(dev, "Failed to prepare pclk\n"); - return ret; - } - } else { + if (PTR_ERR(data->pclk) == -ENOENT) data->pclk = NULL; - } + else if (IS_ERR(data->pclk)) + return PTR_ERR(data->pclk); + ret = clk_prepare(data->pclk); + if (ret) + return ret; if (!data->clk && (!data->aclk || !data->pclk)) { dev_err(dev, "Failed to get device clock(s)!\n"); @@ -640,15 +634,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk_master = devm_clk_get(dev, "master"); - if (!IS_ERR(data->clk_master)) { - ret = clk_prepare(data->clk_master); - if (ret) { - dev_err(dev, "Failed to prepare master's clk\n"); - return ret; - } - } else { + if (PTR_ERR(data->clk_master) == -ENOENT) data->clk_master = NULL; - } + else if (IS_ERR(data->clk_master)) + return PTR_ERR(data->clk_master); + ret = clk_prepare(data->clk_master); + if (ret) + return ret; data->sysmmu = dev; spin_lock_init(&data->lock); -- cgit v1.2.3 From 01324ab2c990bbd39ec0ef388b7c311a54452a0b Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 23 May 2016 11:30:08 +0200 Subject: iommu/exynos: Fix master clock management for inactive SYSMMU If SYSMMU controller is not active, there is no point in enabling master's clock just for doing the the of internal state. This patch moves enabling that clock to the block which actually does the register access. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 989365682d14..018bcd5d5edc 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -524,16 +524,15 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, { unsigned long flags; - clk_enable(data->clk_master); spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) { - if (data->version >= MAKE_MMU_VER(3, 3)) - __sysmmu_tlb_invalidate_entry(data, iova, 1); + if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) { + clk_enable(data->clk_master); + __sysmmu_tlb_invalidate_entry(data, iova, 1); + clk_disable(data->clk_master); } spin_unlock_irqrestore(&data->lock, flags); - clk_disable(data->clk_master); } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, -- cgit v1.2.3 From fecc49db884b902f7abbea4c9287d17a56966be4 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 23 May 2016 11:30:09 +0200 Subject: iommu/exynos: Prepare clocks when needed, not in driver probe Make clock preparation together with clk_enable(). This way inactive SYSMMU controllers will not keep clocks prepared all the time. This change allows more fine graded power management in the future. All the code assumes that clock management doesn't fail, so guard clock_prepare_enable() it with BUG_ON(). Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 54 +++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 28 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 018bcd5d5edc..e0b834375f17 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -322,14 +322,27 @@ static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) __sysmmu_tlb_invalidate(data); } +static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) +{ + BUG_ON(clk_prepare_enable(data->clk_master)); + BUG_ON(clk_prepare_enable(data->clk)); + BUG_ON(clk_prepare_enable(data->pclk)); + BUG_ON(clk_prepare_enable(data->aclk)); +} + +static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) +{ + clk_disable_unprepare(data->aclk); + clk_disable_unprepare(data->pclk); + clk_disable_unprepare(data->clk); + clk_disable_unprepare(data->clk_master); +} + static void __sysmmu_get_version(struct sysmmu_drvdata *data) { u32 ver; - clk_enable(data->clk_master); - clk_enable(data->clk); - clk_enable(data->pclk); - clk_enable(data->aclk); + __sysmmu_enable_clocks(data); ver = readl(data->sfrbase + REG_MMU_VERSION); @@ -342,10 +355,7 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) dev_dbg(data->sysmmu, "hardware version: %d.%d\n", MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); - clk_disable(data->aclk); - clk_disable(data->pclk); - clk_disable(data->clk); - clk_disable(data->clk_master); + __sysmmu_disable_clocks(data); } static void show_fault_information(struct sysmmu_drvdata *data, @@ -427,10 +437,7 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); writel(0, data->sfrbase + REG_MMU_CFG); - clk_disable(data->aclk); - clk_disable(data->pclk); - clk_disable(data->clk); - clk_disable(data->clk_master); + __sysmmu_disable_clocks(data); } static bool __sysmmu_disable(struct sysmmu_drvdata *data) @@ -475,10 +482,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { - clk_enable(data->clk_master); - clk_enable(data->clk); - clk_enable(data->pclk); - clk_enable(data->aclk); + __sysmmu_enable_clocks(data); writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); @@ -488,6 +492,12 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + /* + * SYSMMU driver keeps master's clock enabled only for the short + * time, while accessing the registers. For performing address + * translation during DMA transaction it relies on the client + * driver to enable it. + */ clk_disable(data->clk_master); } @@ -605,27 +615,18 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) data->clk = NULL; else if (IS_ERR(data->clk)) return PTR_ERR(data->clk); - ret = clk_prepare(data->clk); - if (ret) - return ret; data->aclk = devm_clk_get(dev, "aclk"); if (PTR_ERR(data->aclk) == -ENOENT) data->aclk = NULL; else if (IS_ERR(data->aclk)) return PTR_ERR(data->aclk); - ret = clk_prepare(data->aclk); - if (ret) - return ret; data->pclk = devm_clk_get(dev, "pclk"); if (PTR_ERR(data->pclk) == -ENOENT) data->pclk = NULL; else if (IS_ERR(data->pclk)) return PTR_ERR(data->pclk); - ret = clk_prepare(data->pclk); - if (ret) - return ret; if (!data->clk && (!data->aclk || !data->pclk)) { dev_err(dev, "Failed to get device clock(s)!\n"); @@ -637,9 +638,6 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) data->clk_master = NULL; else if (IS_ERR(data->clk_master)) return PTR_ERR(data->clk_master); - ret = clk_prepare(data->clk_master); - if (ret) - return ret; data->sysmmu = dev; spin_lock_init(&data->lock); -- cgit v1.2.3 From 96f6655700d16d1bae8ef49db6ce28607ebee6f3 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 23 May 2016 13:01:27 +0200 Subject: iommu/exynos: Prepare for deferred probe support Register iommu_ops at the end of successful probe instead of doing that unconditionally. This makes Exynos IOMMU driver ready for deferred probe caused by not-yet-available clocks. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index e0b834375f17..633e6d023c0d 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -581,6 +581,8 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, spin_unlock_irqrestore(&data->lock, flags); } +static struct iommu_ops exynos_iommu_ops; + static int __init exynos_sysmmu_probe(struct platform_device *pdev) { int irq, ret; @@ -654,6 +656,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) pm_runtime_enable(dev); + of_iommu_set_ops(dev->of_node, &exynos_iommu_ops); + return 0; } @@ -1347,7 +1351,6 @@ static int __init exynos_iommu_of_setup(struct device_node *np) if (!dma_dev) dma_dev = &pdev->dev; - of_iommu_set_ops(np, &exynos_iommu_ops); return 0; } -- cgit v1.2.3 From ffb2d1eb88c3262f7a7001c5afeec6babe144b73 Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Thu, 2 Jun 2016 17:46:10 -0700 Subject: iommu/vt-d: Don't reject NTB devices due to scope mismatch On a system with an Intel PCIe port configured as an NTB device, iommu initialization fails with DMAR: Device scope type does not match for 0000:80:03.0 This is because the DMAR table reports this device as having scope 2 (ACPI_DMAR_SCOPE_TYPE_BRIDGE): [0A0h 0160 1] Device Scope Entry Type : 02 [0A1h 0161 1] Entry Length : 08 [0A2h 0162 2] Reserved : 0000 [0A4h 0164 1] Enumeration ID : 00 [0A5h 0165 1] PCI Bus Number : 80 [0A6h 0166 2] PCI Path : 03,00 but the device has a type 0 PCI header: 80:03.0 Bridge [0680]: Intel Corporation Device [8086:2f0d] (rev 02) 00: 86 80 0d 2f 00 00 10 00 02 00 80 06 10 00 80 00 10: 0c 00 c0 00 c0 38 00 00 0c 00 00 00 80 38 00 00 20: 00 00 00 c8 00 00 10 c8 00 00 00 00 86 80 00 00 30: 00 00 00 00 60 00 00 00 00 00 00 00 ff 01 00 00 VT-d works perfectly on this system, so there's no reason to bail out on initialization due to this apparent scope mismatch. Use the class 0x0680 ("Other bridge device") as a heuristic for allowing DMAR initialization for non-bridge PCI devices listed with scope bridge. Signed-off-by: Roland Dreier Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6a86b5d1defa..2eff7b6c6c98 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -241,8 +241,20 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, if (!dmar_match_pci_path(info, scope->bus, path, level)) continue; - if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^ - (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { + /* + * We expect devices with endpoint scope to have normal PCI + * headers, and devices with bridge scope to have bridge PCI + * headers. However PCI NTB devices may be listed in the + * DMAR table with bridge scope, even though they have a + * normal PCI header. NTB devices are identified by class + * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch + * for this special case. + */ + if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && + info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) || + (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE && + (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL && + info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) { pr_warn("Device scope type does not match for %s\n", pci_name(info->dev)); return -EINVAL; -- cgit v1.2.3 From a4c34ff1c029e90e7d5f8dd8d29b0a93b31c3cb2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 17 Jun 2016 11:29:48 +0200 Subject: iommu/vt-d: Enable QI on all IOMMUs before setting root entry This seems to be required on some X58 chipsets on systems with more than one IOMMU. QI does not work until it is enabled on all IOMMUs in the system. Reported-by: Dheeraj CVR Tested-by: Dheeraj CVR Fixes: 5f0a7f7614a9 ('iommu/vt-d: Make root entry visible for hardware right after allocation') Cc: stable@vger.kernel.org Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a644d0cec2d8..10700945994e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3222,11 +3222,6 @@ static int __init init_dmars(void) } } - iommu_flush_write_buffer(iommu); - iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); - if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; #ifdef CONFIG_INTEL_IOMMU_SVM @@ -3235,6 +3230,18 @@ static int __init init_dmars(void) #endif } + /* + * Now that qi is enabled on all iommus, set the root entry and flush + * caches. This is required on some Intel X58 chipsets, otherwise the + * flush_context function will loop forever and the boot hangs. + */ + for_each_active_iommu(iommu, drhd) { + iommu_flush_write_buffer(iommu); + iommu_set_root_entry(iommu); + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + } + if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; -- cgit v1.2.3 From cf7513e759d19908c29cf2c129587ecda34a5c19 Mon Sep 17 00:00:00 2001 From: Bhaktipriya Shridhar Date: Sat, 18 Jun 2016 13:58:30 +0530 Subject: iommu/amd: Remove create_workqueue alloc_workqueue replaces deprecated create_workqueue(). A dedicated workqueue has been used since the workitem (viz &fault->work), is involved in IO page-fault handling. WQ_MEM_RECLAIM has been set to guarantee forward progress under memory pressure, which is a requirement here. Since there are only a fixed number of work items, explicit concurrency limit is unnecessary. Signed-off-by: Bhaktipriya Shridhar Acked-by: Tejun Heo Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 56999d2fac07..0d52ceb0846d 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -961,7 +961,7 @@ static int __init amd_iommu_v2_init(void) spin_lock_init(&state_lock); ret = -ENOMEM; - iommu_wq = create_workqueue("amd_iommu_v2"); + iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0); if (iommu_wq == NULL) goto out; -- cgit v1.2.3 From 9fec79df898602f5a20c0ab489bf9780aa98185d Mon Sep 17 00:00:00 2001 From: Honghui Zhang Date: Wed, 8 Jun 2016 17:50:44 +0800 Subject: iommu/mediatek: Do not call of_node_put in mtk_iommu_of_xlate The device_node will be released in of_iommu_configure, it may be double released if call of_node_put in mtk_iommu_of_xlate. Signed-off-by: Honghui Zhang Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index c3043d8754e3..493bd3e0f8db 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -455,7 +455,6 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) if (!dev->archdata.iommu) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); - of_node_put(args->np); if (WARN_ON(!m4updev)) return -EINVAL; -- cgit v1.2.3 From 9ca340c98c0dc6cb60b5ebd7847302f57648f0ba Mon Sep 17 00:00:00 2001 From: Honghui Zhang Date: Wed, 8 Jun 2016 17:50:58 +0800 Subject: iommu/mediatek: move the common struct into header file Move the struct defines of mtk iommu into a new header files for common use. Signed-off-by: Honghui Zhang Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 48 +---------------------------- drivers/iommu/mtk_iommu.h | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 47 deletions(-) create mode 100644 drivers/iommu/mtk_iommu.h (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 493bd3e0f8db..b12c12d74c33 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -34,7 +34,7 @@ #include #include -#include "io-pgtable.h" +#include "mtk_iommu.h" #define REG_MMU_PT_BASE_ADDR 0x000 @@ -93,20 +93,6 @@ #define MTK_PROTECT_PA_ALIGN 128 -struct mtk_iommu_suspend_reg { - u32 standard_axi_mode; - u32 dcm_dis; - u32 ctrl_reg; - u32 int_control0; - u32 int_main_control; -}; - -struct mtk_iommu_client_priv { - struct list_head client; - unsigned int mtk_m4u_id; - struct device *m4udev; -}; - struct mtk_iommu_domain { spinlock_t pgtlock; /* lock for page table */ @@ -116,19 +102,6 @@ struct mtk_iommu_domain { struct iommu_domain domain; }; -struct mtk_iommu_data { - void __iomem *base; - int irq; - struct device *dev; - struct clk *bclk; - phys_addr_t protect_base; /* protect memory base */ - struct mtk_iommu_suspend_reg reg; - struct mtk_iommu_domain *m4u_dom; - struct iommu_group *m4u_group; - struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ - bool enable_4GB; -}; - static struct iommu_ops mtk_iommu_ops; static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) @@ -551,25 +524,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return 0; } -static int compare_of(struct device *dev, void *data) -{ - return dev->of_node == data; -} - -static int mtk_iommu_bind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - return component_bind_all(dev, &data->smi_imu); -} - -static void mtk_iommu_unbind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - component_unbind_all(dev, &data->smi_imu); -} - static const struct component_master_ops mtk_iommu_com_ops = { .bind = mtk_iommu_bind, .unbind = mtk_iommu_unbind, diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h new file mode 100644 index 000000000000..9ed0a8462ccf --- /dev/null +++ b/drivers/iommu/mtk_iommu.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Honghui Zhang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _MTK_IOMMU_H_ +#define _MTK_IOMMU_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io-pgtable.h" + +struct mtk_iommu_suspend_reg { + u32 standard_axi_mode; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; + u32 int_main_control; +}; + +struct mtk_iommu_client_priv { + struct list_head client; + unsigned int mtk_m4u_id; + struct device *m4udev; +}; + +struct mtk_iommu_domain; + +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_domain *m4u_dom; + struct iommu_group *m4u_group; + struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ + bool enable_4GB; +}; + +static int compare_of(struct device *dev, void *data) +{ + return dev->of_node == data; +} + +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->smi_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->smi_imu); +} + +#endif -- cgit v1.2.3 From b17336c55d8928c4c693d3feb6245508e562aab5 Mon Sep 17 00:00:00 2001 From: Honghui Zhang Date: Wed, 8 Jun 2016 17:51:00 +0800 Subject: iommu/mediatek: add support for mtk iommu generation one HW Mediatek SoC's M4U has two generations of HW architcture. Generation one uses flat, one layer pagetable, and was shipped with ARM architecture, it only supports 4K size page mapping. MT2701 SoC uses this generation one m4u HW. Generation two uses the ARM short-descriptor translation table format for address translation, and was shipped with ARM64 architecture, MT8173 uses this generation two m4u HW. All the two generation iommu HW only have one iommu domain, and all its iommu clients share the same iova address. These two generation m4u HW have slit different register groups and register offset, but most register names are the same. This patch add iommu support for mediatek SoC mt2701. Signed-off-by: Honghui Zhang Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 18 ++ drivers/iommu/Makefile | 1 + drivers/iommu/mtk_iommu_v1.c | 727 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 746 insertions(+) create mode 100644 drivers/iommu/mtk_iommu_v1.c (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index ad0860383cb3..32bb1e5a4091 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -343,4 +343,22 @@ config MTK_IOMMU If unsure, say N here. +config MTK_IOMMU_V1 + bool "MTK IOMMU Version 1 (M4U gen1) Support" + depends on ARM + depends on ARCH_MEDIATEK || COMPILE_TEST + select ARM_DMA_USE_IOMMU + select IOMMU_API + select MEMORY + select MTK_SMI + select COMMON_CLK_MT2701_MMSYS + select COMMON_CLK_MT2701_IMGSYS + select COMMON_CLK_MT2701_VDECSYS + help + Support for the M4U on certain Mediatek SoCs. M4U generation 1 HW is + Multimedia Memory Managememt Unit. This option enables remapping of + DMA memory accesses for the multimedia subsystem. + + if unsure, say N here. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index c6edb31bf8c6..778baf5714e1 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -18,6 +18,7 @@ obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o +obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c new file mode 100644 index 000000000000..294485ddac2b --- /dev/null +++ b/drivers/iommu/mtk_iommu_v1.c @@ -0,0 +1,727 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Honghui Zhang + * + * Based on driver/iommu/mtk_iommu.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mtk_iommu.h" + +#define REG_MMU_PT_BASE_ADDR 0x000 + +#define F_ALL_INVLD 0x2 +#define F_MMU_INV_RANGE 0x1 +#define F_INVLD_EN0 BIT(0) +#define F_INVLD_EN1 BIT(1) + +#define F_MMU_FAULT_VA_MSK 0xfffff000 +#define MTK_PROTECT_PA_ALIGN 128 + +#define REG_MMU_CTRL_REG 0x210 +#define F_MMU_CTRL_COHERENT_EN BIT(8) +#define REG_MMU_IVRP_PADDR 0x214 +#define REG_MMU_INT_CONTROL 0x220 +#define F_INT_TRANSLATION_FAULT BIT(0) +#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) +#define F_INT_INVALID_PA_FAULT BIT(2) +#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) +#define F_INT_TABLE_WALK_FAULT BIT(4) +#define F_INT_TLB_MISS_FAULT BIT(5) +#define F_INT_PFH_DMA_FIFO_OVERFLOW BIT(6) +#define F_INT_MISS_DMA_FIFO_OVERFLOW BIT(7) + +#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) +#define F_INT_CLR_BIT BIT(12) + +#define REG_MMU_FAULT_ST 0x224 +#define REG_MMU_FAULT_VA 0x228 +#define REG_MMU_INVLD_PA 0x22C +#define REG_MMU_INT_ID 0x388 +#define REG_MMU_INVALIDATE 0x5c0 +#define REG_MMU_INVLD_START_A 0x5c4 +#define REG_MMU_INVLD_END_A 0x5c8 + +#define REG_MMU_INV_SEL 0x5d8 +#define REG_MMU_STANDARD_AXI_MODE 0x5e8 + +#define REG_MMU_DCM 0x5f0 +#define F_MMU_DCM_ON BIT(1) +#define REG_MMU_CPE_DONE 0x60c +#define F_DESC_VALID 0x2 +#define F_DESC_NONSEC BIT(3) +#define MT2701_M4U_TF_LARB(TF) (6 - (((TF) >> 13) & 0x7)) +#define MT2701_M4U_TF_PORT(TF) (((TF) >> 8) & 0xF) +/* MTK generation one iommu HW only support 4K size mapping */ +#define MT2701_IOMMU_PAGE_SHIFT 12 +#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) + +/* + * MTK m4u support 4GB iova address space, and only support 4K page + * mapping. So the pagetable size should be exactly as 4M. + */ +#define M2701_IOMMU_PGT_SIZE SZ_4M + +struct mtk_iommu_domain { + spinlock_t pgtlock; /* lock for page table */ + struct iommu_domain domain; + u32 *pgt_va; + dma_addr_t pgt_pa; + struct mtk_iommu_data *data; +}; + +static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct mtk_iommu_domain, domain); +} + +static const int mt2701_m4u_in_larb[] = { + LARB0_PORT_OFFSET, LARB1_PORT_OFFSET, + LARB2_PORT_OFFSET, LARB3_PORT_OFFSET +}; + +static inline int mt2701_m4u_to_larb(int id) +{ + int i; + + for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--) + if ((id) >= mt2701_m4u_in_larb[i]) + return i; + + return 0; +} + +static inline int mt2701_m4u_to_port(int id) +{ + int larb = mt2701_m4u_to_larb(id); + + return id - mt2701_m4u_in_larb[larb]; +} + +static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) +{ + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, + data->base + REG_MMU_INV_SEL); + writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); + wmb(); /* Make sure the tlb flush all done */ +} + +static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data, + unsigned long iova, size_t size) +{ + int ret; + u32 tmp; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, + data->base + REG_MMU_INV_SEL); + writel_relaxed(iova & F_MMU_FAULT_VA_MSK, + data->base + REG_MMU_INVLD_START_A); + writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK, + data->base + REG_MMU_INVLD_END_A); + writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); + + ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, + tmp, tmp != 0, 10, 100000); + if (ret) { + dev_warn(data->dev, + "Partial TLB flush timed out, falling back to full flush\n"); + mtk_iommu_tlb_flush_all(data); + } + /* Clear the CPE status */ + writel_relaxed(0, data->base + REG_MMU_CPE_DONE); +} + +static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) +{ + struct mtk_iommu_data *data = dev_id; + struct mtk_iommu_domain *dom = data->m4u_dom; + u32 int_state, regval, fault_iova, fault_pa; + unsigned int fault_larb, fault_port; + + /* Read error information from registers */ + int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST); + fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); + + fault_iova &= F_MMU_FAULT_VA_MSK; + fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); + regval = readl_relaxed(data->base + REG_MMU_INT_ID); + fault_larb = MT2701_M4U_TF_LARB(regval); + fault_port = MT2701_M4U_TF_PORT(regval); + + /* + * MTK v1 iommu HW could not determine whether the fault is read or + * write fault, report as read fault. + */ + if (report_iommu_fault(&dom->domain, data->dev, fault_iova, + IOMMU_FAULT_READ)) + dev_err_ratelimited(data->dev, + "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n", + int_state, fault_iova, fault_pa, + fault_larb, fault_port); + + /* Interrupt clear */ + regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL); + regval |= F_INT_CLR_BIT; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); + + mtk_iommu_tlb_flush_all(data); + + return IRQ_HANDLED; +} + +static void mtk_iommu_config(struct mtk_iommu_data *data, + struct device *dev, bool enable) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + struct mtk_smi_larb_iommu *larb_mmu; + unsigned int larbid, portid; + + head = dev->archdata.iommu; + list_for_each_entry_safe(cur, next, &head->client, client) { + larbid = mt2701_m4u_to_larb(cur->mtk_m4u_id); + portid = mt2701_m4u_to_port(cur->mtk_m4u_id); + larb_mmu = &data->smi_imu.larb_imu[larbid]; + + dev_dbg(dev, "%s iommu port: %d\n", + enable ? "enable" : "disable", portid); + + if (enable) + larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); + else + larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } +} + +static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) +{ + struct mtk_iommu_domain *dom = data->m4u_dom; + + spin_lock_init(&dom->pgtlock); + + dom->pgt_va = dma_zalloc_coherent(data->dev, + M2701_IOMMU_PGT_SIZE, + &dom->pgt_pa, GFP_KERNEL); + if (!dom->pgt_va) + return -ENOMEM; + + writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR); + + dom->data = data; + + return 0; +} + +static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +{ + struct mtk_iommu_domain *dom; + + if (type != IOMMU_DOMAIN_UNMANAGED) + return NULL; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + return &dom->domain; +} + +static void mtk_iommu_domain_free(struct iommu_domain *domain) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_data *data = dom->data; + + dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE, + dom->pgt_va, dom->pgt_pa); + kfree(to_mtk_domain(domain)); +} + +static int mtk_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + int ret; + + if (!priv) + return -ENODEV; + + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_dom) { + data->m4u_dom = dom; + ret = mtk_iommu_domain_finalise(data); + if (ret) { + data->m4u_dom = NULL; + return ret; + } + } + + mtk_iommu_config(data, dev, true); + return 0; +} + +static void mtk_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + + if (!priv) + return; + + data = dev_get_drvdata(priv->m4udev); + mtk_iommu_config(data, dev, false); +} + +static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; + unsigned long flags; + unsigned int i; + u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); + u32 pabase = (u32)paddr; + int map_size = 0; + + spin_lock_irqsave(&dom->pgtlock, flags); + for (i = 0; i < page_num; i++) { + if (pgt_base_iova[i]) { + memset(pgt_base_iova, 0, i * sizeof(u32)); + break; + } + pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC; + pabase += MT2701_IOMMU_PAGE_SIZE; + map_size += MT2701_IOMMU_PAGE_SIZE; + } + + spin_unlock_irqrestore(&dom->pgtlock, flags); + + mtk_iommu_tlb_flush_range(dom->data, iova, size); + + return map_size == size ? 0 : -EEXIST; +} + +static size_t mtk_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); + unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; + + spin_lock_irqsave(&dom->pgtlock, flags); + memset(pgt_base_iova, 0, page_num * sizeof(u32)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + mtk_iommu_tlb_flush_range(dom->data, iova, size); + + return size; +} + +static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + phys_addr_t pa; + + spin_lock_irqsave(&dom->pgtlock, flags); + pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT)); + pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return pa; +} + +/* + * MTK generation one iommu HW only support one iommu domain, and all the client + * sharing the same iova address space. + */ +static int mtk_iommu_create_mapping(struct device *dev, + struct of_phandle_args *args) +{ + struct mtk_iommu_client_priv *head, *priv, *next; + struct platform_device *m4updev; + struct dma_iommu_mapping *mtk_mapping; + struct device *m4udev; + int ret; + + if (args->args_count != 1) { + dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", + args->args_count); + return -EINVAL; + } + + if (!dev->archdata.iommu) { + /* Get the m4u device */ + m4updev = of_find_device_by_node(args->np); + if (WARN_ON(!m4updev)) + return -EINVAL; + + head = kzalloc(sizeof(*head), GFP_KERNEL); + if (!head) + return -ENOMEM; + + dev->archdata.iommu = head; + INIT_LIST_HEAD(&head->client); + head->m4udev = &m4updev->dev; + } else { + head = dev->archdata.iommu; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + ret = -ENOMEM; + goto err_free_mem; + } + priv->mtk_m4u_id = args->args[0]; + list_add_tail(&priv->client, &head->client); + + m4udev = head->m4udev; + mtk_mapping = m4udev->archdata.iommu; + if (!mtk_mapping) { + /* MTK iommu support 4GB iova address space. */ + mtk_mapping = arm_iommu_create_mapping(&platform_bus_type, + 0, 1ULL << 32); + if (IS_ERR(mtk_mapping)) { + ret = PTR_ERR(mtk_mapping); + goto err_free_mem; + } + m4udev->archdata.iommu = mtk_mapping; + } + + ret = arm_iommu_attach_device(dev, mtk_mapping); + if (ret) + goto err_release_mapping; + + return 0; + +err_release_mapping: + arm_iommu_release_mapping(mtk_mapping); + m4udev->archdata.iommu = NULL; +err_free_mem: + list_for_each_entry_safe(priv, next, &head->client, client) + kfree(priv); + kfree(head); + dev->archdata.iommu = NULL; + return ret; +} + +static int mtk_iommu_add_device(struct device *dev) +{ + struct iommu_group *group; + struct of_phandle_args iommu_spec; + struct of_phandle_iterator it; + int err; + + of_for_each_phandle(&it, err, dev->of_node, "iommus", + "#iommu-cells", 0) { + int count = of_phandle_iterator_args(&it, iommu_spec.args, + MAX_PHANDLE_ARGS); + iommu_spec.np = of_node_get(it.node); + iommu_spec.args_count = count; + + mtk_iommu_create_mapping(dev, &iommu_spec); + of_node_put(iommu_spec.np); + } + + if (!dev->archdata.iommu) /* Not a iommu client device */ + return -ENODEV; + + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + iommu_group_put(group); + return 0; +} + +static void mtk_iommu_remove_device(struct device *dev) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + + head = dev->archdata.iommu; + if (!head) + return; + + list_for_each_entry_safe(cur, next, &head->client, client) { + list_del(&cur->client); + kfree(cur); + } + kfree(head); + dev->archdata.iommu = NULL; + + iommu_group_remove_device(dev); +} + +static struct iommu_group *mtk_iommu_device_group(struct device *dev) +{ + struct mtk_iommu_data *data; + struct mtk_iommu_client_priv *priv; + + priv = dev->archdata.iommu; + if (!priv) + return ERR_PTR(-ENODEV); + + /* All the client devices are in the same m4u iommu-group */ + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_group) { + data->m4u_group = iommu_group_alloc(); + if (IS_ERR(data->m4u_group)) + dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } + return data->m4u_group; +} + +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +{ + u32 regval; + int ret; + + ret = clk_prepare_enable(data->bclk); + if (ret) { + dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); + return ret; + } + + regval = F_MMU_CTRL_COHERENT_EN | F_MMU_TF_PROTECT_SEL(2); + writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); + + regval = F_INT_TRANSLATION_FAULT | + F_INT_MAIN_MULTI_HIT_FAULT | + F_INT_INVALID_PA_FAULT | + F_INT_ENTRY_REPLACEMENT_FAULT | + F_INT_TABLE_WALK_FAULT | + F_INT_TLB_MISS_FAULT | + F_INT_PFH_DMA_FIFO_OVERFLOW | + F_INT_MISS_DMA_FIFO_OVERFLOW; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); + + /* protect memory,hw will write here while translation fault */ + writel_relaxed(data->protect_base, + data->base + REG_MMU_IVRP_PADDR); + + writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM); + + if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, + dev_name(data->dev), (void *)data)) { + writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); + clk_disable_unprepare(data->bclk); + dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); + return -ENODEV; + } + + return 0; +} + +static struct iommu_ops mtk_iommu_ops = { + .domain_alloc = mtk_iommu_domain_alloc, + .domain_free = mtk_iommu_domain_free, + .attach_dev = mtk_iommu_attach_device, + .detach_dev = mtk_iommu_detach_device, + .map = mtk_iommu_map, + .unmap = mtk_iommu_unmap, + .map_sg = default_iommu_map_sg, + .iova_to_phys = mtk_iommu_iova_to_phys, + .add_device = mtk_iommu_add_device, + .remove_device = mtk_iommu_remove_device, + .device_group = mtk_iommu_device_group, + .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, +}; + +static const struct of_device_id mtk_iommu_of_ids[] = { + { .compatible = "mediatek,mt2701-m4u", }, + {} +}; + +static const struct component_master_ops mtk_iommu_com_ops = { + .bind = mtk_iommu_bind, + .unbind = mtk_iommu_unbind, +}; + +static int mtk_iommu_probe(struct platform_device *pdev) +{ + struct mtk_iommu_data *data; + struct device *dev = &pdev->dev; + struct resource *res; + struct component_match *match = NULL; + struct of_phandle_args larb_spec; + struct of_phandle_iterator it; + void *protect; + int larb_nr, ret, err; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->dev = dev; + + /* Protect memory. HW will access here while translation fault.*/ + protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, + GFP_KERNEL | GFP_DMA); + if (!protect) + return -ENOMEM; + data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->irq = platform_get_irq(pdev, 0); + if (data->irq < 0) + return data->irq; + + data->bclk = devm_clk_get(dev, "bclk"); + if (IS_ERR(data->bclk)) + return PTR_ERR(data->bclk); + + larb_nr = 0; + of_for_each_phandle(&it, err, dev->of_node, + "mediatek,larbs", NULL, 0) { + struct platform_device *plarbdev; + int count = of_phandle_iterator_args(&it, larb_spec.args, + MAX_PHANDLE_ARGS); + + if (count) + continue; + + larb_spec.np = of_node_get(it.node); + if (!of_device_is_available(larb_spec.np)) + continue; + + plarbdev = of_find_device_by_node(larb_spec.np); + of_node_put(larb_spec.np); + if (!plarbdev) { + plarbdev = of_platform_device_create( + larb_spec.np, NULL, + platform_bus_type.dev_root); + if (!plarbdev) + return -EPROBE_DEFER; + } + + data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev; + component_match_add(dev, &match, compare_of, larb_spec.np); + larb_nr++; + } + + data->smi_imu.larb_nr = larb_nr; + + platform_set_drvdata(pdev, data); + + ret = mtk_iommu_hw_init(data); + if (ret) + return ret; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + + return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); +} + +static int mtk_iommu_remove(struct platform_device *pdev) +{ + struct mtk_iommu_data *data = platform_get_drvdata(pdev); + + if (iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, NULL); + + clk_disable_unprepare(data->bclk); + devm_free_irq(&pdev->dev, data->irq, data); + component_master_del(&pdev->dev, &mtk_iommu_com_ops); + return 0; +} + +static int __maybe_unused mtk_iommu_suspend(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + reg->standard_axi_mode = readl_relaxed(base + + REG_MMU_STANDARD_AXI_MODE); + reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM); + reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); + reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL); + return 0; +} + +static int __maybe_unused mtk_iommu_resume(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR); + writel_relaxed(reg->standard_axi_mode, + base + REG_MMU_STANDARD_AXI_MODE); + writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM); + writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); + writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL); + writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR); + return 0; +} + +const struct dev_pm_ops mtk_iommu_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) +}; + +static struct platform_driver mtk_iommu_driver = { + .probe = mtk_iommu_probe, + .remove = mtk_iommu_remove, + .driver = { + .name = "mtk-iommu", + .of_match_table = mtk_iommu_of_ids, + .pm = &mtk_iommu_pm_ops, + } +}; + +static int __init m4u_init(void) +{ + return platform_driver_register(&mtk_iommu_driver); +} + +static void __exit m4u_exit(void) +{ + return platform_driver_unregister(&mtk_iommu_driver); +} + +subsys_initcall(m4u_init); +module_exit(m4u_exit); + +MODULE_DESCRIPTION("IOMMU API for MTK architected m4u v1 implementations"); +MODULE_AUTHOR("Honghui Zhang "); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 131bc8ebb46a84f93d90a6d95347cfb9402f58a3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jun 2016 11:52:13 +0200 Subject: iommu/mediatek: Make mtk_iommu_pm_ops static The symbol exists elsewhere already, so that is fails to link if the symbol is non-static. Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 294485ddac2b..b8aeb0768483 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -695,7 +695,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) return 0; } -const struct dev_pm_ops mtk_iommu_pm_ops = { +static const struct dev_pm_ops mtk_iommu_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) }; -- cgit v1.2.3 From 6ae5343c26f9cba5e9ef8ba6f23b5bb255ebc798 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Wed, 8 Jun 2016 19:31:10 +0100 Subject: iommu/exynos: update to use iommu big-endian Add initial support for big endian by always writing the pte in le32. Note, revisit if hardware capable of doing big endian fetches. Signed-off-by: Ben Dooks Acked-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 633e6d023c0d..33dcc29ec200 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -54,6 +54,10 @@ typedef u32 sysmmu_pte_t; #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) +#ifdef CONFIG_BIG_ENDIAN +#warning "revisit driver if we can enable big-endian ptes" +#endif + /* * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces * v5.0 introduced support for 36bit physical address space by shifting @@ -710,7 +714,7 @@ static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); - *ent = val; + *ent = cpu_to_le32(val); dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); } -- cgit v1.2.3 From 109bd48ea2e1fb4e924712018397a51c7b2aaadd Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 13 Jun 2016 17:06:02 +0530 Subject: iommu/msm: Add DT adaptation The driver currently works based on platform data. Remove this and add support for DT. A single master can have multiple ports connected to more than one iommu. master | | | ------------------------ | | IOMMU0 IOMMU1 | | ctx0 ctx1 ctx0 ctx1 This association of master and iommus/contexts were previously represented by platform data parent/child device details. The client drivers were responsible for programming all of the iommus/contexts for the device. Now while adapting to generic DT bindings we maintain the list of iommus, contexts that each master domain is connected to and program all of them on attach/detach. Signed-off-by: Sricharan R Tested-by: Archit Taneja Tested-by: Srinivas Kandagatla Signed-off-by: Joerg Roedel --- drivers/iommu/msm_iommu.c | 252 ++++++++++++++++++--------------- drivers/iommu/msm_iommu.h | 73 ++++------ drivers/iommu/msm_iommu_dev.c | 315 ++++++++++-------------------------------- 3 files changed, 237 insertions(+), 403 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index e321fa517a45..bc1a4e3a7042 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -48,6 +48,7 @@ __asm__ __volatile__ ( \ static int msm_iommu_tex_class[4]; DEFINE_SPINLOCK(msm_iommu_lock); +static LIST_HEAD(qcom_iommu_devices); struct msm_priv { unsigned long *pgtable; @@ -60,35 +61,37 @@ static struct msm_priv *to_msm_priv(struct iommu_domain *dom) return container_of(dom, struct msm_priv, domain); } -static int __enable_clocks(struct msm_iommu_drvdata *drvdata) +static int __enable_clocks(struct msm_iommu_dev *iommu) { int ret; - ret = clk_enable(drvdata->pclk); + ret = clk_enable(iommu->pclk); if (ret) goto fail; - if (drvdata->clk) { - ret = clk_enable(drvdata->clk); + if (iommu->clk) { + ret = clk_enable(iommu->clk); if (ret) - clk_disable(drvdata->pclk); + clk_disable(iommu->pclk); } fail: return ret; } -static void __disable_clocks(struct msm_iommu_drvdata *drvdata) +static void __disable_clocks(struct msm_iommu_dev *iommu) { - clk_disable(drvdata->clk); - clk_disable(drvdata->pclk); + if (iommu->clk) + clk_disable(iommu->clk); + clk_disable(iommu->pclk); } static int __flush_iotlb(struct iommu_domain *domain) { struct msm_priv *priv = to_msm_priv(domain); - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; + struct msm_iommu_dev *iommu = NULL; + struct msm_iommu_ctx_dev *master; int ret = 0; + #ifndef CONFIG_IOMMU_PGTABLES_L2 unsigned long *fl_table = priv->pgtable; int i; @@ -105,24 +108,67 @@ static int __flush_iotlb(struct iommu_domain *domain) } #endif - list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { - - BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent); - - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); - BUG_ON(!iommu_drvdata); - - ret = __enable_clocks(iommu_drvdata); + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); if (ret) goto fail; - SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); - __disable_clocks(iommu_drvdata); + list_for_each_entry(master, &iommu->ctx_list, list) + SET_CTX_TLBIALL(iommu->base, master->num, 0); + + __disable_clocks(iommu); } fail: return ret; } +static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end) +{ + int idx; + + do { + idx = find_next_zero_bit(map, end, start); + if (idx == end) + return -ENOSPC; + } while (test_and_set_bit(idx, map)); + + return idx; +} + +static void msm_iommu_free_ctx(unsigned long *map, int idx) +{ + clear_bit(idx, map); +} + +static void config_mids(struct msm_iommu_dev *iommu, + struct msm_iommu_ctx_dev *master) +{ + int mid, ctx, i; + + for (i = 0; i < master->num_mids; i++) { + mid = master->mids[i]; + ctx = master->num; + + SET_M2VCBR_N(iommu->base, mid, 0); + SET_CBACR_N(iommu->base, ctx, 0); + + /* Set VMID = 0 */ + SET_VMID(iommu->base, mid, 0); + + /* Set the context number for that MID to this context */ + SET_CBNDX(iommu->base, mid, ctx); + + /* Set MID associated with this context bank to 0*/ + SET_CBVMID(iommu->base, ctx, 0); + + /* Set the ASID for TLB tagging for this context */ + SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx); + + /* Set security bit override to be Non-secure */ + SET_NSCFG(iommu->base, mid, 3); + } +} + static void __reset_context(void __iomem *base, int ctx) { SET_BPRCOSH(base, ctx, 0); @@ -272,94 +318,76 @@ static void msm_iommu_domain_free(struct iommu_domain *domain) static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - struct msm_iommu_ctx_drvdata *tmp_drvdata; int ret = 0; unsigned long flags; + struct msm_iommu_dev *iommu; + struct msm_priv *priv = to_msm_priv(domain); + struct msm_iommu_ctx_dev *master; spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = to_msm_priv(domain); - - if (!dev) { - ret = -EINVAL; - goto fail; - } - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { - ret = -EINVAL; - goto fail; - } - - if (!list_empty(&ctx_drvdata->attached_elm)) { - ret = -EBUSY; - goto fail; - } - - list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) - if (tmp_drvdata == ctx_drvdata) { - ret = -EBUSY; - goto fail; + list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { + master = list_first_entry(&iommu->ctx_list, + struct msm_iommu_ctx_dev, + list); + if (master->of_node == dev->of_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; + + list_for_each_entry(master, &iommu->ctx_list, list) { + if (master->num) { + dev_err(dev, "domain already attached"); + ret = -EEXIST; + goto fail; + } + master->num = + msm_iommu_alloc_ctx(iommu->context_map, + 0, iommu->ncb); + if (IS_ERR_VALUE(master->num)) { + ret = -ENODEV; + goto fail; + } + config_mids(iommu, master); + __program_context(iommu->base, master->num, + __pa(priv->pgtable)); + } + __disable_clocks(iommu); + list_add(&iommu->dom_node, &priv->list_attached); } + } - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - __program_context(iommu_drvdata->base, ctx_dev->num, - __pa(priv->pgtable)); - - __disable_clocks(iommu_drvdata); - list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); ret = __flush_iotlb(domain); - fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); + return ret; } static void msm_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) { - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; + struct msm_iommu_dev *iommu; + struct msm_iommu_ctx_dev *master; int ret; spin_lock_irqsave(&msm_iommu_lock, flags); - priv = to_msm_priv(domain); - - if (!dev) - goto fail; - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) - goto fail; - ret = __flush_iotlb(domain); if (ret) goto fail; - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - __reset_context(iommu_drvdata->base, ctx_dev->num); - __disable_clocks(iommu_drvdata); - list_del_init(&ctx_drvdata->attached_elm); + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; + list_for_each_entry(master, &iommu->ctx_list, list) { + msm_iommu_free_ctx(iommu->context_map, master->num); + __reset_context(iommu->base, master->num); + } + __disable_clocks(iommu); + } fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); } @@ -555,47 +583,46 @@ static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t va) { struct msm_priv *priv; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; + struct msm_iommu_dev *iommu; + struct msm_iommu_ctx_dev *master; unsigned int par; unsigned long flags; - void __iomem *base; phys_addr_t ret = 0; - int ctx; spin_lock_irqsave(&msm_iommu_lock, flags); priv = to_msm_priv(domain); - if (list_empty(&priv->list_attached)) - goto fail; + iommu = list_first_entry(&priv->list_attached, + struct msm_iommu_dev, dom_node); - ctx_drvdata = list_entry(priv->list_attached.next, - struct msm_iommu_ctx_drvdata, attached_elm); - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); + if (list_empty(&iommu->ctx_list)) + goto fail; - base = iommu_drvdata->base; - ctx = ctx_drvdata->num; + master = list_first_entry(&iommu->ctx_list, + struct msm_iommu_ctx_dev, list); + if (!master) + goto fail; - ret = __enable_clocks(iommu_drvdata); + ret = __enable_clocks(iommu); if (ret) goto fail; /* Invalidate context TLB */ - SET_CTX_TLBIALL(base, ctx, 0); - SET_V2PPR(base, ctx, va & V2Pxx_VA); + SET_CTX_TLBIALL(iommu->base, master->num, 0); + SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA); - par = GET_PAR(base, ctx); + par = GET_PAR(iommu->base, master->num); /* We are dealing with a supersection */ - if (GET_NOFAULT_SS(base, ctx)) + if (GET_NOFAULT_SS(iommu->base, master->num)) ret = (par & 0xFF000000) | (va & 0x00FFFFFF); else /* Upper 20 bits from PAR, lower 12 from VA */ ret = (par & 0xFFFFF000) | (va & 0x00000FFF); - if (GET_FAULT(base, ctx)) + if (GET_FAULT(iommu->base, master->num)) ret = 0; - __disable_clocks(iommu_drvdata); + __disable_clocks(iommu); fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; @@ -635,37 +662,34 @@ static void print_ctx_regs(void __iomem *base, int ctx) irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) { - struct msm_iommu_drvdata *drvdata = dev_id; - void __iomem *base; + struct msm_iommu_dev *iommu = dev_id; unsigned int fsr; int i, ret; spin_lock(&msm_iommu_lock); - if (!drvdata) { + if (!iommu) { pr_err("Invalid device ID in context interrupt handler\n"); goto fail; } - base = drvdata->base; - pr_err("Unexpected IOMMU page fault!\n"); - pr_err("base = %08x\n", (unsigned int) base); + pr_err("base = %08x\n", (unsigned int)iommu->base); - ret = __enable_clocks(drvdata); + ret = __enable_clocks(iommu); if (ret) goto fail; - for (i = 0; i < drvdata->ncb; i++) { - fsr = GET_FSR(base, i); + for (i = 0; i < iommu->ncb; i++) { + fsr = GET_FSR(iommu->base, i); if (fsr) { pr_err("Fault occurred in context %d.\n", i); pr_err("Interesting registers:\n"); - print_ctx_regs(base, i); - SET_FSR(base, i, 0x4000000F); + print_ctx_regs(iommu->base, i); + SET_FSR(iommu->base, i, 0x4000000F); } } - __disable_clocks(drvdata); + __disable_clocks(iommu); fail: spin_unlock(&msm_iommu_lock); return 0; diff --git a/drivers/iommu/msm_iommu.h b/drivers/iommu/msm_iommu.h index 5c7c955e6d25..4ca25d50d679 100644 --- a/drivers/iommu/msm_iommu.h +++ b/drivers/iommu/msm_iommu.h @@ -42,74 +42,53 @@ */ #define MAX_NUM_MIDS 32 +/* Maximum number of context banks that can be present in IOMMU */ +#define IOMMU_MAX_CBS 128 + /** * struct msm_iommu_dev - a single IOMMU hardware instance - * name Human-readable name given to this IOMMU HW instance * ncb Number of context banks present on this IOMMU HW instance + * dev: IOMMU device + * irq: Interrupt number + * clk: The bus clock for this IOMMU hardware instance + * pclk: The clock for the IOMMU bus interconnect + * dev_node: list head in qcom_iommu_device_list + * dom_node: list head for domain + * ctx_list: list of 'struct msm_iommu_ctx_dev' + * context_map: Bitmap to track allocated context banks */ struct msm_iommu_dev { - const char *name; + void __iomem *base; int ncb; + struct device *dev; + int irq; + struct clk *clk; + struct clk *pclk; + struct list_head dev_node; + struct list_head dom_node; + struct list_head ctx_list; + DECLARE_BITMAP(context_map, IOMMU_MAX_CBS); }; /** * struct msm_iommu_ctx_dev - an IOMMU context bank instance - * name Human-readable name given to this context bank + * of_node node ptr of client device * num Index of this context bank within the hardware * mids List of Machine IDs that are to be mapped into this context * bank, terminated by -1. The MID is a set of signals on the * AXI bus that identifies the function associated with a specific * memory request. (See ARM spec). + * num_mids Total number of mids + * node list head in ctx_list */ struct msm_iommu_ctx_dev { - const char *name; + struct device_node *of_node; int num; int mids[MAX_NUM_MIDS]; + int num_mids; + struct list_head list; }; - -/** - * struct msm_iommu_drvdata - A single IOMMU hardware instance - * @base: IOMMU config port base address (VA) - * @ncb The number of contexts on this IOMMU - * @irq: Interrupt number - * @clk: The bus clock for this IOMMU hardware instance - * @pclk: The clock for the IOMMU bus interconnect - * - * A msm_iommu_drvdata holds the global driver data about a single piece - * of an IOMMU hardware instance. - */ -struct msm_iommu_drvdata { - void __iomem *base; - int irq; - int ncb; - struct clk *clk; - struct clk *pclk; -}; - -/** - * struct msm_iommu_ctx_drvdata - an IOMMU context bank instance - * @num: Hardware context number of this context - * @pdev: Platform device associated wit this HW instance - * @attached_elm: List element for domains to track which devices are - * attached to them - * - * A msm_iommu_ctx_drvdata holds the driver data for a single context bank - * within each IOMMU hardware instance - */ -struct msm_iommu_ctx_drvdata { - int num; - struct platform_device *pdev; - struct list_head attached_elm; -}; - -/* - * Look up an IOMMU context device by its context name. NULL if none found. - * Useful for testing and drivers that do not yet fully have IOMMU stuff in - * their platform devices. - */ -struct device *msm_iommu_get_ctx(const char *ctx_name); - /* * Interrupt handler for the IOMMU context fault interrupt. Hooking the * interrupt is not supported in the API yet, but this will print an error diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c index 4b09e815accf..be01cc47c285 100644 --- a/drivers/iommu/msm_iommu_dev.c +++ b/drivers/iommu/msm_iommu_dev.c @@ -30,60 +30,6 @@ #include "msm_iommu_hw-8xxx.h" #include "msm_iommu.h" -struct iommu_ctx_iter_data { - /* input */ - const char *name; - - /* output */ - struct device *dev; -}; - -static struct platform_device *msm_iommu_root_dev; - -static int each_iommu_ctx(struct device *dev, void *data) -{ - struct iommu_ctx_iter_data *res = data; - struct msm_iommu_ctx_dev *c = dev->platform_data; - - if (!res || !c || !c->name || !res->name) - return -EINVAL; - - if (!strcmp(res->name, c->name)) { - res->dev = dev; - return 1; - } - return 0; -} - -static int each_iommu(struct device *dev, void *data) -{ - return device_for_each_child(dev, data, each_iommu_ctx); -} - -struct device *msm_iommu_get_ctx(const char *ctx_name) -{ - struct iommu_ctx_iter_data r; - int found; - - if (!msm_iommu_root_dev) { - pr_err("No root IOMMU device.\n"); - goto fail; - } - - r.name = ctx_name; - found = device_for_each_child(&msm_iommu_root_dev->dev, &r, each_iommu); - - if (!found) { - pr_err("Could not find context <%s>\n", ctx_name); - goto fail; - } - - return r.dev; -fail: - return NULL; -} -EXPORT_SYMBOL(msm_iommu_get_ctx); - static void msm_iommu_reset(void __iomem *base, int ncb) { int ctx; @@ -128,237 +74,122 @@ static void msm_iommu_reset(void __iomem *base, int ncb) static int msm_iommu_probe(struct platform_device *pdev) { struct resource *r; - struct clk *iommu_clk; - struct clk *iommu_pclk; - struct msm_iommu_drvdata *drvdata; - struct msm_iommu_dev *iommu_dev = dev_get_platdata(&pdev->dev); - void __iomem *regs_base; - int ret, irq, par; + struct msm_iommu_dev *iommu; + int ret, par, val; - if (pdev->id == -1) { - msm_iommu_root_dev = pdev; - return 0; - } + iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENODEV; - drvdata = kzalloc(sizeof(*drvdata), GFP_KERNEL); + iommu->dev = &pdev->dev; + INIT_LIST_HEAD(&iommu->ctx_list); - if (!drvdata) { - ret = -ENOMEM; - goto fail; + iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); + if (IS_ERR(iommu->pclk)) { + dev_err(iommu->dev, "could not get smmu_pclk\n"); + return PTR_ERR(iommu->pclk); } - if (!iommu_dev) { - ret = -ENODEV; - goto fail; + ret = clk_prepare(iommu->pclk); + if (ret) { + dev_err(iommu->dev, "could not prepare smmu_pclk\n"); + return ret; } - iommu_pclk = clk_get(NULL, "smmu_pclk"); - if (IS_ERR(iommu_pclk)) { - ret = -ENODEV; - goto fail; + iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); + if (IS_ERR(iommu->clk)) { + dev_err(iommu->dev, "could not get iommu_clk\n"); + clk_unprepare(iommu->pclk); + return PTR_ERR(iommu->clk); } - ret = clk_prepare_enable(iommu_pclk); - if (ret) - goto fail_enable; - - iommu_clk = clk_get(&pdev->dev, "iommu_clk"); - - if (!IS_ERR(iommu_clk)) { - if (clk_get_rate(iommu_clk) == 0) - clk_set_rate(iommu_clk, 1); - - ret = clk_prepare_enable(iommu_clk); - if (ret) { - clk_put(iommu_clk); - goto fail_pclk; - } - } else - iommu_clk = NULL; + ret = clk_prepare(iommu->clk); + if (ret) { + dev_err(iommu->dev, "could not prepare iommu_clk\n"); + clk_unprepare(iommu->pclk); + return ret; + } - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase"); - regs_base = devm_ioremap_resource(&pdev->dev, r); - if (IS_ERR(regs_base)) { - ret = PTR_ERR(regs_base); - goto fail_clk; + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + iommu->base = devm_ioremap_resource(iommu->dev, r); + if (IS_ERR(iommu->base)) { + dev_err(iommu->dev, "could not get iommu base\n"); + ret = PTR_ERR(iommu->base); + goto fail; } - irq = platform_get_irq_byname(pdev, "secure_irq"); - if (irq < 0) { + iommu->irq = platform_get_irq(pdev, 0); + if (iommu->irq < 0) { + dev_err(iommu->dev, "could not get iommu irq\n"); ret = -ENODEV; - goto fail_clk; + goto fail; } - msm_iommu_reset(regs_base, iommu_dev->ncb); + ret = of_property_read_u32(iommu->dev->of_node, "ncb", &val); + if (ret) { + dev_err(iommu->dev, "could not get ncb\n"); + goto fail; + } + iommu->ncb = val; - SET_M(regs_base, 0, 1); - SET_PAR(regs_base, 0, 0); - SET_V2PCFG(regs_base, 0, 1); - SET_V2PPR(regs_base, 0, 0); - par = GET_PAR(regs_base, 0); - SET_V2PCFG(regs_base, 0, 0); - SET_M(regs_base, 0, 0); + msm_iommu_reset(iommu->base, iommu->ncb); + SET_M(iommu->base, 0, 1); + SET_PAR(iommu->base, 0, 0); + SET_V2PCFG(iommu->base, 0, 1); + SET_V2PPR(iommu->base, 0, 0); + par = GET_PAR(iommu->base, 0); + SET_V2PCFG(iommu->base, 0, 0); + SET_M(iommu->base, 0, 0); if (!par) { - pr_err("%s: Invalid PAR value detected\n", iommu_dev->name); + pr_err("Invalid PAR value detected\n"); ret = -ENODEV; - goto fail_clk; + goto fail; } - ret = request_irq(irq, msm_iommu_fault_handler, 0, - "msm_iommu_secure_irpt_handler", drvdata); + ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, + msm_iommu_fault_handler, + IRQF_ONESHOT | IRQF_SHARED, + "msm_iommu_secure_irpt_handler", + iommu); if (ret) { - pr_err("Request IRQ %d failed with ret=%d\n", irq, ret); - goto fail_clk; + pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); + goto fail; } + list_add(&iommu->dev_node, &qcom_iommu_devices); - drvdata->pclk = iommu_pclk; - drvdata->clk = iommu_clk; - drvdata->base = regs_base; - drvdata->irq = irq; - drvdata->ncb = iommu_dev->ncb; - - pr_info("device %s mapped at %p, irq %d with %d ctx banks\n", - iommu_dev->name, regs_base, irq, iommu_dev->ncb); - - platform_set_drvdata(pdev, drvdata); - - clk_disable(iommu_clk); - - clk_disable(iommu_pclk); - - return 0; -fail_clk: - if (iommu_clk) { - clk_disable(iommu_clk); - clk_put(iommu_clk); - } -fail_pclk: - clk_disable_unprepare(iommu_pclk); -fail_enable: - clk_put(iommu_pclk); + pr_info("device mapped at %p, irq %d with %d ctx banks\n", + iommu->base, iommu->irq, iommu->ncb); fail: - kfree(drvdata); + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); return ret; } +static const struct of_device_id msm_iommu_dt_match[] = { + { .compatible = "qcom,apq8064-iommu" }, + {} +}; + static int msm_iommu_remove(struct platform_device *pdev) { - struct msm_iommu_drvdata *drv = NULL; + struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); - drv = platform_get_drvdata(pdev); - if (drv) { - if (drv->clk) { - clk_unprepare(drv->clk); - clk_put(drv->clk); - } - clk_unprepare(drv->pclk); - clk_put(drv->pclk); - memset(drv, 0, sizeof(*drv)); - kfree(drv); - } - return 0; -} - -static int msm_iommu_ctx_probe(struct platform_device *pdev) -{ - struct msm_iommu_ctx_dev *c = dev_get_platdata(&pdev->dev); - struct msm_iommu_drvdata *drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - int i, ret; - - if (!c || !pdev->dev.parent) - return -EINVAL; - - drvdata = dev_get_drvdata(pdev->dev.parent); - if (!drvdata) - return -ENODEV; - - ctx_drvdata = kzalloc(sizeof(*ctx_drvdata), GFP_KERNEL); - if (!ctx_drvdata) - return -ENOMEM; - - ctx_drvdata->num = c->num; - ctx_drvdata->pdev = pdev; - - INIT_LIST_HEAD(&ctx_drvdata->attached_elm); - platform_set_drvdata(pdev, ctx_drvdata); - - ret = clk_prepare_enable(drvdata->pclk); - if (ret) - goto fail; - - if (drvdata->clk) { - ret = clk_prepare_enable(drvdata->clk); - if (ret) { - clk_disable_unprepare(drvdata->pclk); - goto fail; - } - } - - /* Program the M2V tables for this context */ - for (i = 0; i < MAX_NUM_MIDS; i++) { - int mid = c->mids[i]; - if (mid == -1) - break; - - SET_M2VCBR_N(drvdata->base, mid, 0); - SET_CBACR_N(drvdata->base, c->num, 0); - - /* Set VMID = 0 */ - SET_VMID(drvdata->base, mid, 0); - - /* Set the context number for that MID to this context */ - SET_CBNDX(drvdata->base, mid, c->num); - - /* Set MID associated with this context bank to 0*/ - SET_CBVMID(drvdata->base, c->num, 0); - - /* Set the ASID for TLB tagging for this context */ - SET_CONTEXTIDR_ASID(drvdata->base, c->num, c->num); - - /* Set security bit override to be Non-secure */ - SET_NSCFG(drvdata->base, mid, 3); - } - - clk_disable(drvdata->clk); - clk_disable(drvdata->pclk); - - dev_info(&pdev->dev, "context %s using bank %d\n", c->name, c->num); - return 0; -fail: - kfree(ctx_drvdata); - return ret; -} - -static int msm_iommu_ctx_remove(struct platform_device *pdev) -{ - struct msm_iommu_ctx_drvdata *drv = NULL; - drv = platform_get_drvdata(pdev); - if (drv) { - memset(drv, 0, sizeof(struct msm_iommu_ctx_drvdata)); - kfree(drv); - } + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); return 0; } static struct platform_driver msm_iommu_driver = { .driver = { .name = "msm_iommu", + .of_match_table = msm_iommu_dt_match, }, .probe = msm_iommu_probe, .remove = msm_iommu_remove, }; -static struct platform_driver msm_iommu_ctx_driver = { - .driver = { - .name = "msm_iommu_ctx", - }, - .probe = msm_iommu_ctx_probe, - .remove = msm_iommu_ctx_remove, -}; - static struct platform_driver * const drivers[] = { &msm_iommu_driver, &msm_iommu_ctx_driver, -- cgit v1.2.3 From f7f125ef0b0210a2eb269148df9a1d641521857b Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 13 Jun 2016 17:06:04 +0530 Subject: iommu/msm: Move the contents from msm_iommu_dev.c to msm_iommu.c There are only two functions left in msm_iommu_dev.c. Move it to msm_iommu.c and delete the file. Signed-off-by: Sricharan R Tested-by: Archit Taneja Tested-by: Srinivas Kandagatla Signed-off-by: Joerg Roedel --- drivers/iommu/Makefile | 2 +- drivers/iommu/msm_iommu.c | 182 ++++++++++++++++++++++++++++++++++++ drivers/iommu/msm_iommu_dev.c | 212 ------------------------------------------ 3 files changed, 183 insertions(+), 213 deletions(-) delete mode 100644 drivers/iommu/msm_iommu_dev.c (limited to 'drivers/iommu') diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index c6edb31bf8c6..7fe479fb2770 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -7,7 +7,7 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o -obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o +obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index bc1a4e3a7042..792b3526b4a5 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,47 @@ static void __disable_clocks(struct msm_iommu_dev *iommu) clk_disable(iommu->pclk); } +static void msm_iommu_reset(void __iomem *base, int ncb) +{ + int ctx; + + SET_RPUE(base, 0); + SET_RPUEIE(base, 0); + SET_ESRRESTORE(base, 0); + SET_TBE(base, 0); + SET_CR(base, 0); + SET_SPDMBE(base, 0); + SET_TESTBUSCR(base, 0); + SET_TLBRSW(base, 0); + SET_GLOBAL_TLBIALL(base, 0); + SET_RPU_ACR(base, 0); + SET_TLBLKCRWE(base, 1); + + for (ctx = 0; ctx < ncb; ctx++) { + SET_BPRCOSH(base, ctx, 0); + SET_BPRCISH(base, ctx, 0); + SET_BPRCNSH(base, ctx, 0); + SET_BPSHCFG(base, ctx, 0); + SET_BPMTCFG(base, ctx, 0); + SET_ACTLR(base, ctx, 0); + SET_SCTLR(base, ctx, 0); + SET_FSRRESTORE(base, ctx, 0); + SET_TTBR0(base, ctx, 0); + SET_TTBR1(base, ctx, 0); + SET_TTBCR(base, ctx, 0); + SET_BFBCR(base, ctx, 0); + SET_PAR(base, ctx, 0); + SET_FAR(base, ctx, 0); + SET_CTX_TLBIALL(base, ctx, 0); + SET_TLBFLPTER(base, ctx, 0); + SET_TLBSLPTER(base, ctx, 0); + SET_TLBLKCR(base, ctx, 0); + SET_PRRR(base, ctx, 0); + SET_NMRR(base, ctx, 0); + SET_CONTEXTIDR(base, ctx, 0); + } +} + static int __flush_iotlb(struct iommu_domain *domain) { struct msm_priv *priv = to_msm_priv(domain); @@ -708,6 +750,146 @@ static const struct iommu_ops msm_iommu_ops = { .pgsize_bitmap = MSM_IOMMU_PGSIZES, }; +static int msm_iommu_probe(struct platform_device *pdev) +{ + struct resource *r; + struct msm_iommu_dev *iommu; + int ret, par, val; + + iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENODEV; + + iommu->dev = &pdev->dev; + INIT_LIST_HEAD(&iommu->ctx_list); + + iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); + if (IS_ERR(iommu->pclk)) { + dev_err(iommu->dev, "could not get smmu_pclk\n"); + return PTR_ERR(iommu->pclk); + } + + ret = clk_prepare(iommu->pclk); + if (ret) { + dev_err(iommu->dev, "could not prepare smmu_pclk\n"); + return ret; + } + + iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); + if (IS_ERR(iommu->clk)) { + dev_err(iommu->dev, "could not get iommu_clk\n"); + clk_unprepare(iommu->pclk); + return PTR_ERR(iommu->clk); + } + + ret = clk_prepare(iommu->clk); + if (ret) { + dev_err(iommu->dev, "could not prepare iommu_clk\n"); + clk_unprepare(iommu->pclk); + return ret; + } + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + iommu->base = devm_ioremap_resource(iommu->dev, r); + if (IS_ERR(iommu->base)) { + dev_err(iommu->dev, "could not get iommu base\n"); + ret = PTR_ERR(iommu->base); + goto fail; + } + + iommu->irq = platform_get_irq(pdev, 0); + if (iommu->irq < 0) { + dev_err(iommu->dev, "could not get iommu irq\n"); + ret = -ENODEV; + goto fail; + } + + ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val); + if (ret) { + dev_err(iommu->dev, "could not get ncb\n"); + goto fail; + } + iommu->ncb = val; + + msm_iommu_reset(iommu->base, iommu->ncb); + SET_M(iommu->base, 0, 1); + SET_PAR(iommu->base, 0, 0); + SET_V2PCFG(iommu->base, 0, 1); + SET_V2PPR(iommu->base, 0, 0); + par = GET_PAR(iommu->base, 0); + SET_V2PCFG(iommu->base, 0, 0); + SET_M(iommu->base, 0, 0); + + if (!par) { + pr_err("Invalid PAR value detected\n"); + ret = -ENODEV; + goto fail; + } + + ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, + msm_iommu_fault_handler, + IRQF_ONESHOT | IRQF_SHARED, + "msm_iommu_secure_irpt_handler", + iommu); + if (ret) { + pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); + goto fail; + } + + list_add(&iommu->dev_node, &qcom_iommu_devices); + + pr_info("device mapped at %p, irq %d with %d ctx banks\n", + iommu->base, iommu->irq, iommu->ncb); + + return ret; +fail: + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); + return ret; +} + +static const struct of_device_id msm_iommu_dt_match[] = { + { .compatible = "qcom,apq8064-iommu" }, + {} +}; + +static int msm_iommu_remove(struct platform_device *pdev) +{ + struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); + + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); + return 0; +} + +static struct platform_driver msm_iommu_driver = { + .driver = { + .name = "msm_iommu", + .of_match_table = msm_iommu_dt_match, + }, + .probe = msm_iommu_probe, + .remove = msm_iommu_remove, +}; + +static int __init msm_iommu_driver_init(void) +{ + int ret; + + ret = platform_driver_register(&msm_iommu_driver); + if (ret != 0) + pr_err("Failed to register IOMMU driver\n"); + + return ret; +} + +static void __exit msm_iommu_driver_exit(void) +{ + platform_driver_unregister(&msm_iommu_driver); +} + +subsys_initcall(msm_iommu_driver_init); +module_exit(msm_iommu_driver_exit); + static int __init get_tex_class(int icp, int ocp, int mt, int nos) { int i = 0; diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c deleted file mode 100644 index be01cc47c285..000000000000 --- a/drivers/iommu/msm_iommu_dev.c +++ /dev/null @@ -1,212 +0,0 @@ -/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "msm_iommu_hw-8xxx.h" -#include "msm_iommu.h" - -static void msm_iommu_reset(void __iomem *base, int ncb) -{ - int ctx; - - SET_RPUE(base, 0); - SET_RPUEIE(base, 0); - SET_ESRRESTORE(base, 0); - SET_TBE(base, 0); - SET_CR(base, 0); - SET_SPDMBE(base, 0); - SET_TESTBUSCR(base, 0); - SET_TLBRSW(base, 0); - SET_GLOBAL_TLBIALL(base, 0); - SET_RPU_ACR(base, 0); - SET_TLBLKCRWE(base, 1); - - for (ctx = 0; ctx < ncb; ctx++) { - SET_BPRCOSH(base, ctx, 0); - SET_BPRCISH(base, ctx, 0); - SET_BPRCNSH(base, ctx, 0); - SET_BPSHCFG(base, ctx, 0); - SET_BPMTCFG(base, ctx, 0); - SET_ACTLR(base, ctx, 0); - SET_SCTLR(base, ctx, 0); - SET_FSRRESTORE(base, ctx, 0); - SET_TTBR0(base, ctx, 0); - SET_TTBR1(base, ctx, 0); - SET_TTBCR(base, ctx, 0); - SET_BFBCR(base, ctx, 0); - SET_PAR(base, ctx, 0); - SET_FAR(base, ctx, 0); - SET_CTX_TLBIALL(base, ctx, 0); - SET_TLBFLPTER(base, ctx, 0); - SET_TLBSLPTER(base, ctx, 0); - SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); - SET_CONTEXTIDR(base, ctx, 0); - } -} - -static int msm_iommu_probe(struct platform_device *pdev) -{ - struct resource *r; - struct msm_iommu_dev *iommu; - int ret, par, val; - - iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); - if (!iommu) - return -ENODEV; - - iommu->dev = &pdev->dev; - INIT_LIST_HEAD(&iommu->ctx_list); - - iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); - if (IS_ERR(iommu->pclk)) { - dev_err(iommu->dev, "could not get smmu_pclk\n"); - return PTR_ERR(iommu->pclk); - } - - ret = clk_prepare(iommu->pclk); - if (ret) { - dev_err(iommu->dev, "could not prepare smmu_pclk\n"); - return ret; - } - - iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); - if (IS_ERR(iommu->clk)) { - dev_err(iommu->dev, "could not get iommu_clk\n"); - clk_unprepare(iommu->pclk); - return PTR_ERR(iommu->clk); - } - - ret = clk_prepare(iommu->clk); - if (ret) { - dev_err(iommu->dev, "could not prepare iommu_clk\n"); - clk_unprepare(iommu->pclk); - return ret; - } - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iommu->base = devm_ioremap_resource(iommu->dev, r); - if (IS_ERR(iommu->base)) { - dev_err(iommu->dev, "could not get iommu base\n"); - ret = PTR_ERR(iommu->base); - goto fail; - } - - iommu->irq = platform_get_irq(pdev, 0); - if (iommu->irq < 0) { - dev_err(iommu->dev, "could not get iommu irq\n"); - ret = -ENODEV; - goto fail; - } - - ret = of_property_read_u32(iommu->dev->of_node, "ncb", &val); - if (ret) { - dev_err(iommu->dev, "could not get ncb\n"); - goto fail; - } - iommu->ncb = val; - - msm_iommu_reset(iommu->base, iommu->ncb); - SET_M(iommu->base, 0, 1); - SET_PAR(iommu->base, 0, 0); - SET_V2PCFG(iommu->base, 0, 1); - SET_V2PPR(iommu->base, 0, 0); - par = GET_PAR(iommu->base, 0); - SET_V2PCFG(iommu->base, 0, 0); - SET_M(iommu->base, 0, 0); - - if (!par) { - pr_err("Invalid PAR value detected\n"); - ret = -ENODEV; - goto fail; - } - - ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, - msm_iommu_fault_handler, - IRQF_ONESHOT | IRQF_SHARED, - "msm_iommu_secure_irpt_handler", - iommu); - if (ret) { - pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); - goto fail; - } - - list_add(&iommu->dev_node, &qcom_iommu_devices); - - pr_info("device mapped at %p, irq %d with %d ctx banks\n", - iommu->base, iommu->irq, iommu->ncb); -fail: - clk_unprepare(iommu->clk); - clk_unprepare(iommu->pclk); - return ret; -} - -static const struct of_device_id msm_iommu_dt_match[] = { - { .compatible = "qcom,apq8064-iommu" }, - {} -}; - -static int msm_iommu_remove(struct platform_device *pdev) -{ - struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); - - clk_unprepare(iommu->clk); - clk_unprepare(iommu->pclk); - return 0; -} - -static struct platform_driver msm_iommu_driver = { - .driver = { - .name = "msm_iommu", - .of_match_table = msm_iommu_dt_match, - }, - .probe = msm_iommu_probe, - .remove = msm_iommu_remove, -}; - -static struct platform_driver * const drivers[] = { - &msm_iommu_driver, - &msm_iommu_ctx_driver, -}; - -static int __init msm_iommu_driver_init(void) -{ - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} - -static void __exit msm_iommu_driver_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} - -subsys_initcall(msm_iommu_driver_init); -module_exit(msm_iommu_driver_exit); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Stepan Moskovchenko "); -- cgit v1.2.3 From f78ebca8ff3d61fb45fef1274595a72d1314d955 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 13 Jun 2016 17:06:05 +0530 Subject: iommu/msm: Add support for generic master bindings This adds the xlate callback which gets invoked during device registration from DT. The master devices gets added through this. Signed-off-by: Sricharan R Tested-by: Archit Taneja Tested-by: Srinivas Kandagatla Signed-off-by: Joerg Roedel --- drivers/iommu/msm_iommu.c | 61 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 792b3526b4a5..8ab064324a68 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -702,6 +703,54 @@ static void print_ctx_regs(void __iomem *base, int ctx) GET_PRRR(base, ctx), GET_NMRR(base, ctx)); } +static void insert_iommu_master(struct device *dev, + struct msm_iommu_dev **iommu, + struct of_phandle_args *spec) +{ + struct msm_iommu_ctx_dev *master = dev->archdata.iommu; + int sid; + + if (list_empty(&(*iommu)->ctx_list)) { + master = kzalloc(sizeof(*master), GFP_ATOMIC); + master->of_node = dev->of_node; + list_add(&master->list, &(*iommu)->ctx_list); + dev->archdata.iommu = master; + } + + for (sid = 0; sid < master->num_mids; sid++) + if (master->mids[sid] == spec->args[0]) { + dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n", + sid); + return; + } + + master->mids[master->num_mids++] = spec->args[0]; +} + +static int qcom_iommu_of_xlate(struct device *dev, + struct of_phandle_args *spec) +{ + struct msm_iommu_dev *iommu; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&msm_iommu_lock, flags); + list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) + if (iommu->dev->of_node == spec->np) + break; + + if (!iommu || iommu->dev->of_node != spec->np) { + ret = -ENODEV; + goto fail; + } + + insert_iommu_master(dev, &iommu, spec); +fail: + spin_unlock_irqrestore(&msm_iommu_lock, flags); + + return ret; +} + irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) { struct msm_iommu_dev *iommu = dev_id; @@ -737,7 +786,7 @@ fail: return 0; } -static const struct iommu_ops msm_iommu_ops = { +static struct iommu_ops msm_iommu_ops = { .capable = msm_iommu_capable, .domain_alloc = msm_iommu_domain_alloc, .domain_free = msm_iommu_domain_free, @@ -748,6 +797,7 @@ static const struct iommu_ops msm_iommu_ops = { .map_sg = default_iommu_map_sg, .iova_to_phys = msm_iommu_iova_to_phys, .pgsize_bitmap = MSM_IOMMU_PGSIZES, + .of_xlate = qcom_iommu_of_xlate, }; static int msm_iommu_probe(struct platform_device *pdev) @@ -837,6 +887,7 @@ static int msm_iommu_probe(struct platform_device *pdev) } list_add(&iommu->dev_node, &qcom_iommu_devices); + of_iommu_set_ops(pdev->dev.of_node, &msm_iommu_ops); pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); @@ -935,7 +986,13 @@ static int __init msm_iommu_init(void) return 0; } -subsys_initcall(msm_iommu_init); +static int __init msm_iommu_of_setup(struct device_node *np) +{ + msm_iommu_init(); + return 0; +} + +IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu", msm_iommu_of_setup); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Stepan Moskovchenko "); -- cgit v1.2.3 From c9220fbd7741861294dede37465243ee7efdb7bd Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 13 Jun 2016 17:06:06 +0530 Subject: iommu/msm: use generic ARMV7S short descriptor pagetable ops This iommu uses the armv7 short descriptor format. So use the generic ARMV7S pagetable ops instead of rewriting the same stuff in the driver. Signed-off-by: Sricharan R Tested-by: Archit Taneja Tested-by: Srinivas Kandagatla Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 + drivers/iommu/msm_iommu.c | 405 +++++++++++++--------------------------------- 2 files changed, 109 insertions(+), 297 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index ad0860383cb3..b60e72b78a5f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -91,6 +91,7 @@ config MSM_IOMMU depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST depends on BROKEN select IOMMU_API + select IOMMU_IO_PGTABLE_ARMV7S help Support for the IOMMUs found on certain Qualcomm SOCs. These IOMMUs allow virtualization of the address space used by most diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 8ab064324a68..b09692bb5b0a 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -35,27 +35,27 @@ #include "msm_iommu_hw-8xxx.h" #include "msm_iommu.h" +#include "io-pgtable.h" #define MRC(reg, processor, op1, crn, crm, op2) \ __asm__ __volatile__ ( \ " mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ : "=r" (reg)) -#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) -#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) - /* bitmap of the page sizes currently supported */ #define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) -static int msm_iommu_tex_class[4]; - DEFINE_SPINLOCK(msm_iommu_lock); static LIST_HEAD(qcom_iommu_devices); +static struct iommu_ops msm_iommu_ops; struct msm_priv { - unsigned long *pgtable; struct list_head list_attached; struct iommu_domain domain; + struct io_pgtable_cfg cfg; + struct io_pgtable_ops *iop; + struct device *dev; + spinlock_t pgtlock; /* pagetable lock */ }; static struct msm_priv *to_msm_priv(struct iommu_domain *dom) @@ -122,49 +122,79 @@ static void msm_iommu_reset(void __iomem *base, int ncb) SET_TLBFLPTER(base, ctx, 0); SET_TLBSLPTER(base, ctx, 0); SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); SET_CONTEXTIDR(base, ctx, 0); } } -static int __flush_iotlb(struct iommu_domain *domain) +static void __flush_iotlb(void *cookie) { - struct msm_priv *priv = to_msm_priv(domain); + struct msm_priv *priv = cookie; struct msm_iommu_dev *iommu = NULL; struct msm_iommu_ctx_dev *master; int ret = 0; -#ifndef CONFIG_IOMMU_PGTABLES_L2 - unsigned long *fl_table = priv->pgtable; - int i; + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; - if (!list_empty(&priv->list_attached)) { - dmac_flush_range(fl_table, fl_table + SZ_16K); + list_for_each_entry(master, &iommu->ctx_list, list) + SET_CTX_TLBIALL(iommu->base, master->num, 0); - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { - void *sl_table = __va(fl_table[i] & - FL_BASE_MASK); - dmac_flush_range(sl_table, sl_table + SZ_4K); - } + __disable_clocks(iommu); } -#endif +fail: + return; +} + +static void __flush_iotlb_range(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) +{ + struct msm_priv *priv = cookie; + struct msm_iommu_dev *iommu = NULL; + struct msm_iommu_ctx_dev *master; + int ret = 0; + int temp_size; list_for_each_entry(iommu, &priv->list_attached, dom_node) { ret = __enable_clocks(iommu); if (ret) goto fail; - list_for_each_entry(master, &iommu->ctx_list, list) - SET_CTX_TLBIALL(iommu->base, master->num, 0); + list_for_each_entry(master, &iommu->ctx_list, list) { + temp_size = size; + do { + iova &= TLBIVA_VA; + iova |= GET_CONTEXTIDR_ASID(iommu->base, + master->num); + SET_TLBIVA(iommu->base, master->num, iova); + iova += granule; + } while (temp_size -= granule); + } __disable_clocks(iommu); } + fail: - return ret; + return; } +static void __flush_iotlb_sync(void *cookie) +{ + /* + * Nothing is needed here, the barrier to guarantee + * completion of the tlb sync operation is implicitly + * taken care when the iommu client does a writel before + * kick starting the other master. + */ +} + +static const struct iommu_gather_ops msm_iommu_gather_ops = { + .tlb_flush_all = __flush_iotlb, + .tlb_add_flush = __flush_iotlb_range, + .tlb_sync = __flush_iotlb_sync, +}; + static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end) { int idx; @@ -232,15 +262,17 @@ static void __reset_context(void __iomem *base, int ctx) SET_TLBFLPTER(base, ctx, 0); SET_TLBSLPTER(base, ctx, 0); SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); } -static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) +static void __program_context(void __iomem *base, int ctx, + struct msm_priv *priv) { - unsigned int prrr, nmrr; __reset_context(base, ctx); + /* Turn on TEX Remap */ + SET_TRE(base, ctx, 1); + SET_AFE(base, ctx, 1); + /* Set up HTW mode */ /* TLB miss configuration: perform HTW on miss */ SET_TLBMCFG(base, ctx, 0x3); @@ -248,8 +280,13 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) /* V2P configuration: HTW for access */ SET_V2PCFG(base, ctx, 0x3); - SET_TTBCR(base, ctx, 0); - SET_TTBR0_PA(base, ctx, (pgtable >> 14)); + SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr); + SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]); + SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]); + + /* Set prrr and nmrr */ + SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr); + SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr); /* Invalidate the TLB for this context */ SET_CTX_TLBIALL(base, ctx, 0); @@ -268,38 +305,9 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) SET_RCOSH(base, ctx, 1); SET_RCNSH(base, ctx, 1); - /* Turn on TEX Remap */ - SET_TRE(base, ctx, 1); - - /* Set TEX remap attributes */ - RCP15_PRRR(prrr); - RCP15_NMRR(nmrr); - SET_PRRR(base, ctx, prrr); - SET_NMRR(base, ctx, nmrr); - /* Turn on BFB prefetch */ SET_BFBDFE(base, ctx, 1); -#ifdef CONFIG_IOMMU_PGTABLES_L2 - /* Configure page tables as inner-cacheable and shareable to reduce - * the TLB miss penalty. - */ - SET_TTBR0_SH(base, ctx, 1); - SET_TTBR1_SH(base, ctx, 1); - - SET_TTBR0_NOS(base, ctx, 1); - SET_TTBR1_NOS(base, ctx, 1); - - SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR0_IRGNL(base, ctx, 1); - - SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR1_IRGNL(base, ctx, 1); - - SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ - SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ -#endif - /* Enable the MMU */ SET_M(base, ctx, 1); } @@ -316,13 +324,6 @@ static struct iommu_domain *msm_iommu_domain_alloc(unsigned type) goto fail_nomem; INIT_LIST_HEAD(&priv->list_attached); - priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, - get_order(SZ_16K)); - - if (!priv->pgtable) - goto fail_nomem; - - memset(priv->pgtable, 0, SZ_16K); priv->domain.geometry.aperture_start = 0; priv->domain.geometry.aperture_end = (1ULL << 32) - 1; @@ -339,24 +340,35 @@ static void msm_iommu_domain_free(struct iommu_domain *domain) { struct msm_priv *priv; unsigned long flags; - unsigned long *fl_table; - int i; spin_lock_irqsave(&msm_iommu_lock, flags); priv = to_msm_priv(domain); + kfree(priv); + spin_unlock_irqrestore(&msm_iommu_lock, flags); +} - fl_table = priv->pgtable; +static int msm_iommu_domain_config(struct msm_priv *priv) +{ + spin_lock_init(&priv->pgtlock); - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) - free_page((unsigned long) __va(((fl_table[i]) & - FL_BASE_MASK))); + priv->cfg = (struct io_pgtable_cfg) { + .quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP, + .pgsize_bitmap = msm_iommu_ops.pgsize_bitmap, + .ias = 32, + .oas = 32, + .tlb = &msm_iommu_gather_ops, + .iommu_dev = priv->dev, + }; - free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); - priv->pgtable = NULL; + priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv); + if (!priv->iop) { + dev_err(priv->dev, "Failed to allocate pgtable\n"); + return -EINVAL; + } - kfree(priv); - spin_unlock_irqrestore(&msm_iommu_lock, flags); + msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap; + + return 0; } static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) @@ -367,6 +379,9 @@ static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) struct msm_priv *priv = to_msm_priv(domain); struct msm_iommu_ctx_dev *master; + priv->dev = dev; + msm_iommu_domain_config(priv); + spin_lock_irqsave(&msm_iommu_lock, flags); list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { master = list_first_entry(&iommu->ctx_list, @@ -392,14 +407,13 @@ static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) } config_mids(iommu, master); __program_context(iommu->base, master->num, - __pa(priv->pgtable)); + priv); } __disable_clocks(iommu); list_add(&iommu->dom_node, &priv->list_attached); } } - ret = __flush_iotlb(domain); fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); @@ -415,11 +429,9 @@ static void msm_iommu_detach_dev(struct iommu_domain *domain, struct msm_iommu_ctx_dev *master; int ret; - spin_lock_irqsave(&msm_iommu_lock, flags); - ret = __flush_iotlb(domain); - if (ret) - goto fail; + free_io_pgtable_ops(priv->iop); + spin_lock_irqsave(&msm_iommu_lock, flags); list_for_each_entry(iommu, &priv->list_attached, dom_node) { ret = __enable_clocks(iommu); if (ret) @@ -435,190 +447,30 @@ fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); } -static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, +static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t pa, size_t len, int prot) { - struct msm_priv *priv; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - unsigned int pgprot; - int ret = 0, tex, sh; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; - tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; - - if (tex < 0 || tex > NUM_TEX_CLASS - 1) { - ret = -EINVAL; - goto fail; - } - - priv = to_msm_priv(domain); - - fl_table = priv->pgtable; - - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad size: %d\n", len); - ret = -EINVAL; - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - ret = -EINVAL; - goto fail; - } - - if (len == SZ_16M || len == SZ_1M) { - pgprot = sh ? FL_SHARED : 0; - pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? FL_TEX0 : 0; - } else { - pgprot = sh ? SL_SHARED : 0; - pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? SL_TEX0 : 0; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (len == SZ_16M) { - int i = 0; - for (i = 0; i < 16; i++) - *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | - FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | - FL_SHARED | FL_NG | pgprot; - } - - if (len == SZ_1M) - *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | - FL_TYPE_SECT | FL_SHARED | pgprot; - - /* Need a 2nd level table */ - if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { - unsigned long *sl; - sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, - get_order(SZ_4K)); - - if (!sl) { - pr_debug("Could not allocate second level table\n"); - ret = -ENOMEM; - goto fail; - } - - memset(sl, 0, SZ_4K); - *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); - } - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - - if (len == SZ_4K) - *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | - SL_SHARED | SL_TYPE_SMALL | pgprot; - - if (len == SZ_64K) { - int i; + int ret; - for (i = 0; i < 16; i++) - *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | - SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; - } + spin_lock_irqsave(&priv->pgtlock, flags); + ret = priv->iop->map(priv->iop, iova, pa, len, prot); + spin_unlock_irqrestore(&priv->pgtlock, flags); - ret = __flush_iotlb(domain); -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; } -static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, - size_t len) +static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t len) { - struct msm_priv *priv; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - int i, ret = 0; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = to_msm_priv(domain); - fl_table = priv->pgtable; + spin_lock_irqsave(&priv->pgtlock, flags); + len = priv->iop->unmap(priv->iop, iova, len); + spin_unlock_irqrestore(&priv->pgtlock, flags); - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad length: %d\n", len); - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - goto fail; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (*fl_pte == 0) { - pr_debug("First level PTE is 0\n"); - goto fail; - } - - /* Unmap supersection */ - if (len == SZ_16M) - for (i = 0; i < 16; i++) - *(fl_pte+i) = 0; - - if (len == SZ_1M) - *fl_pte = 0; - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - if (len == SZ_64K) { - for (i = 0; i < 16; i++) - *(sl_pte+i) = 0; - } - - if (len == SZ_4K) - *sl_pte = 0; - - if (len == SZ_4K || len == SZ_64K) { - int used = 0; - - for (i = 0; i < NUM_SL_PTE; i++) - if (sl_table[i]) - used = 1; - if (!used) { - free_page((unsigned long)sl_table); - *fl_pte = 0; - } - } - - ret = __flush_iotlb(domain); - -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - - /* the IOMMU API requires us to return how many bytes were unmapped */ - len = ret ? 0 : len; return len; } @@ -699,8 +551,6 @@ static void print_ctx_regs(void __iomem *base, int ctx) GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); pr_err("SCTLR = %08x ACTLR = %08x\n", GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); - pr_err("PRRR = %08x NMRR = %08x\n", - GET_PRRR(base, ctx), GET_NMRR(base, ctx)); } static void insert_iommu_master(struct device *dev, @@ -941,47 +791,8 @@ static void __exit msm_iommu_driver_exit(void) subsys_initcall(msm_iommu_driver_init); module_exit(msm_iommu_driver_exit); -static int __init get_tex_class(int icp, int ocp, int mt, int nos) -{ - int i = 0; - unsigned int prrr = 0; - unsigned int nmrr = 0; - int c_icp, c_ocp, c_mt, c_nos; - - RCP15_PRRR(prrr); - RCP15_NMRR(nmrr); - - for (i = 0; i < NUM_TEX_CLASS; i++) { - c_nos = PRRR_NOS(prrr, i); - c_mt = PRRR_MT(prrr, i); - c_icp = NMRR_ICP(nmrr, i); - c_ocp = NMRR_OCP(nmrr, i); - - if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) - return i; - } - - return -ENODEV; -} - -static void __init setup_iommu_tex_classes(void) -{ - msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = - get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); - - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = - get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); - - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = - get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); - - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = - get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); -} - static int __init msm_iommu_init(void) { - setup_iommu_tex_classes(); bus_set_iommu(&platform_bus_type, &msm_iommu_ops); return 0; } -- cgit v1.2.3 From 1cb13f78329ca2f1f7ef4e6024901cc26d611ee1 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Mon, 13 Jun 2016 17:06:07 +0530 Subject: iommu/msm: Remove driver BROKEN Now that the driver is DT adapted, bus_set_iommu gets called only when on compatible matching. So the driver should not break multiplatform builds now. So remove the BROKEN config. Signed-off-by: Sricharan R Tested-by: Archit Taneja Tested-by: Srinivas Kandagatla Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index b60e72b78a5f..f5f4d1472771 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -89,7 +89,6 @@ config MSM_IOMMU bool "MSM IOMMU Support" depends on ARM depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST - depends on BROKEN select IOMMU_API select IOMMU_IO_PGTABLE_ARMV7S help -- cgit v1.2.3 From bb8e15d60462a84a25a3bf33e8bc29b46c6d470a Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 1 Jun 2016 14:06:15 +0800 Subject: of: iommu: make of_iommu_init() postcore_initcall_sync The of_iommu_init() is called multiple times by arch code, make it postcore_initcall_sync, then we can drop relevant calls fully. Note, the IOMMUs should have a chance to perform some basic initialisation before we start adding masters to them. So postcore_initcall_sync is good choice, it ensures of_iommu_init() called before of_platform_populate. Acked-by: Rich Felker Tested-by: Marek Szyprowski Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Marek Szyprowski Cc: Rich Felker Cc: Rob Herring Cc: Robin Murphy Cc: Will Deacon Signed-off-by: Kefeng Wang Acked-by: Joerg Roedel Signed-off-by: Rob Herring --- arch/arm/kernel/setup.c | 2 -- arch/arm64/kernel/setup.c | 2 -- arch/sh/boards/of-generic.c | 2 -- drivers/iommu/of_iommu.c | 5 ++++- include/linux/of_iommu.h | 2 -- 5 files changed, 4 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 7b5350060612..7e455339789b 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -903,7 +902,6 @@ static int __init customize_machine(void) * machine from the device tree, if no callback is provided, * otherwise we would always need an init_machine callback. */ - of_iommu_init(); if (machine_desc->init_machine) machine_desc->init_machine(); #ifdef CONFIG_OF diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 3279defabaa2..8412520ef62d 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -305,7 +304,6 @@ void __init setup_arch(char **cmdline_p) static int __init arm64_device_init(void) { if (of_have_populated_dt()) { - of_iommu_init(); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); } else if (acpi_disabled) { diff --git a/arch/sh/boards/of-generic.c b/arch/sh/boards/of-generic.c index bf3a166a5407..b4d4313af6f1 100644 --- a/arch/sh/boards/of-generic.c +++ b/arch/sh/boards/of-generic.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -185,7 +184,6 @@ static int __init sh_of_device_init(void) { pr_info("SH generic board support: populating platform devices\n"); if (of_have_populated_dt()) { - of_iommu_init(); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); } else { diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index af499aea0a1a..57f23eaaa2f9 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -174,7 +174,7 @@ err_put_node: return NULL; } -void __init of_iommu_init(void) +static int __init of_iommu_init(void) { struct device_node *np; const struct of_device_id *match, *matches = &__iommu_of_table; @@ -186,4 +186,7 @@ void __init of_iommu_init(void) pr_err("Failed to initialise IOMMU %s\n", of_node_full_name(np)); } + + return 0; } +postcore_initcall_sync(of_iommu_init); diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h index bd02b44902d0..e80b9c762a03 100644 --- a/include/linux/of_iommu.h +++ b/include/linux/of_iommu.h @@ -11,7 +11,6 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix, int index, unsigned long *busno, dma_addr_t *addr, size_t *size); -extern void of_iommu_init(void); extern const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np); @@ -24,7 +23,6 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix, return -EINVAL; } -static inline void of_iommu_init(void) { } static inline const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np) { -- cgit v1.2.3 From 583248e6620a4726093295e2d6785fcbc2e86428 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 1 Jun 2016 12:10:08 +0100 Subject: iommu/iova: Disable preemption around use of this_cpu_ptr() Between acquiring the this_cpu_ptr() and using it, ideally we don't want to be preempted and work on another CPU's private data. this_cpu_ptr() checks whether or not preemption is disable, and get_cpu_ptr() provides a convenient wrapper for operating on the cpu ptr inside a preemption disabled critical section (which currently is provided by the spinlock). [ 167.997877] BUG: using smp_processor_id() in preemptible [00000000] code: usb-storage/216 [ 167.997940] caller is debug_smp_processor_id+0x17/0x20 [ 167.997945] CPU: 7 PID: 216 Comm: usb-storage Tainted: G U 4.7.0-rc1-gfxbench-RO_Patchwork_1057+ #1 [ 167.997948] Hardware name: Hewlett-Packard HP Pro 3500 Series/2ABF, BIOS 8.11 10/24/2012 [ 167.997951] 0000000000000000 ffff880118b7f9c8 ffffffff8140dca5 0000000000000007 [ 167.997958] ffffffff81a3a7e9 ffff880118b7f9f8 ffffffff8142a927 0000000000000000 [ 167.997965] ffff8800d499ed58 0000000000000001 00000000000fffff ffff880118b7fa08 [ 167.997971] Call Trace: [ 167.997977] [] dump_stack+0x67/0x92 [ 167.997981] [] check_preemption_disabled+0xd7/0xe0 [ 167.997985] [] debug_smp_processor_id+0x17/0x20 [ 167.997990] [] alloc_iova_fast+0xb7/0x210 [ 167.997994] [] intel_alloc_iova+0x7f/0xd0 [ 167.997998] [] intel_map_sg+0xbd/0x240 [ 167.998002] [] ? debug_lockdep_rcu_enabled+0x1d/0x20 [ 167.998009] [] usb_hcd_map_urb_for_dma+0x4b9/0x5a0 [ 167.998013] [] usb_hcd_submit_urb+0xe9/0xaa0 [ 167.998017] [] ? mark_held_locks+0x6f/0xa0 [ 167.998022] [] ? __raw_spin_lock_init+0x1c/0x50 [ 167.998025] [] ? debug_lockdep_rcu_enabled+0x1d/0x20 [ 167.998028] [] usb_submit_urb+0x3f3/0x5a0 [ 167.998032] [] ? trace_hardirqs_on_caller+0x122/0x1b0 [ 167.998035] [] usb_sg_wait+0x67/0x150 [ 167.998039] [] usb_stor_bulk_transfer_sglist.part.3+0x82/0xd0 [ 167.998042] [] usb_stor_bulk_srb+0x4c/0x60 [ 167.998045] [] usb_stor_Bulk_transport+0x17e/0x420 [ 167.998049] [] usb_stor_invoke_transport+0x242/0x540 [ 167.998052] [] ? debug_lockdep_rcu_enabled+0x1d/0x20 [ 167.998058] [] usb_stor_transparent_scsi_command+0x9/0x10 [ 167.998061] [] usb_stor_control_thread+0x158/0x260 [ 167.998064] [] ? fill_inquiry_response+0x20/0x20 [ 167.998067] [] ? fill_inquiry_response+0x20/0x20 [ 167.998071] [] kthread+0xea/0x100 [ 167.998078] [] ret_from_fork+0x1f/0x40 [ 167.998081] [] ? kthread_create_on_node+0x1f0/0x1f0 Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=96293 Signed-off-by: Chris Wilson Cc: Joonas Lahtinen Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org Fixes: 9257b4a206fc ('iommu/iova: introduce per-cpu caching to iova allocation') Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ba764a0835d3..e23001bfcfee 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -420,8 +420,10 @@ retry: /* Try replenishing IOVAs by flushing rcache. */ flushed_rcache = true; + preempt_disable(); for_each_online_cpu(cpu) free_cpu_cached_iovas(cpu, iovad); + preempt_enable(); goto retry; } @@ -749,7 +751,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, bool can_insert = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_full(cpu_rcache->loaded)) { @@ -779,6 +781,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, iova_magazine_push(cpu_rcache->loaded, iova_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); if (mag_to_free) { iova_magazine_free_pfns(mag_to_free, iovad); @@ -812,7 +815,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, bool has_pfn = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_empty(cpu_rcache->loaded)) { @@ -834,6 +837,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); return iova_pfn; } -- cgit v1.2.3 From 3bd4f9112f87a9c65fe6e817272806167f0bc9ed Mon Sep 17 00:00:00 2001 From: Jan Niehusmann Date: Mon, 6 Jun 2016 14:20:11 +0200 Subject: iommu/vt-d: Fix overflow of iommu->domains array The valid range of 'did' in get_iommu_domain(*iommu, did) is 0..cap_ndoms(iommu->cap), so don't exceed that range in free_all_cpu_cached_iovas(). The user-visible impact of the out-of-bounds access is the machine hanging on suspend-to-ram. It is, in fact, a kernel panic, but due to already suspended devices, that's often not visible to the user. Fixes: 22e2f9fa63b0 ("iommu/vt-d: Use per-cpu IOVA caching") Signed-off-by: Jan Niehusmann Tested-By: Marius Vlad Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 10700945994e..cfe410eedaf0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4607,7 +4607,7 @@ static void free_all_cpu_cached_iovas(unsigned int cpu) if (!iommu) continue; - for (did = 0; did < 0xffff; did++) { + for (did = 0; did < cap_ndoms(iommu->cap); did++) { domain = get_iommu_domain(iommu, did); if (!domain) -- cgit v1.2.3 From 6082ee72e9d89e80a664418be06f47d728243e85 Mon Sep 17 00:00:00 2001 From: Nicolas Iooss Date: Sun, 26 Jun 2016 10:33:29 +0200 Subject: iommu/amd: Initialize devid variable before using it Commit 2a0cb4e2d423 ("iommu/amd: Add new map for storing IVHD dev entry type HID") added a call to DUMP_printk in init_iommu_from_acpi() which used the value of devid before this variable was initialized. Fixes: 2a0cb4e2d423 ('iommu/amd: Add new map for storing IVHD dev entry type HID') Signed-off-by: Nicolas Iooss Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 9e0034196e10..d091defc3426 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1107,13 +1107,13 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } + devid = e->devid; DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", hid, uid, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); - devid = e->devid; flags = e->flags; ret = add_acpi_hid_device(hid, uid, &devid, false); -- cgit v1.2.3 From fec3b217ff19978650121103d849b5889b04780a Mon Sep 17 00:00:00 2001 From: Simon Xue Date: Fri, 24 Jun 2016 10:13:26 +0800 Subject: iommu/rockchip: Fix devm_{request,free}_irq parameter Even though the IOMMU shares IRQ with its master, the struct device passed to {request,free}_irq is supposed to represent the device that is signalling the interrupt. This patch makes the driver use IOMMU device instead of master's device to make things clear. Signed-off-by: Simon Xue Signed-off-by: Shunqian Zheng Reviewed-by: Douglas Anderson Signed-off-by: Tomasz Figa Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 25b4627cb57f..5a9659a14e97 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -807,7 +807,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, iommu->domain = domain; - ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, + ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq, IRQF_SHARED, dev_name(dev), iommu); if (ret) return ret; @@ -860,7 +860,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, } rk_iommu_disable_stall(iommu); - devm_free_irq(dev, iommu->irq, iommu); + devm_free_irq(iommu->dev, iommu->irq, iommu); iommu->domain = NULL; -- cgit v1.2.3 From e6d0f4737c468d1889aba06801c490988cf66ad7 Mon Sep 17 00:00:00 2001 From: Simon Xue Date: Fri, 24 Jun 2016 10:13:27 +0800 Subject: iommu/rockchip: Add map_sg callback for rk_iommu_ops The iommu_dma_alloc() in iommu/dma-iommu.c calls iommu_map_sg() that requires the callback iommu_ops .map_sg(). Adding the default_iommu_map_sg() to Rockchip IOMMU accordingly. Signed-off-by: Simon Xue Signed-off-by: Shunqian Zheng Reviewed-by: Douglas Anderson Signed-off-by: Tomasz Figa Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 5a9659a14e97..53fa0d939c9b 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1022,6 +1022,7 @@ static const struct iommu_ops rk_iommu_ops = { .detach_dev = rk_iommu_detach_device, .map = rk_iommu_map, .unmap = rk_iommu_unmap, + .map_sg = default_iommu_map_sg, .add_device = rk_iommu_add_device, .remove_device = rk_iommu_remove_device, .iova_to_phys = rk_iommu_iova_to_phys, -- cgit v1.2.3 From 3d08f434bd58656ae630376d0b5afd6ca1ffb013 Mon Sep 17 00:00:00 2001 From: Shunqian Zheng Date: Fri, 24 Jun 2016 10:13:28 +0800 Subject: iommu/rockchip: Fix allocation of bases array in driver probe In .probe(), devm_kzalloc() is called with size == 0 and works only by luck, due to internal behavior of the allocator and the fact that the proper allocation size is small. Let's use proper value for calculating the size. Fixes: cd6438c5f844 ("iommu/rockchip: Reconstruct to support multi slaves") Signed-off-by: Shunqian Zheng Signed-off-by: Tomasz Figa Reviewed-by: Douglas Anderson Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 53fa0d939c9b..8a5bac7ff0b7 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1034,6 +1034,7 @@ static int rk_iommu_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct rk_iommu *iommu; struct resource *res; + int num_res = pdev->num_resources; int i; iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); @@ -1043,12 +1044,13 @@ static int rk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, iommu); iommu->dev = dev; iommu->num_mmu = 0; - iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * iommu->num_mmu, + + iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res, GFP_KERNEL); if (!iommu->bases) return -ENOMEM; - for (i = 0; i < pdev->num_resources; i++) { + for (i = 0; i < num_res; i++) { res = platform_get_resource(pdev, IORESOURCE_MEM, i); if (!res) continue; -- cgit v1.2.3 From 4f0aba676735c653b4e739b760c1e66cd520d3e3 Mon Sep 17 00:00:00 2001 From: Shunqian Zheng Date: Fri, 24 Jun 2016 10:13:29 +0800 Subject: iommu/rockchip: Use DMA API to manage coherency Use DMA API instead of architecture internal functions like __cpuc_flush_dcache_area() etc. The biggest difficulty here is that dma_map and _sync calls require some struct device, while there is no real 1:1 relation between an IOMMU domain and some device. To overcome this, a simple platform device is registered for each allocated IOMMU domain. With this patch, this driver can be used on both ARM and ARM64 platforms, such as RK3288 and RK3399 respectively. Signed-off-by: Shunqian Zheng Signed-off-by: Tomasz Figa Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 162 +++++++++++++++++++++++++++++++---------- 1 file changed, 123 insertions(+), 39 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 8a5bac7ff0b7..712ed7591783 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -4,11 +4,10 @@ * published by the Free Software Foundation. */ -#include -#include #include #include #include +#include #include #include #include @@ -77,7 +76,9 @@ struct rk_iommu_domain { struct list_head iommus; + struct platform_device *pdev; u32 *dt; /* page directory table */ + dma_addr_t dt_dma; spinlock_t iommus_lock; /* lock for iommus list */ spinlock_t dt_lock; /* lock for modifying page directory table */ @@ -93,14 +94,12 @@ struct rk_iommu { struct iommu_domain *domain; /* domain to which iommu is attached */ }; -static inline void rk_table_flush(u32 *va, unsigned int count) +static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, + unsigned int count) { - phys_addr_t pa_start = virt_to_phys(va); - phys_addr_t pa_end = virt_to_phys(va + count); - size_t size = pa_end - pa_start; + size_t size = count * sizeof(u32); /* count of u32 entry */ - __cpuc_flush_dcache_area(va, size); - outer_flush_range(pa_start, pa_end); + dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE); } static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) @@ -183,10 +182,9 @@ static inline bool rk_dte_is_pt_valid(u32 dte) return dte & RK_DTE_PT_VALID; } -static u32 rk_mk_dte(u32 *pt) +static inline u32 rk_mk_dte(dma_addr_t pt_dma) { - phys_addr_t pt_phys = virt_to_phys(pt); - return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; + return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; } /* @@ -603,13 +601,16 @@ static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { + struct device *dev = &rk_domain->pdev->dev; u32 *page_table, *dte_addr; - u32 dte; + u32 dte_index, dte; phys_addr_t pt_phys; + dma_addr_t pt_dma; assert_spin_locked(&rk_domain->dt_lock); - dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; + dte_index = rk_iova_dte_index(iova); + dte_addr = &rk_domain->dt[dte_index]; dte = *dte_addr; if (rk_dte_is_pt_valid(dte)) goto done; @@ -618,19 +619,27 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, if (!page_table) return ERR_PTR(-ENOMEM); - dte = rk_mk_dte(page_table); - *dte_addr = dte; + pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pt_dma)) { + dev_err(dev, "DMA mapping error while allocating page table\n"); + free_page((unsigned long)page_table); + return ERR_PTR(-ENOMEM); + } - rk_table_flush(page_table, NUM_PT_ENTRIES); - rk_table_flush(dte_addr, 1); + dte = rk_mk_dte(pt_dma); + *dte_addr = dte; + rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES); + rk_table_flush(rk_domain, + rk_domain->dt_dma + dte_index * sizeof(u32), 1); done: pt_phys = rk_dte_pt_address(dte); return (u32 *)phys_to_virt(pt_phys); } static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, - u32 *pte_addr, dma_addr_t iova, size_t size) + u32 *pte_addr, dma_addr_t pte_dma, + size_t size) { unsigned int pte_count; unsigned int pte_total = size / SPAGE_SIZE; @@ -645,14 +654,14 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, pte_addr[pte_count] = rk_mk_pte_invalid(pte); } - rk_table_flush(pte_addr, pte_count); + rk_table_flush(rk_domain, pte_dma, pte_count); return pte_count * SPAGE_SIZE; } static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, - dma_addr_t iova, phys_addr_t paddr, size_t size, - int prot) + dma_addr_t pte_dma, dma_addr_t iova, + phys_addr_t paddr, size_t size, int prot) { unsigned int pte_count; unsigned int pte_total = size / SPAGE_SIZE; @@ -671,7 +680,7 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, paddr += SPAGE_SIZE; } - rk_table_flush(pte_addr, pte_count); + rk_table_flush(rk_domain, pte_dma, pte_total); /* * Zap the first and last iova to evict from iotlb any previously @@ -684,7 +693,8 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, return 0; unwind: /* Unmap the range of iovas that we just mapped */ - rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); + rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, + pte_count * SPAGE_SIZE); iova += pte_count * SPAGE_SIZE; page_phys = rk_pte_page_address(pte_addr[pte_count]); @@ -699,8 +709,9 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, { struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - dma_addr_t iova = (dma_addr_t)_iova; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; u32 *page_table, *pte_addr; + u32 dte_index, pte_index; int ret; spin_lock_irqsave(&rk_domain->dt_lock, flags); @@ -718,8 +729,13 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, return PTR_ERR(page_table); } - pte_addr = &page_table[rk_iova_pte_index(iova)]; - ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); + dte_index = rk_domain->dt[rk_iova_dte_index(iova)]; + pte_index = rk_iova_pte_index(iova); + pte_addr = &page_table[pte_index]; + pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32); + ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova, + paddr, size, prot); + spin_unlock_irqrestore(&rk_domain->dt_lock, flags); return ret; @@ -730,7 +746,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, { struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - dma_addr_t iova = (dma_addr_t)_iova; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; phys_addr_t pt_phys; u32 dte; u32 *pte_addr; @@ -754,7 +770,8 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, pt_phys = rk_dte_pt_address(dte); pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); - unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); + pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); + unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); spin_unlock_irqrestore(&rk_domain->dt_lock, flags); @@ -787,7 +804,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; int ret, i; - phys_addr_t dte_addr; /* * Allow 'virtual devices' (e.g., drm) to attach to domain. @@ -812,9 +828,9 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, if (ret) return ret; - dte_addr = virt_to_phys(rk_domain->dt); for (i = 0; i < iommu->num_mmu; i++) { - rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr); + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, + rk_domain->dt_dma); rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); } @@ -870,14 +886,30 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) { struct rk_iommu_domain *rk_domain; + struct platform_device *pdev; + struct device *iommu_dev; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; - rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); - if (!rk_domain) + /* Register a pdev per domain, so DMA API can base on this *dev + * even some virtual master doesn't have an iommu slave + */ + pdev = platform_device_register_simple("rk_iommu_domain", + PLATFORM_DEVID_AUTO, NULL, 0); + if (IS_ERR(pdev)) return NULL; + rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL); + if (!rk_domain) + goto err_unreg_pdev; + + rk_domain->pdev = pdev; + + /* To init the iovad which is required by iommu_dma_init_domain() */ + if (iommu_get_dma_cookie(&rk_domain->domain)) + goto err_unreg_pdev; + /* * rk32xx iommus use a 2 level pagetable. * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. @@ -885,9 +917,17 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) */ rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); if (!rk_domain->dt) - goto err_dt; + goto err_put_cookie; + + iommu_dev = &pdev->dev; + rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) { + dev_err(iommu_dev, "DMA map error for DT\n"); + goto err_free_dt; + } - rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); + rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES); spin_lock_init(&rk_domain->iommus_lock); spin_lock_init(&rk_domain->dt_lock); @@ -895,8 +935,13 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) return &rk_domain->domain; -err_dt: - kfree(rk_domain); +err_free_dt: + free_page((unsigned long)rk_domain->dt); +err_put_cookie: + iommu_put_dma_cookie(&rk_domain->domain); +err_unreg_pdev: + platform_device_unregister(pdev); + return NULL; } @@ -912,12 +957,19 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) if (rk_dte_is_pt_valid(dte)) { phys_addr_t pt_phys = rk_dte_pt_address(dte); u32 *page_table = phys_to_virt(pt_phys); + dma_unmap_single(&rk_domain->pdev->dev, pt_phys, + SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)page_table); } } + dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)rk_domain->dt); - kfree(rk_domain); + + iommu_put_dma_cookie(&rk_domain->domain); + + platform_device_unregister(rk_domain->pdev); } static bool rk_iommu_is_dev_iommu_master(struct device *dev) @@ -1029,6 +1081,30 @@ static const struct iommu_ops rk_iommu_ops = { .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, }; +static int rk_iommu_domain_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); + if (!dev->dma_parms) + return -ENOMEM; + + /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */ + arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false); + + dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); + dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); + + return 0; +} + +static struct platform_driver rk_iommu_domain_driver = { + .probe = rk_iommu_domain_probe, + .driver = { + .name = "rk_iommu_domain", + }, +}; + static int rk_iommu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1106,11 +1182,19 @@ static int __init rk_iommu_init(void) if (ret) return ret; - return platform_driver_register(&rk_iommu_driver); + ret = platform_driver_register(&rk_iommu_domain_driver); + if (ret) + return ret; + + ret = platform_driver_register(&rk_iommu_driver); + if (ret) + platform_driver_unregister(&rk_iommu_domain_driver); + return ret; } static void __exit rk_iommu_exit(void) { platform_driver_unregister(&rk_iommu_driver); + platform_driver_unregister(&rk_iommu_domain_driver); } subsys_initcall(rk_iommu_init); -- cgit v1.2.3 From a93db2f22b6b48369acb72f66a0ae47ec17a0b05 Mon Sep 17 00:00:00 2001 From: Shunqian Zheng Date: Fri, 24 Jun 2016 10:13:30 +0800 Subject: iommu/rockchip: Prepare to support generic DMA mapping Set geometry for allocated domains and fix .domain_alloc() callback to work with IOMMU_DOMAIN_DMA domain type, which is used for implicit domains on ARM64. Signed-off-by: Shunqian Zheng Signed-off-by: Tomasz Figa Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 712ed7591783..9afcbf79f0b0 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -889,7 +889,7 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) struct platform_device *pdev; struct device *iommu_dev; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; /* Register a pdev per domain, so DMA API can base on this *dev @@ -906,8 +906,8 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) rk_domain->pdev = pdev; - /* To init the iovad which is required by iommu_dma_init_domain() */ - if (iommu_get_dma_cookie(&rk_domain->domain)) + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&rk_domain->domain)) goto err_unreg_pdev; /* @@ -933,12 +933,17 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) spin_lock_init(&rk_domain->dt_lock); INIT_LIST_HEAD(&rk_domain->iommus); + rk_domain->domain.geometry.aperture_start = 0; + rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); + rk_domain->domain.geometry.force_aperture = true; + return &rk_domain->domain; err_free_dt: free_page((unsigned long)rk_domain->dt); err_put_cookie: - iommu_put_dma_cookie(&rk_domain->domain); + if (type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&rk_domain->domain); err_unreg_pdev: platform_device_unregister(pdev); @@ -967,7 +972,8 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)rk_domain->dt); - iommu_put_dma_cookie(&rk_domain->domain); + if (domain->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&rk_domain->domain); platform_device_unregister(rk_domain->pdev); } -- cgit v1.2.3 From 112c898b59dd5cfd95ee30dfe7cc4fc11a6d484e Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Mon, 13 Jun 2016 17:20:17 +0800 Subject: iommu/arm-smmu: request pcie devices to enable ACS The PCIe ACS capability will affect the layout of iommu groups. Generally speaking, if the path from root port to the PCIe device is ACS enabled, the iommu will create a single iommu group for this PCIe device. If all PCIe devices on the path are ACS enabled then Linux can determine this path is ACS enabled. Linux use two PCIe configuration registers to determine the ACS status of PCIe devices: ACS Capability Register and ACS Control Register. The first register is used to check the implementation of ACS function of a PCIe device, the second register is used to check the enable status of ACS function. If one PCIe device has implemented and enabled the ACS function then Linux will determine this PCIe device enabled ACS. From the Chapter:6.12 of PCI Express Base Specification Revision 3.1a, we can find that when a PCIe device implements ACS function, the enable status is set to disabled by default and can be enabled by ACS-aware software. ACS will affect the iommu groups topology, so, the iommu driver is ACS-aware software. This patch adds a call to pci_request_acs() to the arm-smmu driver to enable the ACS function in PCIe devices that support it, when they get probed. Reviewed-by: Robin Murphy Reviewed-by: Eric Auger Signed-off-by: Wei Chen Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 2 ++ drivers/iommu/arm-smmu.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 94b68213c50d..30ea89945543 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2686,6 +2686,8 @@ static int __init arm_smmu_init(void) if (ret) return ret; + pci_request_acs(); + return bus_set_iommu(&pci_bus_type, &arm_smmu_ops); } diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 9345a3fcb706..ab365ec6184d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -2096,8 +2096,10 @@ static int __init arm_smmu_init(void) #endif #ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) + if (!iommu_present(&pci_bus_type)) { + pci_request_acs(); bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + } #endif return 0; -- cgit v1.2.3 From 7c6d90e2bb1a98b86d73b9e8ab4d97ed5507e37c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 16 Jun 2016 18:21:19 +0100 Subject: iommu/io-pgtable-arm: Fix iova_to_phys for block entries The implementation of iova_to_phys for the long-descriptor ARM io-pgtable code always masks with the granule size when inserting the low virtual address bits into the physical address determined from the page tables. In cases where the leaf entry is found before the final level of table (i.e. due to a block mapping), this results in rounding down to the bottom page of the block mapping. Consequently, the physical address range batching in the vfio_unmap_unpin is defeated and we end up taking the long way home. This patch fixes the problem by masking the virtual address with the appropriate mask for the level at which the leaf descriptor is located. The short-descriptor code already gets this right, so no change is needed there. Cc: Reported-by: Robin Murphy Tested-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index a1ed1b73fed4..f5c90e1366ce 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -576,7 +576,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, return 0; found_translation: - iova &= (ARM_LPAE_GRANULE(data) - 1); + iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; } -- cgit v1.2.3 From 0caa7616a6aca449dd68b58cb29bd491d296c2d5 Mon Sep 17 00:00:00 2001 From: Aaron Campbell Date: Sat, 2 Jul 2016 21:23:24 -0300 Subject: iommu/vt-d: Fix infinite loop in free_all_cpu_cached_iovas Per VT-d spec Section 10.4.2 ("Capability Register"), the maximum number of possible domains is 64K; indeed this is the maximum value that the cap_ndoms() macro will expand to. Since the value 65536 will not fix in a u16, the 'did' variable must be promoted to an int, otherwise the test for < 65536 will always be true and the loop will never end. The symptom, in my case, was a hung machine during suspend. Fixes: 3bd4f9112f87 ("iommu/vt-d: Fix overflow of iommu->domains array") Signed-off-by: Aaron Campbell Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index cfe410eedaf0..323dac9900ba 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4602,13 +4602,13 @@ static void free_all_cpu_cached_iovas(unsigned int cpu) for (i = 0; i < g_num_of_iommus; i++) { struct intel_iommu *iommu = g_iommus[i]; struct dmar_domain *domain; - u16 did; + int did; if (!iommu) continue; for (did = 0; did < cap_ndoms(iommu->cap); did++) { - domain = get_iommu_domain(iommu, did); + domain = get_iommu_domain(iommu, (u16)did); if (!domain) continue; -- cgit v1.2.3 From 522e5cb76d0663c88f96b6a8301451c8efa37207 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 1 Jul 2016 16:42:55 +0200 Subject: iommu/amd: Fix unity mapping initialization race There is a race condition in the AMD IOMMU init code that causes requested unity mappings to be blocked by the IOMMU for a short period of time. This results on boot failures and IO_PAGE_FAULTs on some machines. Fix this by making sure the unity mappings are installed before all other DMA is blocked. Fixes: aafd8ba0ca74 ('iommu/amd: Implement add_device and remove_device') Cc: stable@vger.kernel.org # v4.2+ Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu_init.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index d091defc3426..59741ead7e15 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1568,13 +1568,23 @@ static int __init amd_iommu_init_pci(void) break; } + /* + * Order is important here to make sure any unity map requirements are + * fulfilled. The unity mappings are created and written to the device + * table during the amd_iommu_init_api() call. + * + * After that we call init_device_table_dma() to make sure any + * uninitialized DTE will block DMA, and in the end we flush the caches + * of all IOMMUs to make sure the changes to the device table are + * active. + */ + ret = amd_iommu_init_api(); + init_device_table_dma(); for_each_iommu(iommu) iommu_flush_all_caches(iommu); - ret = amd_iommu_init_api(); - if (!ret) print_iommu_info(); -- cgit v1.2.3 From bee140044579fbfdad5fd98717c0405d7b492226 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 4 Jul 2016 17:38:22 +0800 Subject: iommu/arm-smmu: Use devm_request_irq and devm_free_irq Use devm_request_irq to simplify error handling path, when probe smmu device. Also devm_{request|free}_irq when init or destroy domain context. Signed-off-by: Peng Fan Cc: Will Deacon Cc: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index ab365ec6184d..4f49fe29f202 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -987,8 +987,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, * handler seeing a half-initialised domain state. */ irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, - "arm-smmu-context-fault", domain); + ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, + IRQF_SHARED, "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); @@ -1028,7 +1028,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - free_irq(irq, domain); + devm_free_irq(smmu->dev, irq, domain); } free_io_pgtable_ops(smmu_domain->pgtbl_ops); @@ -1986,15 +1986,15 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } for (i = 0; i < smmu->num_global_irqs; ++i) { - err = request_irq(smmu->irqs[i], - arm_smmu_global_fault, - IRQF_SHARED, - "arm-smmu global fault", - smmu); + err = devm_request_irq(smmu->dev, smmu->irqs[i], + arm_smmu_global_fault, + IRQF_SHARED, + "arm-smmu global fault", + smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - goto out_free_irqs; + goto out_put_masters; } } @@ -2006,10 +2006,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) arm_smmu_device_reset(smmu); return 0; -out_free_irqs: - while (i--) - free_irq(smmu->irqs[i], smmu); - out_put_masters: for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { struct arm_smmu_master *master @@ -2050,7 +2046,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev) dev_err(dev, "removing device with active domains!\n"); for (i = 0; i < smmu->num_global_irqs; ++i) - free_irq(smmu->irqs[i], smmu); + devm_free_irq(smmu->dev, smmu->irqs[i], smmu); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); -- cgit v1.2.3 From 452014d2b4824ed9ca32a3ef6ee745f22431132a Mon Sep 17 00:00:00 2001 From: Nadav Amit Date: Fri, 24 Jun 2016 06:13:14 -0700 Subject: iommu/vt-d: Remove unnecassary qi clflushes According to the manual: "Hardware access to ... invalidation queue ... are always coherent." Remove unnecassary clflushes accordingly. Signed-off-by: Nadav Amit Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 2eff7b6c6c98..1344e29acd86 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1167,8 +1167,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index) (unsigned long long)qi->desc[index].high); memcpy(&qi->desc[index], &qi->desc[wait_index], sizeof(struct qi_desc)); - __iommu_flush_cache(iommu, &qi->desc[index], - sizeof(struct qi_desc)); writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); return -EINVAL; } @@ -1243,9 +1241,6 @@ restart: hw[wait_index] = wait_desc; - __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc)); - __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc)); - qi->free_head = (qi->free_head + 2) % QI_LENGTH; qi->free_cnt -= 2; -- cgit v1.2.3 From e38d1f1312e4e88d1dab1fdf591824e1f3b105a9 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Tue, 28 Jun 2016 20:38:36 +0200 Subject: iommu: Simplify init function iommu_group_ida and iommu_group_mutex can be initialized statically. There's no need to do this dynamically in the init function. Signed-off-by: Heiner Kallweit Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3000051f48b4..debce45b5b8c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -34,8 +34,8 @@ #include static struct kset *iommu_group_kset; -static struct ida iommu_group_ida; -static struct mutex iommu_group_mutex; +static DEFINE_IDA(iommu_group_ida); +static DEFINE_MUTEX(iommu_group_mutex); struct iommu_callback_data { const struct iommu_ops *ops; @@ -1483,9 +1483,6 @@ static int __init iommu_init(void) { iommu_group_kset = kset_create_and_add("iommu_groups", NULL, kernel_kobj); - ida_init(&iommu_group_ida); - mutex_init(&iommu_group_mutex); - BUG_ON(!iommu_group_kset); return 0; -- cgit v1.2.3 From feccf398db631f3b98c4c6572381517d90b5fd87 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 29 Jun 2016 21:13:59 +0200 Subject: iommu: Simplify and fix ida handling Ida handling can be much simplified by using the ida_simple_.. functions. This change also fixes the bug that previously checking for errors returned by ida_get_new() was incomplete. ida_get_new() can return errors other than EAGAIN, e.g. ENOSPC. This case wasn't handled. Signed-off-by: Heiner Kallweit Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index debce45b5b8c..4d3c4a82af03 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -35,7 +35,6 @@ static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); -static DEFINE_MUTEX(iommu_group_mutex); struct iommu_callback_data { const struct iommu_ops *ops; @@ -144,9 +143,7 @@ static void iommu_group_release(struct kobject *kobj) if (group->iommu_data_release) group->iommu_data_release(group->iommu_data); - mutex_lock(&iommu_group_mutex); - ida_remove(&iommu_group_ida, group->id); - mutex_unlock(&iommu_group_mutex); + ida_simple_remove(&iommu_group_ida, group->id); if (group->default_domain) iommu_domain_free(group->default_domain); @@ -186,26 +183,17 @@ struct iommu_group *iommu_group_alloc(void) INIT_LIST_HEAD(&group->devices); BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); - mutex_lock(&iommu_group_mutex); - -again: - if (unlikely(0 == ida_pre_get(&iommu_group_ida, GFP_KERNEL))) { + ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); + if (ret < 0) { kfree(group); - mutex_unlock(&iommu_group_mutex); - return ERR_PTR(-ENOMEM); + return ERR_PTR(ret); } - - if (-EAGAIN == ida_get_new(&iommu_group_ida, &group->id)) - goto again; - - mutex_unlock(&iommu_group_mutex); + group->id = ret; ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, NULL, "%d", group->id); if (ret) { - mutex_lock(&iommu_group_mutex); - ida_remove(&iommu_group_ida, group->id); - mutex_unlock(&iommu_group_mutex); + ida_simple_remove(&iommu_group_ida, group->id); kfree(group); return ERR_PTR(ret); } -- cgit v1.2.3 From b548e786ce47017107765bbeb0f100202525ea83 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 13 Jul 2016 12:35:24 +0200 Subject: iommu/amd: Init unity mappings only for dma_ops domains The default domain for a device might also be identity-mapped. In this case the kernel would crash when unity mappings are defined for the device. Fix that by making sure the domain is a dma_ops domain. Fixes: 0bb6e243d7fb ('iommu/amd: Support IOMMU_DOMAIN_DMA type allocation') Cc: stable@vger.kernel.org # v4.2+ Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 921111e65c8c..b938a4ac3a37 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -467,9 +467,11 @@ static void init_iommu_group(struct device *dev) if (!domain) goto out; - dma_domain = to_pdomain(domain)->priv; + if (to_pdomain(domain)->flags == PD_DMA_OPS_MASK) { + dma_domain = to_pdomain(domain)->priv; + init_unity_mappings_for_device(dev, dma_domain); + } - init_unity_mappings_for_device(dev, dma_domain); out: iommu_group_put(group); } -- cgit v1.2.3 From 33b21a6b203f70e2012b02753134e59c3ab38779 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 13:07:53 +0200 Subject: iommu: Add apply_dm_region call-back to iommu-ops This new call-back will be used by the iommu driver to do reserve the given dm_region in its iova space before the mapping is created. The call-back is temporary until the dma-ops implementation is part of the common iommu code. Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 3 +++ include/linux/iommu.h | 3 +++ 2 files changed, 6 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3000051f48b4..e8d2fb02d88e 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -348,6 +348,9 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group, list_for_each_entry(entry, &mappings, list) { dma_addr_t start, end, addr; + if (domain->ops->apply_dm_region) + domain->ops->apply_dm_region(dev, domain, entry); + start = ALIGN(entry->start, pg_size); end = ALIGN(entry->start + entry->length, pg_size); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 664683aedcce..a35fb8b42e1a 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -152,6 +152,7 @@ struct iommu_dm_region { * @domain_set_attr: Change domain attributes * @get_dm_regions: Request list of direct mapping requirements for a device * @put_dm_regions: Free list of direct mapping requirements for a device + * @apply_dm_region: Temporary helper call-back for iova reserved ranges * @domain_window_enable: Configure and enable a particular window for a domain * @domain_window_disable: Disable a particular window for a domain * @domain_set_windows: Set the number of windows for a domain @@ -186,6 +187,8 @@ struct iommu_ops { /* Request/Free a list of direct mapping requirements for a device */ void (*get_dm_regions)(struct device *dev, struct list_head *list); void (*put_dm_regions)(struct device *dev, struct list_head *list); + void (*apply_dm_region)(struct device *dev, struct iommu_domain *domain, + struct iommu_dm_region *region); /* Window handling functions */ int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, -- cgit v1.2.3 From a72c4225447bae5d435671446168876bcea4de59 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 11:12:49 +0200 Subject: iommu/amd: Select IOMMU_IOVA for AMD IOMMU Include the generic IOVA code to make use of it in the AMD IOMMU driver too. Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index ad0860383cb3..0b1aabb12de9 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -111,6 +111,7 @@ config AMD_IOMMU select PCI_PRI select PCI_PASID select IOMMU_API + select IOMMU_IOVA depends on X86_64 && PCI && ACPI ---help--- With this option you can enable support for AMD IOMMU hardware in -- cgit v1.2.3 From 307d5851ef74c6c06a68f50302bf19809966d345 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 11:54:04 +0200 Subject: iommu/amd: Allocate iova_domain for dma_ops_domain Use it later for allocating the IO virtual addresses. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index b938a4ac3a37..e7825b25c62f 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,11 @@ #define LOOP_TIMEOUT 100000 +/* IO virtual address start page frame number */ +#define IOVA_START_PFN (1) +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) +#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) + /* * This bitmap is used to advertise the page sizes our hardware support * to the IOMMU core, which will then use this information to split @@ -158,6 +164,9 @@ struct dma_ops_domain { /* address space relevant data */ struct aperture_range *aperture[APERTURE_MAX_RANGES]; + + /* IOVA RB-Tree */ + struct iova_domain iovad; }; /**************************************************************************** @@ -1969,6 +1978,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) if (!dom) return; + put_iova_domain(&dom->iovad); + free_percpu(dom->next_index); del_domain_from_list(&dom->domain); @@ -2044,6 +2055,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) for_each_possible_cpu(cpu) *per_cpu_ptr(dma_dom->next_index, cpu) = 0; + init_iova_domain(&dma_dom->iovad, PAGE_SIZE, + IOVA_START_PFN, DMA_32BIT_PFN); + return dma_dom; free_dma_dom: @@ -2951,7 +2965,11 @@ static struct dma_map_ops amd_iommu_dma_ops = { int __init amd_iommu_init_api(void) { - int err = 0; + int ret, err = 0; + + ret = iova_cache_get(); + if (ret) + return ret; err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); if (err) -- cgit v1.2.3 From 81cd07b9c92ad446fcde18db7de961def0dbcfd3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 7 Jul 2016 18:01:10 +0200 Subject: iommu/amd: Create a list of reserved iova addresses Put the MSI-range, the HT-range and the MMIO ranges of PCI devices into that range, so that these addresses are not allocated for DMA. Copy this address list into every created dma_ops_domain. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index e7825b25c62f..1bb59ae2d586 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -63,6 +63,12 @@ #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) +/* Reserved IOVA ranges */ +#define MSI_RANGE_START (0xfee00000) +#define MSI_RANGE_END (0xfeefffff) +#define HT_RANGE_START (0xfd00000000ULL) +#define HT_RANGE_END (0xffffffffffULL) + /* * This bitmap is used to advertise the page sizes our hardware support * to the IOMMU core, which will then use this information to split @@ -169,6 +175,9 @@ struct dma_ops_domain { struct iova_domain iovad; }; +static struct iova_domain reserved_iova_ranges; +static struct lock_class_key reserved_rbtree_key; + /**************************************************************************** * * Helper functions @@ -2058,6 +2067,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_START_PFN, DMA_32BIT_PFN); + /* Initialize reserved ranges */ + copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad); + return dma_dom; free_dma_dom: @@ -2963,6 +2975,59 @@ static struct dma_map_ops amd_iommu_dma_ops = { .set_dma_mask = set_dma_mask, }; +static int init_reserved_iova_ranges(void) +{ + struct pci_dev *pdev = NULL; + struct iova *val; + + init_iova_domain(&reserved_iova_ranges, PAGE_SIZE, + IOVA_START_PFN, DMA_32BIT_PFN); + + lockdep_set_class(&reserved_iova_ranges.iova_rbtree_lock, + &reserved_rbtree_key); + + /* MSI memory range */ + val = reserve_iova(&reserved_iova_ranges, + IOVA_PFN(MSI_RANGE_START), IOVA_PFN(MSI_RANGE_END)); + if (!val) { + pr_err("Reserving MSI range failed\n"); + return -ENOMEM; + } + + /* HT memory range */ + val = reserve_iova(&reserved_iova_ranges, + IOVA_PFN(HT_RANGE_START), IOVA_PFN(HT_RANGE_END)); + if (!val) { + pr_err("Reserving HT range failed\n"); + return -ENOMEM; + } + + /* + * Memory used for PCI resources + * FIXME: Check whether we can reserve the PCI-hole completly + */ + for_each_pci_dev(pdev) { + int i; + + for (i = 0; i < PCI_NUM_RESOURCES; ++i) { + struct resource *r = &pdev->resource[i]; + + if (!(r->flags & IORESOURCE_MEM)) + continue; + + val = reserve_iova(&reserved_iova_ranges, + IOVA_PFN(r->start), + IOVA_PFN(r->end)); + if (!val) { + pr_err("Reserve pci-resource range failed\n"); + return -ENOMEM; + } + } + } + + return 0; +} + int __init amd_iommu_init_api(void) { int ret, err = 0; @@ -2971,6 +3036,10 @@ int __init amd_iommu_init_api(void) if (ret) return ret; + ret = init_reserved_iova_ranges(); + if (ret) + return ret; + err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); if (err) return err; -- cgit v1.2.3 From 8d54d6c8b8f3e40dadd15b1dad6f190c2ccf7473 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 13:32:20 +0200 Subject: iommu/amd: Implement apply_dm_region call-back It is used to reserve the dm-regions in the iova-tree. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 1bb59ae2d586..4c9063ae442a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3385,6 +3385,20 @@ static void amd_iommu_put_dm_regions(struct device *dev, kfree(entry); } +static void amd_iommu_apply_dm_region(struct device *dev, + struct iommu_domain *domain, + struct iommu_dm_region *region) +{ + struct protection_domain *pdomain = to_pdomain(domain); + struct dma_ops_domain *dma_dom = pdomain->priv; + unsigned long start, end; + + start = IOVA_PFN(region->start); + end = IOVA_PFN(region->start + region->length); + + WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL); +} + static const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -3400,6 +3414,7 @@ static const struct iommu_ops amd_iommu_ops = { .device_group = amd_iommu_device_group, .get_dm_regions = amd_iommu_get_dm_regions, .put_dm_regions = amd_iommu_put_dm_regions, + .apply_dm_region = amd_iommu_apply_dm_region, .pgsize_bitmap = AMD_IOMMU_PGSIZES, }; -- cgit v1.2.3 From b911b89b6d0112b5e81b74ce25d894b03023344e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 14:29:11 +0200 Subject: iommu/amd: Pass gfp-flags to iommu_map_page() Make this function ready to be used in the DMA-API path. Reorder parameters a bit while at it. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 4c9063ae442a..c421c0605901 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1418,8 +1418,9 @@ static u64 *fetch_pte(struct protection_domain *domain, static int iommu_map_page(struct protection_domain *dom, unsigned long bus_addr, unsigned long phys_addr, + unsigned long page_size, int prot, - unsigned long page_size) + gfp_t gfp) { u64 __pte, *pte; int i, count; @@ -1431,7 +1432,7 @@ static int iommu_map_page(struct protection_domain *dom, return -EINVAL; count = PAGE_SIZE_PTE_COUNT(page_size); - pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); + pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp); if (!pte) return -ENOMEM; @@ -3283,7 +3284,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, prot |= IOMMU_PROT_IW; mutex_lock(&domain->api_lock); - ret = iommu_map_page(domain, iova, paddr, prot, page_size); + ret = iommu_map_page(domain, iova, paddr, page_size, prot, GFP_KERNEL); mutex_unlock(&domain->api_lock); return ret; -- cgit v1.2.3 From 518d9b450387a3508363af58d1f62db9fc92d438 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 14:39:47 +0200 Subject: iommu/amd: Remove special mapping code for dma_ops path Use the iommu-api map/unmap functions instead. This will be required anyway when IOVA code is used for address allocation. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 107 ++++++---------------------------------------- 1 file changed, 14 insertions(+), 93 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c421c0605901..1cd57a392558 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2559,94 +2559,6 @@ static void update_domain(struct protection_domain *domain) domain->updated = false; } -/* - * This function fetches the PTE for a given address in the aperture - */ -static u64* dma_ops_get_pte(struct dma_ops_domain *dom, - unsigned long address) -{ - struct aperture_range *aperture; - u64 *pte, *pte_page; - - aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; - if (!aperture) - return NULL; - - pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; - if (!pte) { - pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page, - GFP_ATOMIC); - aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page; - } else - pte += PM_LEVEL_INDEX(0, address); - - update_domain(&dom->domain); - - return pte; -} - -/* - * This is the generic map function. It maps one 4kb page at paddr to - * the given address in the DMA address space for the domain. - */ -static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom, - unsigned long address, - phys_addr_t paddr, - int direction) -{ - u64 *pte, __pte; - - WARN_ON(address > dom->aperture_size); - - paddr &= PAGE_MASK; - - pte = dma_ops_get_pte(dom, address); - if (!pte) - return DMA_ERROR_CODE; - - __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC; - - if (direction == DMA_TO_DEVICE) - __pte |= IOMMU_PTE_IR; - else if (direction == DMA_FROM_DEVICE) - __pte |= IOMMU_PTE_IW; - else if (direction == DMA_BIDIRECTIONAL) - __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW; - - WARN_ON_ONCE(*pte); - - *pte = __pte; - - return (dma_addr_t)address; -} - -/* - * The generic unmapping function for on page in the DMA address space. - */ -static void dma_ops_domain_unmap(struct dma_ops_domain *dom, - unsigned long address) -{ - struct aperture_range *aperture; - u64 *pte; - - if (address >= dom->aperture_size) - return; - - aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; - if (!aperture) - return; - - pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; - if (!pte) - return; - - pte += PM_LEVEL_INDEX(0, address); - - WARN_ON_ONCE(!*pte); - - *pte = 0ULL; -} - /* * This function contains common code for mapping of a physically * contiguous memory region into DMA address space. It is used by all @@ -2657,7 +2569,7 @@ static dma_addr_t __map_single(struct device *dev, struct dma_ops_domain *dma_dom, phys_addr_t paddr, size_t size, - int dir, + int direction, bool align, u64 dma_mask) { @@ -2665,6 +2577,7 @@ static dma_addr_t __map_single(struct device *dev, dma_addr_t address, start, ret; unsigned int pages; unsigned long align_mask = 0; + int prot = 0; int i; pages = iommu_num_pages(paddr, size, PAGE_SIZE); @@ -2679,10 +2592,18 @@ static dma_addr_t __map_single(struct device *dev, if (address == DMA_ERROR_CODE) goto out; + if (direction == DMA_TO_DEVICE) + prot = IOMMU_PROT_IR; + else if (direction == DMA_FROM_DEVICE) + prot = IOMMU_PROT_IW; + else if (direction == DMA_BIDIRECTIONAL) + prot = IOMMU_PROT_IW | IOMMU_PROT_IR; + start = address; for (i = 0; i < pages; ++i) { - ret = dma_ops_domain_map(dma_dom, start, paddr, dir); - if (ret == DMA_ERROR_CODE) + ret = iommu_map_page(&dma_dom->domain, start, paddr, + PAGE_SIZE, prot, GFP_ATOMIC); + if (ret) goto out_unmap; paddr += PAGE_SIZE; @@ -2702,7 +2623,7 @@ out_unmap: for (--i; i >= 0; --i) { start -= PAGE_SIZE; - dma_ops_domain_unmap(dma_dom, start); + iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE); } dma_ops_free_addresses(dma_dom, address, pages); @@ -2733,7 +2654,7 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, start = dma_addr; for (i = 0; i < pages; ++i) { - dma_ops_domain_unmap(dma_dom, start); + iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE); start += PAGE_SIZE; } -- cgit v1.2.3 From 256e4621c21aa1bf704e1a12e643923fdb732d04 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 14:23:01 +0200 Subject: iommu/amd: Make use of the generic IOVA allocator Remove the old address allocation code and make use of the generic IOVA allocator that is also used by other dma-ops implementations. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 183 +++++++--------------------------------------- 1 file changed, 26 insertions(+), 157 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 1cd57a392558..77be2d0558bd 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1649,167 +1649,32 @@ out_free: return -ENOMEM; } -static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom, - struct aperture_range *range, - unsigned long pages, - unsigned long dma_mask, - unsigned long boundary_size, - unsigned long align_mask, - bool trylock) -{ - unsigned long offset, limit, flags; - dma_addr_t address; - bool flush = false; - - offset = range->offset >> PAGE_SHIFT; - limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset, - dma_mask >> PAGE_SHIFT); - - if (trylock) { - if (!spin_trylock_irqsave(&range->bitmap_lock, flags)) - return -1; - } else { - spin_lock_irqsave(&range->bitmap_lock, flags); - } - - address = iommu_area_alloc(range->bitmap, limit, range->next_bit, - pages, offset, boundary_size, align_mask); - if (address == -1) { - /* Nothing found, retry one time */ - address = iommu_area_alloc(range->bitmap, limit, - 0, pages, offset, boundary_size, - align_mask); - flush = true; - } - - if (address != -1) - range->next_bit = address + pages; - - spin_unlock_irqrestore(&range->bitmap_lock, flags); - - if (flush) { - domain_flush_tlb(&dom->domain); - domain_flush_complete(&dom->domain); - } - - return address; -} - -static unsigned long dma_ops_area_alloc(struct device *dev, - struct dma_ops_domain *dom, - unsigned int pages, - unsigned long align_mask, - u64 dma_mask) +static unsigned long dma_ops_alloc_iova(struct device *dev, + struct dma_ops_domain *dma_dom, + unsigned int pages, u64 dma_mask) { - unsigned long boundary_size, mask; - unsigned long address = -1; - bool first = true; - u32 start, i; - - preempt_disable(); - - mask = dma_get_seg_boundary(dev); - -again: - start = this_cpu_read(*dom->next_index); - - /* Sanity check - is it really necessary? */ - if (unlikely(start > APERTURE_MAX_RANGES)) { - start = 0; - this_cpu_write(*dom->next_index, 0); - } - - boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : - 1UL << (BITS_PER_LONG - PAGE_SHIFT); + unsigned long pfn = 0; - for (i = 0; i < APERTURE_MAX_RANGES; ++i) { - struct aperture_range *range; - int index; - - index = (start + i) % APERTURE_MAX_RANGES; + pages = __roundup_pow_of_two(pages); - range = dom->aperture[index]; - - if (!range || range->offset >= dma_mask) - continue; - - address = dma_ops_aperture_alloc(dom, range, pages, - dma_mask, boundary_size, - align_mask, first); - if (address != -1) { - address = range->offset + (address << PAGE_SHIFT); - this_cpu_write(*dom->next_index, index); - break; - } - } - - if (address == -1 && first) { - first = false; - goto again; - } + if (dma_mask > DMA_BIT_MASK(32)) + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(DMA_BIT_MASK(32))); - preempt_enable(); + if (!pfn) + pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(dma_mask)); - return address; + return (pfn << PAGE_SHIFT); } -static unsigned long dma_ops_alloc_addresses(struct device *dev, - struct dma_ops_domain *dom, - unsigned int pages, - unsigned long align_mask, - u64 dma_mask) +static void dma_ops_free_iova(struct dma_ops_domain *dma_dom, + unsigned long address, + unsigned int pages) { - unsigned long address = -1; - - while (address == -1) { - address = dma_ops_area_alloc(dev, dom, pages, - align_mask, dma_mask); - - if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC)) - break; - } - - if (unlikely(address == -1)) - address = DMA_ERROR_CODE; - - WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size); - - return address; -} - -/* - * The address free function. - * - * called with domain->lock held - */ -static void dma_ops_free_addresses(struct dma_ops_domain *dom, - unsigned long address, - unsigned int pages) -{ - unsigned i = address >> APERTURE_RANGE_SHIFT; - struct aperture_range *range = dom->aperture[i]; - unsigned long flags; - - BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL); - -#ifdef CONFIG_IOMMU_STRESS - if (i < 4) - return; -#endif - - if (amd_iommu_unmap_flush) { - domain_flush_tlb(&dom->domain); - domain_flush_complete(&dom->domain); - } - - address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT; - - spin_lock_irqsave(&range->bitmap_lock, flags); - if (address + pages > range->next_bit) - range->next_bit = address + pages; - bitmap_clear(range->bitmap, address, pages); - spin_unlock_irqrestore(&range->bitmap_lock, flags); + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + free_iova_fast(&dma_dom->iovad, address, pages); } /**************************************************************************** @@ -2586,9 +2451,7 @@ static dma_addr_t __map_single(struct device *dev, if (align) align_mask = (1UL << get_order(size)) - 1; - address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask, - dma_mask); - + address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask); if (address == DMA_ERROR_CODE) goto out; @@ -2626,7 +2489,10 @@ out_unmap: iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE); } - dma_ops_free_addresses(dma_dom, address, pages); + domain_flush_tlb(&dma_dom->domain); + domain_flush_complete(&dma_dom->domain); + + dma_ops_free_iova(dma_dom, address, pages); return DMA_ERROR_CODE; } @@ -2658,7 +2524,10 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, start += PAGE_SIZE; } - dma_ops_free_addresses(dma_dom, dma_addr, pages); + domain_flush_tlb(&dma_dom->domain); + domain_flush_complete(&dma_dom->domain); + + dma_ops_free_iova(dma_dom, dma_addr, pages); } /* -- cgit v1.2.3 From 2d4c515bf06c9bce87b546279413621f847ef6a3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 16:21:32 +0200 Subject: iommu/amd: Remove other remains of old address allocator There are other remains in the code from the old allocatore. Remove them all. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 316 +--------------------------------------------- 1 file changed, 5 insertions(+), 311 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 77be2d0558bd..2ba8b464ea1a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -133,28 +133,6 @@ static void update_domain(struct protection_domain *domain); static int protection_domain_init(struct protection_domain *domain); static void detach_device(struct device *dev); -/* - * For dynamic growth the aperture size is split into ranges of 128MB of - * DMA address space each. This struct represents one such range. - */ -struct aperture_range { - - spinlock_t bitmap_lock; - - /* address allocation bitmap */ - unsigned long *bitmap; - unsigned long offset; - unsigned long next_bit; - - /* - * Array of PTE pages for the aperture. In this array we save all the - * leaf pages of the domain page table used for the aperture. This way - * we don't need to walk the page table to find a specific PTE. We can - * just calculate its address in constant time. - */ - u64 *pte_pages[64]; -}; - /* * Data container for a dma_ops specific protection domain */ @@ -162,15 +140,6 @@ struct dma_ops_domain { /* generic protection domain information */ struct protection_domain domain; - /* size of the aperture for the mappings */ - unsigned long aperture_size; - - /* aperture index we start searching for free addresses */ - u32 __percpu *next_index; - - /* address space relevant data */ - struct aperture_range *aperture[APERTURE_MAX_RANGES]; - /* IOVA RB-Tree */ struct iova_domain iovad; }; @@ -409,43 +378,6 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) return dev_data->errata & (1 << erratum) ? true : false; } -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } -} - -/* - * Inits the unity mappings required for a specific device - */ -static void init_unity_mappings_for_device(struct device *dev, - struct dma_ops_domain *dma_dom) -{ - struct unity_map_entry *e; - int devid; - - devid = get_device_id(dev); - if (devid < 0) - return; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - alloc_unity_mapping(dma_dom, e); - } -} - /* * This function checks if the driver got a valid device from the caller to * avoid dereferencing invalid pointers. @@ -473,24 +405,12 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { - struct dma_ops_domain *dma_domain; - struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) return; - domain = iommu_group_default_domain(group); - if (!domain) - goto out; - - if (to_pdomain(domain)->flags == PD_DMA_OPS_MASK) { - dma_domain = to_pdomain(domain)->priv; - init_unity_mappings_for_device(dev, dma_domain); - } - -out: iommu_group_put(group); } @@ -1496,158 +1416,10 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops - * interface functions. They work like the allocators in the other IOMMU - * drivers. Its basically a bitmap which marks the allocated pages in - * the aperture. Maybe it could be enhanced in the future to a more - * efficient allocator. + * interface functions. * ****************************************************************************/ -/* - * The address allocator core functions. - * - * called with domain->lock held - */ - -/* - * Used to reserve address ranges in the aperture (e.g. for exclusion - * ranges. - */ -static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, - unsigned long start_page, - unsigned int pages) -{ - unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT; - - if (start_page + pages > last_page) - pages = last_page - start_page; - - for (i = start_page; i < start_page + pages; ++i) { - int index = i / APERTURE_RANGE_PAGES; - int page = i % APERTURE_RANGE_PAGES; - __set_bit(page, dom->aperture[index]->bitmap); - } -} - -/* - * This function is used to add a new aperture range to an existing - * aperture in case of dma_ops domain allocation or address allocation - * failure. - */ -static int alloc_new_range(struct dma_ops_domain *dma_dom, - bool populate, gfp_t gfp) -{ - int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; - unsigned long i, old_size, pte_pgsize; - struct aperture_range *range; - struct amd_iommu *iommu; - unsigned long flags; - -#ifdef CONFIG_IOMMU_STRESS - populate = false; -#endif - - if (index >= APERTURE_MAX_RANGES) - return -ENOMEM; - - range = kzalloc(sizeof(struct aperture_range), gfp); - if (!range) - return -ENOMEM; - - range->bitmap = (void *)get_zeroed_page(gfp); - if (!range->bitmap) - goto out_free; - - range->offset = dma_dom->aperture_size; - - spin_lock_init(&range->bitmap_lock); - - if (populate) { - unsigned long address = dma_dom->aperture_size; - int i, num_ptes = APERTURE_RANGE_PAGES / 512; - u64 *pte, *pte_page; - - for (i = 0; i < num_ptes; ++i) { - pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE, - &pte_page, gfp); - if (!pte) - goto out_free; - - range->pte_pages[i] = pte_page; - - address += APERTURE_RANGE_SIZE / 64; - } - } - - spin_lock_irqsave(&dma_dom->domain.lock, flags); - - /* First take the bitmap_lock and then publish the range */ - spin_lock(&range->bitmap_lock); - - old_size = dma_dom->aperture_size; - dma_dom->aperture[index] = range; - dma_dom->aperture_size += APERTURE_RANGE_SIZE; - - /* Reserve address range used for MSI messages */ - if (old_size < MSI_ADDR_BASE_LO && - dma_dom->aperture_size > MSI_ADDR_BASE_LO) { - unsigned long spage; - int pages; - - pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE); - spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT; - - dma_ops_reserve_addresses(dma_dom, spage, pages); - } - - /* Initialize the exclusion range if necessary */ - for_each_iommu(iommu) { - if (iommu->exclusion_start && - iommu->exclusion_start >= dma_dom->aperture[index]->offset - && iommu->exclusion_start < dma_dom->aperture_size) { - unsigned long startpage; - int pages = iommu_num_pages(iommu->exclusion_start, - iommu->exclusion_length, - PAGE_SIZE); - startpage = iommu->exclusion_start >> PAGE_SHIFT; - dma_ops_reserve_addresses(dma_dom, startpage, pages); - } - } - - /* - * Check for areas already mapped as present in the new aperture - * range and mark those pages as reserved in the allocator. Such - * mappings may already exist as a result of requested unity - * mappings for devices. - */ - for (i = dma_dom->aperture[index]->offset; - i < dma_dom->aperture_size; - i += pte_pgsize) { - u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize); - if (!pte || !IOMMU_PTE_PRESENT(*pte)) - continue; - - dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, - pte_pgsize >> 12); - } - - update_domain(&dma_dom->domain); - - spin_unlock(&range->bitmap_lock); - - spin_unlock_irqrestore(&dma_dom->domain.lock, flags); - - return 0; - -out_free: - update_domain(&dma_dom->domain); - - free_page((unsigned long)range->bitmap); - - kfree(range); - - return -ENOMEM; -} static unsigned long dma_ops_alloc_iova(struct device *dev, struct dma_ops_domain *dma_dom, @@ -1848,46 +1620,18 @@ static void free_gcr3_table(struct protection_domain *domain) */ static void dma_ops_domain_free(struct dma_ops_domain *dom) { - int i; - if (!dom) return; - put_iova_domain(&dom->iovad); - - free_percpu(dom->next_index); - del_domain_from_list(&dom->domain); - free_pagetable(&dom->domain); + put_iova_domain(&dom->iovad); - for (i = 0; i < APERTURE_MAX_RANGES; ++i) { - if (!dom->aperture[i]) - continue; - free_page((unsigned long)dom->aperture[i]->bitmap); - kfree(dom->aperture[i]); - } + free_pagetable(&dom->domain); kfree(dom); } -static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom, - int max_apertures) -{ - int ret, i, apertures; - - apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; - ret = 0; - - for (i = apertures; i < max_apertures; ++i) { - ret = alloc_new_range(dma_dom, false, GFP_KERNEL); - if (ret) - break; - } - - return ret; -} - /* * Allocates a new protection domain usable for the dma_ops functions. * It also initializes the page table and the address allocator data @@ -1896,7 +1640,6 @@ static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom, static struct dma_ops_domain *dma_ops_domain_alloc(void) { struct dma_ops_domain *dma_dom; - int cpu; dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL); if (!dma_dom) @@ -1905,10 +1648,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (protection_domain_init(&dma_dom->domain)) goto free_dma_dom; - dma_dom->next_index = alloc_percpu(u32); - if (!dma_dom->next_index) - goto free_dma_dom; - dma_dom->domain.mode = PAGE_MODE_2_LEVEL; dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); dma_dom->domain.flags = PD_DMA_OPS_MASK; @@ -1916,26 +1655,14 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (!dma_dom->domain.pt_root) goto free_dma_dom; - add_domain_to_list(&dma_dom->domain); - - if (alloc_new_range(dma_dom, true, GFP_KERNEL)) - goto free_dma_dom; - - /* - * mark the first page as allocated so we never return 0 as - * a valid dma-address. So we can use 0 as error value - */ - dma_dom->aperture[0]->bitmap[0] = 1; - - for_each_possible_cpu(cpu) - *per_cpu_ptr(dma_dom->next_index, cpu) = 0; - init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_START_PFN, DMA_32BIT_PFN); /* Initialize reserved ranges */ copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad); + add_domain_to_list(&dma_dom->domain); + return dma_dom; free_dma_dom: @@ -2510,10 +2237,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, dma_addr_t i, start; unsigned int pages; - if ((dma_addr == DMA_ERROR_CODE) || - (dma_addr + size > dma_dom->aperture_size)) - return; - flush_addr = dma_addr; pages = iommu_num_pages(dma_addr, size, PAGE_SIZE); dma_addr &= PAGE_MASK; @@ -2727,34 +2450,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } -static int set_dma_mask(struct device *dev, u64 mask) -{ - struct protection_domain *domain; - int max_apertures = 1; - - domain = get_domain(dev); - if (IS_ERR(domain)) - return PTR_ERR(domain); - - if (mask == DMA_BIT_MASK(64)) - max_apertures = 8; - else if (mask > DMA_BIT_MASK(32)) - max_apertures = 4; - - /* - * To prevent lock contention it doesn't make sense to allocate more - * apertures than online cpus - */ - if (max_apertures > num_online_cpus()) - max_apertures = num_online_cpus(); - - if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures)) - dev_err(dev, "Can't allocate %d iommu apertures\n", - max_apertures); - - return 0; -} - static struct dma_map_ops amd_iommu_dma_ops = { .alloc = alloc_coherent, .free = free_coherent, @@ -2763,7 +2458,6 @@ static struct dma_map_ops amd_iommu_dma_ops = { .map_sg = map_sg, .unmap_sg = unmap_sg, .dma_supported = amd_iommu_dma_supported, - .set_dma_mask = set_dma_mask, }; static int init_reserved_iova_ranges(void) -- cgit v1.2.3 From bda350dbdbc1ad8655ece0ec3d41bebc3ee7a77b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 5 Jul 2016 16:28:02 +0200 Subject: iommu/amd: Remove align-parameter from __map_single() This parameter is not required anymore because the iova-allocations are always aligned to its size. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 2ba8b464ea1a..d218e35ed3e3 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2162,22 +2162,17 @@ static dma_addr_t __map_single(struct device *dev, phys_addr_t paddr, size_t size, int direction, - bool align, u64 dma_mask) { dma_addr_t offset = paddr & ~PAGE_MASK; dma_addr_t address, start, ret; unsigned int pages; - unsigned long align_mask = 0; int prot = 0; int i; pages = iommu_num_pages(paddr, size, PAGE_SIZE); paddr &= PAGE_MASK; - if (align) - align_mask = (1UL << get_order(size)) - 1; - address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask); if (address == DMA_ERROR_CODE) goto out; @@ -2273,8 +2268,7 @@ static dma_addr_t map_page(struct device *dev, struct page *page, dma_mask = *dev->dma_mask; - return __map_single(dev, domain->priv, paddr, size, dir, false, - dma_mask); + return __map_single(dev, domain->priv, paddr, size, dir, dma_mask); } /* @@ -2317,8 +2311,7 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, paddr = sg_phys(s); s->dma_address = __map_single(dev, domain->priv, - paddr, s->length, dir, false, - dma_mask); + paddr, s->length, dir, dma_mask); if (s->dma_address) { s->dma_length = s->length; @@ -2402,7 +2395,7 @@ static void *alloc_coherent(struct device *dev, size_t size, dma_mask = *dev->dma_mask; *dma_addr = __map_single(dev, domain->priv, page_to_phys(page), - size, DMA_BIDIRECTIONAL, true, dma_mask); + size, DMA_BIDIRECTIONAL, dma_mask); if (*dma_addr == DMA_ERROR_CODE) goto out_free; -- cgit v1.2.3 From c5b5da9c79bb2d88fa3c5163ccf1a7a9e89cfa49 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 6 Jul 2016 11:55:37 +0200 Subject: iommu/amd: Set up data structures for flush queue The flush queue is the equivalent to defered-flushing in the Intel VT-d driver. This patch sets up the data structures needed for this. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d218e35ed3e3..38f8a5e461fc 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -89,6 +89,22 @@ LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); LIST_HEAD(acpihid_map); +#define FLUSH_QUEUE_SIZE 256 + +struct flush_queue_entry { + unsigned long iova_pfn; + unsigned long pages; + struct dma_ops_domain *dma_dom; +}; + +struct flush_queue { + spinlock_t lock; + unsigned next; + struct flush_queue_entry *entries; +}; + +DEFINE_PER_CPU(struct flush_queue, flush_queue); + /* * Domain for untranslated devices - only allocated * if iommu=pt passed on kernel cmd line. @@ -2508,7 +2524,7 @@ static int init_reserved_iova_ranges(void) int __init amd_iommu_init_api(void) { - int ret, err = 0; + int ret, cpu, err = 0; ret = iova_cache_get(); if (ret) @@ -2518,6 +2534,18 @@ int __init amd_iommu_init_api(void) if (ret) return ret; + for_each_possible_cpu(cpu) { + struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu); + + queue->entries = kzalloc(FLUSH_QUEUE_SIZE * + sizeof(*queue->entries), + GFP_KERNEL); + if (!queue->entries) + goto out_put_iova; + + spin_lock_init(&queue->lock); + } + err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); if (err) return err; @@ -2530,6 +2558,15 @@ int __init amd_iommu_init_api(void) if (err) return err; return 0; + +out_put_iova: + for_each_possible_cpu(cpu) { + struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu); + + kfree(queue->entries); + } + + return -ENOMEM; } int __init amd_iommu_init_dma_ops(void) @@ -2552,6 +2589,7 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; + } /***************************************************************************** -- cgit v1.2.3 From f1eae7c58036b30d434d84a34c6c3154bfb5b496 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 6 Jul 2016 12:50:35 +0200 Subject: iommu/amd: Allow NULL pointer parameter for domain_flush_complete() If domain == NULL is passed to the function, it will queue a completion-wait command on all IOMMUs in the system. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 38f8a5e461fc..0884b126d932 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1177,7 +1177,7 @@ static void domain_flush_complete(struct protection_domain *domain) int i; for (i = 0; i < amd_iommus_present; ++i) { - if (!domain->dev_iommu[i]) + if (domain && !domain->dev_iommu[i]) continue; /* -- cgit v1.2.3 From b1516a14657acf81a587e9a6e733a881625eee53 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 6 Jul 2016 13:07:22 +0200 Subject: iommu/amd: Implement flush queue With the flush queue the IOMMU TLBs will not be flushed at every dma-ops unmap operation. The unmapped ranges will be queued and flushed at once, when the queue is full. This makes unmapping operations a lot faster (on average) and restores the performance of the old address allocator. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 71 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0884b126d932..a8e4c5adade7 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2120,6 +2120,66 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev) * *****************************************************************************/ +static void __queue_flush(struct flush_queue *queue) +{ + struct protection_domain *domain; + unsigned long flags; + int idx; + + /* First flush TLB of all known domains */ + spin_lock_irqsave(&amd_iommu_pd_lock, flags); + list_for_each_entry(domain, &amd_iommu_pd_list, list) + domain_flush_tlb(domain); + spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); + + /* Wait until flushes have completed */ + domain_flush_complete(NULL); + + for (idx = 0; idx < queue->next; ++idx) { + struct flush_queue_entry *entry; + + entry = queue->entries + idx; + + free_iova_fast(&entry->dma_dom->iovad, + entry->iova_pfn, + entry->pages); + + /* Not really necessary, just to make sure we catch any bugs */ + entry->dma_dom = NULL; + } + + queue->next = 0; +} + +static void queue_add(struct dma_ops_domain *dma_dom, + unsigned long address, unsigned long pages) +{ + struct flush_queue_entry *entry; + struct flush_queue *queue; + unsigned long flags; + int idx; + + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + + queue = get_cpu_ptr(&flush_queue); + spin_lock_irqsave(&queue->lock, flags); + + if (queue->next == FLUSH_QUEUE_SIZE) + __queue_flush(queue); + + idx = queue->next++; + entry = queue->entries + idx; + + entry->iova_pfn = address; + entry->pages = pages; + entry->dma_dom = dma_dom; + + spin_unlock_irqrestore(&queue->lock, flags); + put_cpu_ptr(&flush_queue); +} + + /* * In the dma_ops path we only have the struct device. This function * finds the corresponding IOMMU, the protection domain and the @@ -2258,10 +2318,13 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, start += PAGE_SIZE; } - domain_flush_tlb(&dma_dom->domain); - domain_flush_complete(&dma_dom->domain); - - dma_ops_free_iova(dma_dom, dma_addr, pages); + if (amd_iommu_unmap_flush) { + dma_ops_free_iova(dma_dom, dma_addr, pages); + domain_flush_tlb(&dma_dom->domain); + domain_flush_complete(&dma_dom->domain); + } else { + queue_add(dma_dom, dma_addr, pages); + } } /* -- cgit v1.2.3 From bb279475db4d0bb07e4dbc99e060362b9f3b5093 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 6 Jul 2016 13:56:36 +0200 Subject: iommu/amd: Implement timeout to flush unmap queues In case the queue doesn't fill up, we flush the TLB at least 10ms after the unmap happened to make sure that the TLB is cleaned up. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index a8e4c5adade7..c0b2f4fc6bfc 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -105,6 +105,9 @@ struct flush_queue { DEFINE_PER_CPU(struct flush_queue, flush_queue); +static atomic_t queue_timer_on; +static struct timer_list queue_timer; + /* * Domain for untranslated devices - only allocated * if iommu=pt passed on kernel cmd line. @@ -2151,6 +2154,24 @@ static void __queue_flush(struct flush_queue *queue) queue->next = 0; } +void queue_flush_timeout(unsigned long unsused) +{ + int cpu; + + atomic_set(&queue_timer_on, 0); + + for_each_possible_cpu(cpu) { + struct flush_queue *queue; + unsigned long flags; + + queue = per_cpu_ptr(&flush_queue, cpu); + spin_lock_irqsave(&queue->lock, flags); + if (queue->next > 0) + __queue_flush(queue); + spin_unlock_irqrestore(&queue->lock, flags); + } +} + static void queue_add(struct dma_ops_domain *dma_dom, unsigned long address, unsigned long pages) { @@ -2176,6 +2197,10 @@ static void queue_add(struct dma_ops_domain *dma_dom, entry->dma_dom = dma_dom; spin_unlock_irqrestore(&queue->lock, flags); + + if (atomic_cmpxchg(&queue_timer_on, 0, 1) == 0) + mod_timer(&queue_timer, jiffies + msecs_to_jiffies(10)); + put_cpu_ptr(&flush_queue); } @@ -2634,6 +2659,9 @@ out_put_iova: int __init amd_iommu_init_dma_ops(void) { + setup_timer(&queue_timer, queue_flush_timeout, 0); + atomic_set(&queue_timer_on, 0); + swiotlb = iommu_pass_through ? 1 : 0; iommu_detected = 1; -- cgit v1.2.3 From f37f7f33d561901d599e98a72bbf44af1f162732 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 8 Jul 2016 11:47:22 +0200 Subject: iommu/amd: Introduce dir2prot() helper This function converts dma_data_direction to iommu-protection flags. This will be needed on multiple places in the code, so this will save some code. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c0b2f4fc6bfc..281cacbd5816 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2252,6 +2252,17 @@ static void update_domain(struct protection_domain *domain) domain->updated = false; } +static int dir2prot(enum dma_data_direction direction) +{ + if (direction == DMA_TO_DEVICE) + return IOMMU_PROT_IR; + else if (direction == DMA_FROM_DEVICE) + return IOMMU_PROT_IW; + else if (direction == DMA_BIDIRECTIONAL) + return IOMMU_PROT_IW | IOMMU_PROT_IR; + else + return 0; +} /* * This function contains common code for mapping of a physically * contiguous memory region into DMA address space. It is used by all @@ -2262,7 +2273,7 @@ static dma_addr_t __map_single(struct device *dev, struct dma_ops_domain *dma_dom, phys_addr_t paddr, size_t size, - int direction, + enum dma_data_direction direction, u64 dma_mask) { dma_addr_t offset = paddr & ~PAGE_MASK; @@ -2278,12 +2289,7 @@ static dma_addr_t __map_single(struct device *dev, if (address == DMA_ERROR_CODE) goto out; - if (direction == DMA_TO_DEVICE) - prot = IOMMU_PROT_IR; - else if (direction == DMA_FROM_DEVICE) - prot = IOMMU_PROT_IW; - else if (direction == DMA_BIDIRECTIONAL) - prot = IOMMU_PROT_IW | IOMMU_PROT_IR; + prot = dir2prot(direction); start = address; for (i = 0; i < pages; ++i) { -- cgit v1.2.3 From 80187fd39dcb30e3aa39e93a87b2d2f7fc8f4fd5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 6 Jul 2016 17:20:54 +0200 Subject: iommu/amd: Optimize map_sg and unmap_sg Optimize these functions so that they need only one call into the address alloctor. This also saves a couple of io-tlb flushes in the unmap_sg path. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 111 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 26 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 281cacbd5816..acad37c3f0a1 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2396,50 +2396,110 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, __unmap_single(domain->priv, dma_addr, size, dir); } +static int sg_num_pages(struct device *dev, + struct scatterlist *sglist, + int nelems) +{ + unsigned long mask, boundary_size; + struct scatterlist *s; + int i, npages = 0; + + mask = dma_get_seg_boundary(dev); + boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : + 1UL << (BITS_PER_LONG - PAGE_SHIFT); + + for_each_sg(sglist, s, nelems, i) { + int p, n; + + s->dma_address = npages << PAGE_SHIFT; + p = npages % boundary_size; + n = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE); + if (p + n > boundary_size) + npages += boundary_size - p; + npages += n; + } + + return npages; +} + /* * The exported map_sg function for dma_ops (handles scatter-gather * lists). */ static int map_sg(struct device *dev, struct scatterlist *sglist, - int nelems, enum dma_data_direction dir, + int nelems, enum dma_data_direction direction, struct dma_attrs *attrs) { + int mapped_pages = 0, npages = 0, prot = 0, i; struct protection_domain *domain; - int i; + struct dma_ops_domain *dma_dom; struct scatterlist *s; - phys_addr_t paddr; - int mapped_elems = 0; + unsigned long address; u64 dma_mask; domain = get_domain(dev); if (IS_ERR(domain)) return 0; + dma_dom = domain->priv; dma_mask = *dev->dma_mask; + npages = sg_num_pages(dev, sglist, nelems); + + address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask); + if (address == DMA_ERROR_CODE) + goto out_err; + + prot = dir2prot(direction); + + /* Map all sg entries */ for_each_sg(sglist, s, nelems, i) { - paddr = sg_phys(s); + int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE); + + for (j = 0; j < pages; ++j) { + unsigned long bus_addr, phys_addr; + int ret; - s->dma_address = __map_single(dev, domain->priv, - paddr, s->length, dir, dma_mask); + bus_addr = address + s->dma_address + (j << PAGE_SHIFT); + phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT); + ret = iommu_map_page(domain, bus_addr, phys_addr, PAGE_SIZE, prot, GFP_ATOMIC); + if (ret) + goto out_unmap; - if (s->dma_address) { - s->dma_length = s->length; - mapped_elems++; - } else - goto unmap; + mapped_pages += 1; + } } - return mapped_elems; + /* Everything is mapped - write the right values into s->dma_address */ + for_each_sg(sglist, s, nelems, i) { + s->dma_address += address + s->offset; + s->dma_length = s->length; + } + + return nelems; + +out_unmap: + pr_err("%s: IOMMU mapping error in map_sg (io-pages: %d)\n", + dev_name(dev), npages); + + for_each_sg(sglist, s, nelems, i) { + int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE); + + for (j = 0; j < pages; ++j) { + unsigned long bus_addr; -unmap: - for_each_sg(sglist, s, mapped_elems, i) { - if (s->dma_address) - __unmap_single(domain->priv, s->dma_address, - s->dma_length, dir); - s->dma_address = s->dma_length = 0; + bus_addr = address + s->dma_address + (j << PAGE_SHIFT); + iommu_unmap_page(domain, bus_addr, PAGE_SIZE); + + if (--mapped_pages) + goto out_free_iova; + } } +out_free_iova: + free_iova_fast(&dma_dom->iovad, address, npages); + +out_err: return 0; } @@ -2452,18 +2512,17 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, struct dma_attrs *attrs) { struct protection_domain *domain; - struct scatterlist *s; - int i; + unsigned long startaddr; + int npages = 2; domain = get_domain(dev); if (IS_ERR(domain)) return; - for_each_sg(sglist, s, nelems, i) { - __unmap_single(domain->priv, s->dma_address, - s->dma_length, dir); - s->dma_address = s->dma_length = 0; - } + startaddr = sg_dma_address(sglist) & PAGE_MASK; + npages = sg_num_pages(dev, sglist, nelems); + + __unmap_single(domain->priv, startaddr, npages << PAGE_SHIFT, dir); } /* -- cgit v1.2.3 From d26592a93ddb7a73cbd82b08e2e0d2985d68d750 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 7 Jul 2016 15:31:13 +0200 Subject: iommu/amd: Use dev_data->domain in get_domain() Using the cached value is much more efficient than calling into the IOMMU core code. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index acad37c3f0a1..155a3e6f1373 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2215,16 +2215,11 @@ static void queue_add(struct dma_ops_domain *dma_dom, static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; - struct iommu_domain *io_domain; if (!check_device(dev)) return ERR_PTR(-EINVAL); - io_domain = iommu_get_domain_for_dev(dev); - if (!io_domain) - return NULL; - - domain = to_pdomain(io_domain); + domain = get_dev_data(dev)->domain; if (!dma_ops_domain(domain)) return ERR_PTR(-EBUSY); -- cgit v1.2.3 From cda7005ba2cbd0744fea343dd5b2aa637eba5b9e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 7 Jul 2016 15:57:04 +0200 Subject: iommu/amd: Handle IOMMU_DOMAIN_DMA in ops->domain_free call-back This domain type is not yet handled in the iommu_ops->domain_free() call-back. Fix that. Fixes: 0bb6e243d7fb ('iommu/amd: Support IOMMU_DOMAIN_DMA type allocation') Cc: stable@vger.kernel.org # v4.2+ Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 155a3e6f1373..5c72d124403c 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2863,9 +2863,7 @@ static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) static void amd_iommu_domain_free(struct iommu_domain *dom) { struct protection_domain *domain; - - if (!dom) - return; + struct dma_ops_domain *dma_dom; domain = to_pdomain(dom); @@ -2874,13 +2872,24 @@ static void amd_iommu_domain_free(struct iommu_domain *dom) BUG_ON(domain->dev_cnt != 0); - if (domain->mode != PAGE_MODE_NONE) - free_pagetable(domain); + if (!dom) + return; - if (domain->flags & PD_IOMMUV2_MASK) - free_gcr3_table(domain); + switch (dom->type) { + case IOMMU_DOMAIN_DMA: + dma_dom = domain->priv; + dma_ops_domain_free(dma_dom); + break; + default: + if (domain->mode != PAGE_MODE_NONE) + free_pagetable(domain); - protection_domain_free(domain); + if (domain->flags & PD_IOMMUV2_MASK) + free_gcr3_table(domain); + + protection_domain_free(domain); + break; + } } static void amd_iommu_detach_device(struct iommu_domain *dom, -- cgit v1.2.3 From 281e8ccbff172899a60579773e72ad63d58b3770 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 7 Jul 2016 16:12:02 +0200 Subject: iommu/amd: Flush iova queue before releasing dma_ops_domain Before a dma_ops_domain can be freed, we need to make sure it is not longer referenced by the flush queue. So empty the queue before a dma_ops_domain can be freed. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 5c72d124403c..d13a18633dce 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2154,12 +2154,10 @@ static void __queue_flush(struct flush_queue *queue) queue->next = 0; } -void queue_flush_timeout(unsigned long unsused) +static void queue_flush_all(void) { int cpu; - atomic_set(&queue_timer_on, 0); - for_each_possible_cpu(cpu) { struct flush_queue *queue; unsigned long flags; @@ -2172,6 +2170,12 @@ void queue_flush_timeout(unsigned long unsused) } } +static void queue_flush_timeout(unsigned long unsused) +{ + atomic_set(&queue_timer_on, 0); + queue_flush_all(); +} + static void queue_add(struct dma_ops_domain *dma_dom, unsigned long address, unsigned long pages) { @@ -2877,6 +2881,13 @@ static void amd_iommu_domain_free(struct iommu_domain *dom) switch (dom->type) { case IOMMU_DOMAIN_DMA: + /* + * First make sure the domain is no longer referenced from the + * flush queue + */ + queue_flush_all(); + + /* Now release the domain */ dma_dom = domain->priv; dma_ops_domain_free(dma_dom); break; -- cgit v1.2.3 From b3311b061de2e51db683a67092546876839df532 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 8 Jul 2016 13:31:31 +0200 Subject: iommu/amd: Use container_of to get dma_ops_domain This is better than storing an extra pointer in struct protection_domain, because this pointer can now be removed from the struct. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 36 ++++++++++++++++++++++++++---------- drivers/iommu/amd_iommu_types.h | 1 - 2 files changed, 26 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d13a18633dce..fb43cc5857c7 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -231,6 +231,12 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) return container_of(dom, struct protection_domain, domain); } +static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domain) +{ + BUG_ON(domain->flags != PD_DMA_OPS_MASK); + return container_of(domain, struct dma_ops_domain, domain); +} + static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; @@ -1670,7 +1676,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) dma_dom->domain.mode = PAGE_MODE_2_LEVEL; dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); dma_dom->domain.flags = PD_DMA_OPS_MASK; - dma_dom->domain.priv = dma_dom; if (!dma_dom->domain.pt_root) goto free_dma_dom; @@ -2367,6 +2372,7 @@ static dma_addr_t map_page(struct device *dev, struct page *page, { phys_addr_t paddr = page_to_phys(page) + offset; struct protection_domain *domain; + struct dma_ops_domain *dma_dom; u64 dma_mask; domain = get_domain(dev); @@ -2376,8 +2382,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page, return DMA_ERROR_CODE; dma_mask = *dev->dma_mask; + dma_dom = to_dma_ops_domain(domain); - return __map_single(dev, domain->priv, paddr, size, dir, dma_mask); + return __map_single(dev, dma_dom, paddr, size, dir, dma_mask); } /* @@ -2387,12 +2394,15 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { struct protection_domain *domain; + struct dma_ops_domain *dma_dom; domain = get_domain(dev); if (IS_ERR(domain)) return; - __unmap_single(domain->priv, dma_addr, size, dir); + dma_dom = to_dma_ops_domain(domain); + + __unmap_single(dma_dom, dma_addr, size, dir); } static int sg_num_pages(struct device *dev, @@ -2440,7 +2450,7 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, if (IS_ERR(domain)) return 0; - dma_dom = domain->priv; + dma_dom = to_dma_ops_domain(domain); dma_mask = *dev->dma_mask; npages = sg_num_pages(dev, sglist, nelems); @@ -2511,6 +2521,7 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, struct dma_attrs *attrs) { struct protection_domain *domain; + struct dma_ops_domain *dma_dom; unsigned long startaddr; int npages = 2; @@ -2519,9 +2530,10 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, return; startaddr = sg_dma_address(sglist) & PAGE_MASK; + dma_dom = to_dma_ops_domain(domain); npages = sg_num_pages(dev, sglist, nelems); - __unmap_single(domain->priv, startaddr, npages << PAGE_SHIFT, dir); + __unmap_single(dma_dom, startaddr, npages << PAGE_SHIFT, dir); } /* @@ -2533,6 +2545,7 @@ static void *alloc_coherent(struct device *dev, size_t size, { u64 dma_mask = dev->coherent_dma_mask; struct protection_domain *domain; + struct dma_ops_domain *dma_dom; struct page *page; domain = get_domain(dev); @@ -2543,6 +2556,7 @@ static void *alloc_coherent(struct device *dev, size_t size, } else if (IS_ERR(domain)) return NULL; + dma_dom = to_dma_ops_domain(domain); size = PAGE_ALIGN(size); dma_mask = dev->coherent_dma_mask; flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); @@ -2562,7 +2576,7 @@ static void *alloc_coherent(struct device *dev, size_t size, if (!dma_mask) dma_mask = *dev->dma_mask; - *dma_addr = __map_single(dev, domain->priv, page_to_phys(page), + *dma_addr = __map_single(dev, dma_dom, page_to_phys(page), size, DMA_BIDIRECTIONAL, dma_mask); if (*dma_addr == DMA_ERROR_CODE) @@ -2586,6 +2600,7 @@ static void free_coherent(struct device *dev, size_t size, struct dma_attrs *attrs) { struct protection_domain *domain; + struct dma_ops_domain *dma_dom; struct page *page; page = virt_to_page(virt_addr); @@ -2595,7 +2610,9 @@ static void free_coherent(struct device *dev, size_t size, if (IS_ERR(domain)) goto free_mem; - __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); + dma_dom = to_dma_ops_domain(domain); + + __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL); free_mem: if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) @@ -2888,7 +2905,7 @@ static void amd_iommu_domain_free(struct iommu_domain *dom) queue_flush_all(); /* Now release the domain */ - dma_dom = domain->priv; + dma_dom = to_dma_ops_domain(domain); dma_ops_domain_free(dma_dom); break; default: @@ -3076,8 +3093,7 @@ static void amd_iommu_apply_dm_region(struct device *dev, struct iommu_domain *domain, struct iommu_dm_region *region) { - struct protection_domain *pdomain = to_pdomain(domain); - struct dma_ops_domain *dma_dom = pdomain->priv; + struct dma_ops_domain *dma_dom = to_dma_ops_domain(to_pdomain(domain)); unsigned long start, end; start = IOVA_PFN(region->start); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 590956ac704e..caf5e3822715 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -421,7 +421,6 @@ struct protection_domain { bool updated; /* complete domain flush required */ unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ - void *priv; /* private data */ }; /* -- cgit v1.2.3 From 5c365d18a73d3979db37006eaacefc0008869c0f Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 13 Jul 2016 13:53:21 +0000 Subject: iommu/vt-d: Return error code in domain_context_mapping_one() In 'commit <55d940430ab9> ("iommu/vt-d: Get rid of domain->iommu_lock")', the error handling path is changed a little, which makes the function always return 0. This path fixes this. Signed-off-by: Wei Yang Fixes: 55d940430ab9 ('iommu/vt-d: Get rid of domain->iommu_lock') Cc: stable@vger.kernel.org # v4.3+ Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 748e5e4b3144..0a97a6da407c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2076,7 +2076,7 @@ out_unlock: spin_unlock(&iommu->lock); spin_unlock_irqrestore(&device_domain_lock, flags); - return 0; + return ret; } struct domain_context_mapping_data { -- cgit v1.2.3 From 3254de6bf74fe94c197c9f819fe62a3a3c36f073 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 26 Jul 2016 15:18:54 +0200 Subject: iommu/amd: Update Alias-DTE in update_device_table() Not doing so might cause IO-Page-Faults when a device uses an alias request-id and the alias-dte is left in a lower page-mode which does not cover the address allocated from the iova-allocator. Fixes: 492667dacc0a ('x86/amd-iommu: Remove amd_iommu_pd_table') Cc: stable@vger.kernel.org Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index fb43cc5857c7..9703913f6bdd 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2239,8 +2239,15 @@ static void update_device_table(struct protection_domain *domain) { struct iommu_dev_data *dev_data; - list_for_each_entry(dev_data, &domain->dev_list, list) + list_for_each_entry(dev_data, &domain->dev_list, list) { set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled); + + if (dev_data->devid == dev_data->alias) + continue; + + /* There is an alias, update device table entry for it */ + set_dte_entry(dev_data->alias, domain, dev_data->ats.enabled); + } } static void update_domain(struct protection_domain *domain) -- cgit v1.2.3 From ffec219770dadd32171e6dd927e1d83d3218529f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 26 Jul 2016 15:31:23 +0200 Subject: iommu/amd: Initialize dma-ops domains with 3-level page-table A two-level page-table can map up to 1GB of address space. With the IOVA allocator now in use, the allocated addresses are often more closely to 4G, which requires the address space to be increased much more often. Avoid that by using a three-level page-table by default. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 9703913f6bdd..33c177ba93be 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1673,7 +1673,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (protection_domain_init(&dma_dom->domain)) goto free_dma_dom; - dma_dom->domain.mode = PAGE_MODE_2_LEVEL; + dma_dom->domain.mode = PAGE_MODE_3_LEVEL; dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); dma_dom->domain.flags = PD_DMA_OPS_MASK; if (!dma_dom->domain.pt_root) -- cgit v1.2.3 From dcddffd41d3f1d3bdcc1dce3f1cd142779b6d4c1 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 26 Jul 2016 15:25:18 -0700 Subject: mm: do not pass mm_struct into handle_mm_fault We always have vma->vm_mm around. Link: http://lkml.kernel.org/r/1466021202-61880-8-git-send-email-kirill.shutemov@linux.intel.com Signed-off-by: Kirill A. Shutemov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/mm/fault.c | 2 +- arch/arc/mm/fault.c | 2 +- arch/arm/mm/fault.c | 2 +- arch/arm64/mm/fault.c | 2 +- arch/avr32/mm/fault.c | 2 +- arch/cris/mm/fault.c | 2 +- arch/frv/mm/fault.c | 2 +- arch/hexagon/mm/vm_fault.c | 2 +- arch/ia64/mm/fault.c | 2 +- arch/m32r/mm/fault.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/metag/mm/fault.c | 2 +- arch/microblaze/mm/fault.c | 2 +- arch/mips/mm/fault.c | 2 +- arch/mn10300/mm/fault.c | 2 +- arch/nios2/mm/fault.c | 2 +- arch/openrisc/mm/fault.c | 2 +- arch/parisc/mm/fault.c | 2 +- arch/powerpc/mm/copro_fault.c | 2 +- arch/powerpc/mm/fault.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/score/mm/fault.c | 2 +- arch/sh/mm/fault.c | 2 +- arch/sparc/mm/fault_32.c | 4 ++-- arch/sparc/mm/fault_64.c | 2 +- arch/tile/mm/fault.c | 2 +- arch/um/kernel/trap.c | 2 +- arch/unicore32/mm/fault.c | 2 +- arch/x86/mm/fault.c | 2 +- arch/xtensa/mm/fault.c | 2 +- drivers/iommu/amd_iommu_v2.c | 3 +-- drivers/iommu/intel-svm.c | 2 +- include/linux/mm.h | 9 ++++----- mm/gup.c | 5 ++--- mm/ksm.c | 5 ++--- mm/memory.c | 13 +++++++------ 36 files changed, 48 insertions(+), 51 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 4a905bd667e2..83e9eee57a55 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -147,7 +147,7 @@ retry: /* If for any reason at all we couldn't handle the fault, make sure we exit gracefully rather than endlessly redo the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index af63f4a13e60..e94e5aa33985 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -137,7 +137,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); /* If Pagefault was interrupted by SIGKILL, exit page fault "early" */ if (unlikely(fatal_signal_pending(current))) { diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index ad5841856007..3a2e678b8d30 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -243,7 +243,7 @@ good_area: goto out; } - return handle_mm_fault(mm, vma, addr & PAGE_MASK, flags); + return handle_mm_fault(vma, addr & PAGE_MASK, flags); check_stack: /* Don't allow expansion below FIRST_USER_ADDRESS */ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index b1166d1e5955..031820d989a8 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -233,7 +233,7 @@ good_area: goto out; } - return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags); + return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); check_stack: if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c index c03533937a9f..a4b7edac8f10 100644 --- a/arch/avr32/mm/fault.c +++ b/arch/avr32/mm/fault.c @@ -134,7 +134,7 @@ good_area: * sure we exit gracefully rather than endlessly redo the * fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c index 3066d40a6db1..112ef26c7f2e 100644 --- a/arch/cris/mm/fault.c +++ b/arch/cris/mm/fault.c @@ -168,7 +168,7 @@ retry: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index 61d99767fe16..614a46c413d2 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c @@ -164,7 +164,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, ear0, flags); + fault = handle_mm_fault(vma, ear0, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index 8704c9320032..bd7c251e2bce 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -101,7 +101,7 @@ good_area: break; } - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 70b40d1205a6..fa6ad95e992e 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -159,7 +159,7 @@ retry: * sure we exit gracefully rather than endlessly redo the * fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c index 8f9875b7933d..a3785d3644c2 100644 --- a/arch/m32r/mm/fault.c +++ b/arch/m32r/mm/fault.c @@ -196,7 +196,7 @@ good_area: */ addr = (address & PAGE_MASK); set_thread_fault_code(error_code); - fault = handle_mm_fault(mm, vma, addr, flags); + fault = handle_mm_fault(vma, addr, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 6a94cdd0c830..bd66a0b20c6b 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -136,7 +136,7 @@ good_area: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); pr_debug("handle_mm_fault returns %d\n", fault); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) diff --git a/arch/metag/mm/fault.c b/arch/metag/mm/fault.c index f57edca63609..372783a67dda 100644 --- a/arch/metag/mm/fault.c +++ b/arch/metag/mm/fault.c @@ -133,7 +133,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index 177dfc003643..abb678ccde6f 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -216,7 +216,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 4b88fa031891..9560ad731120 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -153,7 +153,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c index 4a1d181ed32f..f23781d6bbb3 100644 --- a/arch/mn10300/mm/fault.c +++ b/arch/mn10300/mm/fault.c @@ -254,7 +254,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index b51878b0c6b8..affc4eb3f89e 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -131,7 +131,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index 230ac20ae794..e94cd225e816 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -163,7 +163,7 @@ good_area: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 16dbe81c97c9..163af2c31d76 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -239,7 +239,7 @@ good_area: * fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index 6527882ce05e..bb0354222b11 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -75,7 +75,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, } ret = 0; - *flt = handle_mm_fault(mm, vma, ea, is_write ? FAULT_FLAG_WRITE : 0); + *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0); if (unlikely(*flt & VM_FAULT_ERROR)) { if (*flt & VM_FAULT_OOM) { ret = -ENOMEM; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index a67c6d781c52..a4db22f65021 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -429,7 +429,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & (VM_FAULT_RETRY|VM_FAULT_ERROR))) { if (fault & VM_FAULT_SIGSEGV) goto bad_area; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 19288c1b36d3..6c47488745ae 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -456,7 +456,7 @@ retry: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); /* No reason to continue if interrupted by SIGKILL. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { fault = VM_FAULT_SIGNAL; diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c index 37a6c2e0e969..995b71e4db4b 100644 --- a/arch/score/mm/fault.c +++ b/arch/score/mm/fault.c @@ -111,7 +111,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 79d8276377d1..9bf876780cef 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -487,7 +487,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR))) if (mm_fault_error(regs, error_code, address, fault)) diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index b6c559cbd64d..4714061d6cd3 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -241,7 +241,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; @@ -411,7 +411,7 @@ good_area: if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, flags)) { + switch (handle_mm_fault(vma, address, flags)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index cb841a33da59..6c43b924a7a2 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -436,7 +436,7 @@ good_area: goto bad_area; } - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto exit_exception; diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c index 26734214818c..beba986589e5 100644 --- a/arch/tile/mm/fault.c +++ b/arch/tile/mm/fault.c @@ -434,7 +434,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 98783dd0fa2e..ad8f206ab5e8 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -73,7 +73,7 @@ good_area: do { int fault; - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto out_nosemaphore; diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 2ec3d3adcefc..6c7f70bcaae3 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c @@ -194,7 +194,7 @@ good_area: * If for any reason at all we couldn't handle the fault, make * sure we exit gracefully rather than endlessly redo the fault. */ - fault = handle_mm_fault(mm, vma, addr & PAGE_MASK, flags); + fault = handle_mm_fault(vma, addr & PAGE_MASK, flags); return fault; check_stack: diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index d22161ab941d..dc8023060456 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1353,7 +1353,7 @@ good_area: * the fault. Since we never set FAULT_FLAG_RETRY_NOWAIT, if * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); major |= fault & VM_FAULT_MAJOR; /* diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 7f4a1fdb1502..2725e08ef353 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -110,7 +110,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 56999d2fac07..fbdaf81ae925 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -538,8 +538,7 @@ static void do_fault(struct work_struct *work) if (access_error(vma, fault)) goto out; - ret = handle_mm_fault(mm, vma, address, flags); - + ret = handle_mm_fault(vma, address, flags); out: up_read(&mm->mmap_sem); diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index d9939fa9b588..8ebb3530afa7 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -583,7 +583,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (access_error(vma, req)) goto invalid; - ret = handle_mm_fault(svm->mm, vma, address, + ret = handle_mm_fault(vma, address, req->wr_req ? FAULT_FLAG_WRITE : 0); if (ret & VM_FAULT_ERROR) goto invalid; diff --git a/include/linux/mm.h b/include/linux/mm.h index 6c9a394b2979..646bc36b4d1b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1215,15 +1215,14 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page); int invalidate_inode_page(struct page *page); #ifdef CONFIG_MMU -extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long address, unsigned int flags); +extern int handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + unsigned int flags); extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked); #else -static inline int handle_mm_fault(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long address, - unsigned int flags) +static inline int handle_mm_fault(struct vm_area_struct *vma, + unsigned long address, unsigned int flags) { /* should never happen if there's no MMU */ BUG(); diff --git a/mm/gup.c b/mm/gup.c index dee142e100f4..9671e29f8ffd 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -352,7 +352,6 @@ unmap: static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, unsigned long address, unsigned int *flags, int *nonblocking) { - struct mm_struct *mm = vma->vm_mm; unsigned int fault_flags = 0; int ret; @@ -377,7 +376,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, fault_flags |= FAULT_FLAG_TRIED; } - ret = handle_mm_fault(mm, vma, address, fault_flags); + ret = handle_mm_fault(vma, address, fault_flags); if (ret & VM_FAULT_ERROR) { if (ret & VM_FAULT_OOM) return -ENOMEM; @@ -692,7 +691,7 @@ retry: if (!vma_permits_fault(vma, fault_flags)) return -EFAULT; - ret = handle_mm_fault(mm, vma, address, fault_flags); + ret = handle_mm_fault(vma, address, fault_flags); major |= ret & VM_FAULT_MAJOR; if (ret & VM_FAULT_ERROR) { if (ret & VM_FAULT_OOM) diff --git a/mm/ksm.c b/mm/ksm.c index 35b8aef867a9..73d43bafd9fb 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -376,9 +376,8 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) if (IS_ERR_OR_NULL(page)) break; if (PageKsm(page)) - ret = handle_mm_fault(vma->vm_mm, vma, addr, - FAULT_FLAG_WRITE | - FAULT_FLAG_REMOTE); + ret = handle_mm_fault(vma, addr, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE); else ret = VM_FAULT_WRITE; put_page(page); diff --git a/mm/memory.c b/mm/memory.c index 5e6eadd127e7..6bf2b8564376 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3420,9 +3420,10 @@ unlock: * The mmap_sem may have been released depending on flags and our * return value. See filemap_fault() and __lock_page_or_retry(). */ -static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long address, unsigned int flags) +static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + unsigned int flags) { + struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; pud_t *pud; pmd_t *pmd; @@ -3509,15 +3510,15 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, * The mmap_sem may have been released depending on flags and our * return value. See filemap_fault() and __lock_page_or_retry(). */ -int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long address, unsigned int flags) +int handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + unsigned int flags) { int ret; __set_current_state(TASK_RUNNING); count_vm_event(PGFAULT); - mem_cgroup_count_vm_event(mm, PGFAULT); + mem_cgroup_count_vm_event(vma->vm_mm, PGFAULT); /* do counter updates before entering really critical section. */ check_sync_rss_stat(current); @@ -3529,7 +3530,7 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, if (flags & FAULT_FLAG_USER) mem_cgroup_oom_enable(); - ret = __handle_mm_fault(mm, vma, address, flags); + ret = __handle_mm_fault(vma, address, flags); if (flags & FAULT_FLAG_USER) { mem_cgroup_oom_disable(); -- cgit v1.2.3 From 194dc870a5890e855ecffb30f3b80ba7c88f96d6 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Wed, 27 Jul 2016 20:03:31 -0700 Subject: Add braces to avoid "ambiguous ‘else’" compiler warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of our "for_each_xyz()" macro constructs make gcc unhappy about lack of braces around if-statements inside or outside the loop, because the loop construct itself has a "if-then-else" statement inside of it. The resulting warnings look something like this: drivers/gpu/drm/i915/i915_debugfs.c: In function ‘i915_dump_lrc’: drivers/gpu/drm/i915/i915_debugfs.c:2103:6: warning: suggest explicit braces to avoid ambiguous ‘else’ [-Wparentheses] if (ctx != dev_priv->kernel_context) ^ even if the code itself is fine. Since the warning is fairly easy to avoid by adding a braces around the if-statement near the for_each_xyz() construct, do so, rather than disabling the otherwise potentially useful warning. (The if-then-else statements used in the "for_each_xyz()" constructs are designed to be inherently safe even with no braces, but in this case it's quite understandable that gcc isn't really able to tell that). This finally leaves the standard "allmodconfig" build with just a handful of remaining warnings, so new and valid warnings hopefully will stand out. Signed-off-by: Linus Torvalds --- drivers/gpu/drm/i915/i915_debugfs.c | 3 ++- drivers/iommu/dmar.c | 3 ++- drivers/iommu/intel-iommu.c | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 103546834b60..2a6e12956baf 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2100,9 +2100,10 @@ static int i915_dump_lrc(struct seq_file *m, void *unused) return ret; list_for_each_entry(ctx, &dev_priv->context_list, link) - if (ctx != dev_priv->kernel_context) + if (ctx != dev_priv->kernel_context) { for_each_engine(engine, dev_priv) i915_dump_lrc_obj(m, ctx, engine); + } mutex_unlock(&dev->struct_mutex); diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6a86b5d1defa..7330a66e2b7e 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1871,10 +1871,11 @@ static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg) /* * All PCI devices managed by this unit should have been destroyed. */ - if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) + if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) { for_each_active_dev_scope(dmaru->devices, dmaru->devices_cnt, i, dev) return -EBUSY; + } ret = dmar_ir_hotplug(dmaru, false); if (ret == 0) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 323dac9900ba..4b9040bb2f1c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4272,10 +4272,11 @@ int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg) if (!atsru) return 0; - if (!atsru->include_all && atsru->devices && atsru->devices_cnt) + if (!atsru->include_all && atsru->devices && atsru->devices_cnt) { for_each_active_dev_scope(atsru->devices, atsru->devices_cnt, i, dev) return -EBUSY; + } return 0; } -- cgit v1.2.3 From 00085f1efa387a8ce100e3734920f7639c80caa3 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 3 Aug 2016 13:46:00 -0700 Subject: dma-mapping: use unsigned long for dma_attrs The dma-mapping core and the implementations do not change the DMA attributes passed by pointer. Thus the pointer can point to const data. However the attributes do not have to be a bitfield. Instead unsigned long will do fine: 1. This is just simpler. Both in terms of reading the code and setting attributes. Instead of initializing local attributes on the stack and passing pointer to it to dma_set_attr(), just set the bits. 2. It brings safeness and checking for const correctness because the attributes are passed by value. Semantic patches for this change (at least most of them): virtual patch virtual context @r@ identifier f, attrs; @@ f(..., - struct dma_attrs *attrs + unsigned long attrs , ...) { ... } @@ identifier r.f; @@ f(..., - NULL + 0 ) and // Options: --all-includes virtual patch virtual context @r@ identifier f, attrs; type t; @@ t f(..., struct dma_attrs *attrs); @@ identifier r.f; @@ f(..., - NULL + 0 ) Link: http://lkml.kernel.org/r/1468399300-5399-2-git-send-email-k.kozlowski@samsung.com Signed-off-by: Krzysztof Kozlowski Acked-by: Vineet Gupta Acked-by: Robin Murphy Acked-by: Hans-Christian Noren Egtvedt Acked-by: Mark Salter [c6x] Acked-by: Jesper Nilsson [cris] Acked-by: Daniel Vetter [drm] Reviewed-by: Bart Van Assche Acked-by: Joerg Roedel [iommu] Acked-by: Fabien Dessenne [bdisp] Reviewed-by: Marek Szyprowski [vb2-core] Acked-by: David Vrabel [xen] Acked-by: Konrad Rzeszutek Wilk [xen swiotlb] Acked-by: Joerg Roedel [iommu] Acked-by: Richard Kuo [hexagon] Acked-by: Geert Uytterhoeven [m68k] Acked-by: Gerald Schaefer [s390] Acked-by: Bjorn Andersson Acked-by: Hans-Christian Noren Egtvedt [avr32] Acked-by: Vineet Gupta [arc] Acked-by: Robin Murphy [arm64 and dma-iommu] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/DMA-API.txt | 33 +++--- Documentation/DMA-attributes.txt | 2 +- arch/alpha/include/asm/dma-mapping.h | 2 - arch/alpha/kernel/pci-noop.c | 2 +- arch/alpha/kernel/pci_iommu.c | 12 +- arch/arc/mm/dma.c | 12 +- arch/arm/common/dmabounce.c | 4 +- arch/arm/include/asm/dma-mapping.h | 13 +-- arch/arm/include/asm/xen/page-coherent.h | 16 +-- arch/arm/mm/dma-mapping.c | 129 ++++++++++----------- arch/arm/xen/mm.c | 8 +- arch/arm64/mm/dma-mapping.c | 66 +++++------ arch/avr32/mm/dma-coherent.c | 12 +- arch/blackfin/kernel/dma-mapping.c | 8 +- arch/c6x/include/asm/dma-mapping.h | 4 +- arch/c6x/kernel/dma.c | 9 +- arch/c6x/mm/dma-coherent.c | 4 +- arch/cris/arch-v32/drivers/pci/dma.c | 9 +- arch/frv/mb93090-mb00/pci-dma-nommu.c | 8 +- arch/frv/mb93090-mb00/pci-dma.c | 9 +- arch/h8300/kernel/dma.c | 8 +- arch/hexagon/include/asm/dma-mapping.h | 1 - arch/hexagon/kernel/dma.c | 8 +- arch/ia64/hp/common/sba_iommu.c | 22 ++-- arch/ia64/include/asm/machvec.h | 1 - arch/ia64/kernel/pci-swiotlb.c | 4 +- arch/ia64/sn/pci/pci_dma.c | 22 ++-- arch/m68k/kernel/dma.c | 12 +- arch/metag/kernel/dma.c | 16 +-- arch/microblaze/include/asm/dma-mapping.h | 1 - arch/microblaze/kernel/dma.c | 12 +- arch/mips/cavium-octeon/dma-octeon.c | 8 +- arch/mips/loongson64/common/dma-swiotlb.c | 10 +- arch/mips/mm/dma-default.c | 20 ++-- arch/mips/netlogic/common/nlm-dma.c | 4 +- arch/mn10300/mm/dma-alloc.c | 8 +- arch/nios2/mm/dma-mapping.c | 12 +- arch/openrisc/kernel/dma.c | 21 ++-- arch/parisc/kernel/pci-dma.c | 18 +-- arch/powerpc/include/asm/dma-mapping.h | 7 +- arch/powerpc/include/asm/iommu.h | 10 +- arch/powerpc/kernel/dma-iommu.c | 12 +- arch/powerpc/kernel/dma.c | 18 +-- arch/powerpc/kernel/ibmebus.c | 12 +- arch/powerpc/kernel/iommu.c | 12 +- arch/powerpc/kernel/vio.c | 12 +- arch/powerpc/platforms/cell/iommu.c | 28 ++--- arch/powerpc/platforms/pasemi/iommu.c | 2 +- arch/powerpc/platforms/powernv/npu-dma.c | 8 +- arch/powerpc/platforms/powernv/pci-ioda.c | 4 +- arch/powerpc/platforms/powernv/pci.c | 2 +- arch/powerpc/platforms/powernv/pci.h | 2 +- arch/powerpc/platforms/ps3/system-bus.c | 18 +-- arch/powerpc/platforms/pseries/iommu.c | 6 +- arch/powerpc/sysdev/dart_iommu.c | 2 +- arch/s390/include/asm/dma-mapping.h | 1 - arch/s390/pci/pci_dma.c | 23 ++-- arch/sh/include/asm/dma-mapping.h | 4 +- arch/sh/kernel/dma-nommu.c | 4 +- arch/sh/mm/consistent.c | 4 +- arch/sparc/kernel/iommu.c | 12 +- arch/sparc/kernel/ioport.c | 24 ++-- arch/sparc/kernel/pci_sun4v.c | 12 +- arch/tile/kernel/pci-dma.c | 28 ++--- arch/unicore32/mm/dma-swiotlb.c | 4 +- arch/x86/include/asm/dma-mapping.h | 5 +- arch/x86/include/asm/swiotlb.h | 4 +- arch/x86/include/asm/xen/page-coherent.h | 9 +- arch/x86/kernel/amd_gart_64.c | 20 ++-- arch/x86/kernel/pci-calgary_64.c | 14 +-- arch/x86/kernel/pci-dma.c | 4 +- arch/x86/kernel/pci-nommu.c | 4 +- arch/x86/kernel/pci-swiotlb.c | 4 +- arch/x86/pci/sta2x11-fixup.c | 2 +- arch/x86/pci/vmd.c | 16 +-- arch/xtensa/kernel/pci-dma.c | 12 +- drivers/gpu/drm/exynos/exynos_drm_fbdev.c | 2 +- drivers/gpu/drm/exynos/exynos_drm_g2d.c | 12 +- drivers/gpu/drm/exynos/exynos_drm_gem.c | 20 ++-- drivers/gpu/drm/exynos/exynos_drm_gem.h | 2 +- drivers/gpu/drm/mediatek/mtk_drm_gem.c | 13 +-- drivers/gpu/drm/mediatek/mtk_drm_gem.h | 2 +- drivers/gpu/drm/msm/msm_drv.c | 13 +-- .../gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c | 13 +-- drivers/gpu/drm/rockchip/rockchip_drm_gem.c | 17 ++- drivers/gpu/drm/rockchip/rockchip_drm_gem.h | 2 +- drivers/infiniband/core/umem.c | 7 +- drivers/iommu/amd_iommu.c | 12 +- drivers/iommu/dma-iommu.c | 8 +- drivers/iommu/intel-iommu.c | 12 +- drivers/media/platform/sti/bdisp/bdisp-hw.c | 26 ++--- drivers/media/v4l2-core/videobuf2-dma-contig.c | 28 ++--- drivers/media/v4l2-core/videobuf2-dma-sg.c | 21 +--- drivers/media/v4l2-core/videobuf2-vmalloc.c | 2 +- drivers/misc/mic/host/mic_boot.c | 20 ++-- drivers/parisc/ccio-dma.c | 16 +-- drivers/parisc/sba_iommu.c | 16 +-- drivers/remoteproc/qcom_q6v5_pil.c | 7 +- drivers/video/fbdev/omap2/omapfb/omapfb-main.c | 12 +- drivers/video/fbdev/omap2/omapfb/omapfb.h | 3 +- drivers/xen/swiotlb-xen.c | 14 +-- include/linux/dma-attrs.h | 71 ------------ include/linux/dma-iommu.h | 6 +- include/linux/dma-mapping.h | 128 +++++++++++++------- include/linux/swiotlb.h | 10 +- include/media/videobuf2-core.h | 6 +- include/media/videobuf2-dma-contig.h | 2 - include/rdma/ib_verbs.h | 17 +-- include/xen/swiotlb-xen.h | 12 +- lib/dma-noop.c | 9 +- lib/swiotlb.c | 13 ++- 111 files changed, 705 insertions(+), 804 deletions(-) delete mode 100644 include/linux/dma-attrs.h (limited to 'drivers/iommu') diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt index 45ef3f279c3b..1d26eeb6b5f6 100644 --- a/Documentation/DMA-API.txt +++ b/Documentation/DMA-API.txt @@ -369,35 +369,32 @@ See also dma_map_single(). dma_addr_t dma_map_single_attrs(struct device *dev, void *cpu_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) void dma_unmap_single_attrs(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) int dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) The four functions above are just like the counterpart functions without the _attrs suffixes, except that they pass an optional -struct dma_attrs*. - -struct dma_attrs encapsulates a set of "DMA attributes". For the -definition of struct dma_attrs see linux/dma-attrs.h. +dma_attrs. The interpretation of DMA attributes is architecture-specific, and each attribute should be documented in Documentation/DMA-attributes.txt. -If struct dma_attrs* is NULL, the semantics of each of these -functions is identical to those of the corresponding function +If dma_attrs are 0, the semantics of each of these functions +is identical to those of the corresponding function without the _attrs suffix. As a result dma_map_single_attrs() can generally replace dma_map_single(), etc. @@ -405,15 +402,15 @@ As an example of the use of the *_attrs functions, here's how you could pass an attribute DMA_ATTR_FOO when mapping memory for DMA: -#include -/* DMA_ATTR_FOO should be defined in linux/dma-attrs.h and +#include +/* DMA_ATTR_FOO should be defined in linux/dma-mapping.h and * documented in Documentation/DMA-attributes.txt */ ... - DEFINE_DMA_ATTRS(attrs); - dma_set_attr(DMA_ATTR_FOO, &attrs); + unsigned long attr; + attr |= DMA_ATTR_FOO; .... - n = dma_map_sg_attrs(dev, sg, nents, DMA_TO_DEVICE, &attr); + n = dma_map_sg_attrs(dev, sg, nents, DMA_TO_DEVICE, attr); .... Architectures that care about DMA_ATTR_FOO would check for its @@ -422,12 +419,10 @@ routines, e.g.: void whizco_dma_map_sg_attrs(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { .... - int foo = dma_get_attr(DMA_ATTR_FOO, attrs); - .... - if (foo) + if (attrs & DMA_ATTR_FOO) /* twizzle the frobnozzle */ .... diff --git a/Documentation/DMA-attributes.txt b/Documentation/DMA-attributes.txt index e8cf9cf873b3..2d455a5cf671 100644 --- a/Documentation/DMA-attributes.txt +++ b/Documentation/DMA-attributes.txt @@ -2,7 +2,7 @@ ============== This document describes the semantics of the DMA attributes that are -defined in linux/dma-attrs.h. +defined in linux/dma-mapping.h. DMA_ATTR_WRITE_BARRIER ---------------------- diff --git a/arch/alpha/include/asm/dma-mapping.h b/arch/alpha/include/asm/dma-mapping.h index 3c3451f58ff4..c63b6ac19ee5 100644 --- a/arch/alpha/include/asm/dma-mapping.h +++ b/arch/alpha/include/asm/dma-mapping.h @@ -1,8 +1,6 @@ #ifndef _ALPHA_DMA_MAPPING_H #define _ALPHA_DMA_MAPPING_H -#include - extern struct dma_map_ops *dma_ops; static inline struct dma_map_ops *get_dma_ops(struct device *dev) diff --git a/arch/alpha/kernel/pci-noop.c b/arch/alpha/kernel/pci-noop.c index 8e735b5e56bd..bb152e21e5ae 100644 --- a/arch/alpha/kernel/pci-noop.c +++ b/arch/alpha/kernel/pci-noop.c @@ -109,7 +109,7 @@ sys_pciconfig_write(unsigned long bus, unsigned long dfn, static void *alpha_noop_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret; diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index 8969bf2dfe3a..451fc9cdd323 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -349,7 +349,7 @@ static struct pci_dev *alpha_gendev_to_pci(struct device *dev) static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); int dac_allowed; @@ -369,7 +369,7 @@ static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags; struct pci_dev *pdev = alpha_gendev_to_pci(dev); @@ -433,7 +433,7 @@ static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, static void *alpha_pci_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); void *cpu_addr; @@ -478,7 +478,7 @@ try_again: static void alpha_pci_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); @@ -651,7 +651,7 @@ sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); struct scatterlist *start, *end, *out; @@ -729,7 +729,7 @@ static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); unsigned long flags; diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index ab74b5d9186c..20afc65e22dc 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -22,7 +22,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { unsigned long order = get_order(size); struct page *page; @@ -46,7 +46,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size, * (vs. always going to memory - thus are faster) */ if ((is_isa_arcv2() && ioc_exists) || - dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) + (attrs & DMA_ATTR_NON_CONSISTENT)) need_coh = 0; /* @@ -90,13 +90,13 @@ static void *arc_dma_alloc(struct device *dev, size_t size, } static void arc_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { phys_addr_t paddr = plat_dma_to_phys(dev, dma_handle); struct page *page = virt_to_page(paddr); int is_non_coh = 1; - is_non_coh = dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs) || + is_non_coh = (attrs & DMA_ATTR_NON_CONSISTENT) || (is_isa_arcv2() && ioc_exists); if (PageHighMem(page) || !is_non_coh) @@ -130,7 +130,7 @@ static void _dma_cache_sync(phys_addr_t paddr, size_t size, static dma_addr_t arc_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { phys_addr_t paddr = page_to_phys(page) + offset; _dma_cache_sync(paddr, size, dir); @@ -138,7 +138,7 @@ static dma_addr_t arc_dma_map_page(struct device *dev, struct page *page, } static int arc_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *s; int i; diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c index 1143c4d5c567..301281645d08 100644 --- a/arch/arm/common/dmabounce.c +++ b/arch/arm/common/dmabounce.c @@ -310,7 +310,7 @@ static inline void unmap_single(struct device *dev, struct safe_buffer *buf, */ static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t dma_addr; int ret; @@ -344,7 +344,7 @@ static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page, * should be) */ static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct safe_buffer *buf; diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index a83570f10124..d009f7911ffc 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -174,7 +173,7 @@ static inline void dma_mark_clean(void *addr, size_t size) { } * to be the device-viewed address. */ extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, struct dma_attrs *attrs); + gfp_t gfp, unsigned long attrs); /** * arm_dma_free - free memory allocated by arm_dma_alloc @@ -191,7 +190,7 @@ extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, * during and after this call executing are illegal. */ extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs); + dma_addr_t handle, unsigned long attrs); /** * arm_dma_mmap - map a coherent DMA allocation into user space @@ -208,7 +207,7 @@ extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, */ extern int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs); + unsigned long attrs); /* * This can be called during early boot to increase the size of the atomic @@ -262,16 +261,16 @@ extern void dmabounce_unregister_dev(struct device *); * The scatter list versions of the above methods. */ extern int arm_dma_map_sg(struct device *, struct scatterlist *, int, - enum dma_data_direction, struct dma_attrs *attrs); + enum dma_data_direction, unsigned long attrs); extern void arm_dma_unmap_sg(struct device *, struct scatterlist *, int, - enum dma_data_direction, struct dma_attrs *attrs); + enum dma_data_direction, unsigned long attrs); extern void arm_dma_sync_sg_for_cpu(struct device *, struct scatterlist *, int, enum dma_data_direction); extern void arm_dma_sync_sg_for_device(struct device *, struct scatterlist *, int, enum dma_data_direction); extern int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs); + unsigned long attrs); #endif /* __KERNEL__ */ #endif diff --git a/arch/arm/include/asm/xen/page-coherent.h b/arch/arm/include/asm/xen/page-coherent.h index 9408a994cc91..95ce6ac3a971 100644 --- a/arch/arm/include/asm/xen/page-coherent.h +++ b/arch/arm/include/asm/xen/page-coherent.h @@ -2,15 +2,14 @@ #define _ASM_ARM_XEN_PAGE_COHERENT_H #include -#include #include void __xen_dma_map_page(struct device *hwdev, struct page *page, dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs); + enum dma_data_direction dir, unsigned long attrs); void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs); + unsigned long attrs); void __xen_dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir); @@ -18,22 +17,20 @@ void __xen_dma_sync_single_for_device(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir); static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs) { return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs); } static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { __generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs); } static inline void xen_dma_map_page(struct device *hwdev, struct page *page, dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { unsigned long page_pfn = page_to_xen_pfn(page); unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr); @@ -58,8 +55,7 @@ static inline void xen_dma_map_page(struct device *hwdev, struct page *page, } static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + size_t size, enum dma_data_direction dir, unsigned long attrs) { unsigned long pfn = PFN_DOWN(handle); /* diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index b7eed75960fe..c6834c0cfd1c 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -128,16 +128,16 @@ static void __dma_page_dev_to_cpu(struct page *, unsigned long, */ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_cpu_to_dev(page, offset, size, dir); return pfn_to_dma(dev, page_to_pfn(page)) + offset; } static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return pfn_to_dma(dev, page_to_pfn(page)) + offset; } @@ -157,10 +157,9 @@ static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *pag * whatever the device wrote there. */ static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + size_t size, enum dma_data_direction dir, unsigned long attrs) { - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), handle & ~PAGE_MASK, size, dir); } @@ -198,12 +197,12 @@ struct dma_map_ops arm_dma_ops = { EXPORT_SYMBOL(arm_dma_ops); static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); + dma_addr_t *handle, gfp_t gfp, unsigned long attrs); static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs); + dma_addr_t handle, unsigned long attrs); static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs); + unsigned long attrs); struct dma_map_ops arm_coherent_dma_ops = { .alloc = arm_coherent_dma_alloc, @@ -639,11 +638,11 @@ static void __free_from_contiguous(struct device *dev, struct page *page, dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } -static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) +static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot) { - prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ? - pgprot_writecombine(prot) : - pgprot_dmacoherent(prot); + prot = (attrs & DMA_ATTR_WRITE_COMBINE) ? + pgprot_writecombine(prot) : + pgprot_dmacoherent(prot); return prot; } @@ -751,7 +750,7 @@ static struct arm_dma_allocator remap_allocator = { static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot, bool is_coherent, - struct dma_attrs *attrs, const void *caller) + unsigned long attrs, const void *caller) { u64 mask = get_coherent_dma_mask(dev); struct page *page = NULL; @@ -764,7 +763,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, .gfp = gfp, .prot = prot, .caller = caller, - .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), + .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), .coherent_flag = is_coherent ? COHERENT : NORMAL, }; @@ -834,7 +833,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, * virtual and bus address for that space. */ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, struct dma_attrs *attrs) + gfp_t gfp, unsigned long attrs) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); @@ -843,7 +842,7 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, attrs, __builtin_return_address(0)); @@ -851,7 +850,7 @@ static void *arm_coherent_dma_alloc(struct device *dev, size_t size, static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { int ret = -ENXIO; #ifdef CONFIG_MMU @@ -879,14 +878,14 @@ static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, */ static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef CONFIG_MMU vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); @@ -898,7 +897,7 @@ int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, * Free a buffer as defined by the above mapping. */ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs, + dma_addr_t handle, unsigned long attrs, bool is_coherent) { struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); @@ -908,7 +907,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, .size = PAGE_ALIGN(size), .cpu_addr = cpu_addr, .page = page, - .want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs), + .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), }; buf = arm_dma_buffer_find(cpu_addr); @@ -920,20 +919,20 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, } void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs) + dma_addr_t handle, unsigned long attrs) { __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); } static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs) + dma_addr_t handle, unsigned long attrs) { __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); } int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t handle, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); int ret; @@ -1066,7 +1065,7 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, * here. */ int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct dma_map_ops *ops = get_dma_ops(dev); struct scatterlist *s; @@ -1100,7 +1099,7 @@ int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, * rules concerning calls here are the same as for dma_unmap_single(). */ void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct dma_map_ops *ops = get_dma_ops(dev); struct scatterlist *s; @@ -1273,7 +1272,7 @@ static inline void __free_iova(struct dma_iommu_mapping *mapping, static const int iommu_order_array[] = { 9, 8, 4, 0 }; static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, - gfp_t gfp, struct dma_attrs *attrs, + gfp_t gfp, unsigned long attrs, int coherent_flag) { struct page **pages; @@ -1289,7 +1288,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, if (!pages) return NULL; - if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) + if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { unsigned long order = get_order(size); struct page *page; @@ -1307,7 +1306,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, } /* Go straight to 4K chunks if caller says it's OK. */ - if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) + if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) order_idx = ARRAY_SIZE(iommu_order_array) - 1; /* @@ -1363,12 +1362,12 @@ error: } static int __iommu_free_buffer(struct device *dev, struct page **pages, - size_t size, struct dma_attrs *attrs) + size_t size, unsigned long attrs) { int count = size >> PAGE_SHIFT; int i; - if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { + if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { dma_release_from_contiguous(dev, pages[0], count); } else { for (i = 0; i < count; i++) @@ -1460,14 +1459,14 @@ static struct page **__atomic_get_pages(void *addr) return (struct page **)page; } -static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) +static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs) { struct vm_struct *area; if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) return __atomic_get_pages(cpu_addr); - if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) return cpu_addr; area = find_vm_area(cpu_addr); @@ -1511,7 +1510,7 @@ static void __iommu_free_atomic(struct device *dev, void *cpu_addr, } static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs, int coherent_flag) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); @@ -1542,7 +1541,7 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, if (*handle == DMA_ERROR_CODE) goto err_buffer; - if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) return pages; addr = __iommu_alloc_remap(pages, size, gfp, prot, @@ -1560,20 +1559,20 @@ err_buffer: } static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL); } static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT); } static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long uaddr = vma->vm_start; unsigned long usize = vma->vm_end - vma->vm_start; @@ -1603,7 +1602,7 @@ static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma } static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs) + dma_addr_t dma_addr, size_t size, unsigned long attrs) { vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); @@ -1612,7 +1611,7 @@ static int arm_iommu_mmap_attrs(struct device *dev, static int arm_coherent_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs) + dma_addr_t dma_addr, size_t size, unsigned long attrs) { return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); } @@ -1622,7 +1621,7 @@ static int arm_coherent_iommu_mmap_attrs(struct device *dev, * Must not be called with IRQs disabled. */ void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs, int coherent_flag) + dma_addr_t handle, unsigned long attrs, int coherent_flag) { struct page **pages; size = PAGE_ALIGN(size); @@ -1638,7 +1637,7 @@ void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, return; } - if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { + if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0) { dma_common_free_remap(cpu_addr, size, VM_ARM_DMA_CONSISTENT | VM_USERMAP); } @@ -1648,20 +1647,20 @@ void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, } void arm_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, struct dma_attrs *attrs) + void *cpu_addr, dma_addr_t handle, unsigned long attrs) { __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL); } void arm_coherent_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, struct dma_attrs *attrs) + void *cpu_addr, dma_addr_t handle, unsigned long attrs) { __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT); } static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, - size_t size, struct dma_attrs *attrs) + size_t size, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page **pages = __iommu_get_pages(cpu_addr, attrs); @@ -1699,7 +1698,7 @@ static int __dma_direction_to_prot(enum dma_data_direction dir) */ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, size_t size, dma_addr_t *handle, - enum dma_data_direction dir, struct dma_attrs *attrs, + enum dma_data_direction dir, unsigned long attrs, bool is_coherent) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); @@ -1720,8 +1719,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, phys_addr_t phys = page_to_phys(sg_page(s)); unsigned int len = PAGE_ALIGN(s->offset + s->length); - if (!is_coherent && - !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); prot = __dma_direction_to_prot(dir); @@ -1742,7 +1740,7 @@ fail: } static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs, + enum dma_data_direction dir, unsigned long attrs, bool is_coherent) { struct scatterlist *s = sg, *dma = sg, *start = sg; @@ -1800,7 +1798,7 @@ bad_mapping: * obtained via sg_dma_{address,length}. */ int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { return __iommu_map_sg(dev, sg, nents, dir, attrs, true); } @@ -1818,14 +1816,14 @@ int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, * sg_dma_{address,length}. */ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { return __iommu_map_sg(dev, sg, nents, dir, attrs, false); } static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs, - bool is_coherent) + int nents, enum dma_data_direction dir, + unsigned long attrs, bool is_coherent) { struct scatterlist *s; int i; @@ -1834,8 +1832,7 @@ static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, if (sg_dma_len(s)) __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); - if (!is_coherent && - !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); } @@ -1852,7 +1849,8 @@ static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, * rules concerning calls here are the same as for dma_unmap_single(). */ void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, + unsigned long attrs) { __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); } @@ -1868,7 +1866,8 @@ void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, * rules concerning calls here are the same as for dma_unmap_single(). */ void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, + unsigned long attrs) { __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); } @@ -1921,7 +1920,7 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, */ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t dma_addr; @@ -1955,9 +1954,9 @@ fail: */ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_cpu_to_dev(page, offset, size, dir); return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); @@ -1973,8 +1972,7 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, * Coherent IOMMU aware version of arm_dma_unmap_page() */ static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; @@ -1998,8 +1996,7 @@ static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, * IOMMU aware version of arm_dma_unmap_page() */ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; @@ -2010,7 +2007,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, if (!iova) return; - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_dev_to_cpu(page, offset, size, dir); iommu_unmap(mapping->domain, iova, len); diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index c5f9a9e3d1f3..d062f08f5020 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -98,11 +98,11 @@ static void __xen_dma_page_cpu_to_dev(struct device *hwdev, dma_addr_t handle, void __xen_dma_map_page(struct device *hwdev, struct page *page, dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { if (is_device_dma_coherent(hwdev)) return; - if (dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if (attrs & DMA_ATTR_SKIP_CPU_SYNC) return; __xen_dma_page_cpu_to_dev(hwdev, dev_addr, size, dir); @@ -110,12 +110,12 @@ void __xen_dma_map_page(struct device *hwdev, struct page *page, void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { if (is_device_dma_coherent(hwdev)) return; - if (dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if (attrs & DMA_ATTR_SKIP_CPU_SYNC) return; __xen_dma_page_dev_to_cpu(hwdev, handle, size, dir); diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index f6c55afab3e2..c4284c432ae8 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -32,10 +32,10 @@ static int swiotlb __read_mostly; -static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, +static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, bool coherent) { - if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + if (!coherent || (attrs & DMA_ATTR_WRITE_COMBINE)) return pgprot_writecombine(prot); return prot; } @@ -91,7 +91,7 @@ static int __free_from_pool(void *start, size_t size) static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); @@ -121,7 +121,7 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size, static void __dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { bool freed; phys_addr_t paddr = dma_to_phys(dev, dma_handle); @@ -140,7 +140,7 @@ static void __dma_free_coherent(struct device *dev, size_t size, static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { struct page *page; void *ptr, *coherent_ptr; @@ -188,7 +188,7 @@ no_mem: static void __dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle)); @@ -205,7 +205,7 @@ static void __dma_free(struct device *dev, size_t size, static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t dev_addr; @@ -219,7 +219,7 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page, static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { if (!is_device_dma_coherent(dev)) __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); @@ -228,7 +228,7 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr, static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i, ret; @@ -245,7 +245,7 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl, static void __swiotlb_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -306,7 +306,7 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, static int __swiotlb_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { int ret = -ENXIO; unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> @@ -333,7 +333,7 @@ static int __swiotlb_mmap(struct device *dev, static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t handle, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); @@ -435,21 +435,21 @@ out: static void *__dummy_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { return NULL; } static void __dummy_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { } static int __dummy_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { return -ENXIO; } @@ -457,20 +457,20 @@ static int __dummy_mmap(struct device *dev, static dma_addr_t __dummy_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return DMA_ERROR_CODE; } static void __dummy_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { } static int __dummy_map_sg(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return 0; } @@ -478,7 +478,7 @@ static int __dummy_map_sg(struct device *dev, struct scatterlist *sgl, static void __dummy_unmap_sg(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { } @@ -553,7 +553,7 @@ static void flush_page(struct device *dev, const void *virt, phys_addr_t phys) static void *__iommu_alloc_attrs(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { bool coherent = is_device_dma_coherent(dev); int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent); @@ -613,7 +613,7 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, } static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, struct dma_attrs *attrs) + dma_addr_t handle, unsigned long attrs) { size_t iosize = size; @@ -629,7 +629,7 @@ static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, * Hence how dodgy the below logic looks... */ if (__in_atomic_pool(cpu_addr, size)) { - iommu_dma_unmap_page(dev, handle, iosize, 0, NULL); + iommu_dma_unmap_page(dev, handle, iosize, 0, 0); __free_from_pool(cpu_addr, size); } else if (is_vmalloc_addr(cpu_addr)){ struct vm_struct *area = find_vm_area(cpu_addr); @@ -639,14 +639,14 @@ static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, iommu_dma_free(dev, area->pages, iosize, &handle); dma_common_free_remap(cpu_addr, size, VM_USERMAP); } else { - iommu_dma_unmap_page(dev, handle, iosize, 0, NULL); + iommu_dma_unmap_page(dev, handle, iosize, 0, 0); __free_pages(virt_to_page(cpu_addr), get_order(size)); } } static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { struct vm_struct *area; int ret; @@ -666,7 +666,7 @@ static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, - size_t size, struct dma_attrs *attrs) + size_t size, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct vm_struct *area = find_vm_area(cpu_addr); @@ -707,14 +707,14 @@ static void __iommu_sync_single_for_device(struct device *dev, static dma_addr_t __iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { bool coherent = is_device_dma_coherent(dev); int prot = dma_direction_to_prot(dir, coherent); dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); if (!iommu_dma_mapping_error(dev, dev_addr) && - !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __iommu_sync_single_for_device(dev, dev_addr, size, dir); return dev_addr; @@ -722,9 +722,9 @@ static dma_addr_t __iommu_map_page(struct device *dev, struct page *page, static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __iommu_sync_single_for_cpu(dev, dev_addr, size, dir); iommu_dma_unmap_page(dev, dev_addr, size, dir, attrs); @@ -760,11 +760,11 @@ static void __iommu_sync_sg_for_device(struct device *dev, static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { bool coherent = is_device_dma_coherent(dev); - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __iommu_sync_sg_for_device(dev, sgl, nelems, dir); return iommu_dma_map_sg(dev, sgl, nelems, @@ -774,9 +774,9 @@ static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl, static void __iommu_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { - if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __iommu_sync_sg_for_cpu(dev, sgl, nelems, dir); iommu_dma_unmap_sg(dev, sgl, nelems, dir, attrs); diff --git a/arch/avr32/mm/dma-coherent.c b/arch/avr32/mm/dma-coherent.c index 92cf1fb2b3e6..58610d0df7ed 100644 --- a/arch/avr32/mm/dma-coherent.c +++ b/arch/avr32/mm/dma-coherent.c @@ -99,7 +99,7 @@ static void __dma_free(struct device *dev, size_t size, } static void *avr32_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { struct page *page; dma_addr_t phys; @@ -109,7 +109,7 @@ static void *avr32_dma_alloc(struct device *dev, size_t size, return NULL; phys = page_to_phys(page); - if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) { + if (attrs & DMA_ATTR_WRITE_COMBINE) { /* Now, map the page into P3 with write-combining turned on */ *handle = phys; return __ioremap(phys, size, _PAGE_BUFFER); @@ -119,11 +119,11 @@ static void *avr32_dma_alloc(struct device *dev, size_t size, } static void avr32_dma_free(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, struct dma_attrs *attrs) + void *cpu_addr, dma_addr_t handle, unsigned long attrs) { struct page *page; - if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) { + if (attrs & DMA_ATTR_WRITE_COMBINE) { iounmap(cpu_addr); page = phys_to_page(handle); @@ -142,7 +142,7 @@ static void avr32_dma_free(struct device *dev, size_t size, static dma_addr_t avr32_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { void *cpu_addr = page_address(page) + offset; @@ -152,7 +152,7 @@ static dma_addr_t avr32_dma_map_page(struct device *dev, struct page *page, static int avr32_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct scatterlist *sg; diff --git a/arch/blackfin/kernel/dma-mapping.c b/arch/blackfin/kernel/dma-mapping.c index 771afe6e4264..53fbbb61aa86 100644 --- a/arch/blackfin/kernel/dma-mapping.c +++ b/arch/blackfin/kernel/dma-mapping.c @@ -79,7 +79,7 @@ static void __free_dma_pages(unsigned long addr, unsigned int pages) } static void *bfin_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -94,7 +94,7 @@ static void *bfin_dma_alloc(struct device *dev, size_t size, } static void bfin_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { __free_dma_pages((unsigned long)vaddr, get_pages(size)); } @@ -111,7 +111,7 @@ EXPORT_SYMBOL(__dma_sync); static int bfin_dma_map_sg(struct device *dev, struct scatterlist *sg_list, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -139,7 +139,7 @@ static void bfin_dma_sync_sg_for_device(struct device *dev, static dma_addr_t bfin_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t handle = (dma_addr_t)(page_address(page) + offset); diff --git a/arch/c6x/include/asm/dma-mapping.h b/arch/c6x/include/asm/dma-mapping.h index 6b5cd7b0cf32..5717b1e52d96 100644 --- a/arch/c6x/include/asm/dma-mapping.h +++ b/arch/c6x/include/asm/dma-mapping.h @@ -26,8 +26,8 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) extern void coherent_mem_init(u32 start, u32 size); void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, struct dma_attrs *attrs); + gfp_t gfp, unsigned long attrs); void c6x_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs); + dma_addr_t dma_handle, unsigned long attrs); #endif /* _ASM_C6X_DMA_MAPPING_H */ diff --git a/arch/c6x/kernel/dma.c b/arch/c6x/kernel/dma.c index 8a80f3a250c0..db4a6a301f5e 100644 --- a/arch/c6x/kernel/dma.c +++ b/arch/c6x/kernel/dma.c @@ -38,7 +38,7 @@ static void c6x_dma_sync(dma_addr_t handle, size_t size, static dma_addr_t c6x_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t handle = virt_to_phys(page_address(page) + offset); @@ -47,13 +47,13 @@ static dma_addr_t c6x_dma_map_page(struct device *dev, struct page *page, } static void c6x_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) + size_t size, enum dma_data_direction dir, unsigned long attrs) { c6x_dma_sync(handle, size, dir); } static int c6x_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *sg; int i; @@ -67,8 +67,7 @@ static int c6x_dma_map_sg(struct device *dev, struct scatterlist *sglist, } static void c6x_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *sg; int i; diff --git a/arch/c6x/mm/dma-coherent.c b/arch/c6x/mm/dma-coherent.c index f7ee63af2541..95e38ad27c69 100644 --- a/arch/c6x/mm/dma-coherent.c +++ b/arch/c6x/mm/dma-coherent.c @@ -74,7 +74,7 @@ static void __free_dma_pages(u32 addr, int order) * virtual and DMA address for that space. */ void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, struct dma_attrs *attrs) + gfp_t gfp, unsigned long attrs) { u32 paddr; int order; @@ -99,7 +99,7 @@ void *c6x_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, * Free DMA coherent memory as defined by the above mapping. */ void c6x_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { int order; diff --git a/arch/cris/arch-v32/drivers/pci/dma.c b/arch/cris/arch-v32/drivers/pci/dma.c index 8d5efa58cce1..1f0636793f0c 100644 --- a/arch/cris/arch-v32/drivers/pci/dma.c +++ b/arch/cris/arch-v32/drivers/pci/dma.c @@ -17,7 +17,7 @@ #include static void *v32_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -37,22 +37,21 @@ static void *v32_dma_alloc(struct device *dev, size_t size, } static void v32_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { free_pages((unsigned long)vaddr, get_order(size)); } static inline dma_addr_t v32_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, - struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { return page_to_phys(page) + offset; } static inline int v32_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { printk("Map sg\n"); return nents; diff --git a/arch/frv/mb93090-mb00/pci-dma-nommu.c b/arch/frv/mb93090-mb00/pci-dma-nommu.c index 082be49b5df0..90f2e4cb33d6 100644 --- a/arch/frv/mb93090-mb00/pci-dma-nommu.c +++ b/arch/frv/mb93090-mb00/pci-dma-nommu.c @@ -35,7 +35,7 @@ static DEFINE_SPINLOCK(dma_alloc_lock); static LIST_HEAD(dma_alloc_list); static void *frv_dma_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, struct dma_attrs *attrs) + gfp_t gfp, unsigned long attrs) { struct dma_alloc_record *new; struct list_head *this = &dma_alloc_list; @@ -86,7 +86,7 @@ static void *frv_dma_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_ha } static void frv_dma_free(struct device *hwdev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { struct dma_alloc_record *rec; unsigned long flags; @@ -107,7 +107,7 @@ static void frv_dma_free(struct device *hwdev, size_t size, void *vaddr, static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct scatterlist *sg; @@ -124,7 +124,7 @@ static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist, static dma_addr_t frv_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { BUG_ON(direction == DMA_NONE); flush_dcache_page(page); diff --git a/arch/frv/mb93090-mb00/pci-dma.c b/arch/frv/mb93090-mb00/pci-dma.c index 316b7b65348d..f585745b1abc 100644 --- a/arch/frv/mb93090-mb00/pci-dma.c +++ b/arch/frv/mb93090-mb00/pci-dma.c @@ -19,8 +19,7 @@ #include static void *frv_dma_alloc(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -32,14 +31,14 @@ static void *frv_dma_alloc(struct device *hwdev, size_t size, } static void frv_dma_free(struct device *hwdev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { consistent_free(vaddr); } static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long dampr2; void *vaddr; @@ -69,7 +68,7 @@ static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist, static dma_addr_t frv_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { flush_dcache_page(page); return (dma_addr_t) page_to_phys(page) + offset; diff --git a/arch/h8300/kernel/dma.c b/arch/h8300/kernel/dma.c index eeb13d3f2424..3651da045806 100644 --- a/arch/h8300/kernel/dma.c +++ b/arch/h8300/kernel/dma.c @@ -12,7 +12,7 @@ static void *dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret; @@ -32,7 +32,7 @@ static void *dma_alloc(struct device *dev, size_t size, static void dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { free_pages((unsigned long)vaddr, get_order(size)); @@ -41,14 +41,14 @@ static void dma_free(struct device *dev, size_t size, static dma_addr_t map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return page_to_phys(page) + offset; } static int map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; diff --git a/arch/hexagon/include/asm/dma-mapping.h b/arch/hexagon/include/asm/dma-mapping.h index aa6203464520..7ef58df909fc 100644 --- a/arch/hexagon/include/asm/dma-mapping.h +++ b/arch/hexagon/include/asm/dma-mapping.h @@ -26,7 +26,6 @@ #include #include #include -#include #include struct device; diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c index 9e3ddf792bd3..b9017785fb71 100644 --- a/arch/hexagon/kernel/dma.c +++ b/arch/hexagon/kernel/dma.c @@ -51,7 +51,7 @@ static struct gen_pool *coherent_pool; static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret; @@ -84,7 +84,7 @@ static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, } static void hexagon_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_addr, struct dma_attrs *attrs) + dma_addr_t dma_addr, unsigned long attrs) { gen_pool_free(coherent_pool, (unsigned long) vaddr, size); } @@ -105,7 +105,7 @@ static int check_addr(const char *name, struct device *hwdev, static int hexagon_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; @@ -172,7 +172,7 @@ static inline void dma_sync(void *addr, size_t size, static dma_addr_t hexagon_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t bus = page_to_phys(page) + offset; WARN_ON(size == 0); diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c index a6d6190c9d24..630ee8073899 100644 --- a/arch/ia64/hp/common/sba_iommu.c +++ b/arch/ia64/hp/common/sba_iommu.c @@ -919,7 +919,7 @@ sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt) static dma_addr_t sba_map_page(struct device *dev, struct page *page, unsigned long poff, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct ioc *ioc; void *addr = page_address(page) + poff; @@ -1005,7 +1005,7 @@ static dma_addr_t sba_map_page(struct device *dev, struct page *page, static dma_addr_t sba_map_single_attrs(struct device *dev, void *addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return sba_map_page(dev, virt_to_page(addr), (unsigned long)addr & ~PAGE_MASK, size, dir, attrs); @@ -1046,7 +1046,7 @@ sba_mark_clean(struct ioc *ioc, dma_addr_t iova, size_t size) * See Documentation/DMA-API-HOWTO.txt */ static void sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct ioc *ioc; #if DELAYED_RESOURCE_CNT > 0 @@ -1115,7 +1115,7 @@ static void sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size, } void sba_unmap_single_attrs(struct device *dev, dma_addr_t iova, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { sba_unmap_page(dev, iova, size, dir, attrs); } @@ -1130,7 +1130,7 @@ void sba_unmap_single_attrs(struct device *dev, dma_addr_t iova, size_t size, */ static void * sba_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t flags, struct dma_attrs *attrs) + gfp_t flags, unsigned long attrs) { struct ioc *ioc; void *addr; @@ -1175,7 +1175,7 @@ sba_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, * device to map single to get an iova mapping. */ *dma_handle = sba_map_single_attrs(&ioc->sac_only_dev->dev, addr, - size, 0, NULL); + size, 0, 0); return addr; } @@ -1191,9 +1191,9 @@ sba_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, * See Documentation/DMA-API-HOWTO.txt */ static void sba_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { - sba_unmap_single_attrs(dev, dma_handle, size, 0, NULL); + sba_unmap_single_attrs(dev, dma_handle, size, 0, 0); free_pages((unsigned long) vaddr, get_order(size)); } @@ -1442,7 +1442,7 @@ sba_coalesce_chunks(struct ioc *ioc, struct device *dev, static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs); + unsigned long attrs); /** * sba_map_sg - map Scatter/Gather list * @dev: instance of PCI owned by the driver that's asking. @@ -1455,7 +1455,7 @@ static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist, */ static int sba_map_sg_attrs(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct ioc *ioc; int coalesced, filled = 0; @@ -1551,7 +1551,7 @@ static int sba_map_sg_attrs(struct device *dev, struct scatterlist *sglist, */ static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef ASSERT_PDIR_SANITY struct ioc *ioc; diff --git a/arch/ia64/include/asm/machvec.h b/arch/ia64/include/asm/machvec.h index 9c39bdfc2da8..ed7f09089f12 100644 --- a/arch/ia64/include/asm/machvec.h +++ b/arch/ia64/include/asm/machvec.h @@ -22,7 +22,6 @@ struct pci_bus; struct task_struct; struct pci_dev; struct msi_desc; -struct dma_attrs; typedef void ia64_mv_setup_t (char **); typedef void ia64_mv_cpu_init_t (void); diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c index 939260aeac98..2933208c0285 100644 --- a/arch/ia64/kernel/pci-swiotlb.c +++ b/arch/ia64/kernel/pci-swiotlb.c @@ -16,7 +16,7 @@ EXPORT_SYMBOL(swiotlb); static void *ia64_swiotlb_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { if (dev->coherent_dma_mask != DMA_BIT_MASK(64)) gfp |= GFP_DMA; @@ -25,7 +25,7 @@ static void *ia64_swiotlb_alloc_coherent(struct device *dev, size_t size, static void ia64_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_addr); } diff --git a/arch/ia64/sn/pci/pci_dma.c b/arch/ia64/sn/pci/pci_dma.c index 8f59907007cb..74c934a997bb 100644 --- a/arch/ia64/sn/pci/pci_dma.c +++ b/arch/ia64/sn/pci/pci_dma.c @@ -77,7 +77,7 @@ EXPORT_SYMBOL(sn_dma_set_mask); */ static void *sn_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t * dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { void *cpuaddr; unsigned long phys_addr; @@ -138,7 +138,7 @@ static void *sn_dma_alloc_coherent(struct device *dev, size_t size, * any associated IOMMU mappings. */ static void sn_dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { struct pci_dev *pdev = to_pci_dev(dev); struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev); @@ -176,21 +176,18 @@ static void sn_dma_free_coherent(struct device *dev, size_t size, void *cpu_addr static dma_addr_t sn_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *cpu_addr = page_address(page) + offset; dma_addr_t dma_addr; unsigned long phys_addr; struct pci_dev *pdev = to_pci_dev(dev); struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev); - int dmabarr; - - dmabarr = dma_get_attr(DMA_ATTR_WRITE_BARRIER, attrs); BUG_ON(!dev_is_pci(dev)); phys_addr = __pa(cpu_addr); - if (dmabarr) + if (attrs & DMA_ATTR_WRITE_BARRIER) dma_addr = provider->dma_map_consistent(pdev, phys_addr, size, SN_DMA_ADDR_PHYS); else @@ -218,7 +215,7 @@ static dma_addr_t sn_dma_map_page(struct device *dev, struct page *page, */ static void sn_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_dev *pdev = to_pci_dev(dev); struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev); @@ -240,7 +237,7 @@ static void sn_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, */ static void sn_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, int nhwentries, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct pci_dev *pdev = to_pci_dev(dev); @@ -273,16 +270,13 @@ static void sn_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, */ static int sn_dma_map_sg(struct device *dev, struct scatterlist *sgl, int nhwentries, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long phys_addr; struct scatterlist *saved_sg = sgl, *sg; struct pci_dev *pdev = to_pci_dev(dev); struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev); int i; - int dmabarr; - - dmabarr = dma_get_attr(DMA_ATTR_WRITE_BARRIER, attrs); BUG_ON(!dev_is_pci(dev)); @@ -292,7 +286,7 @@ static int sn_dma_map_sg(struct device *dev, struct scatterlist *sgl, for_each_sg(sgl, sg, nhwentries, i) { dma_addr_t dma_addr; phys_addr = SG_ENT_PHYS_ADDRESS(sg); - if (dmabarr) + if (attrs & DMA_ATTR_WRITE_BARRIER) dma_addr = provider->dma_map_consistent(pdev, phys_addr, sg->length, diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c index cbc78b4117b5..8cf97cbadc91 100644 --- a/arch/m68k/kernel/dma.c +++ b/arch/m68k/kernel/dma.c @@ -19,7 +19,7 @@ #if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE) static void *m68k_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t flag, struct dma_attrs *attrs) + gfp_t flag, unsigned long attrs) { struct page *page, **map; pgprot_t pgprot; @@ -62,7 +62,7 @@ static void *m68k_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } static void m68k_dma_free(struct device *dev, size_t size, void *addr, - dma_addr_t handle, struct dma_attrs *attrs) + dma_addr_t handle, unsigned long attrs) { pr_debug("dma_free_coherent: %p, %x\n", addr, handle); vfree(addr); @@ -73,7 +73,7 @@ static void m68k_dma_free(struct device *dev, size_t size, void *addr, #include static void *m68k_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; /* ignore region specifiers */ @@ -91,7 +91,7 @@ static void *m68k_dma_alloc(struct device *dev, size_t size, } static void m68k_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { free_pages((unsigned long)vaddr, get_order(size)); } @@ -130,7 +130,7 @@ static void m68k_dma_sync_sg_for_device(struct device *dev, static dma_addr_t m68k_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t handle = page_to_phys(page) + offset; @@ -139,7 +139,7 @@ static dma_addr_t m68k_dma_map_page(struct device *dev, struct page *page, } static int m68k_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction dir, struct dma_attrs *attrs) + int nents, enum dma_data_direction dir, unsigned long attrs) { int i; struct scatterlist *sg; diff --git a/arch/metag/kernel/dma.c b/arch/metag/kernel/dma.c index e12368d02155..0db31e24c541 100644 --- a/arch/metag/kernel/dma.c +++ b/arch/metag/kernel/dma.c @@ -172,7 +172,7 @@ out: * virtual and bus address for that space. */ static void *metag_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { struct page *page; struct metag_vm_region *c; @@ -268,7 +268,7 @@ no_page: * free a page as defined by the above mapping. */ static void metag_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { struct metag_vm_region *c; unsigned long flags, addr; @@ -331,13 +331,13 @@ no_area: static int metag_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, user_size, kern_size; struct metag_vm_region *c; int ret = -ENXIO; - if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + if (attrs & DMA_ATTR_WRITE_COMBINE) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); else vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -482,7 +482,7 @@ static void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction) static dma_addr_t metag_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { dma_sync_for_device((void *)(page_to_phys(page) + offset), size, direction); @@ -491,14 +491,14 @@ static dma_addr_t metag_dma_map_page(struct device *dev, struct page *page, static void metag_dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { dma_sync_for_cpu(phys_to_virt(dma_address), size, direction); } static int metag_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -516,7 +516,7 @@ static int metag_dma_map_sg(struct device *dev, struct scatterlist *sglist, static void metag_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nhwentries, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; diff --git a/arch/microblaze/include/asm/dma-mapping.h b/arch/microblaze/include/asm/dma-mapping.h index 1884783d15c0..1768d4bdc8d3 100644 --- a/arch/microblaze/include/asm/dma-mapping.h +++ b/arch/microblaze/include/asm/dma-mapping.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c index bf4dec229437..ec04dc1e2527 100644 --- a/arch/microblaze/kernel/dma.c +++ b/arch/microblaze/kernel/dma.c @@ -17,7 +17,7 @@ static void *dma_direct_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef NOT_COHERENT_CACHE return consistent_alloc(flag, size, dma_handle); @@ -42,7 +42,7 @@ static void *dma_direct_alloc_coherent(struct device *dev, size_t size, static void dma_direct_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef NOT_COHERENT_CACHE consistent_free(size, vaddr); @@ -53,7 +53,7 @@ static void dma_direct_free_coherent(struct device *dev, size_t size, static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -78,7 +78,7 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { __dma_sync(page_to_phys(page) + offset, size, direction); return page_to_phys(page) + offset; @@ -88,7 +88,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { /* There is not necessary to do cache cleanup * @@ -157,7 +157,7 @@ dma_direct_sync_sg_for_device(struct device *dev, static int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t handle, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef CONFIG_MMU unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c index 2cd45f5f9481..fd69528b24fb 100644 --- a/arch/mips/cavium-octeon/dma-octeon.c +++ b/arch/mips/cavium-octeon/dma-octeon.c @@ -125,7 +125,7 @@ static phys_addr_t octeon_small_dma_to_phys(struct device *dev, static dma_addr_t octeon_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size, direction, attrs); @@ -135,7 +135,7 @@ static dma_addr_t octeon_dma_map_page(struct device *dev, struct page *page, } static int octeon_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction direction, struct dma_attrs *attrs) + int nents, enum dma_data_direction direction, unsigned long attrs) { int r = swiotlb_map_sg_attrs(dev, sg, nents, direction, attrs); mb(); @@ -157,7 +157,7 @@ static void octeon_dma_sync_sg_for_device(struct device *dev, } static void *octeon_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -189,7 +189,7 @@ static void *octeon_dma_alloc_coherent(struct device *dev, size_t size, } static void octeon_dma_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) + void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_handle); } diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c index 4ffa6fc81c8f..1a80b6f73ab2 100644 --- a/arch/mips/loongson64/common/dma-swiotlb.c +++ b/arch/mips/loongson64/common/dma-swiotlb.c @@ -10,7 +10,7 @@ #include static void *loongson_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -41,7 +41,7 @@ static void *loongson_dma_alloc_coherent(struct device *dev, size_t size, } static void loongson_dma_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) + void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_handle); } @@ -49,7 +49,7 @@ static void loongson_dma_free_coherent(struct device *dev, size_t size, static dma_addr_t loongson_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size, dir, attrs); @@ -59,9 +59,9 @@ static dma_addr_t loongson_dma_map_page(struct device *dev, struct page *page, static int loongson_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { - int r = swiotlb_map_sg_attrs(dev, sg, nents, dir, NULL); + int r = swiotlb_map_sg_attrs(dev, sg, nents, dir, 0); mb(); return r; diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c index cb557d28cb21..b2eadd6fa9a1 100644 --- a/arch/mips/mm/dma-default.c +++ b/arch/mips/mm/dma-default.c @@ -131,7 +131,7 @@ static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size, } static void *mips_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; struct page *page = NULL; @@ -141,7 +141,7 @@ static void *mips_dma_alloc_coherent(struct device *dev, size_t size, * XXX: seems like the coherent and non-coherent implementations could * be consolidated. */ - if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) + if (attrs & DMA_ATTR_NON_CONSISTENT) return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp); gfp = massage_gfp_flags(dev, gfp); @@ -176,13 +176,13 @@ static void mips_dma_free_noncoherent(struct device *dev, size_t size, } static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { unsigned long addr = (unsigned long) vaddr; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = NULL; - if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) { + if (attrs & DMA_ATTR_NON_CONSISTENT) { mips_dma_free_noncoherent(dev, size, vaddr, dma_handle); return; } @@ -200,7 +200,7 @@ static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -214,7 +214,7 @@ static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma, pfn = page_to_pfn(virt_to_page((void *)addr)); - if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + if (attrs & DMA_ATTR_WRITE_COMBINE) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); else vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -291,7 +291,7 @@ static inline void __dma_sync(struct page *page, } static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) + size_t size, enum dma_data_direction direction, unsigned long attrs) { if (cpu_needs_post_dma_flush(dev)) __dma_sync(dma_addr_to_page(dev, dma_addr), @@ -301,7 +301,7 @@ static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, } static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction direction, struct dma_attrs *attrs) + int nents, enum dma_data_direction direction, unsigned long attrs) { int i; struct scatterlist *sg; @@ -322,7 +322,7 @@ static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist, static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { if (!plat_device_is_coherent(dev)) __dma_sync(page, offset, size, direction); @@ -332,7 +332,7 @@ static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page, static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nhwentries, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct scatterlist *sg; diff --git a/arch/mips/netlogic/common/nlm-dma.c b/arch/mips/netlogic/common/nlm-dma.c index 3758715d4ab6..0630693bec2a 100644 --- a/arch/mips/netlogic/common/nlm-dma.c +++ b/arch/mips/netlogic/common/nlm-dma.c @@ -45,7 +45,7 @@ static char *nlm_swiotlb; static void *nlm_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { /* ignore region specifiers */ gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); @@ -62,7 +62,7 @@ static void *nlm_dma_alloc_coherent(struct device *dev, size_t size, } static void nlm_dma_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) + void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_handle); } diff --git a/arch/mn10300/mm/dma-alloc.c b/arch/mn10300/mm/dma-alloc.c index 8842394cb49a..4f4b9029f0ea 100644 --- a/arch/mn10300/mm/dma-alloc.c +++ b/arch/mn10300/mm/dma-alloc.c @@ -21,7 +21,7 @@ static unsigned long pci_sram_allocated = 0xbc000000; static void *mn10300_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { unsigned long addr; void *ret; @@ -63,7 +63,7 @@ done: } static void mn10300_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { unsigned long addr = (unsigned long) vaddr & ~0x20000000; @@ -75,7 +75,7 @@ static void mn10300_dma_free(struct device *dev, size_t size, void *vaddr, static int mn10300_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -92,7 +92,7 @@ static int mn10300_dma_map_sg(struct device *dev, struct scatterlist *sglist, static dma_addr_t mn10300_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { return page_to_bus(page) + offset; } diff --git a/arch/nios2/mm/dma-mapping.c b/arch/nios2/mm/dma-mapping.c index 90422c367ed3..d800fad87896 100644 --- a/arch/nios2/mm/dma-mapping.c +++ b/arch/nios2/mm/dma-mapping.c @@ -59,7 +59,7 @@ static inline void __dma_sync_for_cpu(void *vaddr, size_t size, } static void *nios2_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { void *ret; @@ -84,7 +84,7 @@ static void *nios2_dma_alloc(struct device *dev, size_t size, } static void nios2_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { unsigned long addr = (unsigned long) CAC_ADDR((unsigned long) vaddr); @@ -93,7 +93,7 @@ static void nios2_dma_free(struct device *dev, size_t size, void *vaddr, static int nios2_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; @@ -113,7 +113,7 @@ static int nios2_dma_map_sg(struct device *dev, struct scatterlist *sg, static dma_addr_t nios2_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { void *addr = page_address(page) + offset; @@ -123,14 +123,14 @@ static dma_addr_t nios2_dma_map_page(struct device *dev, struct page *page, static void nios2_dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { __dma_sync_for_cpu(phys_to_virt(dma_address), size, direction); } static void nios2_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { void *addr; int i; diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c index 0b77ddb1ee07..140c99140649 100644 --- a/arch/openrisc/kernel/dma.c +++ b/arch/openrisc/kernel/dma.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -83,7 +82,7 @@ page_clear_nocache(pte_t *pte, unsigned long addr, static void * or1k_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long va; void *page; @@ -101,7 +100,7 @@ or1k_dma_alloc(struct device *dev, size_t size, va = (unsigned long)page; - if (!dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) { + if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) { /* * We need to iterate through the pages, clearing the dcache for * them and setting the cache-inhibit bit. @@ -117,7 +116,7 @@ or1k_dma_alloc(struct device *dev, size_t size, static void or1k_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { unsigned long va = (unsigned long)vaddr; struct mm_walk walk = { @@ -125,7 +124,7 @@ or1k_dma_free(struct device *dev, size_t size, void *vaddr, .mm = &init_mm }; - if (!dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) { + if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) { /* walk_page_range shouldn't be able to fail here */ WARN_ON(walk_page_range(va, va + size, &walk)); } @@ -137,7 +136,7 @@ static dma_addr_t or1k_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long cl; dma_addr_t addr = page_to_phys(page) + offset; @@ -170,7 +169,7 @@ or1k_map_page(struct device *dev, struct page *page, static void or1k_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { /* Nothing special to do here... */ } @@ -178,14 +177,14 @@ or1k_unmap_page(struct device *dev, dma_addr_t dma_handle, static int or1k_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; for_each_sg(sg, s, nents, i) { s->dma_address = or1k_map_page(dev, sg_page(s), s->offset, - s->length, dir, NULL); + s->length, dir, 0); } return nents; @@ -194,13 +193,13 @@ or1k_map_sg(struct device *dev, struct scatterlist *sg, static void or1k_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; for_each_sg(sg, s, nents, i) { - or1k_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, NULL); + or1k_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, 0); } } diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index a27e4928bf73..02d9ed0f3949 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -414,7 +414,7 @@ pcxl_dma_init(void) __initcall(pcxl_dma_init); static void *pa11_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { unsigned long vaddr; unsigned long paddr; @@ -441,7 +441,7 @@ static void *pa11_dma_alloc(struct device *dev, size_t size, } static void pa11_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { int order; @@ -454,7 +454,7 @@ static void pa11_dma_free(struct device *dev, size_t size, void *vaddr, static dma_addr_t pa11_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { void *addr = page_address(page) + offset; BUG_ON(direction == DMA_NONE); @@ -465,7 +465,7 @@ static dma_addr_t pa11_dma_map_page(struct device *dev, struct page *page, static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(direction == DMA_NONE); @@ -484,7 +484,7 @@ static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct scatterlist *sg; @@ -503,7 +503,7 @@ static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist, static void pa11_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; struct scatterlist *sg; @@ -577,11 +577,11 @@ struct dma_map_ops pcxl_dma_ops = { }; static void *pcx_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { void *addr; - if (!dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) + if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) return NULL; addr = (void *)__get_free_pages(flag, get_order(size)); @@ -592,7 +592,7 @@ static void *pcx_dma_alloc(struct device *dev, size_t size, } static void pcx_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t iova, struct dma_attrs *attrs) + dma_addr_t iova, unsigned long attrs) { free_pages((unsigned long)vaddr, get_order(size)); return; diff --git a/arch/powerpc/include/asm/dma-mapping.h b/arch/powerpc/include/asm/dma-mapping.h index 77816acd4fd9..84e3f8dd5e4f 100644 --- a/arch/powerpc/include/asm/dma-mapping.h +++ b/arch/powerpc/include/asm/dma-mapping.h @@ -13,7 +13,6 @@ /* need struct page definitions */ #include #include -#include #include #include #include @@ -25,14 +24,14 @@ /* Some dma direct funcs must be visible for use in other dma_ops */ extern void *__dma_direct_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs); + unsigned long attrs); extern void __dma_direct_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs); + unsigned long attrs); extern int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t handle, - size_t size, struct dma_attrs *attrs); + size_t size, unsigned long attrs); #ifdef CONFIG_NOT_COHERENT_CACHE /* diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index f49a72a9062d..2c1d50792944 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -53,7 +53,7 @@ struct iommu_table_ops { long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); #ifdef CONFIG_IOMMU_API /* * Exchanges existing TCE with new TCE plus direction bits; @@ -248,12 +248,12 @@ extern int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, struct scatterlist *sglist, int nelems, unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); extern void ppc_iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); extern void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, size_t size, dma_addr_t *dma_handle, @@ -264,10 +264,10 @@ extern dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, struct page *page, unsigned long offset, size_t size, unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); extern void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); extern void iommu_init_early_pSeries(void); extern void iommu_init_early_dart(struct pci_controller_ops *controller_ops); diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 41a7d9d49a5a..fb7cbaa37658 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -18,7 +18,7 @@ */ static void *dma_iommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size, dma_handle, dev->coherent_dma_mask, flag, @@ -27,7 +27,7 @@ static void *dma_iommu_alloc_coherent(struct device *dev, size_t size, static void dma_iommu_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { iommu_free_coherent(get_iommu_table_base(dev), size, vaddr, dma_handle); } @@ -40,7 +40,7 @@ static void dma_iommu_free_coherent(struct device *dev, size_t size, static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return iommu_map_page(dev, get_iommu_table_base(dev), page, offset, size, device_to_mask(dev), direction, attrs); @@ -49,7 +49,7 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction, attrs); @@ -58,7 +58,7 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems, device_to_mask(dev), direction, attrs); @@ -66,7 +66,7 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems, direction, attrs); diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c index 3f1472a78f39..e64a6016fba7 100644 --- a/arch/powerpc/kernel/dma.c +++ b/arch/powerpc/kernel/dma.c @@ -64,7 +64,7 @@ static int dma_direct_dma_supported(struct device *dev, u64 mask) void *__dma_direct_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret; #ifdef CONFIG_NOT_COHERENT_CACHE @@ -121,7 +121,7 @@ void *__dma_direct_alloc_coherent(struct device *dev, size_t size, void __dma_direct_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { #ifdef CONFIG_NOT_COHERENT_CACHE __dma_free_coherent(size, vaddr); @@ -132,7 +132,7 @@ void __dma_direct_free_coherent(struct device *dev, size_t size, static void *dma_direct_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu_table *iommu; @@ -156,7 +156,7 @@ static void *dma_direct_alloc_coherent(struct device *dev, size_t size, static void dma_direct_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu_table *iommu; @@ -177,7 +177,7 @@ static void dma_direct_free_coherent(struct device *dev, size_t size, int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t handle, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long pfn; @@ -195,7 +195,7 @@ int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -211,7 +211,7 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { } @@ -232,7 +232,7 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(dir == DMA_NONE); __dma_sync_page(page, offset, size, dir); @@ -243,7 +243,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { } diff --git a/arch/powerpc/kernel/ibmebus.c b/arch/powerpc/kernel/ibmebus.c index a89f4f7a66bd..c1ca9282f4a0 100644 --- a/arch/powerpc/kernel/ibmebus.c +++ b/arch/powerpc/kernel/ibmebus.c @@ -65,7 +65,7 @@ static void *ibmebus_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { void *mem; @@ -78,7 +78,7 @@ static void *ibmebus_alloc_coherent(struct device *dev, static void ibmebus_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { kfree(vaddr); } @@ -88,7 +88,7 @@ static dma_addr_t ibmebus_map_page(struct device *dev, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return (dma_addr_t)(page_address(page) + offset); } @@ -97,7 +97,7 @@ static void ibmebus_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return; } @@ -105,7 +105,7 @@ static void ibmebus_unmap_page(struct device *dev, static int ibmebus_map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -121,7 +121,7 @@ static int ibmebus_map_sg(struct device *dev, static void ibmebus_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return; } diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index a8e3490b54e3..37d6e741be82 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -307,7 +307,7 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, void *page, unsigned int npages, enum dma_data_direction direction, unsigned long mask, unsigned int align_order, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long entry; dma_addr_t ret = DMA_ERROR_CODE; @@ -431,7 +431,7 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, struct scatterlist *sglist, int nelems, unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t dma_next = 0, dma_addr; struct scatterlist *s, *outs, *segstart; @@ -574,7 +574,7 @@ int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, void ppc_iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; @@ -753,7 +753,7 @@ void iommu_free_table(struct iommu_table *tbl, const char *node_name) dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, struct page *page, unsigned long offset, size_t size, unsigned long mask, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t dma_handle = DMA_ERROR_CODE; void *vaddr; @@ -790,7 +790,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned int npages; @@ -845,7 +845,7 @@ void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, nio_pages = size >> tbl->it_page_shift; io_order = get_iommu_order(size, tbl); mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, - mask >> tbl->it_page_shift, io_order, NULL); + mask >> tbl->it_page_shift, io_order, 0); if (mapping == DMA_ERROR_CODE) { free_pages((unsigned long)ret, order); return NULL; diff --git a/arch/powerpc/kernel/vio.c b/arch/powerpc/kernel/vio.c index 8d7358f3a273..b3813ddb2fb4 100644 --- a/arch/powerpc/kernel/vio.c +++ b/arch/powerpc/kernel/vio.c @@ -482,7 +482,7 @@ static void vio_cmo_balance(struct work_struct *work) static void *vio_dma_iommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); void *ret; @@ -503,7 +503,7 @@ static void *vio_dma_iommu_alloc_coherent(struct device *dev, size_t size, static void vio_dma_iommu_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); @@ -515,7 +515,7 @@ static void vio_dma_iommu_free_coherent(struct device *dev, size_t size, static dma_addr_t vio_dma_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl; @@ -539,7 +539,7 @@ static dma_addr_t vio_dma_iommu_map_page(struct device *dev, struct page *page, static void vio_dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl; @@ -552,7 +552,7 @@ static void vio_dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl; @@ -588,7 +588,7 @@ static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, static void vio_dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl; diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c index 9027d7c48507..f7d1a4953ea0 100644 --- a/arch/powerpc/platforms/cell/iommu.c +++ b/arch/powerpc/platforms/cell/iommu.c @@ -166,7 +166,7 @@ static void invalidate_tce_cache(struct cbe_iommu *iommu, unsigned long *pte, static int tce_build_cell(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int i; unsigned long *io_pte, base_pte; @@ -193,7 +193,7 @@ static int tce_build_cell(struct iommu_table *tbl, long index, long npages, base_pte = CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_M | CBE_IOPTE_SO_RW | (window->ioid & CBE_IOPTE_IOID_Mask); #endif - if (unlikely(dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs))) + if (unlikely(attrs & DMA_ATTR_WEAK_ORDERING)) base_pte &= ~CBE_IOPTE_SO_RW; io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset); @@ -526,7 +526,7 @@ cell_iommu_setup_window(struct cbe_iommu *iommu, struct device_node *np, __set_bit(0, window->table.it_map); tce_build_cell(&window->table, window->table.it_offset, 1, - (unsigned long)iommu->pad_page, DMA_TO_DEVICE, NULL); + (unsigned long)iommu->pad_page, DMA_TO_DEVICE, 0); return window; } @@ -572,7 +572,7 @@ static struct iommu_table *cell_get_iommu_table(struct device *dev) static void *dma_fixed_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { if (iommu_fixed_is_weak) return iommu_alloc_coherent(dev, cell_get_iommu_table(dev), @@ -586,7 +586,7 @@ static void *dma_fixed_alloc_coherent(struct device *dev, size_t size, static void dma_fixed_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { if (iommu_fixed_is_weak) iommu_free_coherent(cell_get_iommu_table(dev), size, vaddr, @@ -598,9 +598,9 @@ static void dma_fixed_free_coherent(struct device *dev, size_t size, static dma_addr_t dma_fixed_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { - if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs)) + if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING)) return dma_direct_ops.map_page(dev, page, offset, size, direction, attrs); else @@ -611,9 +611,9 @@ static dma_addr_t dma_fixed_map_page(struct device *dev, struct page *page, static void dma_fixed_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { - if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs)) + if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING)) dma_direct_ops.unmap_page(dev, dma_addr, size, direction, attrs); else @@ -623,9 +623,9 @@ static void dma_fixed_unmap_page(struct device *dev, dma_addr_t dma_addr, static int dma_fixed_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { - if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs)) + if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING)) return dma_direct_ops.map_sg(dev, sg, nents, direction, attrs); else return ppc_iommu_map_sg(dev, cell_get_iommu_table(dev), sg, @@ -635,9 +635,9 @@ static int dma_fixed_map_sg(struct device *dev, struct scatterlist *sg, static void dma_fixed_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { - if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs)) + if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING)) dma_direct_ops.unmap_sg(dev, sg, nents, direction, attrs); else ppc_iommu_unmap_sg(cell_get_iommu_table(dev), sg, nents, @@ -1162,7 +1162,7 @@ static int __init setup_iommu_fixed(char *str) pciep = of_find_node_by_type(NULL, "pcie-endpoint"); if (strcmp(str, "weak") == 0 || (pciep && strcmp(str, "strong") != 0)) - iommu_fixed_is_weak = 1; + iommu_fixed_is_weak = DMA_ATTR_WEAK_ORDERING; of_node_put(pciep); diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c index 43dd3fb514e0..309d9ccccd50 100644 --- a/arch/powerpc/platforms/pasemi/iommu.c +++ b/arch/powerpc/platforms/pasemi/iommu.c @@ -88,7 +88,7 @@ static int iommu_table_iobmap_inited; static int iobmap_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { u32 *ip; u32 rpn; diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c index 4383a5ff82ba..00e1a0195c78 100644 --- a/arch/powerpc/platforms/powernv/npu-dma.c +++ b/arch/powerpc/platforms/powernv/npu-dma.c @@ -73,7 +73,7 @@ EXPORT_SYMBOL(pnv_pci_get_npu_dev); static void *dma_npu_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { NPU_DMA_OP_UNSUPPORTED(); return NULL; @@ -81,7 +81,7 @@ static void *dma_npu_alloc(struct device *dev, size_t size, static void dma_npu_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { NPU_DMA_OP_UNSUPPORTED(); } @@ -89,7 +89,7 @@ static void dma_npu_free(struct device *dev, size_t size, static dma_addr_t dma_npu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { NPU_DMA_OP_UNSUPPORTED(); return 0; @@ -97,7 +97,7 @@ static dma_addr_t dma_npu_map_page(struct device *dev, struct page *page, static int dma_npu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { NPU_DMA_OP_UNSUPPORTED(); return 0; diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 891fc4a453df..6b9528307f62 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1806,7 +1806,7 @@ static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl, static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int ret = pnv_tce_build(tbl, index, npages, uaddr, direction, attrs); @@ -1950,7 +1950,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl, static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int ret = pnv_tce_build(tbl, index, npages, uaddr, direction, attrs); diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 6701dd5ded20..a21d831c1114 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -704,7 +704,7 @@ static __be64 *pnv_tce(struct iommu_table *tbl, long idx) int pnv_tce_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { u64 proto_tce = iommu_direction_to_tce_perm(direction); u64 rpn = __pa(uaddr) >> tbl->it_page_shift; diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index d088d4f06116..e64df7894d6e 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -181,7 +181,7 @@ struct pnv_phb { extern struct pci_ops pnv_pci_ops; extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs); + unsigned long attrs); extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages); extern int pnv_tce_xchg(struct iommu_table *tbl, long index, unsigned long *hpa, enum dma_data_direction *direction); diff --git a/arch/powerpc/platforms/ps3/system-bus.c b/arch/powerpc/platforms/ps3/system-bus.c index 5606fe36faf2..8af1c15aef85 100644 --- a/arch/powerpc/platforms/ps3/system-bus.c +++ b/arch/powerpc/platforms/ps3/system-bus.c @@ -516,7 +516,7 @@ core_initcall(ps3_system_bus_init); */ static void * ps3_alloc_coherent(struct device *_dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { int result; struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); @@ -553,7 +553,7 @@ clean_none: } static void ps3_free_coherent(struct device *_dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); @@ -569,7 +569,7 @@ static void ps3_free_coherent(struct device *_dev, size_t size, void *vaddr, static dma_addr_t ps3_sb_map_page(struct device *_dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); int result; @@ -592,7 +592,7 @@ static dma_addr_t ps3_sb_map_page(struct device *_dev, struct page *page, static dma_addr_t ps3_ioc0_map_page(struct device *_dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); int result; @@ -626,7 +626,7 @@ static dma_addr_t ps3_ioc0_map_page(struct device *_dev, struct page *page, } static void ps3_unmap_page(struct device *_dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) + size_t size, enum dma_data_direction direction, unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); int result; @@ -640,7 +640,7 @@ static void ps3_unmap_page(struct device *_dev, dma_addr_t dma_addr, } static int ps3_sb_map_sg(struct device *_dev, struct scatterlist *sgl, - int nents, enum dma_data_direction direction, struct dma_attrs *attrs) + int nents, enum dma_data_direction direction, unsigned long attrs) { #if defined(CONFIG_PS3_DYNAMIC_DMA) BUG_ON("do"); @@ -670,14 +670,14 @@ static int ps3_sb_map_sg(struct device *_dev, struct scatterlist *sgl, static int ps3_ioc0_map_sg(struct device *_dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG(); return 0; } static void ps3_sb_unmap_sg(struct device *_dev, struct scatterlist *sg, - int nents, enum dma_data_direction direction, struct dma_attrs *attrs) + int nents, enum dma_data_direction direction, unsigned long attrs) { #if defined(CONFIG_PS3_DYNAMIC_DMA) BUG_ON("do"); @@ -686,7 +686,7 @@ static void ps3_sb_unmap_sg(struct device *_dev, struct scatterlist *sg, static void ps3_ioc0_unmap_sg(struct device *_dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG(); } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 770a753b52c9..0024e451bb36 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -123,7 +123,7 @@ static void iommu_pseries_free_group(struct iommu_table_group *table_group, static int tce_build_pSeries(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { u64 proto_tce; __be64 *tcep, *tces; @@ -173,7 +173,7 @@ static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long); static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { u64 rc = 0; u64 proto_tce, tce; @@ -216,7 +216,7 @@ static DEFINE_PER_CPU(__be64 *, tce_page); static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { u64 rc = 0; u64 proto_tce; diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 26904f4879ec..3573d54b2770 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -185,7 +185,7 @@ static void dart_flush(struct iommu_table *tbl) static int dart_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned int *dp, *orig_dp; unsigned int rpn; diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h index 3249b7464889..ffaba07f50ab 100644 --- a/arch/s390/include/asm/dma-mapping.h +++ b/arch/s390/include/asm/dma-mapping.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 070f1ae5cfad..7297fce9bf80 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -286,7 +286,7 @@ static inline void zpci_err_dma(unsigned long rc, unsigned long addr) static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); unsigned long nr_pages, iommu_page_index; @@ -332,7 +332,7 @@ out_err: static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); unsigned long iommu_page_index; @@ -355,7 +355,7 @@ static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr, static void *s390_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); struct page *page; @@ -370,7 +370,7 @@ static void *s390_dma_alloc(struct device *dev, size_t size, pa = page_to_phys(page); memset((void *) pa, 0, size); - map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, NULL); + map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, 0); if (dma_mapping_error(dev, map)) { free_pages(pa, get_order(size)); return NULL; @@ -384,19 +384,19 @@ static void *s390_dma_alloc(struct device *dev, size_t size, static void s390_dma_free(struct device *dev, size_t size, void *pa, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); size = PAGE_ALIGN(size); atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages); - s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL); + s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, 0); free_pages((unsigned long) pa, get_order(size)); } static int s390_dma_map_sg(struct device *dev, struct scatterlist *sg, int nr_elements, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { int mapped_elements = 0; struct scatterlist *s; @@ -405,7 +405,7 @@ static int s390_dma_map_sg(struct device *dev, struct scatterlist *sg, for_each_sg(sg, s, nr_elements, i) { struct page *page = sg_page(s); s->dma_address = s390_dma_map_pages(dev, page, s->offset, - s->length, dir, NULL); + s->length, dir, 0); if (!dma_mapping_error(dev, s->dma_address)) { s->dma_length = s->length; mapped_elements++; @@ -419,7 +419,7 @@ unmap: for_each_sg(sg, s, mapped_elements, i) { if (s->dma_address) s390_dma_unmap_pages(dev, s->dma_address, s->dma_length, - dir, NULL); + dir, 0); s->dma_address = 0; s->dma_length = 0; } @@ -429,13 +429,14 @@ unmap: static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nr_elements, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; for_each_sg(sg, s, nr_elements, i) { - s390_dma_unmap_pages(dev, s->dma_address, s->dma_length, dir, NULL); + s390_dma_unmap_pages(dev, s->dma_address, s->dma_length, dir, + 0); s->dma_address = 0; s->dma_length = 0; } diff --git a/arch/sh/include/asm/dma-mapping.h b/arch/sh/include/asm/dma-mapping.h index e11cf0c8206b..0052ad40e86d 100644 --- a/arch/sh/include/asm/dma-mapping.h +++ b/arch/sh/include/asm/dma-mapping.h @@ -17,9 +17,9 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size, /* arch/sh/mm/consistent.c */ extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, - struct dma_attrs *attrs); + unsigned long attrs); extern void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs); + unsigned long attrs); #endif /* __ASM_SH_DMA_MAPPING_H */ diff --git a/arch/sh/kernel/dma-nommu.c b/arch/sh/kernel/dma-nommu.c index 5b0bfcda6d0b..eadb669a7329 100644 --- a/arch/sh/kernel/dma-nommu.c +++ b/arch/sh/kernel/dma-nommu.c @@ -13,7 +13,7 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t addr = page_to_phys(page) + offset; @@ -25,7 +25,7 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page, static int nommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c index b81d9dbf9fef..92b6976fde59 100644 --- a/arch/sh/mm/consistent.c +++ b/arch/sh/mm/consistent.c @@ -34,7 +34,7 @@ fs_initcall(dma_init); void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret, *ret_nocache; int order = get_order(size); @@ -66,7 +66,7 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size, void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { int order = get_order(size); unsigned long pfn = dma_handle >> PAGE_SHIFT; diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 37686828c3d9..5c615abff030 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -196,7 +196,7 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) static void *dma_4u_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long order, first_page; struct iommu *iommu; @@ -245,7 +245,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, static void dma_4u_free_coherent(struct device *dev, size_t size, void *cpu, dma_addr_t dvma, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; unsigned long order, npages; @@ -263,7 +263,7 @@ static void dma_4u_free_coherent(struct device *dev, size_t size, static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, unsigned long offset, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; @@ -385,7 +385,7 @@ do_flush_sync: static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; @@ -431,7 +431,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot, ctx; @@ -607,7 +607,7 @@ static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, ctx; struct scatterlist *sg; diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index ffd5ff4678cf..2344103414d1 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -260,7 +260,7 @@ EXPORT_SYMBOL(sbus_set_sbus64); */ static void *sbus_alloc_coherent(struct device *dev, size_t len, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { struct platform_device *op = to_platform_device(dev); unsigned long len_total = PAGE_ALIGN(len); @@ -315,7 +315,7 @@ err_nopages: } static void sbus_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) + dma_addr_t ba, unsigned long attrs) { struct resource *res; struct page *pgv; @@ -355,7 +355,7 @@ static void sbus_free_coherent(struct device *dev, size_t n, void *p, static dma_addr_t sbus_map_page(struct device *dev, struct page *page, unsigned long offset, size_t len, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *va = page_address(page) + offset; @@ -371,20 +371,20 @@ static dma_addr_t sbus_map_page(struct device *dev, struct page *page, } static void sbus_unmap_page(struct device *dev, dma_addr_t ba, size_t n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_release_scsi_one(dev, ba, n); } static int sbus_map_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_get_scsi_sgl(dev, sg, n); return n; } static void sbus_unmap_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_release_scsi_sgl(dev, sg, n); } @@ -429,7 +429,7 @@ arch_initcall(sparc_register_ioport); */ static void *pci32_alloc_coherent(struct device *dev, size_t len, dma_addr_t *pba, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long len_total = PAGE_ALIGN(len); void *va; @@ -482,7 +482,7 @@ err_nopages: * past this call are illegal. */ static void pci32_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) + dma_addr_t ba, unsigned long attrs) { struct resource *res; @@ -518,14 +518,14 @@ static void pci32_free_coherent(struct device *dev, size_t n, void *p, static dma_addr_t pci32_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { /* IIep is write-through, not flushing. */ return page_to_phys(page) + offset; } static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { if (dir != PCI_DMA_TODEVICE) dma_make_coherent(ba, PAGE_ALIGN(size)); @@ -548,7 +548,7 @@ static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, */ static int pci32_map_sg(struct device *device, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int n; @@ -567,7 +567,7 @@ static int pci32_map_sg(struct device *device, struct scatterlist *sgl, */ static void pci32_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int n; diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 836e8cef47e2..61c6f935accc 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -130,7 +130,7 @@ static inline long iommu_batch_end(void) static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, order, first_page, npages, n; struct iommu *iommu; @@ -213,7 +213,7 @@ static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry, } static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, - dma_addr_t dvma, struct dma_attrs *attrs) + dma_addr_t dvma, unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -235,7 +235,7 @@ static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, unsigned long offset, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; unsigned long flags, npages, oaddr; @@ -294,7 +294,7 @@ iommu_map_fail: static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -322,7 +322,7 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot; @@ -466,7 +466,7 @@ iommu_map_failed: static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct scatterlist *sg; diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c index b6bc0547a4f6..09bb774b39cd 100644 --- a/arch/tile/kernel/pci-dma.c +++ b/arch/tile/kernel/pci-dma.c @@ -34,7 +34,7 @@ static void *tile_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { u64 dma_mask = (dev && dev->coherent_dma_mask) ? dev->coherent_dma_mask : DMA_BIT_MASK(32); @@ -78,7 +78,7 @@ static void *tile_dma_alloc_coherent(struct device *dev, size_t size, */ static void tile_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { homecache_free_pages((unsigned long)vaddr, get_order(size)); } @@ -202,7 +202,7 @@ static void __dma_complete_pa_range(dma_addr_t dma_addr, size_t size, static int tile_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -224,7 +224,7 @@ static int tile_dma_map_sg(struct device *dev, struct scatterlist *sglist, static void tile_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -240,7 +240,7 @@ static void tile_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, static dma_addr_t tile_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(!valid_dma_direction(direction)); @@ -252,7 +252,7 @@ static dma_addr_t tile_dma_map_page(struct device *dev, struct page *page, static void tile_dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(!valid_dma_direction(direction)); @@ -343,7 +343,7 @@ EXPORT_SYMBOL(tile_dma_map_ops); static void *tile_pci_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { int node = dev_to_node(dev); int order = get_order(size); @@ -368,14 +368,14 @@ static void *tile_pci_dma_alloc_coherent(struct device *dev, size_t size, */ static void tile_pci_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { homecache_free_pages((unsigned long)vaddr, get_order(size)); } static int tile_pci_dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -400,7 +400,7 @@ static int tile_pci_dma_map_sg(struct device *dev, struct scatterlist *sglist, static void tile_pci_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int i; @@ -416,7 +416,7 @@ static void tile_pci_dma_unmap_sg(struct device *dev, static dma_addr_t tile_pci_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(!valid_dma_direction(direction)); @@ -429,7 +429,7 @@ static dma_addr_t tile_pci_dma_map_page(struct device *dev, struct page *page, static void tile_pci_dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { BUG_ON(!valid_dma_direction(direction)); @@ -531,7 +531,7 @@ EXPORT_SYMBOL(gx_pci_dma_map_ops); #ifdef CONFIG_SWIOTLB static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { gfp |= GFP_DMA; return swiotlb_alloc_coherent(dev, size, dma_handle, gfp); @@ -539,7 +539,7 @@ static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size, static void tile_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_addr); } diff --git a/arch/unicore32/mm/dma-swiotlb.c b/arch/unicore32/mm/dma-swiotlb.c index 16c08b2143a7..3e9f6489ba38 100644 --- a/arch/unicore32/mm/dma-swiotlb.c +++ b/arch/unicore32/mm/dma-swiotlb.c @@ -19,14 +19,14 @@ static void *unicore_swiotlb_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { return swiotlb_alloc_coherent(dev, size, dma_handle, flags); } static void unicore_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { swiotlb_free_coherent(dev, size, vaddr, dma_addr); } diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h index 3a27b93e6261..44461626830e 100644 --- a/arch/x86/include/asm/dma-mapping.h +++ b/arch/x86/include/asm/dma-mapping.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -48,11 +47,11 @@ extern int dma_supported(struct device *hwdev, u64 mask); extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, - struct dma_attrs *attrs); + unsigned long attrs); extern void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs); + unsigned long attrs); #ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */ extern bool dma_capable(struct device *dev, dma_addr_t addr, size_t size); diff --git a/arch/x86/include/asm/swiotlb.h b/arch/x86/include/asm/swiotlb.h index ab05d73e2bb7..d2f69b9ff732 100644 --- a/arch/x86/include/asm/swiotlb.h +++ b/arch/x86/include/asm/swiotlb.h @@ -31,9 +31,9 @@ static inline void dma_mark_clean(void *addr, size_t size) {} extern void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs); + unsigned long attrs); extern void x86_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs); + unsigned long attrs); #endif /* _ASM_X86_SWIOTLB_H */ diff --git a/arch/x86/include/asm/xen/page-coherent.h b/arch/x86/include/asm/xen/page-coherent.h index acd844c017d3..f02f025ff988 100644 --- a/arch/x86/include/asm/xen/page-coherent.h +++ b/arch/x86/include/asm/xen/page-coherent.h @@ -2,12 +2,11 @@ #define _ASM_X86_XEN_PAGE_COHERENT_H #include -#include #include static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { void *vstart = (void*)__get_free_pages(flags, get_order(size)); *dma_handle = virt_to_phys(vstart); @@ -16,18 +15,18 @@ static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { free_pages((unsigned long) cpu_addr, get_order(size)); } static inline void xen_dma_map_page(struct device *hwdev, struct page *page, dma_addr_t dev_addr, unsigned long offset, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) { } + enum dma_data_direction dir, unsigned long attrs) { } static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) { } + unsigned long attrs) { } static inline void xen_dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { } diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index 42d27a62a404..63ff468a7986 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -241,7 +241,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, static dma_addr_t gart_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long bus; phys_addr_t paddr = page_to_phys(page) + offset; @@ -263,7 +263,7 @@ static dma_addr_t gart_map_page(struct device *dev, struct page *page, */ static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long iommu_page; int npages; @@ -285,7 +285,7 @@ static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, * Wrapper for pci_unmap_single working with scatterlists. */ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *s; int i; @@ -293,7 +293,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, for_each_sg(sg, s, nents, i) { if (!s->dma_length || !s->length) break; - gart_unmap_page(dev, s->dma_address, s->dma_length, dir, NULL); + gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0); } } @@ -315,7 +315,7 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, addr = dma_map_area(dev, addr, s->length, dir, 0); if (addr == bad_dma_addr) { if (i > 0) - gart_unmap_sg(dev, sg, i, dir, NULL); + gart_unmap_sg(dev, sg, i, dir, 0); nents = 0; sg[0].dma_length = 0; break; @@ -386,7 +386,7 @@ dma_map_cont(struct device *dev, struct scatterlist *start, int nelems, * Merge chunks that have page aligned sizes into a continuous mapping. */ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *s, *ps, *start_sg, *sgmap; int need = 0, nextneed, i, out, start; @@ -456,7 +456,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, error: flush_gart(); - gart_unmap_sg(dev, sg, out, dir, NULL); + gart_unmap_sg(dev, sg, out, dir, 0); /* When it was forced or merged try again in a dumb way */ if (force_iommu || iommu_merge) { @@ -476,7 +476,7 @@ error: /* allocate and map a coherent mapping */ static void * gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, - gfp_t flag, struct dma_attrs *attrs) + gfp_t flag, unsigned long attrs) { dma_addr_t paddr; unsigned long align_mask; @@ -508,9 +508,9 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, /* free a coherent mapping */ static void gart_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_addr, struct dma_attrs *attrs) + dma_addr_t dma_addr, unsigned long attrs) { - gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, NULL); + gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs); } diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c index 833b1d329c47..5d400ba1349d 100644 --- a/arch/x86/kernel/pci-calgary_64.c +++ b/arch/x86/kernel/pci-calgary_64.c @@ -340,7 +340,7 @@ static inline struct iommu_table *find_iommu_table(struct device *dev) static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu_table *tbl = find_iommu_table(dev); struct scatterlist *s; @@ -364,7 +364,7 @@ static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist, static int calgary_map_sg(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu_table *tbl = find_iommu_table(dev); struct scatterlist *s; @@ -396,7 +396,7 @@ static int calgary_map_sg(struct device *dev, struct scatterlist *sg, return nelems; error: - calgary_unmap_sg(dev, sg, nelems, dir, NULL); + calgary_unmap_sg(dev, sg, nelems, dir, 0); for_each_sg(sg, s, nelems, i) { sg->dma_address = DMA_ERROR_CODE; sg->dma_length = 0; @@ -407,7 +407,7 @@ error: static dma_addr_t calgary_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *vaddr = page_address(page) + offset; unsigned long uaddr; @@ -422,7 +422,7 @@ static dma_addr_t calgary_map_page(struct device *dev, struct page *page, static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu_table *tbl = find_iommu_table(dev); unsigned int npages; @@ -432,7 +432,7 @@ static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr, } static void* calgary_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, struct dma_attrs *attrs) + dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { void *ret = NULL; dma_addr_t mapping; @@ -466,7 +466,7 @@ error: static void calgary_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned int npages; struct iommu_table *tbl = find_iommu_table(dev); diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 6ba014c61d62..d30c37750765 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -77,7 +77,7 @@ void __init pci_iommu_alloc(void) } void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long dma_mask; struct page *page; @@ -120,7 +120,7 @@ again: } void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_addr, struct dma_attrs *attrs) + dma_addr_t dma_addr, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = virt_to_page(vaddr); diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c index da15918d1c81..00e71ce396a8 100644 --- a/arch/x86/kernel/pci-nommu.c +++ b/arch/x86/kernel/pci-nommu.c @@ -28,7 +28,7 @@ check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size) static dma_addr_t nommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t bus = page_to_phys(page) + offset; WARN_ON(size == 0); @@ -55,7 +55,7 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page, */ static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index 5069ef560d83..b47edb8f5256 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -16,7 +16,7 @@ int swiotlb __read_mostly; void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { void *vaddr; @@ -37,7 +37,7 @@ void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, void x86_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr))) swiotlb_free_coherent(dev, size, vaddr, dma_addr); diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c index 5ceda85b8687..052c1cb76305 100644 --- a/arch/x86/pci/sta2x11-fixup.c +++ b/arch/x86/pci/sta2x11-fixup.c @@ -169,7 +169,7 @@ static void *sta2x11_swiotlb_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { void *vaddr; diff --git a/arch/x86/pci/vmd.c b/arch/x86/pci/vmd.c index e88b4176260f..b814ca675131 100644 --- a/arch/x86/pci/vmd.c +++ b/arch/x86/pci/vmd.c @@ -274,14 +274,14 @@ static struct dma_map_ops *vmd_dma_ops(struct device *dev) } static void *vmd_alloc(struct device *dev, size_t size, dma_addr_t *addr, - gfp_t flag, struct dma_attrs *attrs) + gfp_t flag, unsigned long attrs) { return vmd_dma_ops(dev)->alloc(to_vmd_dev(dev), size, addr, flag, attrs); } static void vmd_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t addr, struct dma_attrs *attrs) + dma_addr_t addr, unsigned long attrs) { return vmd_dma_ops(dev)->free(to_vmd_dev(dev), size, vaddr, addr, attrs); @@ -289,7 +289,7 @@ static void vmd_free(struct device *dev, size_t size, void *vaddr, static int vmd_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { return vmd_dma_ops(dev)->mmap(to_vmd_dev(dev), vma, cpu_addr, addr, size, attrs); @@ -297,7 +297,7 @@ static int vmd_mmap(struct device *dev, struct vm_area_struct *vma, static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t addr, size_t size, - struct dma_attrs *attrs) + unsigned long attrs) { return vmd_dma_ops(dev)->get_sgtable(to_vmd_dev(dev), sgt, cpu_addr, addr, size, attrs); @@ -306,26 +306,26 @@ static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt, static dma_addr_t vmd_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return vmd_dma_ops(dev)->map_page(to_vmd_dev(dev), page, offset, size, dir, attrs); } static void vmd_unmap_page(struct device *dev, dma_addr_t addr, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { vmd_dma_ops(dev)->unmap_page(to_vmd_dev(dev), addr, size, dir, attrs); } static int vmd_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { return vmd_dma_ops(dev)->map_sg(to_vmd_dev(dev), sg, nents, dir, attrs); } static void vmd_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { vmd_dma_ops(dev)->unmap_sg(to_vmd_dev(dev), sg, nents, dir, attrs); } diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c index cd66698348ca..1e68806d6695 100644 --- a/arch/xtensa/kernel/pci-dma.c +++ b/arch/xtensa/kernel/pci-dma.c @@ -142,7 +142,7 @@ static void xtensa_sync_sg_for_device(struct device *dev, static void *xtensa_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long ret; unsigned long uncached = 0; @@ -171,7 +171,7 @@ static void *xtensa_dma_alloc(struct device *dev, size_t size, } static void xtensa_dma_free(struct device *hwdev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { unsigned long addr = (unsigned long)vaddr + XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; @@ -185,7 +185,7 @@ static void xtensa_dma_free(struct device *hwdev, size_t size, void *vaddr, static dma_addr_t xtensa_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t dma_handle = page_to_phys(page) + offset; @@ -195,14 +195,14 @@ static dma_addr_t xtensa_map_page(struct device *dev, struct page *page, static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { xtensa_sync_single_for_cpu(dev, dma_handle, size, dir); } static int xtensa_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; @@ -217,7 +217,7 @@ static int xtensa_map_sg(struct device *dev, struct scatterlist *sg, static void xtensa_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s; int i; diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c index fb49443bfd32..4cfb39d543b4 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c @@ -52,7 +52,7 @@ static int exynos_drm_fb_mmap(struct fb_info *info, ret = dma_mmap_attrs(to_dma_dev(helper->dev), vma, exynos_gem->cookie, exynos_gem->dma_addr, exynos_gem->size, - &exynos_gem->dma_attrs); + exynos_gem->dma_attrs); if (ret < 0) { DRM_ERROR("failed to mmap.\n"); return ret; diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index 8564c3da0d22..4bf00f57ffe8 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -235,7 +234,7 @@ struct g2d_data { struct mutex cmdlist_mutex; dma_addr_t cmdlist_pool; void *cmdlist_pool_virt; - struct dma_attrs cmdlist_dma_attrs; + unsigned long cmdlist_dma_attrs; /* runqueue*/ struct g2d_runqueue_node *runqueue_node; @@ -256,13 +255,12 @@ static int g2d_init_cmdlist(struct g2d_data *g2d) int ret; struct g2d_buf_info *buf_info; - init_dma_attrs(&g2d->cmdlist_dma_attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &g2d->cmdlist_dma_attrs); + g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE, &g2d->cmdlist_pool, GFP_KERNEL, - &g2d->cmdlist_dma_attrs); + g2d->cmdlist_dma_attrs); if (!g2d->cmdlist_pool_virt) { dev_err(dev, "failed to allocate dma memory\n"); return -ENOMEM; @@ -295,7 +293,7 @@ static int g2d_init_cmdlist(struct g2d_data *g2d) err: dma_free_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE, g2d->cmdlist_pool_virt, - g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs); + g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); return ret; } @@ -309,7 +307,7 @@ static void g2d_fini_cmdlist(struct g2d_data *g2d) dma_free_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE, g2d->cmdlist_pool_virt, - g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs); + g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); } } diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c index cdf9f1af4347..f2ae72ba7d5a 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gem.c +++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c @@ -24,7 +24,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) { struct drm_device *dev = exynos_gem->base.dev; - enum dma_attr attr; + unsigned long attr; unsigned int nr_pages; struct sg_table sgt; int ret = -ENOMEM; @@ -34,7 +34,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) return 0; } - init_dma_attrs(&exynos_gem->dma_attrs); + exynos_gem->dma_attrs = 0; /* * if EXYNOS_BO_CONTIG, fully physically contiguous memory @@ -42,7 +42,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) * as possible. */ if (!(exynos_gem->flags & EXYNOS_BO_NONCONTIG)) - dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, &exynos_gem->dma_attrs); + exynos_gem->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS; /* * if EXYNOS_BO_WC or EXYNOS_BO_NONCACHABLE, writecombine mapping @@ -54,8 +54,8 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) else attr = DMA_ATTR_NON_CONSISTENT; - dma_set_attr(attr, &exynos_gem->dma_attrs); - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &exynos_gem->dma_attrs); + exynos_gem->dma_attrs |= attr; + exynos_gem->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING; nr_pages = exynos_gem->size >> PAGE_SHIFT; @@ -67,7 +67,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) exynos_gem->cookie = dma_alloc_attrs(to_dma_dev(dev), exynos_gem->size, &exynos_gem->dma_addr, GFP_KERNEL, - &exynos_gem->dma_attrs); + exynos_gem->dma_attrs); if (!exynos_gem->cookie) { DRM_ERROR("failed to allocate buffer.\n"); goto err_free; @@ -75,7 +75,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem) ret = dma_get_sgtable_attrs(to_dma_dev(dev), &sgt, exynos_gem->cookie, exynos_gem->dma_addr, exynos_gem->size, - &exynos_gem->dma_attrs); + exynos_gem->dma_attrs); if (ret < 0) { DRM_ERROR("failed to get sgtable.\n"); goto err_dma_free; @@ -99,7 +99,7 @@ err_sgt_free: sg_free_table(&sgt); err_dma_free: dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie, - exynos_gem->dma_addr, &exynos_gem->dma_attrs); + exynos_gem->dma_addr, exynos_gem->dma_attrs); err_free: drm_free_large(exynos_gem->pages); @@ -120,7 +120,7 @@ static void exynos_drm_free_buf(struct exynos_drm_gem *exynos_gem) dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie, (dma_addr_t)exynos_gem->dma_addr, - &exynos_gem->dma_attrs); + exynos_gem->dma_attrs); drm_free_large(exynos_gem->pages); } @@ -346,7 +346,7 @@ static int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem *exynos_gem, ret = dma_mmap_attrs(to_dma_dev(drm_dev), vma, exynos_gem->cookie, exynos_gem->dma_addr, exynos_gem->size, - &exynos_gem->dma_attrs); + exynos_gem->dma_attrs); if (ret < 0) { DRM_ERROR("failed to mmap.\n"); return ret; diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h index 78100742281d..df7c543d6558 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gem.h +++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h @@ -50,7 +50,7 @@ struct exynos_drm_gem { void *cookie; void __iomem *kvaddr; dma_addr_t dma_addr; - struct dma_attrs dma_attrs; + unsigned long dma_attrs; struct page **pages; struct sg_table *sgt; }; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c index fa2ec0cd00e8..7abc550ebc00 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c @@ -54,15 +54,14 @@ struct mtk_drm_gem_obj *mtk_drm_gem_create(struct drm_device *dev, obj = &mtk_gem->base; - init_dma_attrs(&mtk_gem->dma_attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &mtk_gem->dma_attrs); + mtk_gem->dma_attrs = DMA_ATTR_WRITE_COMBINE; if (!alloc_kmap) - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &mtk_gem->dma_attrs); + mtk_gem->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING; mtk_gem->cookie = dma_alloc_attrs(priv->dma_dev, obj->size, &mtk_gem->dma_addr, GFP_KERNEL, - &mtk_gem->dma_attrs); + mtk_gem->dma_attrs); if (!mtk_gem->cookie) { DRM_ERROR("failed to allocate %zx byte dma buffer", obj->size); ret = -ENOMEM; @@ -93,7 +92,7 @@ void mtk_drm_gem_free_object(struct drm_gem_object *obj) drm_prime_gem_destroy(obj, mtk_gem->sg); else dma_free_attrs(priv->dma_dev, obj->size, mtk_gem->cookie, - mtk_gem->dma_addr, &mtk_gem->dma_attrs); + mtk_gem->dma_addr, mtk_gem->dma_attrs); /* release file pointer to gem object. */ drm_gem_object_release(obj); @@ -173,7 +172,7 @@ static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj, vma->vm_pgoff = 0; ret = dma_mmap_attrs(priv->dma_dev, vma, mtk_gem->cookie, - mtk_gem->dma_addr, obj->size, &mtk_gem->dma_attrs); + mtk_gem->dma_addr, obj->size, mtk_gem->dma_attrs); if (ret) drm_gem_vm_close(vma); @@ -224,7 +223,7 @@ struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj) ret = dma_get_sgtable_attrs(priv->dma_dev, sgt, mtk_gem->cookie, mtk_gem->dma_addr, obj->size, - &mtk_gem->dma_attrs); + mtk_gem->dma_attrs); if (ret) { DRM_ERROR("failed to allocate sgt, %d\n", ret); kfree(sgt); diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.h b/drivers/gpu/drm/mediatek/mtk_drm_gem.h index 3a2a5624a1cb..2752718fa5b2 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_gem.h +++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.h @@ -35,7 +35,7 @@ struct mtk_drm_gem_obj { void *cookie; void *kvaddr; dma_addr_t dma_addr; - struct dma_attrs dma_attrs; + unsigned long dma_attrs; struct sg_table *sg; }; diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 26f859ec24b3..8a0237008f74 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -238,11 +238,10 @@ static int msm_drm_uninit(struct device *dev) } if (priv->vram.paddr) { - DEFINE_DMA_ATTRS(attrs); - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs); + unsigned long attrs = DMA_ATTR_NO_KERNEL_MAPPING; drm_mm_takedown(&priv->vram.mm); dma_free_attrs(dev, priv->vram.size, NULL, - priv->vram.paddr, &attrs); + priv->vram.paddr, attrs); } component_unbind_all(dev, ddev); @@ -310,21 +309,21 @@ static int msm_init_vram(struct drm_device *dev) } if (size) { - DEFINE_DMA_ATTRS(attrs); + unsigned long attrs = 0; void *p; priv->vram.size = size; drm_mm_init(&priv->vram.mm, 0, (size >> PAGE_SHIFT) - 1); - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + attrs |= DMA_ATTR_NO_KERNEL_MAPPING; + attrs |= DMA_ATTR_WRITE_COMBINE; /* note that for no-kernel-mapping, the vaddr returned * is bogus, but non-null if allocation succeeded: */ p = dma_alloc_attrs(dev->dev, size, - &priv->vram.paddr, GFP_KERNEL, &attrs); + &priv->vram.paddr, GFP_KERNEL, attrs); if (!p) { dev_err(dev->dev, "failed to allocate VRAM\n"); priv->vram.paddr = 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 6b8f2a19b2d9..a6a7fa0d7679 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -109,7 +109,7 @@ struct gk20a_instmem { u16 iommu_bit; /* Only used by DMA API */ - struct dma_attrs attrs; + unsigned long attrs; }; #define gk20a_instmem(p) container_of((p), struct gk20a_instmem, base) @@ -293,7 +293,7 @@ gk20a_instobj_dtor_dma(struct nvkm_memory *memory) goto out; dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->base.vaddr, - node->handle, &imem->attrs); + node->handle, imem->attrs); out: return node; @@ -386,7 +386,7 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align, node->base.vaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT, &node->handle, GFP_KERNEL, - &imem->attrs); + imem->attrs); if (!node->base.vaddr) { nvkm_error(subdev, "cannot allocate DMA memory\n"); return -ENOMEM; @@ -597,10 +597,9 @@ gk20a_instmem_new(struct nvkm_device *device, int index, nvkm_info(&imem->base.subdev, "using IOMMU\n"); } else { - init_dma_attrs(&imem->attrs); - dma_set_attr(DMA_ATTR_NON_CONSISTENT, &imem->attrs); - dma_set_attr(DMA_ATTR_WEAK_ORDERING, &imem->attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &imem->attrs); + imem->attrs = DMA_ATTR_NON_CONSISTENT | + DMA_ATTR_WEAK_ORDERING | + DMA_ATTR_WRITE_COMBINE; nvkm_info(&imem->base.subdev, "using DMA API\n"); } diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c index 059e902f872d..b70f9423379c 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c @@ -17,8 +17,6 @@ #include #include -#include - #include "rockchip_drm_drv.h" #include "rockchip_drm_gem.h" @@ -28,15 +26,14 @@ static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj, struct drm_gem_object *obj = &rk_obj->base; struct drm_device *drm = obj->dev; - init_dma_attrs(&rk_obj->dma_attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &rk_obj->dma_attrs); + rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE; if (!alloc_kmap) - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &rk_obj->dma_attrs); + rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING; rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size, &rk_obj->dma_addr, GFP_KERNEL, - &rk_obj->dma_attrs); + rk_obj->dma_attrs); if (!rk_obj->kvaddr) { DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size); return -ENOMEM; @@ -51,7 +48,7 @@ static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj) struct drm_device *drm = obj->dev; dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr, - &rk_obj->dma_attrs); + rk_obj->dma_attrs); } static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj, @@ -70,7 +67,7 @@ static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj, vma->vm_pgoff = 0; ret = dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr, - obj->size, &rk_obj->dma_attrs); + obj->size, rk_obj->dma_attrs); if (ret) drm_gem_vm_close(vma); @@ -262,7 +259,7 @@ struct sg_table *rockchip_gem_prime_get_sg_table(struct drm_gem_object *obj) ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr, rk_obj->dma_addr, obj->size, - &rk_obj->dma_attrs); + rk_obj->dma_attrs); if (ret) { DRM_ERROR("failed to allocate sgt, %d\n", ret); kfree(sgt); @@ -276,7 +273,7 @@ void *rockchip_gem_prime_vmap(struct drm_gem_object *obj) { struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); - if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, &rk_obj->dma_attrs)) + if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING) return NULL; return rk_obj->kvaddr; diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.h b/drivers/gpu/drm/rockchip/rockchip_drm_gem.h index ad22618473a4..18b3488db4ec 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.h @@ -23,7 +23,7 @@ struct rockchip_gem_object { void *kvaddr; dma_addr_t dma_addr; - struct dma_attrs dma_attrs; + unsigned long dma_attrs; }; struct sg_table *rockchip_gem_prime_get_sg_table(struct drm_gem_object *obj); diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index fe4d2e1a8b58..c68746ce6624 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include @@ -92,12 +91,12 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, unsigned long npages; int ret; int i; - DEFINE_DMA_ATTRS(attrs); + unsigned long dma_attrs = 0; struct scatterlist *sg, *sg_list_start; int need_release = 0; if (dmasync) - dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs); + dma_attrs |= DMA_ATTR_WRITE_BARRIER; if (!size) return ERR_PTR(-EINVAL); @@ -215,7 +214,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, umem->sg_head.sgl, umem->npages, DMA_BIDIRECTIONAL, - &attrs); + dma_attrs); if (umem->nmap <= 0) { ret = -ENOMEM; diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 33c177ba93be..96de97a46079 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2375,7 +2375,7 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, static dma_addr_t map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { phys_addr_t paddr = page_to_phys(page) + offset; struct protection_domain *domain; @@ -2398,7 +2398,7 @@ static dma_addr_t map_page(struct device *dev, struct page *page, * The exported unmap_single function for dma_ops. */ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { struct protection_domain *domain; struct dma_ops_domain *dma_dom; @@ -2444,7 +2444,7 @@ static int sg_num_pages(struct device *dev, */ static int map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { int mapped_pages = 0, npages = 0, prot = 0, i; struct protection_domain *domain; @@ -2525,7 +2525,7 @@ out_err: */ static void unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct protection_domain *domain; struct dma_ops_domain *dma_dom; @@ -2548,7 +2548,7 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, */ static void *alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { u64 dma_mask = dev->coherent_dma_mask; struct protection_domain *domain; @@ -2604,7 +2604,7 @@ out_free: */ static void free_coherent(struct device *dev, size_t size, void *virt_addr, dma_addr_t dma_addr, - struct dma_attrs *attrs) + unsigned long attrs) { struct protection_domain *domain; struct dma_ops_domain *dma_dom; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index ea5a9ebf0f78..08a1e2f3690f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -286,7 +286,7 @@ void iommu_dma_free(struct device *dev, struct page **pages, size_t size, * or NULL on failure. */ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, - struct dma_attrs *attrs, int prot, dma_addr_t *handle, + unsigned long attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); @@ -306,7 +306,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, } else { size = ALIGN(size, min_size); } - if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) + if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) alloc_sizes = min_size; count = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -400,7 +400,7 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, } void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle); } @@ -560,7 +560,7 @@ out_restore_sg: } void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { /* * The scatterlist segments are mapped into a single diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index afbaa2c69a59..ebb5bf3ddbd9 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3552,7 +3552,7 @@ error: static dma_addr_t intel_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { return __intel_map_single(dev, page_to_phys(page) + offset, size, dir, *dev->dma_mask); @@ -3711,14 +3711,14 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { intel_unmap(dev, dev_addr, size); } static void *intel_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) + unsigned long attrs) { struct page *page = NULL; int order; @@ -3764,7 +3764,7 @@ static void *intel_alloc_coherent(struct device *dev, size_t size, } static void intel_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { int order; struct page *page = virt_to_page(vaddr); @@ -3779,7 +3779,7 @@ static void intel_free_coherent(struct device *dev, size_t size, void *vaddr, static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK; unsigned long nrpages = 0; @@ -3808,7 +3808,7 @@ static int intel_nontranslate_map_sg(struct device *hddev, } static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { int i; struct dmar_domain *domain; diff --git a/drivers/media/platform/sti/bdisp/bdisp-hw.c b/drivers/media/platform/sti/bdisp/bdisp-hw.c index 3df66d11c795..b7892f3efd98 100644 --- a/drivers/media/platform/sti/bdisp/bdisp-hw.c +++ b/drivers/media/platform/sti/bdisp/bdisp-hw.c @@ -430,14 +430,11 @@ int bdisp_hw_get_and_clear_irq(struct bdisp_dev *bdisp) */ void bdisp_hw_free_nodes(struct bdisp_ctx *ctx) { - if (ctx && ctx->node[0]) { - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + if (ctx && ctx->node[0]) dma_free_attrs(ctx->bdisp_dev->dev, sizeof(struct bdisp_node) * MAX_NB_NODE, - ctx->node[0], ctx->node_paddr[0], &attrs); - } + ctx->node[0], ctx->node_paddr[0], + DMA_ATTR_WRITE_COMBINE); } /** @@ -455,12 +452,10 @@ int bdisp_hw_alloc_nodes(struct bdisp_ctx *ctx) unsigned int i, node_size = sizeof(struct bdisp_node); void *base; dma_addr_t paddr; - DEFINE_DMA_ATTRS(attrs); /* Allocate all the nodes within a single memory page */ - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); base = dma_alloc_attrs(dev, node_size * MAX_NB_NODE, &paddr, - GFP_KERNEL | GFP_DMA, &attrs); + GFP_KERNEL | GFP_DMA, DMA_ATTR_WRITE_COMBINE); if (!base) { dev_err(dev, "%s no mem\n", __func__); return -ENOMEM; @@ -493,13 +488,9 @@ void bdisp_hw_free_filters(struct device *dev) { int size = (BDISP_HF_NB * NB_H_FILTER) + (BDISP_VF_NB * NB_V_FILTER); - if (bdisp_h_filter[0].virt) { - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + if (bdisp_h_filter[0].virt) dma_free_attrs(dev, size, bdisp_h_filter[0].virt, - bdisp_h_filter[0].paddr, &attrs); - } + bdisp_h_filter[0].paddr, DMA_ATTR_WRITE_COMBINE); } /** @@ -516,12 +507,11 @@ int bdisp_hw_alloc_filters(struct device *dev) unsigned int i, size; void *base; dma_addr_t paddr; - DEFINE_DMA_ATTRS(attrs); /* Allocate all the filters within a single memory page */ size = (BDISP_HF_NB * NB_H_FILTER) + (BDISP_VF_NB * NB_V_FILTER); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); - base = dma_alloc_attrs(dev, size, &paddr, GFP_KERNEL | GFP_DMA, &attrs); + base = dma_alloc_attrs(dev, size, &paddr, GFP_KERNEL | GFP_DMA, + DMA_ATTR_WRITE_COMBINE); if (!base) return -ENOMEM; diff --git a/drivers/media/v4l2-core/videobuf2-dma-contig.c b/drivers/media/v4l2-core/videobuf2-dma-contig.c index 863f658a3fa1..b09b2c9b6b63 100644 --- a/drivers/media/v4l2-core/videobuf2-dma-contig.c +++ b/drivers/media/v4l2-core/videobuf2-dma-contig.c @@ -27,7 +27,7 @@ struct vb2_dc_buf { unsigned long size; void *cookie; dma_addr_t dma_addr; - struct dma_attrs attrs; + unsigned long attrs; enum dma_data_direction dma_dir; struct sg_table *dma_sgt; struct frame_vector *vec; @@ -130,12 +130,12 @@ static void vb2_dc_put(void *buf_priv) kfree(buf->sgt_base); } dma_free_attrs(buf->dev, buf->size, buf->cookie, buf->dma_addr, - &buf->attrs); + buf->attrs); put_device(buf->dev); kfree(buf); } -static void *vb2_dc_alloc(struct device *dev, const struct dma_attrs *attrs, +static void *vb2_dc_alloc(struct device *dev, unsigned long attrs, unsigned long size, enum dma_data_direction dma_dir, gfp_t gfp_flags) { @@ -146,16 +146,16 @@ static void *vb2_dc_alloc(struct device *dev, const struct dma_attrs *attrs, return ERR_PTR(-ENOMEM); if (attrs) - buf->attrs = *attrs; + buf->attrs = attrs; buf->cookie = dma_alloc_attrs(dev, size, &buf->dma_addr, - GFP_KERNEL | gfp_flags, &buf->attrs); + GFP_KERNEL | gfp_flags, buf->attrs); if (!buf->cookie) { dev_err(dev, "dma_alloc_coherent of size %ld failed\n", size); kfree(buf); return ERR_PTR(-ENOMEM); } - if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, &buf->attrs)) + if ((buf->attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0) buf->vaddr = buf->cookie; /* Prevent the device from being released while the buffer is used */ @@ -189,7 +189,7 @@ static int vb2_dc_mmap(void *buf_priv, struct vm_area_struct *vma) vma->vm_pgoff = 0; ret = dma_mmap_attrs(buf->dev, vma, buf->cookie, - buf->dma_addr, buf->size, &buf->attrs); + buf->dma_addr, buf->size, buf->attrs); if (ret) { pr_err("Remapping memory failed, error: %d\n", ret); @@ -372,7 +372,7 @@ static struct sg_table *vb2_dc_get_base_sgt(struct vb2_dc_buf *buf) } ret = dma_get_sgtable_attrs(buf->dev, sgt, buf->cookie, buf->dma_addr, - buf->size, &buf->attrs); + buf->size, buf->attrs); if (ret < 0) { dev_err(buf->dev, "failed to get scatterlist from DMA API\n"); kfree(sgt); @@ -421,15 +421,12 @@ static void vb2_dc_put_userptr(void *buf_priv) struct page **pages; if (sgt) { - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); /* * No need to sync to CPU, it's already synced to the CPU * since the finish() memop will have been called before this. */ dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); pages = frame_vector_pages(buf->vec); /* sgt should exist only if vector contains pages... */ BUG_ON(IS_ERR(pages)); @@ -484,9 +481,6 @@ static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr, struct sg_table *sgt; unsigned long contig_size; unsigned long dma_align = dma_get_cache_alignment(); - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); /* Only cache aligned DMA transfers are reliable */ if (!IS_ALIGNED(vaddr | size, dma_align)) { @@ -548,7 +542,7 @@ static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr, * prepare() memop is called. */ sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); if (sgt->nents <= 0) { pr_err("failed to map scatterlist\n"); ret = -EIO; @@ -572,7 +566,7 @@ out: fail_map_sg: dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); fail_sgt_init: sg_free_table(sgt); diff --git a/drivers/media/v4l2-core/videobuf2-dma-sg.c b/drivers/media/v4l2-core/videobuf2-dma-sg.c index a39db8a6db7a..bd82d709ee82 100644 --- a/drivers/media/v4l2-core/videobuf2-dma-sg.c +++ b/drivers/media/v4l2-core/videobuf2-dma-sg.c @@ -95,7 +95,7 @@ static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf, return 0; } -static void *vb2_dma_sg_alloc(struct device *dev, const struct dma_attrs *dma_attrs, +static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs, unsigned long size, enum dma_data_direction dma_dir, gfp_t gfp_flags) { @@ -103,9 +103,6 @@ static void *vb2_dma_sg_alloc(struct device *dev, const struct dma_attrs *dma_at struct sg_table *sgt; int ret; int num_pages; - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); if (WARN_ON(dev == NULL)) return NULL; @@ -144,7 +141,7 @@ static void *vb2_dma_sg_alloc(struct device *dev, const struct dma_attrs *dma_at * prepare() memop is called. */ sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); if (!sgt->nents) goto fail_map; @@ -179,13 +176,10 @@ static void vb2_dma_sg_put(void *buf_priv) int i = buf->num_pages; if (atomic_dec_and_test(&buf->refcount)) { - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); dprintk(1, "%s: Freeing buffer of %d pages\n", __func__, buf->num_pages); dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(buf->dma_sgt); @@ -228,10 +222,8 @@ static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr, { struct vb2_dma_sg_buf *buf; struct sg_table *sgt; - DEFINE_DMA_ATTRS(attrs); struct frame_vector *vec; - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); buf = kzalloc(sizeof *buf, GFP_KERNEL); if (!buf) return NULL; @@ -262,7 +254,7 @@ static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr, * prepare() memop is called. */ sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, - buf->dma_dir, &attrs); + buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC); if (!sgt->nents) goto userptr_fail_map; @@ -286,14 +278,11 @@ static void vb2_dma_sg_put_userptr(void *buf_priv) struct vb2_dma_sg_buf *buf = buf_priv; struct sg_table *sgt = &buf->sg_table; int i = buf->num_pages; - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); dprintk(1, "%s: Releasing userspace buffer of %d pages\n", __func__, buf->num_pages); dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir, - &attrs); + DMA_ATTR_SKIP_CPU_SYNC); if (buf->vaddr) vm_unmap_ram(buf->vaddr, buf->num_pages); sg_free_table(buf->dma_sgt); diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/v4l2-core/videobuf2-vmalloc.c index 7e8a07ed8d82..c2820a6e164d 100644 --- a/drivers/media/v4l2-core/videobuf2-vmalloc.c +++ b/drivers/media/v4l2-core/videobuf2-vmalloc.c @@ -33,7 +33,7 @@ struct vb2_vmalloc_buf { static void vb2_vmalloc_put(void *buf_priv); -static void *vb2_vmalloc_alloc(struct device *dev, const struct dma_attrs *attrs, +static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs, unsigned long size, enum dma_data_direction dma_dir, gfp_t gfp_flags) { diff --git a/drivers/misc/mic/host/mic_boot.c b/drivers/misc/mic/host/mic_boot.c index e047efd83f57..9599d732aff3 100644 --- a/drivers/misc/mic/host/mic_boot.c +++ b/drivers/misc/mic/host/mic_boot.c @@ -38,7 +38,7 @@ static inline struct mic_device *vpdev_to_mdev(struct device *dev) static dma_addr_t _mic_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { void *va = phys_to_virt(page_to_phys(page)) + offset; struct mic_device *mdev = vpdev_to_mdev(dev); @@ -48,7 +48,7 @@ _mic_dma_map_page(struct device *dev, struct page *page, static void _mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct mic_device *mdev = vpdev_to_mdev(dev); @@ -144,7 +144,7 @@ static inline struct mic_device *scdev_to_mdev(struct scif_hw_dev *scdev) static void *__mic_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); @@ -164,7 +164,7 @@ static void *__mic_dma_alloc(struct device *dev, size_t size, } static void __mic_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); @@ -176,7 +176,7 @@ static void __mic_dma_free(struct device *dev, size_t size, void *vaddr, static dma_addr_t __mic_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *va = phys_to_virt(page_to_phys(page)) + offset; struct scif_hw_dev *scdev = dev_get_drvdata(dev); @@ -188,7 +188,7 @@ __mic_dma_map_page(struct device *dev, struct page *page, unsigned long offset, static void __mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); @@ -198,7 +198,7 @@ __mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, static int __mic_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); @@ -229,7 +229,7 @@ err: static void __mic_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scif_hw_dev *scdev = dev_get_drvdata(dev); struct mic_device *mdev = scdev_to_mdev(scdev); @@ -327,7 +327,7 @@ static inline struct mic_device *mbdev_to_mdev(struct mbus_device *mbdev) static dma_addr_t mic_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *va = phys_to_virt(page_to_phys(page)) + offset; struct mic_device *mdev = dev_get_drvdata(dev->parent); @@ -338,7 +338,7 @@ mic_dma_map_page(struct device *dev, struct page *page, static void mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct mic_device *mdev = dev_get_drvdata(dev->parent); mic_unmap_single(mdev, dma_addr, size); diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index e24b05996a1b..3ed6238f8f6e 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -790,7 +790,7 @@ ccio_map_single(struct device *dev, void *addr, size_t size, static dma_addr_t ccio_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return ccio_map_single(dev, page_address(page) + offset, size, direction); @@ -806,7 +806,7 @@ ccio_map_page(struct device *dev, struct page *page, unsigned long offset, */ static void ccio_unmap_page(struct device *dev, dma_addr_t iova, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; unsigned long flags; @@ -844,7 +844,7 @@ ccio_unmap_page(struct device *dev, dma_addr_t iova, size_t size, */ static void * ccio_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) + unsigned long attrs) { void *ret; #if 0 @@ -878,9 +878,9 @@ ccio_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, */ static void ccio_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { - ccio_unmap_page(dev, dma_handle, size, 0, NULL); + ccio_unmap_page(dev, dma_handle, size, 0, 0); free_pages((unsigned long)cpu_addr, get_order(size)); } @@ -907,7 +907,7 @@ ccio_free(struct device *dev, size_t size, void *cpu_addr, */ static int ccio_map_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; int coalesced, filled = 0; @@ -984,7 +984,7 @@ ccio_map_sg(struct device *dev, struct scatterlist *sglist, int nents, */ static void ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; @@ -1004,7 +1004,7 @@ ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT; #endif ccio_unmap_page(dev, sg_dma_address(sglist), - sg_dma_len(sglist), direction, NULL); + sg_dma_len(sglist), direction, 0); ++sglist; } diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index 42ec4600b7e4..151b86b6d2e2 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -783,7 +783,7 @@ sba_map_single(struct device *dev, void *addr, size_t size, static dma_addr_t sba_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { return sba_map_single(dev, page_address(page) + offset, size, direction); @@ -801,7 +801,7 @@ sba_map_page(struct device *dev, struct page *page, unsigned long offset, */ static void sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; #if DELAYED_RESOURCE_CNT > 0 @@ -876,7 +876,7 @@ sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size, * See Documentation/DMA-API-HOWTO.txt */ static void *sba_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, struct dma_attrs *attrs) + gfp_t gfp, unsigned long attrs) { void *ret; @@ -908,9 +908,9 @@ static void *sba_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle */ static void sba_free(struct device *hwdev, size_t size, void *vaddr, - dma_addr_t dma_handle, struct dma_attrs *attrs) + dma_addr_t dma_handle, unsigned long attrs) { - sba_unmap_page(hwdev, dma_handle, size, 0, NULL); + sba_unmap_page(hwdev, dma_handle, size, 0, 0); free_pages((unsigned long) vaddr, get_order(size)); } @@ -943,7 +943,7 @@ int dump_run_sg = 0; */ static int sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; int coalesced, filled = 0; @@ -1026,7 +1026,7 @@ sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents, */ static void sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction, struct dma_attrs *attrs) + enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; #ifdef ASSERT_PDIR_SANITY @@ -1051,7 +1051,7 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, while (sg_dma_len(sglist) && nents--) { sba_unmap_page(dev, sg_dma_address(sglist), sg_dma_len(sglist), - direction, NULL); + direction, 0); #ifdef SBA_COLLECT_STATS ioc->usg_pages += ((sg_dma_address(sglist) & ~IOVP_MASK) + sg_dma_len(sglist) + IOVP_SIZE - 1) >> PAGE_SHIFT; ioc->usingle_calls--; /* kluge since call is unmap_sg() */ diff --git a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c index 24791886219a..2a1b2c7d8f2c 100644 --- a/drivers/remoteproc/qcom_q6v5_pil.c +++ b/drivers/remoteproc/qcom_q6v5_pil.c @@ -349,13 +349,12 @@ static void q6v5proc_halt_axi_port(struct q6v5 *qproc, static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) { - DEFINE_DMA_ATTRS(attrs); + unsigned long dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS; dma_addr_t phys; void *ptr; int ret; - dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, &attrs); - ptr = dma_alloc_attrs(qproc->dev, fw->size, &phys, GFP_KERNEL, &attrs); + ptr = dma_alloc_attrs(qproc->dev, fw->size, &phys, GFP_KERNEL, dma_attrs); if (!ptr) { dev_err(qproc->dev, "failed to allocate mdt buffer\n"); return -ENOMEM; @@ -372,7 +371,7 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) else if (ret < 0) dev_err(qproc->dev, "MPSS header authentication failed: %d\n", ret); - dma_free_attrs(qproc->dev, fw->size, ptr, phys, &attrs); + dma_free_attrs(qproc->dev, fw->size, ptr, phys, dma_attrs); return ret < 0 ? ret : 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c index 2fb90cb6803f..1d7c012f09db 100644 --- a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c +++ b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c @@ -1332,7 +1332,7 @@ static void omapfb_free_fbmem(struct fb_info *fbi) } dma_free_attrs(fbdev->dev, rg->size, rg->token, rg->dma_handle, - &rg->attrs); + rg->attrs); rg->token = NULL; rg->vaddr = NULL; @@ -1370,7 +1370,7 @@ static int omapfb_alloc_fbmem(struct fb_info *fbi, unsigned long size, struct omapfb2_device *fbdev = ofbi->fbdev; struct omapfb2_mem_region *rg; void *token; - DEFINE_DMA_ATTRS(attrs); + unsigned long attrs; dma_addr_t dma_handle; int r; @@ -1386,15 +1386,15 @@ static int omapfb_alloc_fbmem(struct fb_info *fbi, unsigned long size, size = PAGE_ALIGN(size); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + attrs = DMA_ATTR_WRITE_COMBINE; if (ofbi->rotation_type == OMAP_DSS_ROT_VRFB) - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs); + attrs |= DMA_ATTR_NO_KERNEL_MAPPING; DBG("allocating %lu bytes for fb %d\n", size, ofbi->id); token = dma_alloc_attrs(fbdev->dev, size, &dma_handle, - GFP_KERNEL, &attrs); + GFP_KERNEL, attrs); if (token == NULL) { dev_err(fbdev->dev, "failed to allocate framebuffer\n"); @@ -1408,7 +1408,7 @@ static int omapfb_alloc_fbmem(struct fb_info *fbi, unsigned long size, r = omap_vrfb_request_ctx(&rg->vrfb); if (r) { dma_free_attrs(fbdev->dev, size, token, dma_handle, - &attrs); + attrs); dev_err(fbdev->dev, "vrfb create ctx failed\n"); return r; } diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb.h b/drivers/video/fbdev/omap2/omapfb/omapfb.h index bcb9ff4a607d..555487d6dbea 100644 --- a/drivers/video/fbdev/omap2/omapfb/omapfb.h +++ b/drivers/video/fbdev/omap2/omapfb/omapfb.h @@ -28,7 +28,6 @@ #endif #include -#include #include #include