From b9abb19fa5fd2d8a4be61c6cd4b2a48aa1a17f9c Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Wed, 3 Mar 2021 17:36:11 +0000 Subject: iommu: Check dev->iommu in iommu_dev_xxx functions The device iommu probe/attach might have failed leaving dev->iommu to NULL and device drivers may still invoke these functions resulting in a crash in iommu vendor driver code. Hence make sure we check that. Fixes: a3a195929d40 ("iommu: Add APIs for multiple domains per device") Signed-off-by: Shameer Kolothum Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20210303173611.520-1-shameerali.kolothum.thodi@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d0b0a15dba84..e10cfa99057c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2878,10 +2878,12 @@ EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); */ int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_enable_feat) - return ops->dev_enable_feat(dev, feat); + if (ops->dev_enable_feat) + return ops->dev_enable_feat(dev, feat); + } return -ENODEV; } @@ -2894,10 +2896,12 @@ EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); */ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_disable_feat) - return ops->dev_disable_feat(dev, feat); + if (ops->dev_disable_feat) + return ops->dev_disable_feat(dev, feat); + } return -EBUSY; } @@ -2905,10 +2909,12 @@ EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) { - const struct iommu_ops *ops = dev->bus->iommu_ops; + if (dev->iommu && dev->iommu->iommu_dev) { + const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; - if (ops && ops->dev_feat_enabled) - return ops->dev_feat_enabled(dev, feat); + if (ops->dev_feat_enabled) + return ops->dev_feat_enabled(dev, feat); + } return false; } -- cgit v1.2.3 From b23e4fc4e3faed0b8b604079c44a244da3ec941a Mon Sep 17 00:00:00 2001 From: Chunyan Zhang Date: Fri, 5 Mar 2021 17:32:16 +0800 Subject: iommu: add Unisoc IOMMU basic driver This IOMMU module can be used by Unisoc's multimedia devices, such as display, Image codec(jpeg) and a few signal processors, including VSP(video), GSP(graphic), ISP(image), and CPP(camera pixel processor), etc. Signed-off-by: Chunyan Zhang Link: https://lore.kernel.org/r/20210305093216.201897-3-zhang.lyra@gmail.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 12 + drivers/iommu/Makefile | 1 + drivers/iommu/sprd-iommu.c | 577 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 590 insertions(+) create mode 100644 drivers/iommu/sprd-iommu.c (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 192ef8f61310..2ef0b3918440 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -408,4 +408,16 @@ config VIRTIO_IOMMU Say Y here if you intend to run this kernel as a guest. +config SPRD_IOMMU + tristate "Unisoc IOMMU Support" + depends on ARCH_SPRD || COMPILE_TEST + select IOMMU_API + help + Support for IOMMU on Unisoc's SoCs, this IOMMU can be used by + Unisoc's multimedia devices, such as display, Image codec(jpeg) + and a few signal processors, including VSP(video), GSP(graphic), + ISP(image), and CPP(camera pixel processor), etc. + + Say Y here if you want to use the multimedia devices listed above. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 61bd30cd8369..5925b6af2123 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -28,3 +28,4 @@ obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o obj-$(CONFIG_IOMMU_SVA_LIB) += iommu-sva-lib.o +obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c new file mode 100644 index 000000000000..7100ed17dcce --- /dev/null +++ b/drivers/iommu/sprd-iommu.c @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Unisoc IOMMU driver + * + * Copyright (C) 2020 Unisoc, Inc. + * Author: Chunyan Zhang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SPRD_IOMMU_PAGE_SHIFT 12 +#define SPRD_IOMMU_PAGE_SIZE SZ_4K + +#define SPRD_EX_CFG 0x0 +#define SPRD_IOMMU_VAOR_BYPASS BIT(4) +#define SPRD_IOMMU_GATE_EN BIT(1) +#define SPRD_IOMMU_EN BIT(0) +#define SPRD_EX_UPDATE 0x4 +#define SPRD_EX_FIRST_VPN 0x8 +#define SPRD_EX_VPN_RANGE 0xc +#define SPRD_EX_FIRST_PPN 0x10 +#define SPRD_EX_DEFAULT_PPN 0x14 + +#define SPRD_IOMMU_VERSION 0x0 +#define SPRD_VERSION_MASK GENMASK(15, 8) +#define SPRD_VERSION_SHIFT 0x8 +#define SPRD_VAU_CFG 0x4 +#define SPRD_VAU_UPDATE 0x8 +#define SPRD_VAU_AUTH_CFG 0xc +#define SPRD_VAU_FIRST_PPN 0x10 +#define SPRD_VAU_DEFAULT_PPN_RD 0x14 +#define SPRD_VAU_DEFAULT_PPN_WR 0x18 +#define SPRD_VAU_FIRST_VPN 0x1c +#define SPRD_VAU_VPN_RANGE 0x20 + +enum sprd_iommu_version { + SPRD_IOMMU_EX, + SPRD_IOMMU_VAU, +}; + +/* + * struct sprd_iommu_device - high-level sprd IOMMU device representation, + * including hardware information and configuration, also driver data, etc + * + * @ver: sprd IOMMU IP version + * @prot_page_va: protect page base virtual address + * @prot_page_pa: protect page base physical address, data would be + * written to here while translation fault + * @base: mapped base address for accessing registers + * @dev: pointer to basic device structure + * @iommu: IOMMU core representation + * @group: IOMMU group + * @eb: gate clock which controls IOMMU access + */ +struct sprd_iommu_device { + enum sprd_iommu_version ver; + u32 *prot_page_va; + dma_addr_t prot_page_pa; + void __iomem *base; + struct device *dev; + struct iommu_device iommu; + struct iommu_group *group; + struct clk *eb; +}; + +struct sprd_iommu_domain { + spinlock_t pgtlock; /* lock for page table */ + struct iommu_domain domain; + u32 *pgt_va; /* page table virtual address base */ + dma_addr_t pgt_pa; /* page table physical address base */ + struct sprd_iommu_device *sdev; +}; + +static const struct iommu_ops sprd_iommu_ops; + +static struct sprd_iommu_domain *to_sprd_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct sprd_iommu_domain, domain); +} + +static inline void +sprd_iommu_write(struct sprd_iommu_device *sdev, unsigned int reg, u32 val) +{ + writel_relaxed(val, sdev->base + reg); +} + +static inline u32 +sprd_iommu_read(struct sprd_iommu_device *sdev, unsigned int reg) +{ + return readl_relaxed(sdev->base + reg); +} + +static inline void +sprd_iommu_update_bits(struct sprd_iommu_device *sdev, unsigned int reg, + u32 mask, u32 shift, u32 val) +{ + u32 t = sprd_iommu_read(sdev, reg); + + t = (t & (~(mask << shift))) | ((val & mask) << shift); + sprd_iommu_write(sdev, reg, t); +} + +static inline int +sprd_iommu_get_version(struct sprd_iommu_device *sdev) +{ + int ver = (sprd_iommu_read(sdev, SPRD_IOMMU_VERSION) & + SPRD_VERSION_MASK) >> SPRD_VERSION_SHIFT; + + switch (ver) { + case SPRD_IOMMU_EX: + case SPRD_IOMMU_VAU: + return ver; + default: + return -EINVAL; + } +} + +static size_t +sprd_iommu_pgt_size(struct iommu_domain *domain) +{ + return ((domain->geometry.aperture_end - + domain->geometry.aperture_start + 1) >> + SPRD_IOMMU_PAGE_SHIFT) * sizeof(u32); +} + +static struct iommu_domain *sprd_iommu_domain_alloc(unsigned int domain_type) +{ + struct sprd_iommu_domain *dom; + + if (domain_type != IOMMU_DOMAIN_DMA && domain_type != IOMMU_DOMAIN_UNMANAGED) + return NULL; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + if (iommu_get_dma_cookie(&dom->domain)) { + kfree(dom); + return NULL; + } + + spin_lock_init(&dom->pgtlock); + + dom->domain.geometry.aperture_start = 0; + dom->domain.geometry.aperture_end = SZ_256M - 1; + + return &dom->domain; +} + +static void sprd_iommu_domain_free(struct iommu_domain *domain) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + + iommu_put_dma_cookie(domain); + kfree(dom); +} + +static void sprd_iommu_first_vpn(struct sprd_iommu_domain *dom) +{ + struct sprd_iommu_device *sdev = dom->sdev; + u32 val; + unsigned int reg; + + if (sdev->ver == SPRD_IOMMU_EX) + reg = SPRD_EX_FIRST_VPN; + else + reg = SPRD_VAU_FIRST_VPN; + + val = dom->domain.geometry.aperture_start >> SPRD_IOMMU_PAGE_SHIFT; + sprd_iommu_write(sdev, reg, val); +} + +static void sprd_iommu_vpn_range(struct sprd_iommu_domain *dom) +{ + struct sprd_iommu_device *sdev = dom->sdev; + u32 val; + unsigned int reg; + + if (sdev->ver == SPRD_IOMMU_EX) + reg = SPRD_EX_VPN_RANGE; + else + reg = SPRD_VAU_VPN_RANGE; + + val = (dom->domain.geometry.aperture_end - + dom->domain.geometry.aperture_start) >> SPRD_IOMMU_PAGE_SHIFT; + sprd_iommu_write(sdev, reg, val); +} + +static void sprd_iommu_first_ppn(struct sprd_iommu_domain *dom) +{ + u32 val = dom->pgt_pa >> SPRD_IOMMU_PAGE_SHIFT; + struct sprd_iommu_device *sdev = dom->sdev; + unsigned int reg; + + if (sdev->ver == SPRD_IOMMU_EX) + reg = SPRD_EX_FIRST_PPN; + else + reg = SPRD_VAU_FIRST_PPN; + + sprd_iommu_write(sdev, reg, val); +} + +static void sprd_iommu_default_ppn(struct sprd_iommu_device *sdev) +{ + u32 val = sdev->prot_page_pa >> SPRD_IOMMU_PAGE_SHIFT; + + if (sdev->ver == SPRD_IOMMU_EX) { + sprd_iommu_write(sdev, SPRD_EX_DEFAULT_PPN, val); + } else if (sdev->ver == SPRD_IOMMU_VAU) { + sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_RD, val); + sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_WR, val); + } +} + +static void sprd_iommu_hw_en(struct sprd_iommu_device *sdev, bool en) +{ + unsigned int reg_cfg; + u32 mask, val; + + if (sdev->ver == SPRD_IOMMU_EX) + reg_cfg = SPRD_EX_CFG; + else + reg_cfg = SPRD_VAU_CFG; + + mask = SPRD_IOMMU_EN | SPRD_IOMMU_GATE_EN; + val = en ? mask : 0; + sprd_iommu_update_bits(sdev, reg_cfg, mask, 0, val); +} + +static int sprd_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + size_t pgt_size = sprd_iommu_pgt_size(domain); + + if (dom->sdev) { + pr_err("There's already a device attached to this domain.\n"); + return -EINVAL; + } + + dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL); + if (!dom->pgt_va) + return -ENOMEM; + + dom->sdev = sdev; + + sprd_iommu_first_ppn(dom); + sprd_iommu_first_vpn(dom); + sprd_iommu_vpn_range(dom); + sprd_iommu_default_ppn(sdev); + sprd_iommu_hw_en(sdev, true); + + return 0; +} + +static void sprd_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + struct sprd_iommu_device *sdev = dom->sdev; + size_t pgt_size = sprd_iommu_pgt_size(domain); + + if (!sdev) + return; + + dma_free_coherent(sdev->dev, pgt_size, dom->pgt_va, dom->pgt_pa); + sprd_iommu_hw_en(sdev, false); + dom->sdev = NULL; +} + +static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT; + unsigned long flags; + unsigned int i; + u32 *pgt_base_iova; + u32 pabase = (u32)paddr; + unsigned long start = domain->geometry.aperture_start; + unsigned long end = domain->geometry.aperture_end; + + if (!dom->sdev) { + pr_err("No sprd_iommu_device attached to the domain\n"); + return -EINVAL; + } + + if (iova < start || (iova + size) > (end + 1)) { + dev_err(dom->sdev->dev, "(iova(0x%lx) + sixe(%zx)) are not in the range!\n", + iova, size); + return -EINVAL; + } + + pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); + + spin_lock_irqsave(&dom->pgtlock, flags); + for (i = 0; i < page_num; i++) { + pgt_base_iova[i] = pabase >> SPRD_IOMMU_PAGE_SHIFT; + pabase += SPRD_IOMMU_PAGE_SIZE; + } + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return 0; +} + +static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size, struct iommu_iotlb_gather *iotlb_gather) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + unsigned long flags; + u32 *pgt_base_iova; + unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT; + unsigned long start = domain->geometry.aperture_start; + unsigned long end = domain->geometry.aperture_end; + + if (iova < start || (iova + size) > (end + 1)) + return -EINVAL; + + pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); + + spin_lock_irqsave(&dom->pgtlock, flags); + memset(pgt_base_iova, 0, page_num * sizeof(u32)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return 0; +} + +static void sprd_iommu_sync_map(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + unsigned int reg; + + if (dom->sdev->ver == SPRD_IOMMU_EX) + reg = SPRD_EX_UPDATE; + else + reg = SPRD_VAU_UPDATE; + + /* clear IOMMU TLB buffer after page table updated */ + sprd_iommu_write(dom->sdev, reg, 0xffffffff); +} + +static void sprd_iommu_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + sprd_iommu_sync_map(domain, 0, 0); +} + +static phys_addr_t sprd_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + unsigned long flags; + phys_addr_t pa; + unsigned long start = domain->geometry.aperture_start; + unsigned long end = domain->geometry.aperture_end; + + if (WARN_ON(iova < start || iova > end)) + return 0; + + spin_lock_irqsave(&dom->pgtlock, flags); + pa = *(dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT)); + pa = (pa << SPRD_IOMMU_PAGE_SHIFT) + ((iova - start) & (SPRD_IOMMU_PAGE_SIZE - 1)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return pa; +} + +static struct iommu_device *sprd_iommu_probe_device(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct sprd_iommu_device *sdev; + + if (!fwspec || fwspec->ops != &sprd_iommu_ops) + return ERR_PTR(-ENODEV); + + sdev = dev_iommu_priv_get(dev); + + return &sdev->iommu; +} + +static void sprd_iommu_release_device(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + if (!fwspec || fwspec->ops != &sprd_iommu_ops) + return; + + iommu_fwspec_free(dev); +} + +static struct iommu_group *sprd_iommu_device_group(struct device *dev) +{ + struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); + + return iommu_group_ref_get(sdev->group); +} + +static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + struct platform_device *pdev; + + if (!dev_iommu_priv_get(dev)) { + pdev = of_find_device_by_node(args->np); + dev_iommu_priv_set(dev, platform_get_drvdata(pdev)); + platform_device_put(pdev); + } + + return 0; +} + + +static const struct iommu_ops sprd_iommu_ops = { + .domain_alloc = sprd_iommu_domain_alloc, + .domain_free = sprd_iommu_domain_free, + .attach_dev = sprd_iommu_attach_device, + .detach_dev = sprd_iommu_detach_device, + .map = sprd_iommu_map, + .unmap = sprd_iommu_unmap, + .iotlb_sync_map = sprd_iommu_sync_map, + .iotlb_sync = sprd_iommu_sync, + .iova_to_phys = sprd_iommu_iova_to_phys, + .probe_device = sprd_iommu_probe_device, + .release_device = sprd_iommu_release_device, + .device_group = sprd_iommu_device_group, + .of_xlate = sprd_iommu_of_xlate, + .pgsize_bitmap = ~0UL << SPRD_IOMMU_PAGE_SHIFT, +}; + +static const struct of_device_id sprd_iommu_of_match[] = { + { .compatible = "sprd,iommu-v1" }, + { }, +}; +MODULE_DEVICE_TABLE(of, sprd_iommu_of_match); + +/* + * Clock is not required, access to some of IOMMUs is controlled by gate + * clk, enabled clocks for that kind of IOMMUs before accessing. + * Return 0 for success or no clocks found. + */ +static int sprd_iommu_clk_enable(struct sprd_iommu_device *sdev) +{ + struct clk *eb; + + eb = clk_get_optional(sdev->dev, 0); + if (!eb) + return 0; + + if (IS_ERR(eb)) + return PTR_ERR(eb); + + sdev->eb = eb; + return clk_prepare_enable(eb); +} + +static void sprd_iommu_clk_disable(struct sprd_iommu_device *sdev) +{ + if (sdev->eb) + clk_disable_unprepare(sdev->eb); +} + +static int sprd_iommu_probe(struct platform_device *pdev) +{ + struct sprd_iommu_device *sdev; + struct device *dev = &pdev->dev; + void __iomem *base; + int ret; + + sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) { + dev_err(dev, "Failed to get ioremap resource.\n"); + return PTR_ERR(base); + } + sdev->base = base; + + sdev->prot_page_va = dma_alloc_coherent(dev, SPRD_IOMMU_PAGE_SIZE, + &sdev->prot_page_pa, GFP_KERNEL); + if (!sdev->prot_page_va) + return -ENOMEM; + + platform_set_drvdata(pdev, sdev); + sdev->dev = dev; + + /* All the client devices are in the same iommu-group */ + sdev->group = iommu_group_alloc(); + if (IS_ERR(sdev->group)) { + ret = PTR_ERR(sdev->group); + goto free_page; + } + + ret = iommu_device_sysfs_add(&sdev->iommu, dev, NULL, dev_name(dev)); + if (ret) + goto put_group; + + iommu_device_set_ops(&sdev->iommu, &sprd_iommu_ops); + iommu_device_set_fwnode(&sdev->iommu, &dev->of_node->fwnode); + + ret = iommu_device_register(&sdev->iommu); + if (ret) + goto remove_sysfs; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &sprd_iommu_ops); + + ret = sprd_iommu_clk_enable(sdev); + if (ret) + goto unregister_iommu; + + ret = sprd_iommu_get_version(sdev); + if (ret < 0) { + dev_err(dev, "IOMMU version(%d) is invalid.\n", ret); + goto disable_clk; + } + sdev->ver = ret; + + return 0; + +disable_clk: + sprd_iommu_clk_disable(sdev); +unregister_iommu: + iommu_device_unregister(&sdev->iommu); +remove_sysfs: + iommu_device_sysfs_remove(&sdev->iommu); +put_group: + iommu_group_put(sdev->group); +free_page: + dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa); + return ret; +} + +static int sprd_iommu_remove(struct platform_device *pdev) +{ + struct sprd_iommu_device *sdev = platform_get_drvdata(pdev); + + dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa); + + iommu_group_put(sdev->group); + sdev->group = NULL; + + bus_set_iommu(&platform_bus_type, NULL); + + platform_set_drvdata(pdev, NULL); + iommu_device_sysfs_remove(&sdev->iommu); + iommu_device_unregister(&sdev->iommu); + + return 0; +} + +static struct platform_driver sprd_iommu_driver = { + .driver = { + .name = "sprd-iommu", + .of_match_table = sprd_iommu_of_match, + .suppress_bind_attrs = true, + }, + .probe = sprd_iommu_probe, + .remove = sprd_iommu_remove, +}; +module_platform_driver(sprd_iommu_driver); + +MODULE_DESCRIPTION("IOMMU driver for Unisoc SoCs"); +MODULE_ALIAS("platform:sprd-iommu"); +MODULE_LICENSE("GPL"); -- cgit v1.2.3 From 3542dcb15cef66c0b9e6c3b33168eb657e0d9520 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 5 Mar 2021 16:32:34 +0000 Subject: iommu/dma: Resurrect the "forcedac" option In converting intel-iommu over to the common IOMMU DMA ops, it quietly lost the functionality of its "forcedac" option. Since this is a handy thing both for testing and for performance optimisation on certain platforms, reimplement it under the common IOMMU parameter namespace. For the sake of fixing the inadvertent breakage of the Intel-specific parameter, remove the dmar_forcedac remnants and hook it up as an alias while documenting the transition to the new common parameter. Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops") Signed-off-by: Robin Murphy Acked-by: Lu Baolu Reviewed-by: John Garry Link: https://lore.kernel.org/r/7eece8e0ea7bfbe2cd0e30789e0d46df573af9b0.1614961776.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++------- drivers/iommu/dma-iommu.c | 13 ++++++++++++- drivers/iommu/intel/iommu.c | 5 ++--- include/linux/dma-iommu.h | 2 ++ 4 files changed, 24 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 04545725f187..835f810f2f26 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1869,13 +1869,6 @@ bypassed by not enabling DMAR with this option. In this case, gfx device will use physical address for DMA. - forcedac [X86-64] - With this option iommu will not optimize to look - for io virtual address below 32-bit forcing dual - address cycle on pci bus for cards supporting greater - than 32-bit addressing. The default is to look - for translation below 32-bit and if not available - then look in the higher range. strict [Default Off] With this option on every unmap_single operation will result in a hardware IOTLB flush operation as opposed @@ -1964,6 +1957,14 @@ nobypass [PPC/POWERNV] Disable IOMMU bypass, using IOMMU for PCI devices. + iommu.forcedac= [ARM64, X86] Control IOVA allocation for PCI devices. + Format: { "0" | "1" } + 0 - Try to allocate a 32-bit DMA address first, before + falling back to the full range if needed. + 1 - Allocate directly from the full usable range, + forcing Dual Address Cycle for PCI cards supporting + greater than 32-bit addressing. + iommu.strict= [ARM64] Configure TLB invalidation behaviour Format: { "0" | "1" } 0 - Lazy mode. diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index af765c813cc8..fdd095e1fa52 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -52,6 +52,17 @@ struct iommu_dma_cookie { }; static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); +bool iommu_dma_forcedac __read_mostly; + +static int __init iommu_dma_forcedac_setup(char *str) +{ + int ret = kstrtobool(str, &iommu_dma_forcedac); + + if (!ret && iommu_dma_forcedac) + pr_info("Forcing DAC for PCI devices\n"); + return ret; +} +early_param("iommu.forcedac", iommu_dma_forcedac_setup); void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, struct iommu_domain *domain) @@ -444,7 +455,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); /* Try to get PCI devices a SAC address */ - if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) + if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev)) iova = alloc_iova_fast(iovad, iova_len, DMA_BIT_MASK(32) >> shift, false); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ee0932307d64..1c32522220bc 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -360,7 +360,6 @@ int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); static int dmar_map_gfx = 1; -static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; static int iommu_identity_mapping; @@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str) dmar_map_gfx = 0; pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { - pr_info("Forcing DAC for PCI devices\n"); - dmar_forcedac = 1; + pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n"); + iommu_dma_forcedac = true; } else if (!strncmp(str, "strict", 6)) { pr_info("Disable batched IOTLB flush\n"); intel_iommu_strict = 1; diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 706b68d1359b..13d1f4c14d7b 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, struct iommu_domain *domain); +extern bool iommu_dma_forcedac; + #else /* CONFIG_IOMMU_DMA */ struct iommu_domain; -- cgit v1.2.3 From 7ae31cec5b70e301788b95de543abb56748dcfb6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 5 Mar 2021 16:35:22 +0000 Subject: iommu/iova: Add rbtree entry helper Repeating the rb_entry() boilerplate all over the place gets old fast. Before adding yet more instances, add a little hepler to tidy it up. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/03931d86c0ad71f44b29394e3a8d38bfc32349cd.1614962123.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index e6e2fa85271c..c28003e1d2ee 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -27,6 +27,11 @@ static void fq_destroy_all_entries(struct iova_domain *iovad); static void fq_flush_timeout(struct timer_list *t); static void free_global_cached_iovas(struct iova_domain *iovad); +static struct iova *to_iova(struct rb_node *node) +{ + return rb_entry(node, struct iova, node); +} + void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn) @@ -136,7 +141,7 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) { struct iova *cached_iova; - cached_iova = rb_entry(iovad->cached32_node, struct iova, node); + cached_iova = to_iova(iovad->cached32_node); if (free == cached_iova || (free->pfn_hi < iovad->dma_32bit_pfn && free->pfn_lo >= cached_iova->pfn_lo)) { @@ -144,7 +149,7 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) iovad->max32_alloc_size = iovad->dma_32bit_pfn; } - cached_iova = rb_entry(iovad->cached_node, struct iova, node); + cached_iova = to_iova(iovad->cached_node); if (free->pfn_lo >= cached_iova->pfn_lo) iovad->cached_node = rb_next(&free->node); } @@ -159,7 +164,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova, new = (start) ? &start : &(root->rb_node); /* Figure out where to put new node */ while (*new) { - struct iova *this = rb_entry(*new, struct iova, node); + struct iova *this = to_iova(*new); parent = *new; @@ -198,7 +203,7 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad, goto iova32_full; curr = __get_cached_rbnode(iovad, limit_pfn); - curr_iova = rb_entry(curr, struct iova, node); + curr_iova = to_iova(curr); retry_pfn = curr_iova->pfn_hi + 1; retry: @@ -207,7 +212,7 @@ retry: new_pfn = (high_pfn - size) & align_mask; prev = curr; curr = rb_prev(curr); - curr_iova = rb_entry(curr, struct iova, node); + curr_iova = to_iova(curr); } while (curr && new_pfn <= curr_iova->pfn_hi && new_pfn >= low_pfn); if (high_pfn < size || new_pfn < low_pfn) { @@ -215,7 +220,7 @@ retry: high_pfn = limit_pfn; low_pfn = retry_pfn; curr = &iovad->anchor.node; - curr_iova = rb_entry(curr, struct iova, node); + curr_iova = to_iova(curr); goto retry; } iovad->max32_alloc_size = size; @@ -331,7 +336,7 @@ private_find_iova(struct iova_domain *iovad, unsigned long pfn) assert_spin_locked(&iovad->iova_rbtree_lock); while (node) { - struct iova *iova = rb_entry(node, struct iova, node); + struct iova *iova = to_iova(node); if (pfn < iova->pfn_lo) node = node->rb_left; @@ -617,7 +622,7 @@ static int __is_range_overlap(struct rb_node *node, unsigned long pfn_lo, unsigned long pfn_hi) { - struct iova *iova = rb_entry(node, struct iova, node); + struct iova *iova = to_iova(node); if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo)) return 1; @@ -685,7 +690,7 @@ reserve_iova(struct iova_domain *iovad, spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) { if (__is_range_overlap(node, pfn_lo, pfn_hi)) { - iova = rb_entry(node, struct iova, node); + iova = to_iova(node); __adjust_overlap_range(iova, &pfn_lo, &pfn_hi); if ((pfn_lo >= iova->pfn_lo) && (pfn_hi <= iova->pfn_hi)) -- cgit v1.2.3 From 371d7955e3102fe38daf06de4ed9bfd29864354b Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 5 Mar 2021 16:35:23 +0000 Subject: iommu/iova: Improve restart logic When restarting after searching below the cached node fails, resetting the start point to the anchor node is often overly pessimistic. If allocations are made with mixed limits - particularly in the case of the opportunistic 32-bit allocation for PCI devices - this could mean significant time wasted walking through the whole populated upper range just to reach the initial limit. We can improve on that by implementing a proper tree traversal to find the first node above the relevant limit, and set the exact start point. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/076b3484d1e5057b95d8c387c894bd6ad2514043.1614962123.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index c28003e1d2ee..471c48dd71e7 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -154,6 +154,43 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) iovad->cached_node = rb_next(&free->node); } +static struct rb_node *iova_find_limit(struct iova_domain *iovad, unsigned long limit_pfn) +{ + struct rb_node *node, *next; + /* + * Ideally what we'd like to judge here is whether limit_pfn is close + * enough to the highest-allocated IOVA that starting the allocation + * walk from the anchor node will be quicker than this initial work to + * find an exact starting point (especially if that ends up being the + * anchor node anyway). This is an incredibly crude approximation which + * only really helps the most likely case, but is at least trivially easy. + */ + if (limit_pfn > iovad->dma_32bit_pfn) + return &iovad->anchor.node; + + node = iovad->rbroot.rb_node; + while (to_iova(node)->pfn_hi < limit_pfn) + node = node->rb_right; + +search_left: + while (node->rb_left && to_iova(node->rb_left)->pfn_lo >= limit_pfn) + node = node->rb_left; + + if (!node->rb_left) + return node; + + next = node->rb_left; + while (next->rb_right) { + next = next->rb_right; + if (to_iova(next)->pfn_lo >= limit_pfn) { + node = next; + goto search_left; + } + } + + return node; +} + /* Insert the iova into domain rbtree by holding writer lock */ static void iova_insert_rbtree(struct rb_root *root, struct iova *iova, @@ -219,7 +256,7 @@ retry: if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) { high_pfn = limit_pfn; low_pfn = retry_pfn; - curr = &iovad->anchor.node; + curr = iova_find_limit(iovad, limit_pfn); curr_iova = to_iova(curr); goto retry; } -- cgit v1.2.3 From dec991e4722d763130c8ccd92523f2a173f8a7cd Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Sun, 14 Mar 2021 13:15:34 -0700 Subject: iommu/vt-d: Disable SVM when ATS/PRI/PASID are not enabled in the device Currently, the Intel VT-d supports Shared Virtual Memory (SVM) only when IO page fault is supported. Otherwise, shared memory pages can not be swapped out and need to be pinned. The device needs the Address Translation Service (ATS), Page Request Interface (PRI) and Process Address Space Identifier (PASID) capabilities to be enabled to support IO page fault. Disable SVM when ATS, PRI and PASID are not enabled in the device. Signed-off-by: Kyung Min Park Acked-by: Lu Baolu Link: https://lore.kernel.org/r/20210314201534.918-1-kyung.min.park@intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ee0932307d64..956a02eb40b4 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5380,6 +5380,9 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) if (!info) return -EINVAL; + if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled) + return -EINVAL; + if (info->iommu->flags & VTD_FLAG_SVM_CAPABLE) return 0; } -- cgit v1.2.3 From 6ca69e5841f01ccbfa45e56577e1b33e14e53504 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 18 Mar 2021 08:53:40 +0800 Subject: iommu/vt-d: Report more information about invalidation errors When the invalidation queue errors are encountered, dump the information logged by the VT-d hardware together with the pending queue invalidation descriptors. Signed-off-by: Ashok Raj Tested-by: Guo Kaijie Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210318005340.187311-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 68 ++++++++++++++++++++++++++++++++++++++++++--- include/linux/intel-iommu.h | 6 ++++ 2 files changed, 70 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index d5c51b5c20af..6971397805f3 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1205,6 +1205,63 @@ static inline void reclaim_free_desc(struct q_inval *qi) } } +static const char *qi_type_string(u8 type) +{ + switch (type) { + case QI_CC_TYPE: + return "Context-cache Invalidation"; + case QI_IOTLB_TYPE: + return "IOTLB Invalidation"; + case QI_DIOTLB_TYPE: + return "Device-TLB Invalidation"; + case QI_IEC_TYPE: + return "Interrupt Entry Cache Invalidation"; + case QI_IWD_TYPE: + return "Invalidation Wait"; + case QI_EIOTLB_TYPE: + return "PASID-based IOTLB Invalidation"; + case QI_PC_TYPE: + return "PASID-cache Invalidation"; + case QI_DEIOTLB_TYPE: + return "PASID-based Device-TLB Invalidation"; + case QI_PGRP_RESP_TYPE: + return "Page Group Response"; + default: + return "UNKNOWN"; + } +} + +static void qi_dump_fault(struct intel_iommu *iommu, u32 fault) +{ + unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG); + u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG); + struct qi_desc *desc = iommu->qi->desc + head; + + if (fault & DMA_FSTS_IQE) + pr_err("VT-d detected Invalidation Queue Error: Reason %llx", + DMAR_IQER_REG_IQEI(iqe_err)); + if (fault & DMA_FSTS_ITE) + pr_err("VT-d detected Invalidation Time-out Error: SID %llx", + DMAR_IQER_REG_ITESID(iqe_err)); + if (fault & DMA_FSTS_ICE) + pr_err("VT-d detected Invalidation Completion Error: SID %llx", + DMAR_IQER_REG_ICESID(iqe_err)); + + pr_err("QI HEAD: %s qw0 = 0x%llx, qw1 = 0x%llx\n", + qi_type_string(desc->qw0 & 0xf), + (unsigned long long)desc->qw0, + (unsigned long long)desc->qw1); + + head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH; + head <<= qi_shift(iommu); + desc = iommu->qi->desc + head; + + pr_err("QI PRIOR: %s qw0 = 0x%llx, qw1 = 0x%llx\n", + qi_type_string(desc->qw0 & 0xf), + (unsigned long long)desc->qw0, + (unsigned long long)desc->qw1); +} + static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) { u32 fault; @@ -1216,6 +1273,8 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) return -EAGAIN; fault = readl(iommu->reg + DMAR_FSTS_REG); + if (fault & (DMA_FSTS_IQE | DMA_FSTS_ITE | DMA_FSTS_ICE)) + qi_dump_fault(iommu, fault); /* * If IQE happens, the head points to the descriptor associated @@ -1232,12 +1291,10 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) * used by software as private data. We won't print * out these two qw's for security consideration. */ - pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n", - (unsigned long long)desc->qw0, - (unsigned long long)desc->qw1); memcpy(desc, qi->desc + (wait_index << shift), 1 << shift); writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); + pr_info("Invalidation Queue Error (IQE) cleared\n"); return -EINVAL; } } @@ -1254,6 +1311,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); + pr_info("Invalidation Time-out Error (ITE) cleared\n"); do { if (qi->desc_status[head] == QI_IN_USE) @@ -1265,8 +1323,10 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) return -EAGAIN; } - if (fault & DMA_FSTS_ICE) + if (fault & DMA_FSTS_ICE) { writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG); + pr_info("Invalidation Completion Error (ICE) cleared\n"); + } return 0; } diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 1bc46b88711a..1732298ce888 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,7 @@ #define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */ #define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */ #define DMAR_ICS_REG 0x9c /* Invalidation complete status register */ +#define DMAR_IQER_REG 0xb0 /* Invalidation queue error record register */ #define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */ #define DMAR_PQH_REG 0xc0 /* Page request queue head register */ #define DMAR_PQT_REG 0xc8 /* Page request queue tail register */ @@ -126,6 +128,10 @@ #define DMAR_VCMD_REG 0xe10 /* Virtual command register */ #define DMAR_VCRSP_REG 0xe20 /* Virtual command response register */ +#define DMAR_IQER_REG_IQEI(reg) FIELD_GET(GENMASK_ULL(3, 0), reg) +#define DMAR_IQER_REG_ITESID(reg) FIELD_GET(GENMASK_ULL(47, 32), reg) +#define DMAR_IQER_REG_ICESID(reg) FIELD_GET(GENMASK_ULL(63, 48), reg) + #define OFFSET_STRIDE (9) #define dmar_readq(a) readq(a) -- cgit v1.2.3 From f68c7f539b6e9712e941212ab95a1feb5a0bf3b3 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 2 Mar 2021 02:13:57 -0800 Subject: iommu/vt-d: Enable write protect for supervisor SVM Write protect bit, when set, inhibits supervisor writes to the read-only pages. In supervisor shared virtual addressing (SVA), where page tables are shared between CPU and DMA, IOMMU PASID entry WPE bit should match CR0.WP bit in the CPU. This patch sets WPE bit for supervisor PASIDs if CR0.WP is set. Signed-off-by: Sanjay Kumar Signed-off-by: Jacob Pan Acked-by: Lu Baolu Link: https://lore.kernel.org/r/1614680040-1989-2-git-send-email-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index f26cb6195b2c..629e3aee44fe 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -393,6 +393,15 @@ static inline void pasid_set_sre(struct pasid_entry *pe) pasid_set_bits(&pe->val[2], 1 << 0, 1); } +/* + * Setup the WPE(Write Protect Enable) field (Bit 132) of a + * scalable mode PASID entry. + */ +static inline void pasid_set_wpe(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[2], 1 << 4, 1 << 4); +} + /* * Setup the P(Present) field (Bit 0) of a scalable mode PASID * entry. @@ -522,6 +531,20 @@ static void pasid_flush_caches(struct intel_iommu *iommu, } } +static inline int pasid_enable_wpe(struct pasid_entry *pte) +{ + unsigned long cr0 = read_cr0(); + + /* CR0.WP is normally set but just to be sure */ + if (unlikely(!(cr0 & X86_CR0_WP))) { + pr_err_ratelimited("No CPU write protect!\n"); + return -EINVAL; + } + pasid_set_wpe(pte); + + return 0; +}; + /* * Set up the scalable mode pasid table entry for first only * translation type. @@ -553,6 +576,9 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, return -EINVAL; } pasid_set_sre(pte); + if (pasid_enable_wpe(pte)) + return -EINVAL; + } if (flags & PASID_FLAG_FL5LP) { -- cgit v1.2.3 From bb0f61533dfd6aa815a2719720c77d13f840b683 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 2 Mar 2021 02:13:58 -0800 Subject: iommu/vt-d: Enable write protect propagation from guest Write protect bit, when set, inhibits supervisor writes to the read-only pages. In guest supervisor shared virtual addressing (SVA), write-protect should be honored upon guest bind supervisor PASID request. This patch extends the VT-d portion of the IOMMU UAPI to include WP bit. WPE bit of the supervisor PASID entry will be set to match CPU CR0.WP bit. Signed-off-by: Sanjay Kumar Signed-off-by: Jacob Pan Acked-by: Lu Baolu Link: https://lore.kernel.org/r/1614680040-1989-3-git-send-email-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 3 +++ include/uapi/linux/iommu.h | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 629e3aee44fe..0bf7e0a76890 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -732,6 +732,9 @@ intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte, return -EINVAL; } pasid_set_sre(pte); + /* Enable write protect WP if guest requested */ + if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_WPE) + pasid_set_wpe(pte); } if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) { diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h index e1d9e75f2c94..59178fc229ca 100644 --- a/include/uapi/linux/iommu.h +++ b/include/uapi/linux/iommu.h @@ -288,7 +288,8 @@ struct iommu_gpasid_bind_data_vtd { #define IOMMU_SVA_VTD_GPASID_PWT (1 << 3) /* page-level write through */ #define IOMMU_SVA_VTD_GPASID_EMTE (1 << 4) /* extended mem type enable */ #define IOMMU_SVA_VTD_GPASID_CD (1 << 5) /* PASID-level cache disable */ -#define IOMMU_SVA_VTD_GPASID_LAST (1 << 6) +#define IOMMU_SVA_VTD_GPASID_WPE (1 << 6) /* Write protect enable */ +#define IOMMU_SVA_VTD_GPASID_LAST (1 << 7) __u64 flags; __u32 pat; __u32 emt; -- cgit v1.2.3 From 78a523fe73b81b4447beb2d6c78c9fafae24eebb Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 2 Mar 2021 02:13:59 -0800 Subject: iommu/vt-d: Reject unsupported page request modes When supervisor/privilige mode SVM is used, we bind init_mm.pgd with a supervisor PASID. There should not be any page fault for init_mm. Execution request with DMA read is also not supported. This patch checks PRQ descriptor for both unsupported configurations, reject them both with invalid responses. Fixes: 1c4f88b7f1f92 ("iommu/vt-d: Shared virtual address in scalable mode") Acked-by: Lu Baolu Signed-off-by: Jacob Pan Link: https://lore.kernel.org/r/1614680040-1989-4-git-send-email-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 574a7e657a9a..4627437ae1c0 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -920,7 +920,17 @@ static irqreturn_t prq_event_thread(int irq, void *d) ((unsigned long long *)req)[1]); goto no_pasid; } - + /* We shall not receive page request for supervisor SVM */ + if (req->pm_req && (req->rd_req | req->wr_req)) { + pr_err("Unexpected page request in Privilege Mode"); + /* No need to find the matching sdev as for bad_req */ + goto no_pasid; + } + /* DMA read with exec requeset is not supported. */ + if (req->exe_req && req->rd_req) { + pr_err("Execution request not supported\n"); + goto no_pasid; + } if (!svm || svm->pasid != req->pasid) { rcu_read_lock(); svm = ioasid_find(NULL, req->pasid, NULL); -- cgit v1.2.3 From 396bd6f3d9f659d7ce324806bf3cd6677385f8fd Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 2 Mar 2021 02:14:00 -0800 Subject: iommu/vt-d: Calculate and set flags for handle_mm_fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Page requests are originated from the user page fault. Therefore, we shall set FAULT_FLAG_USER.  FAULT_FLAG_REMOTE indicates that we are walking an mm which is not guaranteed to be the same as the current->mm and should not be subject to protection key enforcement. Therefore, we should set FAULT_FLAG_REMOTE to avoid faults when both SVM and PKEY are used. References: commit 1b2ee1266ea6 ("mm/core: Do not enforce PKEY permissions on remote mm access") Reviewed-by: Raj Ashok Acked-by: Lu Baolu Signed-off-by: Jacob Pan Link: https://lore.kernel.org/r/1614680040-1989-5-git-send-email-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 4627437ae1c0..54db58945c2d 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -895,6 +895,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) struct intel_iommu *iommu = d; struct intel_svm *svm = NULL; int head, tail, handled = 0; + unsigned int flags = 0; /* Clear PPR bit before reading head/tail registers, to * ensure that we get a new interrupt if needed. */ @@ -992,9 +993,11 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (access_error(vma, req)) goto invalid; - ret = handle_mm_fault(vma, address, - req->wr_req ? FAULT_FLAG_WRITE : 0, - NULL); + flags = FAULT_FLAG_USER | FAULT_FLAG_REMOTE; + if (req->wr_req) + flags |= FAULT_FLAG_WRITE; + + ret = handle_mm_fault(vma, address, flags, NULL); if (ret & VM_FAULT_ERROR) goto invalid; -- cgit v1.2.3 From 0bb868e1e3c16827c1b63589b1b0f016ba00140d Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 19 Mar 2021 09:57:50 +0000 Subject: iommu/unisoc: Fix spelling mistake "sixe" -> "size" There is a spelling mistake in a dev_err message. Fix it. Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20210319095750.5624-1-colin.king@canonical.com Signed-off-by: Joerg Roedel --- drivers/iommu/sprd-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 7100ed17dcce..e1dc2f7d5639 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -297,7 +297,7 @@ static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova, } if (iova < start || (iova + size) > (end + 1)) { - dev_err(dom->sdev->dev, "(iova(0x%lx) + sixe(%zx)) are not in the range!\n", + dev_err(dom->sdev->dev, "(iova(0x%lx) + size(%zx)) are not in the range!\n", iova, size); return -EINVAL; } -- cgit v1.2.3 From 655c447c97d7fe462e6cd9e15809037be028bc70 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 24 Mar 2021 16:16:03 +0800 Subject: iommu/arm-smmu-v3: add bit field SFM into GERROR_ERR_MASK In arm_smmu_gerror_handler(), the value of the SMMU_GERROR register is filtered by GERROR_ERR_MASK. However, the GERROR_ERR_MASK does not contain the SFM bit. As a result, the subsequent error processing is not performed when only the SFM error occurs. Fixes: 48ec83bcbcf5 ("iommu/arm-smmu: Add initial driver support for ARM SMMUv3 devices") Reported-by: Rui Zhu Signed-off-by: Zhen Lei Link: https://lore.kernel.org/r/20210324081603.1074-1-thunder.leizhen@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index f985817c967a..230b6f6b3901 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -115,7 +115,7 @@ #define GERROR_PRIQ_ABT_ERR (1 << 3) #define GERROR_EVTQ_ABT_ERR (1 << 2) #define GERROR_CMDQ_ERR (1 << 0) -#define GERROR_ERR_MASK 0xfd +#define GERROR_ERR_MASK 0x1fd #define ARM_SMMU_GERRORN 0x64 -- cgit v1.2.3 From 6cc7e5a9c6b02507b9be5a99b51e970afa91c85f Mon Sep 17 00:00:00 2001 From: Xiang Chen Date: Tue, 30 Mar 2021 20:58:26 +0800 Subject: iommu/arm-smmu-v3: Add a check to avoid invalid iotlb sync It may send a invalid tlb sync for smmuv3 if iotlb_gather is not valid (iotlb_gather->pgsize = 0). So add a check to avoid invalid iotlb sync for it. Signed-off-by: Xiang Chen Link: https://lore.kernel.org/r/1617109106-121844-1-git-send-email-chenxiang66@hisilicon.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8594b4a83043..941ba5484731 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2305,6 +2305,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + if (!gather->pgsize) + return; + arm_smmu_tlb_inv_range_domain(gather->start, gather->end - gather->start + 1, gather->pgsize, true, smmu_domain); -- cgit v1.2.3 From 803766cbf85fb8edbf896729bbefc2d38dcf1e0a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:09:16 +0800 Subject: iommu/vt-d: Fix lockdep splat in intel_pasid_get_entry() The pasid_lock is used to synchronize different threads from modifying a same pasid directory entry at the same time. It causes below lockdep splat. [ 83.296538] ======================================================== [ 83.296538] WARNING: possible irq lock inversion dependency detected [ 83.296539] 5.12.0-rc3+ #25 Tainted: G W [ 83.296539] -------------------------------------------------------- [ 83.296540] bash/780 just changed the state of lock: [ 83.296540] ffffffff82b29c98 (device_domain_lock){..-.}-{2:2}, at: iommu_flush_dev_iotlb.part.0+0x32/0x110 [ 83.296547] but this lock took another, SOFTIRQ-unsafe lock in the past: [ 83.296547] (pasid_lock){+.+.}-{2:2} [ 83.296548] and interrupts could create inverse lock ordering between them. [ 83.296549] other info that might help us debug this: [ 83.296549] Chain exists of: device_domain_lock --> &iommu->lock --> pasid_lock [ 83.296551] Possible interrupt unsafe locking scenario: [ 83.296551] CPU0 CPU1 [ 83.296552] ---- ---- [ 83.296552] lock(pasid_lock); [ 83.296553] local_irq_disable(); [ 83.296553] lock(device_domain_lock); [ 83.296554] lock(&iommu->lock); [ 83.296554] [ 83.296554] lock(device_domain_lock); [ 83.296555] *** DEADLOCK *** Fix it by replacing the pasid_lock with an atomic exchange operation. Reported-and-tested-by: Dave Jiang Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320020916.640115-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 0bf7e0a76890..1a4105a5989b 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -24,7 +24,6 @@ /* * Intel IOMMU system wide PASID name space: */ -static DEFINE_SPINLOCK(pasid_lock); u32 intel_pasid_max_id = PASID_MAX; int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid) @@ -259,19 +258,25 @@ struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) dir_index = pasid >> PASID_PDE_SHIFT; index = pasid & PASID_PTE_MASK; - spin_lock(&pasid_lock); +retry: entries = get_pasid_table_from_pde(&dir[dir_index]); if (!entries) { entries = alloc_pgtable_page(info->iommu->node); - if (!entries) { - spin_unlock(&pasid_lock); + if (!entries) return NULL; - } - WRITE_ONCE(dir[dir_index].val, - (u64)virt_to_phys(entries) | PASID_PTE_PRESENT); + /* + * The pasid directory table entry won't be freed after + * allocation. No worry about the race with free and + * clear. However, this entry might be populated by others + * while we are preparing it. Use theirs with a retry. + */ + if (cmpxchg64(&dir[dir_index].val, 0ULL, + (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { + free_pgtable_page(entries); + goto retry; + } } - spin_unlock(&pasid_lock); return &entries[index]; } -- cgit v1.2.3 From 1d421058c815d54113d9afdf6db3f995c788cf0d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:41:56 +0800 Subject: iommu/vt-d: Don't set then clear private data in prq_event_thread() The VT-d specification (section 7.6) requires that the value in the Private Data field of a Page Group Response Descriptor must match the value in the Private Data field of the respective Page Request Descriptor. The private data field of a page group response descriptor is set then immediately cleared in prq_event_thread(). This breaks the rule defined by the VT-d specification. Fix it by moving clearing code up. Fixes: 5b438f4ba315d ("iommu/vt-d: Support page request in scalable mode") Cc: Jacob Pan Reviewed-by: Liu Yi L Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320024156.640798-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 54db58945c2d..46776db47d16 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -1034,12 +1034,12 @@ no_pasid: QI_PGRP_RESP_TYPE; resp.qw1 = QI_PGRP_IDX(req->prg_index) | QI_PGRP_LPIG(req->lpig); + resp.qw2 = 0; + resp.qw3 = 0; if (req->priv_data_present) memcpy(&resp.qw2, req->priv_data, sizeof(req->priv_data)); - resp.qw2 = 0; - resp.qw3 = 0; qi_submit_sync(iommu, &resp, 1, 0); } prq_advance: -- cgit v1.2.3 From 2e1a44c1c4acf209c0dd7bc04421d101b9e80d11 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 23 Mar 2021 09:05:57 +0800 Subject: iommu/vt-d: Remove svm_dev_ops The svm_dev_ops has never been referenced in the tree, and there's no plan to have anything to use it. Remove it to make the code neat. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210323010600.678627-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 15 +-------------- include/linux/intel-iommu.h | 3 --- include/linux/intel-svm.h | 7 ------- 3 files changed, 1 insertion(+), 24 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 46776db47d16..933fe11797a1 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -462,7 +462,6 @@ static void load_pasid(struct mm_struct *mm, u32 pasid) /* Caller must hold pasid_mutex, mm reference */ static int intel_svm_bind_mm(struct device *dev, unsigned int flags, - struct svm_dev_ops *ops, struct mm_struct *mm, struct intel_svm_dev **sd) { struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); @@ -512,10 +511,6 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, /* Find the matching device in svm list */ for_each_svm_dev(sdev, svm, dev) { - if (sdev->ops != ops) { - ret = -EBUSY; - goto out; - } sdev->users++; goto success; } @@ -550,7 +545,6 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, /* Finish the setup now we know we're keeping it */ sdev->users = 1; - sdev->ops = ops; init_rcu_head(&sdev->rcu); if (!svm) { @@ -1006,13 +1000,6 @@ invalid: mmap_read_unlock(svm->mm); mmput(svm->mm); bad_req: - WARN_ON(!sdev); - if (sdev && sdev->ops && sdev->ops->fault_cb) { - int rwxp = (req->rd_req << 3) | (req->wr_req << 2) | - (req->exe_req << 1) | (req->pm_req); - sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, - req->priv_data, rwxp, result); - } /* We get here in the error case where the PASID lookup failed, and these can be NULL. Do not use them below this point! */ sdev = NULL; @@ -1087,7 +1074,7 @@ intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata) if (drvdata) flags = *(unsigned int *)drvdata; mutex_lock(&pasid_mutex); - ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev); + ret = intel_svm_bind_mm(dev, flags, mm, &sdev); if (ret) sva = ERR_PTR(ret); else if (sdev) diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 1732298ce888..e0f8c2ade3e8 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -769,14 +769,11 @@ u32 intel_svm_get_pasid(struct iommu_sva *handle); int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, struct iommu_page_response *msg); -struct svm_dev_ops; - struct intel_svm_dev { struct list_head list; struct rcu_head rcu; struct device *dev; struct intel_iommu *iommu; - struct svm_dev_ops *ops; struct iommu_sva sva; u32 pasid; int users; diff --git a/include/linux/intel-svm.h b/include/linux/intel-svm.h index 39d368a810b8..6c9d10c0fb1e 100644 --- a/include/linux/intel-svm.h +++ b/include/linux/intel-svm.h @@ -8,13 +8,6 @@ #ifndef __INTEL_SVM_H__ #define __INTEL_SVM_H__ -struct device; - -struct svm_dev_ops { - void (*fault_cb)(struct device *dev, u32 pasid, u64 address, - void *private, int rwxp, int response); -}; - /* Values for rxwp in fault_cb callback */ #define SVM_REQ_READ (1<<3) #define SVM_REQ_WRITE (1<<2) -- cgit v1.2.3 From 06905ea8319731036695cf1a4c53c12b0f9373cb Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 23 Mar 2021 09:05:58 +0800 Subject: iommu/vt-d: Remove SVM_FLAG_PRIVATE_PASID The SVM_FLAG_PRIVATE_PASID has never been referenced in the tree, and there's no plan to have anything to use it. So cleanup it. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210323010600.678627-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 40 ++++++++++++++++++---------------------- include/linux/intel-svm.h | 16 +++------------- 2 files changed, 21 insertions(+), 35 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 933fe11797a1..f77184a9defb 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -465,9 +465,9 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, struct mm_struct *mm, struct intel_svm_dev **sd) { struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); + struct intel_svm *svm = NULL, *t; struct device_domain_info *info; struct intel_svm_dev *sdev; - struct intel_svm *svm = NULL; unsigned long iflags; int pasid_max; int ret; @@ -493,30 +493,26 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, } } - if (!(flags & SVM_FLAG_PRIVATE_PASID)) { - struct intel_svm *t; - - list_for_each_entry(t, &global_svm_list, list) { - if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID)) - continue; - - svm = t; - if (svm->pasid >= pasid_max) { - dev_warn(dev, - "Limited PASID width. Cannot use existing PASID %d\n", - svm->pasid); - ret = -ENOSPC; - goto out; - } + list_for_each_entry(t, &global_svm_list, list) { + if (t->mm != mm) + continue; - /* Find the matching device in svm list */ - for_each_svm_dev(sdev, svm, dev) { - sdev->users++; - goto success; - } + svm = t; + if (svm->pasid >= pasid_max) { + dev_warn(dev, + "Limited PASID width. Cannot use existing PASID %d\n", + svm->pasid); + ret = -ENOSPC; + goto out; + } - break; + /* Find the matching device in svm list */ + for_each_svm_dev(sdev, svm, dev) { + sdev->users++; + goto success; } + + break; } sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); diff --git a/include/linux/intel-svm.h b/include/linux/intel-svm.h index 6c9d10c0fb1e..10fa80eef13a 100644 --- a/include/linux/intel-svm.h +++ b/include/linux/intel-svm.h @@ -14,16 +14,6 @@ #define SVM_REQ_EXEC (1<<1) #define SVM_REQ_PRIV (1<<0) -/* - * The SVM_FLAG_PRIVATE_PASID flag requests a PASID which is *not* the "main" - * PASID for the current process. Even if a PASID already exists, a new one - * will be allocated. And the PASID allocated with SVM_FLAG_PRIVATE_PASID - * will not be given to subsequent callers. This facility allows a driver to - * disambiguate between multiple device contexts which access the same MM, - * if there is no other way to do so. It should be used sparingly, if at all. - */ -#define SVM_FLAG_PRIVATE_PASID (1<<0) - /* * The SVM_FLAG_SUPERVISOR_MODE flag requests a PASID which can be used only * for access to kernel addresses. No IOTLB flushes are automatically done @@ -35,18 +25,18 @@ * It is unlikely that we will ever hook into flush_tlb_kernel_range() to * do such IOTLB flushes automatically. */ -#define SVM_FLAG_SUPERVISOR_MODE (1<<1) +#define SVM_FLAG_SUPERVISOR_MODE BIT(0) /* * The SVM_FLAG_GUEST_MODE flag is used when a PASID bind is for guest * processes. Compared to the host bind, the primary differences are: * 1. mm life cycle management * 2. fault reporting */ -#define SVM_FLAG_GUEST_MODE (1<<2) +#define SVM_FLAG_GUEST_MODE BIT(1) /* * The SVM_FLAG_GUEST_PASID flag is used when a guest has its own PASID space, * which requires guest and host PASID translation at both directions. */ -#define SVM_FLAG_GUEST_PASID (1<<3) +#define SVM_FLAG_GUEST_PASID BIT(2) #endif /* __INTEL_SVM_H__ */ -- cgit v1.2.3 From 1b169fdf427f9401bf9c8544cb9942580c06f8ef Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 23 Mar 2021 09:05:59 +0800 Subject: iommu/vt-d: Remove unused function declarations Some functions have been removed. Remove the remaining function delarations. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210323010600.678627-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 444c0bec221a..90a3268d7a77 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -99,9 +99,6 @@ static inline bool pasid_pte_is_present(struct pasid_entry *pte) } extern unsigned int intel_pasid_max_id; -int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp); -void intel_pasid_free_id(u32 pasid); -void *intel_pasid_lookup_id(u32 pasid); int intel_pasid_alloc_table(struct device *dev); void intel_pasid_free_table(struct device *dev); struct pasid_table *intel_pasid_get_table(struct device *dev); -- cgit v1.2.3 From 442b81836d6fdde1cf7dc5fc437a5f770c84498b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 23 Mar 2021 09:06:00 +0800 Subject: iommu/vt-d: Make unnecessarily global functions static Make some functions static as they are only used inside pasid.c. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210323010600.678627-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 4 ++-- drivers/iommu/intel/pasid.h | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 1a4105a5989b..4754830610a6 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -230,7 +230,7 @@ struct pasid_table *intel_pasid_get_table(struct device *dev) return info->pasid_table; } -int intel_pasid_get_dev_max_id(struct device *dev) +static int intel_pasid_get_dev_max_id(struct device *dev) { struct device_domain_info *info; @@ -241,7 +241,7 @@ int intel_pasid_get_dev_max_id(struct device *dev) return info->pasid_table->max_pasid; } -struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) +static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) { struct device_domain_info *info; struct pasid_table *pasid_table; diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 90a3268d7a77..079534fcf55d 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -102,8 +102,6 @@ extern unsigned int intel_pasid_max_id; int intel_pasid_alloc_table(struct device *dev); void intel_pasid_free_table(struct device *dev); struct pasid_table *intel_pasid_get_table(struct device *dev); -int intel_pasid_get_dev_max_id(struct device *dev); -struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid); int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev, pgd_t *pgd, u32 pasid, u16 did, int flags); -- cgit v1.2.3 From f598a497bc7dfbec60270bca8b8408db3d23ac07 Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 25 Mar 2021 20:29:58 +0800 Subject: iova: Add CPU hotplug handler to flush rcaches Like the Intel IOMMU driver already does, flush the per-IOVA domain CPU rcache when a CPU goes offline - there's no point in keeping it. Reviewed-by: Robin Murphy Signed-off-by: John Garry Link: https://lore.kernel.org/r/1616675401-151997-2-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 30 +++++++++++++++++++++++++++++- include/linux/cpuhotplug.h | 1 + include/linux/iova.h | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 471c48dd71e7..8542059ddef8 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -25,6 +25,17 @@ static void init_iova_rcaches(struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); static void fq_destroy_all_entries(struct iova_domain *iovad); static void fq_flush_timeout(struct timer_list *t); + +static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) +{ + struct iova_domain *iovad; + + iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); + + free_cpu_cached_iovas(cpu, iovad); + return 0; +} + static void free_global_cached_iovas(struct iova_domain *iovad); static struct iova *to_iova(struct rb_node *node) @@ -56,6 +67,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); rb_insert_color(&iovad->anchor.node, &iovad->rbroot); + cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead); init_iova_rcaches(iovad); } EXPORT_SYMBOL_GPL(init_iova_domain); @@ -299,10 +311,21 @@ int iova_cache_get(void) { mutex_lock(&iova_cache_mutex); if (!iova_cache_users) { + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", NULL, + iova_cpuhp_dead); + if (ret) { + mutex_unlock(&iova_cache_mutex); + pr_err("Couldn't register cpuhp handler\n"); + return ret; + } + iova_cache = kmem_cache_create( "iommu_iova", sizeof(struct iova), 0, SLAB_HWCACHE_ALIGN, NULL); if (!iova_cache) { + cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); mutex_unlock(&iova_cache_mutex); pr_err("Couldn't create iova cache\n"); return -ENOMEM; @@ -324,8 +347,10 @@ void iova_cache_put(void) return; } iova_cache_users--; - if (!iova_cache_users) + if (!iova_cache_users) { + cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); kmem_cache_destroy(iova_cache); + } mutex_unlock(&iova_cache_mutex); } EXPORT_SYMBOL_GPL(iova_cache_put); @@ -648,6 +673,9 @@ void put_iova_domain(struct iova_domain *iovad) { struct iova *iova, *tmp; + cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, + &iovad->cpuhp_dead); + free_iova_flush_queue(iovad); free_iova_rcaches(iovad); rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f14adb882338..cedac9986557 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -58,6 +58,7 @@ enum cpuhp_state { CPUHP_NET_DEV_DEAD, CPUHP_PCI_XGENE_DEAD, CPUHP_IOMMU_INTEL_DEAD, + CPUHP_IOMMU_IOVA_DEAD, CPUHP_LUSTRE_CFS_DEAD, CPUHP_AP_ARM_CACHE_B15_RAC_DEAD, CPUHP_PADATA_DEAD, diff --git a/include/linux/iova.h b/include/linux/iova.h index c834c01c0a5b..4be6c0ab4997 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -95,6 +95,7 @@ struct iova_domain { flush-queues */ atomic_t fq_timer_on; /* 1 when timer is active, 0 when not */ + struct hlist_node cpuhp_dead; }; static inline unsigned long iova_size(struct iova *iova) -- cgit v1.2.3 From 363f266eeff6e22a09483dc922dccd7cd0b9fe9c Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 25 Mar 2021 20:29:59 +0800 Subject: iommu/vt-d: Remove IOVA domain rcache flushing for CPU offlining Now that the core code handles flushing per-IOVA domain CPU rcaches, remove the handling here. Reviewed-by: Lu Baolu Signed-off-by: John Garry Link: https://lore.kernel.org/r/1616675401-151997-3-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 31 ------------------------------- include/linux/cpuhotplug.h | 1 - 2 files changed, 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 1c32522220bc..be5ea16ca26f 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4064,35 +4064,6 @@ static struct notifier_block intel_iommu_memory_nb = { .priority = 0 }; -static void free_all_cpu_cached_iovas(unsigned int cpu) -{ - int i; - - for (i = 0; i < g_num_of_iommus; i++) { - struct intel_iommu *iommu = g_iommus[i]; - struct dmar_domain *domain; - int did; - - if (!iommu) - continue; - - for (did = 0; did < cap_ndoms(iommu->cap); did++) { - domain = get_iommu_domain(iommu, (u16)did); - - if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA) - continue; - - iommu_dma_free_cpu_cached_iovas(cpu, &domain->domain); - } - } -} - -static int intel_iommu_cpu_dead(unsigned int cpu) -{ - free_all_cpu_cached_iovas(cpu); - return 0; -} - static void intel_disable_iommus(void) { struct intel_iommu *iommu = NULL; @@ -4387,8 +4358,6 @@ int __init intel_iommu_init(void) bus_set_iommu(&pci_bus_type, &intel_iommu_ops); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); - cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, - intel_iommu_cpu_dead); down_read(&dmar_global_lock); if (probe_acpi_namespace_devices()) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index cedac9986557..85996494bec1 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -57,7 +57,6 @@ enum cpuhp_state { CPUHP_PAGE_ALLOC_DEAD, CPUHP_NET_DEV_DEAD, CPUHP_PCI_XGENE_DEAD, - CPUHP_IOMMU_INTEL_DEAD, CPUHP_IOMMU_IOVA_DEAD, CPUHP_LUSTRE_CFS_DEAD, CPUHP_AP_ARM_CACHE_B15_RAC_DEAD, -- cgit v1.2.3 From 149448b353e2517ecc6eced7d9f46e9f3e08b89e Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 25 Mar 2021 20:30:00 +0800 Subject: iommu: Delete iommu_dma_free_cpu_cached_iovas() Function iommu_dma_free_cpu_cached_iovas() no longer has any caller, so delete it. With that, function free_cpu_cached_iovas() may be made static. Signed-off-by: John Garry Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/1616675401-151997-4-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 9 --------- drivers/iommu/iova.c | 3 ++- include/linux/dma-iommu.h | 5 ----- include/linux/iova.h | 5 ----- 4 files changed, 2 insertions(+), 20 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index fdd095e1fa52..0af4048f9b3f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -64,15 +64,6 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); -void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, - struct iommu_domain *domain) -{ - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - - free_cpu_cached_iovas(cpu, iovad); -} - static void iommu_dma_entry_dtor(unsigned long data) { struct page *freelist = (struct page *)data; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 8542059ddef8..c9726e1b9e11 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -22,6 +22,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn); static void init_iova_rcaches(struct iova_domain *iovad); +static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); static void fq_destroy_all_entries(struct iova_domain *iovad); static void fq_flush_timeout(struct timer_list *t); @@ -1040,7 +1041,7 @@ static void free_iova_rcaches(struct iova_domain *iovad) /* * free all the IOVA ranges cached by a cpu (used when cpu is unplugged) */ -void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) +static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) { struct iova_cpu_rcache *cpu_rcache; struct iova_rcache *rcache; diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 13d1f4c14d7b..6e75a2d689b4 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -83,10 +83,5 @@ static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_he { } -static inline void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, - struct iommu_domain *domain) -{ -} - #endif /* CONFIG_IOMMU_DMA */ #endif /* __DMA_IOMMU_H */ diff --git a/include/linux/iova.h b/include/linux/iova.h index 4be6c0ab4997..71d8a2de6635 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -157,7 +157,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb, iova_entry_dtor entry_dtor); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); -void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); #else static inline int iova_cache_get(void) { @@ -234,10 +233,6 @@ static inline void put_iova_domain(struct iova_domain *iovad) { } -static inline void free_cpu_cached_iovas(unsigned int cpu, - struct iova_domain *iovad) -{ -} #endif #endif -- cgit v1.2.3 From 6e1ea50a065ed1ff8a27cde3e1876ed7dfda97fd Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 25 Mar 2021 20:30:01 +0800 Subject: iommu: Stop exporting free_iova_fast() Function free_iova_fast() is only referenced by dma-iommu.c, which can only be in-built, so stop exporting it. This was missed in an earlier tidy-up patch. Signed-off-by: John Garry Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/1616675401-151997-5-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index c9726e1b9e11..b7ecd5b08039 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -535,7 +535,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) free_iova(iovad, pfn); } -EXPORT_SYMBOL_GPL(free_iova_fast); #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) -- cgit v1.2.3 From 8de000cf0265eaa4f63aff9f2c7a3876d2dda9b6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Fri, 26 Mar 2021 11:23:36 +0800 Subject: iommu/mediatek-v1: Allow building as module This patch only adds support for building the IOMMU-v1 driver as module. Correspondingly switch the config to tristate and update the iommu_ops's owner to THIS_MODULE. Signed-off-by: Yong Wu Link: https://lore.kernel.org/r/20210326032337.24578-1-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 2 +- drivers/iommu/mtk_iommu_v1.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 192ef8f61310..bc93da48bed0 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -364,7 +364,7 @@ config MTK_IOMMU If unsure, say N here. config MTK_IOMMU_V1 - bool "MTK IOMMU Version 1 (M4U gen1) Support" + tristate "MediaTek IOMMU Version 1 (M4U gen1) Support" depends on ARM depends on ARCH_MEDIATEK || COMPILE_TEST select ARM_DMA_USE_IOMMU diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 82ddfe9170d4..be1b20e3f20e 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -529,6 +530,7 @@ static const struct iommu_ops mtk_iommu_ops = { .def_domain_type = mtk_iommu_def_domain_type, .device_group = generic_device_group, .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, + .owner = THIS_MODULE, }; static const struct of_device_id mtk_iommu_of_ids[] = { @@ -691,9 +693,7 @@ static struct platform_driver mtk_iommu_driver = { .pm = &mtk_iommu_pm_ops, } }; +module_platform_driver(mtk_iommu_driver); -static int __init m4u_init(void) -{ - return platform_driver_register(&mtk_iommu_driver); -} -subsys_initcall(m4u_init); +MODULE_DESCRIPTION("IOMMU API for MediaTek M4U v1 implementations"); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 18d8c74ec5987a78bd1e9c1c629dfdd04a151a89 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Fri, 26 Mar 2021 11:23:37 +0800 Subject: iommu/mediatek: Allow building as module The IOMMU in many SoC depends on the MM clocks and power-domain which are device_initcall normally, thus the subsys_init here is not helpful. This patch switches it to module_platform_driver which also allow the driver built as module. Correspondingly switch the config to tristate, and update the iommu_ops's owner. Signed-off-by: Yong Wu Link: https://lore.kernel.org/r/20210326032337.24578-2-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 2 +- drivers/iommu/mtk_iommu.c | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index bc93da48bed0..74f3e15edc14 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -349,7 +349,7 @@ config S390_AP_IOMMU is not implemented as it is not necessary for VFIO. config MTK_IOMMU - bool "MTK IOMMU Support" + tristate "MediaTek IOMMU Support" depends on ARCH_MEDIATEK || COMPILE_TEST select ARM_DMA_USE_IOMMU select IOMMU_API diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 6ecc007f07cd..75375935f301 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -683,6 +684,7 @@ static const struct iommu_ops mtk_iommu_ops = { .get_resv_regions = mtk_iommu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, + .owner = THIS_MODULE, }; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) @@ -1079,16 +1081,7 @@ static struct platform_driver mtk_iommu_driver = { .pm = &mtk_iommu_pm_ops, } }; +module_platform_driver(mtk_iommu_driver); -static int __init mtk_iommu_init(void) -{ - int ret; - - ret = platform_driver_register(&mtk_iommu_driver); - if (ret != 0) - pr_err("Failed to register MTK IOMMU driver\n"); - - return ret; -} - -subsys_initcall(mtk_iommu_init) +MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations"); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 6c00612d0cba10f7d0917cf1f73c945003ed4cd7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 30 Mar 2021 10:11:45 +0800 Subject: iommu/vt-d: Report right snoop capability when using FL for IOVA The Intel VT-d driver checks wrong register to report snoop capablility when using first level page table for GPA to HPA translation. This might lead the IOMMU driver to say that it supports snooping control, but in reality, it does not. Fix this by always setting PASID-table-entry.PGSNP whenever a pasid entry is setting up for GPA to HPA translation so that the IOMMU driver could report snoop capability as long as it runs in the scalable mode. Fixes: b802d070a52a1 ("iommu/vt-d: Use iova over first level") Suggested-by: Rajesh Sankaran Suggested-by: Kevin Tian Suggested-by: Ashok Raj Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210330021145.13824-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 12 +++++++++++- drivers/iommu/intel/pasid.c | 16 ++++++++++++++++ drivers/iommu/intel/pasid.h | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 956a02eb40b4..0ee5f1bd8af2 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -658,7 +658,14 @@ static int domain_update_iommu_snooping(struct intel_iommu *skip) rcu_read_lock(); for_each_active_iommu(iommu, drhd) { if (iommu != skip) { - if (!ecap_sc_support(iommu->ecap)) { + /* + * If the hardware is operating in the scalable mode, + * the snooping control is always supported since we + * always set PASID-table-entry.PGSNP bit if the domain + * is managed outside (UNMANAGED). + */ + if (!sm_supported(iommu) && + !ecap_sc_support(iommu->ecap)) { ret = 0; break; } @@ -2505,6 +2512,9 @@ static int domain_setup_first_level(struct intel_iommu *iommu, flags |= (level == 5) ? PASID_FLAG_FL5LP : 0; + if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) + flags |= PASID_FLAG_PAGE_SNOOP; + return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid, domain->iommu_did[iommu->seq_id], flags); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 4754830610a6..8f2a702ce429 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -425,6 +425,16 @@ static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) pasid_set_bits(&pe->val[1], 1 << 23, value << 23); } +/* + * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode + * PASID entry. + */ +static inline void +pasid_set_pgsnp(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24); +} + /* * Setup the First Level Page table Pointer field (Bit 140~191) * of a scalable mode PASID entry. @@ -596,6 +606,9 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, } } + if (flags & PASID_FLAG_PAGE_SNOOP) + pasid_set_pgsnp(pte); + pasid_set_domain_id(pte, did); pasid_set_address_width(pte, iommu->agaw); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); @@ -674,6 +687,9 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, pasid_set_fault_enable(pte); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); + if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) + pasid_set_pgsnp(pte); + /* * Since it is a second level only translation setup, we should * set SRE bit as well (addresses are expected to be GPAs). diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 079534fcf55d..5ff61c3d401f 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -48,6 +48,7 @@ */ #define PASID_FLAG_SUPERVISOR_MODE BIT(0) #define PASID_FLAG_NESTED BIT(1) +#define PASID_FLAG_PAGE_SNOOP BIT(2) /* * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first- -- cgit v1.2.3 From a56af062ae7cdc9759ed61d1d3d173f28c79510a Mon Sep 17 00:00:00 2001 From: Chunyan Zhang Date: Wed, 31 Mar 2021 11:16:45 +0800 Subject: iommu/sprd: Fix parameter type warning The second parameter of clk_get_optional() is "const char *", so use NULL instead of integer 0 to fix a sparse warning like: ">> drivers/iommu/sprd-iommu.c:456:42: sparse: sparse: Using plain integer as NULL pointer" Also this patch changes to use the resource-managed variant of clk_get_optional(), then there's no need to add clk_put() which is missed in the current driver. Reported-by: kernel test robot Signed-off-by: Chunyan Zhang Link: https://lore.kernel.org/r/20210331031645.1001913-1-zhang.lyra@gmail.com Signed-off-by: Joerg Roedel --- drivers/iommu/sprd-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index e1dc2f7d5639..3811a5085b50 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -453,7 +453,7 @@ static int sprd_iommu_clk_enable(struct sprd_iommu_device *sdev) { struct clk *eb; - eb = clk_get_optional(sdev->dev, 0); + eb = devm_clk_get_optional(sdev->dev, NULL); if (!eb) return 0; -- cgit v1.2.3 From 3e84f878b56b075b9a81de6e73da7b3dc88387d8 Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Thu, 1 Apr 2021 19:19:16 +0800 Subject: iommu/amd: Remove duplicate check of pasids Remove duplicate check of pasids in amd_iommu_domain_enable_v2(), as it has been guaranteed in amd_iommu_init_device(). Signed-off-by: Qi Liu Link: https://lore.kernel.org/r/1617275956-4467-1-git-send-email-liuqi115@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a69a8b573e40..70814ab0b374 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2310,9 +2310,6 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) unsigned long flags; int levels, ret; - if (pasids <= 0 || pasids > (PASID_MASK + 1)) - return -EINVAL; - /* Number of GCR3 table levels required */ for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9) levels += 1; -- cgit v1.2.3 From 434b73e61cc65cdd26618af6fa4736c2ba1eb29b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 1 Apr 2021 17:47:11 +0200 Subject: iommu/arm-smmu-v3: Use device properties for pasid-num-bits The pasid-num-bits property shouldn't need a dedicated fwspec field, it's a job for device properties. Add properties for IORT, and access the number of PASID bits using device_property_read_u32(). Suggested-by: Robin Murphy Acked-by: Jonathan Cameron Acked-by: Will Deacon Reviewed-by: Eric Auger Signed-off-by: Jean-Philippe Brucker Acked-by: Hanjun Guo Link: https://lore.kernel.org/r/20210401154718.307519-3-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 13 +++++++------ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 ++- drivers/iommu/of_iommu.c | 5 ----- include/linux/iommu.h | 2 -- 4 files changed, 9 insertions(+), 14 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 2494138a6905..3912a1f6058e 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -968,15 +968,16 @@ static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data) static void iort_named_component_init(struct device *dev, struct acpi_iort_node *node) { + struct property_entry props[2] = {}; struct acpi_iort_named_component *nc; - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - - if (!fwspec) - return; nc = (struct acpi_iort_named_component *)node->node_data; - fwspec->num_pasid_bits = FIELD_GET(ACPI_IORT_NC_PASID_BITS, - nc->node_flags); + props[0] = PROPERTY_ENTRY_U32("pasid-num-bits", + FIELD_GET(ACPI_IORT_NC_PASID_BITS, + nc->node_flags)); + + if (device_add_properties(dev, props)) + dev_warn(dev, "Could not add device properties\n"); } static int iort_nc_iommu_map(struct device *dev, struct acpi_iort_node *node) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8594b4a83043..7edce914c45e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2392,7 +2392,8 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) } } - master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits); + device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits); + master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); /* * Note that PASID must be enabled before, and disabled after ATS: diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index e505b9130a1c..a9d2df001149 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -210,11 +210,6 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, of_pci_iommu_init, &info); } else { err = of_iommu_configure_device(master_np, dev, id); - - fwspec = dev_iommu_fwspec_get(dev); - if (!err && fwspec) - of_property_read_u32(master_np, "pasid-num-bits", - &fwspec->num_pasid_bits); } /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1d422bf722a1..16ce75693d83 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -572,7 +572,6 @@ struct iommu_group *fsl_mc_device_group(struct device *dev); * @ops: ops for this device's IOMMU * @iommu_fwnode: firmware handle for this device's IOMMU * @flags: IOMMU_FWSPEC_* flags - * @num_pasid_bits: number of PASID bits supported by this device * @num_ids: number of associated device IDs * @ids: IDs which this device may present to the IOMMU */ @@ -580,7 +579,6 @@ struct iommu_fwspec { const struct iommu_ops *ops; struct fwnode_handle *iommu_fwnode; u32 flags; - u32 num_pasid_bits; unsigned int num_ids; u32 ids[]; }; -- cgit v1.2.3 From 9003351cb6bde752de774e6ec874109493413152 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 1 Apr 2021 17:47:13 +0200 Subject: iommu/vt-d: Support IOMMU_DEV_FEAT_IOPF Allow drivers to query and enable IOMMU_DEV_FEAT_IOPF, which amounts to checking whether PRI is enabled. Reviewed-by: Lu Baolu Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20210401154718.307519-5-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index be5ea16ca26f..1ea0f0bd6973 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5311,6 +5311,8 @@ static int siov_find_pci_dvsec(struct pci_dev *pdev) static bool intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat) { + struct device_domain_info *info = get_domain_info(dev); + if (feat == IOMMU_DEV_FEAT_AUX) { int ret; @@ -5325,13 +5327,13 @@ intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat) return !!siov_find_pci_dvsec(to_pci_dev(dev)); } - if (feat == IOMMU_DEV_FEAT_SVA) { - struct device_domain_info *info = get_domain_info(dev); + if (feat == IOMMU_DEV_FEAT_IOPF) + return info && info->pri_supported; + if (feat == IOMMU_DEV_FEAT_SVA) return info && (info->iommu->flags & VTD_FLAG_SVM_CAPABLE) && info->pasid_supported && info->pri_supported && info->ats_supported; - } return false; } @@ -5342,6 +5344,9 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) if (feat == IOMMU_DEV_FEAT_AUX) return intel_iommu_enable_auxd(dev); + if (feat == IOMMU_DEV_FEAT_IOPF) + return intel_iommu_dev_has_feat(dev, feat) ? 0 : -ENODEV; + if (feat == IOMMU_DEV_FEAT_SVA) { struct device_domain_info *info = get_domain_info(dev); -- cgit v1.2.3 From fc36479db74e957c4696b605a32c4afaa15fa6cb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 1 Apr 2021 17:47:15 +0200 Subject: iommu: Add a page fault handler Some systems allow devices to handle I/O Page Faults in the core mm. For example systems implementing the PCIe PRI extension or Arm SMMU stall model. Infrastructure for reporting these recoverable page faults was added to the IOMMU core by commit 0c830e6b3282 ("iommu: Introduce device fault report API"). Add a page fault handler for host SVA. IOMMU driver can now instantiate several fault workqueues and link them to IOPF-capable devices. Drivers can choose between a single global workqueue, one per IOMMU device, one per low-level fault queue, one per domain, etc. When it receives a fault event, most commonly in an IRQ handler, the IOMMU driver reports the fault using iommu_report_device_fault(), which calls the registered handler. The page fault handler then calls the mm fault handler, and reports either success or failure with iommu_page_response(). After the handler succeeds, the hardware retries the access. The iopf_param pointer could be embedded into iommu_fault_param. But putting iopf_param into the iommu_param structure allows us not to care about ordering between calls to iopf_queue_add_device() and iommu_register_device_fault_handler(). Tested-by: Lu Baolu Reviewed-by: Eric Auger Reviewed-by: Jacob Pan Reviewed-by: Jonathan Cameron Reviewed-by: Lu Baolu Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20210401154718.307519-7-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/Makefile | 1 + drivers/iommu/io-pgfault.c | 461 ++++++++++++++++++++++++++++++++++++++++++ drivers/iommu/iommu-sva-lib.h | 53 +++++ include/linux/iommu.h | 2 + 4 files changed, 517 insertions(+) create mode 100644 drivers/iommu/io-pgfault.c (limited to 'drivers/iommu') diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 61bd30cd8369..60fafc23dee6 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -28,3 +28,4 @@ obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o obj-$(CONFIG_IOMMU_SVA_LIB) += iommu-sva-lib.o +obj-$(CONFIG_IOMMU_SVA_LIB) += io-pgfault.o diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c new file mode 100644 index 000000000000..1df8c1dcae77 --- /dev/null +++ b/drivers/iommu/io-pgfault.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Handle device page faults + * + * Copyright (C) 2020 ARM Ltd. + */ + +#include +#include +#include +#include +#include + +#include "iommu-sva-lib.h" + +/** + * struct iopf_queue - IO Page Fault queue + * @wq: the fault workqueue + * @devices: devices attached to this queue + * @lock: protects the device list + */ +struct iopf_queue { + struct workqueue_struct *wq; + struct list_head devices; + struct mutex lock; +}; + +/** + * struct iopf_device_param - IO Page Fault data attached to a device + * @dev: the device that owns this param + * @queue: IOPF queue + * @queue_list: index into queue->devices + * @partial: faults that are part of a Page Request Group for which the last + * request hasn't been submitted yet. + */ +struct iopf_device_param { + struct device *dev; + struct iopf_queue *queue; + struct list_head queue_list; + struct list_head partial; +}; + +struct iopf_fault { + struct iommu_fault fault; + struct list_head list; +}; + +struct iopf_group { + struct iopf_fault last_fault; + struct list_head faults; + struct work_struct work; + struct device *dev; +}; + +static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, + enum iommu_page_response_code status) +{ + struct iommu_page_response resp = { + .version = IOMMU_PAGE_RESP_VERSION_1, + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = status, + }; + + if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && + (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) + resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + + return iommu_page_response(dev, &resp); +} + +static enum iommu_page_response_code +iopf_handle_single(struct iopf_fault *iopf) +{ + vm_fault_t ret; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned int access_flags = 0; + unsigned int fault_flags = FAULT_FLAG_REMOTE; + struct iommu_fault_page_request *prm = &iopf->fault.prm; + enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID; + + if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID)) + return status; + + mm = iommu_sva_find(prm->pasid); + if (IS_ERR_OR_NULL(mm)) + return status; + + mmap_read_lock(mm); + + vma = find_extend_vma(mm, prm->addr); + if (!vma) + /* Unmapped area */ + goto out_put_mm; + + if (prm->perm & IOMMU_FAULT_PERM_READ) + access_flags |= VM_READ; + + if (prm->perm & IOMMU_FAULT_PERM_WRITE) { + access_flags |= VM_WRITE; + fault_flags |= FAULT_FLAG_WRITE; + } + + if (prm->perm & IOMMU_FAULT_PERM_EXEC) { + access_flags |= VM_EXEC; + fault_flags |= FAULT_FLAG_INSTRUCTION; + } + + if (!(prm->perm & IOMMU_FAULT_PERM_PRIV)) + fault_flags |= FAULT_FLAG_USER; + + if (access_flags & ~vma->vm_flags) + /* Access fault */ + goto out_put_mm; + + ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL); + status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID : + IOMMU_PAGE_RESP_SUCCESS; + +out_put_mm: + mmap_read_unlock(mm); + mmput(mm); + + return status; +} + +static void iopf_handle_group(struct work_struct *work) +{ + struct iopf_group *group; + struct iopf_fault *iopf, *next; + enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; + + group = container_of(work, struct iopf_group, work); + + list_for_each_entry_safe(iopf, next, &group->faults, list) { + /* + * For the moment, errors are sticky: don't handle subsequent + * faults in the group if there is an error. + */ + if (status == IOMMU_PAGE_RESP_SUCCESS) + status = iopf_handle_single(iopf); + + if (!(iopf->fault.prm.flags & + IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) + kfree(iopf); + } + + iopf_complete_group(group->dev, &group->last_fault, status); + kfree(group); +} + +/** + * iommu_queue_iopf - IO Page Fault handler + * @fault: fault event + * @cookie: struct device, passed to iommu_register_device_fault_handler. + * + * Add a fault to the device workqueue, to be handled by mm. + * + * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard + * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't + * expect a response. It may be generated when disabling a PASID (issuing a + * PASID stop request) by some PCI devices. + * + * The PASID stop request is issued by the device driver before unbind(). Once + * it completes, no page request is generated for this PASID anymore and + * outstanding ones have been pushed to the IOMMU (as per PCIe 4.0r1.0 - 6.20.1 + * and 10.4.1.2 - Managing PASID TLP Prefix Usage). Some PCI devices will wait + * for all outstanding page requests to come back with a response before + * completing the PASID stop request. Others do not wait for page responses, and + * instead issue this Stop Marker that tells us when the PASID can be + * reallocated. + * + * It is safe to discard the Stop Marker because it is an optimization. + * a. Page requests, which are posted requests, have been flushed to the IOMMU + * when the stop request completes. + * b. The IOMMU driver flushes all fault queues on unbind() before freeing the + * PASID. + * + * So even though the Stop Marker might be issued by the device *after* the stop + * request completes, outstanding faults will have been dealt with by the time + * the PASID is freed. + * + * Return: 0 on success and <0 on error. + */ +int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +{ + int ret; + struct iopf_group *group; + struct iopf_fault *iopf, *next; + struct iopf_device_param *iopf_param; + + struct device *dev = cookie; + struct dev_iommu *param = dev->iommu; + + lockdep_assert_held(¶m->lock); + + if (fault->type != IOMMU_FAULT_PAGE_REQ) + /* Not a recoverable page fault */ + return -EOPNOTSUPP; + + /* + * As long as we're holding param->lock, the queue can't be unlinked + * from the device and therefore cannot disappear. + */ + iopf_param = param->iopf_param; + if (!iopf_param) + return -ENODEV; + + if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); + if (!iopf) + return -ENOMEM; + + iopf->fault = *fault; + + /* Non-last request of a group. Postpone until the last one */ + list_add(&iopf->list, &iopf_param->partial); + + return 0; + } + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) { + /* + * The caller will send a response to the hardware. But we do + * need to clean up before leaving, otherwise partial faults + * will be stuck. + */ + ret = -ENOMEM; + goto cleanup_partial; + } + + group->dev = dev; + group->last_fault.fault = *fault; + INIT_LIST_HEAD(&group->faults); + list_add(&group->last_fault.list, &group->faults); + INIT_WORK(&group->work, iopf_handle_group); + + /* See if we have partial faults for this group */ + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { + if (iopf->fault.prm.grpid == fault->prm.grpid) + /* Insert *before* the last fault */ + list_move(&iopf->list, &group->faults); + } + + queue_work(iopf_param->queue->wq, &group->work); + return 0; + +cleanup_partial: + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { + if (iopf->fault.prm.grpid == fault->prm.grpid) { + list_del(&iopf->list); + kfree(iopf); + } + } + return ret; +} +EXPORT_SYMBOL_GPL(iommu_queue_iopf); + +/** + * iopf_queue_flush_dev - Ensure that all queued faults have been processed + * @dev: the endpoint whose faults need to be flushed. + * + * The IOMMU driver calls this before releasing a PASID, to ensure that all + * pending faults for this PASID have been handled, and won't hit the address + * space of the next process that uses this PASID. The driver must make sure + * that no new fault is added to the queue. In particular it must flush its + * low-level queue before calling this function. + * + * Return: 0 on success and <0 on error. + */ +int iopf_queue_flush_dev(struct device *dev) +{ + int ret = 0; + struct iopf_device_param *iopf_param; + struct dev_iommu *param = dev->iommu; + + if (!param) + return -ENODEV; + + mutex_lock(¶m->lock); + iopf_param = param->iopf_param; + if (iopf_param) + flush_workqueue(iopf_param->queue->wq); + else + ret = -ENODEV; + mutex_unlock(¶m->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); + +/** + * iopf_queue_discard_partial - Remove all pending partial fault + * @queue: the queue whose partial faults need to be discarded + * + * When the hardware queue overflows, last page faults in a group may have been + * lost and the IOMMU driver calls this to discard all partial faults. The + * driver shouldn't be adding new faults to this queue concurrently. + * + * Return: 0 on success and <0 on error. + */ +int iopf_queue_discard_partial(struct iopf_queue *queue) +{ + struct iopf_fault *iopf, *next; + struct iopf_device_param *iopf_param; + + if (!queue) + return -EINVAL; + + mutex_lock(&queue->lock); + list_for_each_entry(iopf_param, &queue->devices, queue_list) { + list_for_each_entry_safe(iopf, next, &iopf_param->partial, + list) { + list_del(&iopf->list); + kfree(iopf); + } + } + mutex_unlock(&queue->lock); + return 0; +} +EXPORT_SYMBOL_GPL(iopf_queue_discard_partial); + +/** + * iopf_queue_add_device - Add producer to the fault queue + * @queue: IOPF queue + * @dev: device to add + * + * Return: 0 on success and <0 on error. + */ +int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) +{ + int ret = -EBUSY; + struct iopf_device_param *iopf_param; + struct dev_iommu *param = dev->iommu; + + if (!param) + return -ENODEV; + + iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL); + if (!iopf_param) + return -ENOMEM; + + INIT_LIST_HEAD(&iopf_param->partial); + iopf_param->queue = queue; + iopf_param->dev = dev; + + mutex_lock(&queue->lock); + mutex_lock(¶m->lock); + if (!param->iopf_param) { + list_add(&iopf_param->queue_list, &queue->devices); + param->iopf_param = iopf_param; + ret = 0; + } + mutex_unlock(¶m->lock); + mutex_unlock(&queue->lock); + + if (ret) + kfree(iopf_param); + + return ret; +} +EXPORT_SYMBOL_GPL(iopf_queue_add_device); + +/** + * iopf_queue_remove_device - Remove producer from fault queue + * @queue: IOPF queue + * @dev: device to remove + * + * Caller makes sure that no more faults are reported for this device. + * + * Return: 0 on success and <0 on error. + */ +int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +{ + int ret = -EINVAL; + struct iopf_fault *iopf, *next; + struct iopf_device_param *iopf_param; + struct dev_iommu *param = dev->iommu; + + if (!param || !queue) + return -EINVAL; + + mutex_lock(&queue->lock); + mutex_lock(¶m->lock); + iopf_param = param->iopf_param; + if (iopf_param && iopf_param->queue == queue) { + list_del(&iopf_param->queue_list); + param->iopf_param = NULL; + ret = 0; + } + mutex_unlock(¶m->lock); + mutex_unlock(&queue->lock); + if (ret) + return ret; + + /* Just in case some faults are still stuck */ + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) + kfree(iopf); + + kfree(iopf_param); + + return 0; +} +EXPORT_SYMBOL_GPL(iopf_queue_remove_device); + +/** + * iopf_queue_alloc - Allocate and initialize a fault queue + * @name: a unique string identifying the queue (for workqueue) + * + * Return: the queue on success and NULL on error. + */ +struct iopf_queue *iopf_queue_alloc(const char *name) +{ + struct iopf_queue *queue; + + queue = kzalloc(sizeof(*queue), GFP_KERNEL); + if (!queue) + return NULL; + + /* + * The WQ is unordered because the low-level handler enqueues faults by + * group. PRI requests within a group have to be ordered, but once + * that's dealt with, the high-level function can handle groups out of + * order. + */ + queue->wq = alloc_workqueue("iopf_queue/%s", WQ_UNBOUND, 0, name); + if (!queue->wq) { + kfree(queue); + return NULL; + } + + INIT_LIST_HEAD(&queue->devices); + mutex_init(&queue->lock); + + return queue; +} +EXPORT_SYMBOL_GPL(iopf_queue_alloc); + +/** + * iopf_queue_free - Free IOPF queue + * @queue: queue to free + * + * Counterpart to iopf_queue_alloc(). The driver must not be queuing faults or + * adding/removing devices on this queue anymore. + */ +void iopf_queue_free(struct iopf_queue *queue) +{ + struct iopf_device_param *iopf_param, *next; + + if (!queue) + return; + + list_for_each_entry_safe(iopf_param, next, &queue->devices, queue_list) + iopf_queue_remove_device(queue, iopf_param->dev); + + destroy_workqueue(queue->wq); + kfree(queue); +} +EXPORT_SYMBOL_GPL(iopf_queue_free); diff --git a/drivers/iommu/iommu-sva-lib.h b/drivers/iommu/iommu-sva-lib.h index b40990aef3fd..031155010ca8 100644 --- a/drivers/iommu/iommu-sva-lib.h +++ b/drivers/iommu/iommu-sva-lib.h @@ -12,4 +12,57 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max); void iommu_sva_free_pasid(struct mm_struct *mm); struct mm_struct *iommu_sva_find(ioasid_t pasid); +/* I/O Page fault */ +struct device; +struct iommu_fault; +struct iopf_queue; + +#ifdef CONFIG_IOMMU_SVA_LIB +int iommu_queue_iopf(struct iommu_fault *fault, void *cookie); + +int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_remove_device(struct iopf_queue *queue, + struct device *dev); +int iopf_queue_flush_dev(struct device *dev); +struct iopf_queue *iopf_queue_alloc(const char *name); +void iopf_queue_free(struct iopf_queue *queue); +int iopf_queue_discard_partial(struct iopf_queue *queue); + +#else /* CONFIG_IOMMU_SVA_LIB */ +static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +{ + return -ENODEV; +} + +static inline int iopf_queue_add_device(struct iopf_queue *queue, + struct device *dev) +{ + return -ENODEV; +} + +static inline int iopf_queue_remove_device(struct iopf_queue *queue, + struct device *dev) +{ + return -ENODEV; +} + +static inline int iopf_queue_flush_dev(struct device *dev) +{ + return -ENODEV; +} + +static inline struct iopf_queue *iopf_queue_alloc(const char *name) +{ + return NULL; +} + +static inline void iopf_queue_free(struct iopf_queue *queue) +{ +} + +static inline int iopf_queue_discard_partial(struct iopf_queue *queue) +{ + return -ENODEV; +} +#endif /* CONFIG_IOMMU_SVA_LIB */ #endif /* _IOMMU_SVA_LIB_H */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 45c4eb372f56..86d688c4418f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -367,6 +367,7 @@ struct iommu_fault_param { * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data + * @iopf_param: I/O Page Fault queue and data * @fwspec: IOMMU fwspec data * @iommu_dev: IOMMU device this device is linked to * @priv: IOMMU Driver private data @@ -377,6 +378,7 @@ struct iommu_fault_param { struct dev_iommu { struct mutex lock; struct iommu_fault_param *fault_param; + struct iopf_device_param *iopf_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; -- cgit v1.2.3 From cdf315f907d46a8cfb60bcc1cb4a73d730303196 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Thu, 1 Apr 2021 17:47:16 +0200 Subject: iommu/arm-smmu-v3: Maintain a SID->device structure When handling faults from the event or PRI queue, we need to find the struct device associated with a SID. Add a rb_tree to keep track of SIDs. Acked-by: Jonathan Cameron Reviewed-by: Eric Auger Reviewed-by: Keqian Zhu Signed-off-by: Jean-Philippe Brucker Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210401154718.307519-8-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 158 +++++++++++++++++++++++----- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 13 ++- 2 files changed, 141 insertions(+), 30 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 7edce914c45e..1c7721b89fec 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -909,8 +909,8 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_for_each_entry(master, &smmu_domain->devices, domain_head) { - for (i = 0; i < master->num_sids; i++) { - cmd.cfgi.sid = master->sids[i]; + for (i = 0; i < master->num_streams; i++) { + cmd.cfgi.sid = master->streams[i].id; arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); } } @@ -1355,6 +1355,29 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return 0; } +__maybe_unused +static struct arm_smmu_master * +arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) +{ + struct rb_node *node; + struct arm_smmu_stream *stream; + + lockdep_assert_held(&smmu->streams_mutex); + + node = smmu->streams.rb_node; + while (node) { + stream = rb_entry(node, struct arm_smmu_stream, node); + if (stream->id < sid) + node = node->rb_right; + else if (stream->id > sid) + node = node->rb_left; + else + return stream->master; + } + + return NULL; +} + /* IRQ and event handlers */ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) { @@ -1588,8 +1611,8 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); - for (i = 0; i < master->num_sids; i++) { - cmd.atc.sid = master->sids[i]; + for (i = 0; i < master->num_streams; i++) { + cmd.atc.sid = master->streams[i].id; arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); } @@ -1632,8 +1655,8 @@ int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, if (!master->ats_enabled) continue; - for (i = 0; i < master->num_sids; i++) { - cmd.atc.sid = master->sids[i]; + for (i = 0; i < master->num_streams; i++) { + cmd.atc.sid = master->streams[i].id; arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); } } @@ -2065,13 +2088,13 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) int i, j; struct arm_smmu_device *smmu = master->smmu; - for (i = 0; i < master->num_sids; ++i) { - u32 sid = master->sids[i]; + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); /* Bridged PCI devices may end up with duplicated IDs */ for (j = 0; j < i; j++) - if (master->sids[j] == sid) + if (master->streams[j].id == sid) break; if (j < i) continue; @@ -2345,11 +2368,101 @@ static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) return sid < limit; } +static int arm_smmu_insert_master(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + int i; + int ret = 0; + struct arm_smmu_stream *new_stream, *cur_stream; + struct rb_node **new_node, *parent_node = NULL; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); + + master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams), + GFP_KERNEL); + if (!master->streams) + return -ENOMEM; + master->num_streams = fwspec->num_ids; + + mutex_lock(&smmu->streams_mutex); + for (i = 0; i < fwspec->num_ids; i++) { + u32 sid = fwspec->ids[i]; + + new_stream = &master->streams[i]; + new_stream->id = sid; + new_stream->master = master; + + /* + * Check the SIDs are in range of the SMMU and our stream table + */ + if (!arm_smmu_sid_in_range(smmu, sid)) { + ret = -ERANGE; + break; + } + + /* Ensure l2 strtab is initialised */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + ret = arm_smmu_init_l2_strtab(smmu, sid); + if (ret) + break; + } + + /* Insert into SID tree */ + new_node = &(smmu->streams.rb_node); + while (*new_node) { + cur_stream = rb_entry(*new_node, struct arm_smmu_stream, + node); + parent_node = *new_node; + if (cur_stream->id > new_stream->id) { + new_node = &((*new_node)->rb_left); + } else if (cur_stream->id < new_stream->id) { + new_node = &((*new_node)->rb_right); + } else { + dev_warn(master->dev, + "stream %u already in tree\n", + cur_stream->id); + ret = -EINVAL; + break; + } + } + if (ret) + break; + + rb_link_node(&new_stream->node, parent_node, new_node); + rb_insert_color(&new_stream->node, &smmu->streams); + } + + if (ret) { + for (i--; i >= 0; i--) + rb_erase(&master->streams[i].node, &smmu->streams); + kfree(master->streams); + } + mutex_unlock(&smmu->streams_mutex); + + return ret; +} + +static void arm_smmu_remove_master(struct arm_smmu_master *master) +{ + int i; + struct arm_smmu_device *smmu = master->smmu; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); + + if (!smmu || !master->streams) + return; + + mutex_lock(&smmu->streams_mutex); + for (i = 0; i < fwspec->num_ids; i++) + rb_erase(&master->streams[i].node, &smmu->streams); + mutex_unlock(&smmu->streams_mutex); + + kfree(master->streams); +} + static struct iommu_ops arm_smmu_ops; static struct iommu_device *arm_smmu_probe_device(struct device *dev) { - int i, ret; + int ret; struct arm_smmu_device *smmu; struct arm_smmu_master *master; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); @@ -2370,27 +2483,12 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) master->dev = dev; master->smmu = smmu; - master->sids = fwspec->ids; - master->num_sids = fwspec->num_ids; INIT_LIST_HEAD(&master->bonds); dev_iommu_priv_set(dev, master); - /* Check the SIDs are in range of the SMMU and our stream table */ - for (i = 0; i < master->num_sids; i++) { - u32 sid = master->sids[i]; - - if (!arm_smmu_sid_in_range(smmu, sid)) { - ret = -ERANGE; - goto err_free_master; - } - - /* Ensure l2 strtab is initialised */ - if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { - ret = arm_smmu_init_l2_strtab(smmu, sid); - if (ret) - goto err_free_master; - } - } + ret = arm_smmu_insert_master(smmu, master); + if (ret) + goto err_free_master; device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits); master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); @@ -2429,6 +2527,7 @@ static void arm_smmu_release_device(struct device *dev) WARN_ON(arm_smmu_master_sva_enabled(master)); arm_smmu_detach_dev(master); arm_smmu_disable_pasid(master); + arm_smmu_remove_master(master); kfree(master); iommu_fwspec_free(dev); } @@ -2852,6 +2951,9 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu) { int ret; + mutex_init(&smmu->streams_mutex); + smmu->streams = RB_ROOT; + ret = arm_smmu_init_queues(smmu); if (ret) return ret; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index f985817c967a..7b15b7580c6e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -639,6 +639,15 @@ struct arm_smmu_device { /* IOMMU core code handle */ struct iommu_device iommu; + + struct rb_root streams; + struct mutex streams_mutex; +}; + +struct arm_smmu_stream { + u32 id; + struct arm_smmu_master *master; + struct rb_node node; }; /* SMMU private data for each master */ @@ -647,8 +656,8 @@ struct arm_smmu_master { struct device *dev; struct arm_smmu_domain *domain; struct list_head domain_head; - u32 *sids; - unsigned int num_sids; + struct arm_smmu_stream *streams; + unsigned int num_streams; bool ats_enabled; bool sva_enabled; struct list_head bonds; -- cgit v1.2.3 From 47685cb202d1aff6f70a2bb91e8271392fefea84 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:37 +0200 Subject: iommu: remove the unused domain_window_disable method domain_window_disable is wired up by fsl_pamu, but never actually called. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-2-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 48 ----------------------------------------- include/linux/iommu.h | 2 -- 2 files changed, 50 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index b2110767caf4..53380cf1fa45 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -473,53 +473,6 @@ static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) return ret; } -static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr) -{ - struct device_domain_info *info; - int ret = 0; - - list_for_each_entry(info, &dma_domain->devices, link) { - if (dma_domain->win_cnt == 1 && dma_domain->enabled) { - ret = pamu_disable_liodn(info->liodn); - if (!ret) - dma_domain->enabled = 0; - } else { - ret = pamu_disable_spaace(info->liodn, wnd_nr); - } - } - - return ret; -} - -static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) -{ - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - unsigned long flags; - int ret; - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - if (!dma_domain->win_arr) { - pr_debug("Number of windows not configured\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return; - } - - if (wnd_nr >= dma_domain->win_cnt) { - pr_debug("Invalid window index\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return; - } - - if (dma_domain->win_arr[wnd_nr].valid) { - ret = disable_domain_win(dma_domain, wnd_nr); - if (!ret) { - dma_domain->win_arr[wnd_nr].valid = 0; - dma_domain->mapped--; - } - } - - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); -} static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot) @@ -1032,7 +985,6 @@ static const struct iommu_ops fsl_pamu_ops = { .attach_dev = fsl_pamu_attach_device, .detach_dev = fsl_pamu_detach_device, .domain_window_enable = fsl_pamu_window_enable, - .domain_window_disable = fsl_pamu_window_disable, .iova_to_phys = fsl_pamu_iova_to_phys, .domain_set_attr = fsl_pamu_set_domain_attr, .domain_get_attr = fsl_pamu_get_domain_attr, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 86d688c4418f..565b8810354d 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -223,7 +223,6 @@ struct iommu_iotlb_gather { * @put_resv_regions: Free list of reserved regions for a device * @apply_resv_region: Temporary helper call-back for iova reserved ranges * @domain_window_enable: Configure and enable a particular window for a domain - * @domain_window_disable: Disable a particular window for a domain * @of_xlate: add OF master IDs to iommu grouping * @is_attach_deferred: Check if domain attach should be deferred from iommu * driver init to device driver init (default no) @@ -284,7 +283,6 @@ struct iommu_ops { /* Window handling functions */ int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot); - void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr); int (*of_xlate)(struct device *dev, struct of_phandle_args *args); bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); -- cgit v1.2.3 From 392825e0c76cf9aca33e5a3bf981cde2a2c87251 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:38 +0200 Subject: iommu/fsl_pamu: remove fsl_pamu_get_domain_attr None of the values returned by this function are ever queried. Also remove the DOMAIN_ATTR_FSL_PAMUV1 enum value that is not otherwise used. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-3-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 30 ------------------------------ include/linux/iommu.h | 4 ---- 2 files changed, 34 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 53380cf1fa45..e587ec43f7e7 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -832,35 +832,6 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, return ret; } -static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, - enum iommu_attr attr_type, void *data) -{ - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - int ret = 0; - - switch (attr_type) { - case DOMAIN_ATTR_FSL_PAMU_STASH: - memcpy(data, &dma_domain->dma_stash, - sizeof(struct pamu_stash_attribute)); - break; - case DOMAIN_ATTR_FSL_PAMU_ENABLE: - *(int *)data = dma_domain->enabled; - break; - case DOMAIN_ATTR_FSL_PAMUV1: - *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; - break; - case DOMAIN_ATTR_WINDOWS: - *(u32 *)data = dma_domain->win_cnt; - break; - default: - pr_debug("Unsupported attribute type\n"); - ret = -EINVAL; - break; - } - - return ret; -} - static struct iommu_group *get_device_iommu_group(struct device *dev) { struct iommu_group *group; @@ -987,7 +958,6 @@ static const struct iommu_ops fsl_pamu_ops = { .domain_window_enable = fsl_pamu_window_enable, .iova_to_phys = fsl_pamu_iova_to_phys, .domain_set_attr = fsl_pamu_set_domain_attr, - .domain_get_attr = fsl_pamu_get_domain_attr, .probe_device = fsl_pamu_probe_device, .release_device = fsl_pamu_release_device, .device_group = fsl_pamu_device_group, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 565b8810354d..95ce205347a9 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -104,9 +104,6 @@ enum iommu_cap { * -the actual size of the mapped region of a window must be power * of 2 starting with 4KB and physical address must be naturally * aligned. - * DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints. - * The caller can invoke iommu_domain_get_attr to check if the underlying - * iommu implementation supports these constraints. */ enum iommu_attr { @@ -115,7 +112,6 @@ enum iommu_attr { DOMAIN_ATTR_WINDOWS, DOMAIN_ATTR_FSL_PAMU_STASH, DOMAIN_ATTR_FSL_PAMU_ENABLE, - DOMAIN_ATTR_FSL_PAMUV1, DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, -- cgit v1.2.3 From f7641bb71d26044403cf44f3f14fd6fd3d16eac3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:39 +0200 Subject: iommu/fsl_pamu: remove support for setting DOMAIN_ATTR_GEOMETRY The default geometry is the same as the one set by qman_port given that FSL_PAMU depends on having 64-bit physical and thus DMA addresses. Remove the support to update the geometry and remove the now pointless geom_size field. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-4-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 55 ++++--------------------------------- drivers/iommu/fsl_pamu_domain.h | 6 ---- drivers/soc/fsl/qbman/qman_portal.c | 12 -------- 3 files changed, 5 insertions(+), 68 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index e587ec43f7e7..7bd08ddad077 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -62,7 +62,7 @@ static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t i geom = &dma_domain->iommu_domain.geometry; - if (!win_cnt || !dma_domain->geom_size) { + if (!win_cnt) { pr_debug("Number of windows/geometry not configured for the domain\n"); return 0; } @@ -72,7 +72,7 @@ static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t i dma_addr_t subwin_iova; u32 wnd; - subwin_size = dma_domain->geom_size >> ilog2(win_cnt); + subwin_size = (geom->aperture_end + 1) >> ilog2(win_cnt); subwin_iova = iova & ~(subwin_size - 1); wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); win_ptr = &dma_domain->win_arr[wnd]; @@ -234,7 +234,7 @@ static int pamu_set_liodn(int liodn, struct device *dev, get_ome_index(&omi_index, dev); window_addr = geom_attr->aperture_start; - window_size = dma_domain->geom_size; + window_size = geom_attr->aperture_end + 1; spin_lock_irqsave(&iommu_lock, flags); ret = pamu_disable_liodn(liodn); @@ -303,7 +303,6 @@ static struct fsl_dma_domain *iommu_alloc_dma_domain(void) domain->stash_id = ~(u32)0; domain->snoop_id = ~(u32)0; domain->win_cnt = pamu_get_max_subwin_cnt(); - domain->geom_size = 0; INIT_LIST_HEAD(&domain->devices); @@ -502,7 +501,8 @@ static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, return -EINVAL; } - win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); + win_size = (domain->geometry.aperture_end + 1) >> + ilog2(dma_domain->win_cnt); if (size > win_size) { pr_debug("Invalid window size\n"); spin_unlock_irqrestore(&dma_domain->domain_lock, flags); @@ -665,41 +665,6 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain, pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); } -static int configure_domain_geometry(struct iommu_domain *domain, void *data) -{ - struct iommu_domain_geometry *geom_attr = data; - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - dma_addr_t geom_size; - unsigned long flags; - - geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1; - /* - * Sanity check the geometry size. Also, we do not support - * DMA outside of the geometry. - */ - if (check_size(geom_size, geom_attr->aperture_start) || - !geom_attr->force_aperture) { - pr_debug("Invalid PAMU geometry attributes\n"); - return -EINVAL; - } - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - if (dma_domain->enabled) { - pr_debug("Can't set geometry attributes as domain is active\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EBUSY; - } - - /* Copy the domain geometry information */ - memcpy(&domain->geometry, geom_attr, - sizeof(struct iommu_domain_geometry)); - dma_domain->geom_size = geom_size; - - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - - return 0; -} - /* Set the domain stash attribute */ static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) { @@ -769,13 +734,6 @@ static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) return -EBUSY; } - /* Ensure that the geometry has been set for the domain */ - if (!dma_domain->geom_size) { - pr_debug("Please configure geometry before setting the number of windows\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } - /* * Ensure we have valid window count i.e. it should be less than * maximum permissible limit and should be a power of two. @@ -811,9 +769,6 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, int ret = 0; switch (attr_type) { - case DOMAIN_ATTR_GEOMETRY: - ret = configure_domain_geometry(domain, data); - break; case DOMAIN_ATTR_FSL_PAMU_STASH: ret = configure_domain_stash(dma_domain, data); break; diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index 2865d42782e8..53d359d66fe5 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -17,12 +17,6 @@ struct dma_window { }; struct fsl_dma_domain { - /* - * Indicates the geometry size for the domain. - * This would be set when the geometry is - * configured for the domain. - */ - dma_addr_t geom_size; /* * Number of windows assocaited with this domain. * During domain initialization, it is set to the diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 5685b6706893..c958e6310d30 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -47,7 +47,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) #ifdef CONFIG_FSL_PAMU struct device *dev = pcfg->dev; int window_count = 1; - struct iommu_domain_geometry geom_attr; struct pamu_stash_attribute stash_attr; int ret; @@ -56,17 +55,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) dev_err(dev, "%s(): iommu_domain_alloc() failed", __func__); goto no_iommu; } - geom_attr.aperture_start = 0; - geom_attr.aperture_end = - ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1; - geom_attr.force_aperture = true; - ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY, - &geom_attr); - if (ret < 0) { - dev_err(dev, "%s(): iommu_domain_set_attr() = %d", __func__, - ret); - goto out_domain_free; - } ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, &window_count); if (ret < 0) { -- cgit v1.2.3 From c8224508074e7d13fc31e8ca3d24e4f159983cb3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:40 +0200 Subject: iommu/fsl_pamu: merge iommu_alloc_dma_domain into fsl_pamu_domain_alloc Keep the functionality to allocate the domain together. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-5-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 7bd08ddad077..a4da5597755d 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -292,25 +292,6 @@ static int check_size(u64 size, dma_addr_t iova) return 0; } -static struct fsl_dma_domain *iommu_alloc_dma_domain(void) -{ - struct fsl_dma_domain *domain; - - domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); - if (!domain) - return NULL; - - domain->stash_id = ~(u32)0; - domain->snoop_id = ~(u32)0; - domain->win_cnt = pamu_get_max_subwin_cnt(); - - INIT_LIST_HEAD(&domain->devices); - - spin_lock_init(&domain->domain_lock); - - return domain; -} - static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) { unsigned long flags; @@ -412,12 +393,17 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; - dma_domain = iommu_alloc_dma_domain(); - if (!dma_domain) { - pr_debug("dma_domain allocation failed\n"); + dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); + if (!dma_domain) return NULL; - } - /* defaul geometry 64 GB i.e. maximum system address */ + + dma_domain->stash_id = ~(u32)0; + dma_domain->snoop_id = ~(u32)0; + dma_domain->win_cnt = pamu_get_max_subwin_cnt(); + INIT_LIST_HEAD(&dma_domain->devices); + spin_lock_init(&dma_domain->domain_lock); + + /* default geometry 64 GB i.e. maximum system address */ dma_domain->iommu_domain. geometry.aperture_start = 0; dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1; dma_domain->iommu_domain.geometry.force_aperture = true; -- cgit v1.2.3 From ba58d1216e2b2d2320b50591b767f50b13c623a8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:41 +0200 Subject: iommu/fsl_pamu: remove support for multiple windows The only domains allocated forces use of a single window. Remove all the code related to multiple window support, as well as the need for qman_portal to force a single window. Remove the now unused DOMAIN_ATTR_WINDOWS iommu_attr. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-6-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu.c | 264 ++-------------------------------- drivers/iommu/fsl_pamu.h | 10 +- drivers/iommu/fsl_pamu_domain.c | 275 +++++++----------------------------- drivers/iommu/fsl_pamu_domain.h | 16 +-- drivers/soc/fsl/qbman/qman_portal.c | 7 - include/linux/iommu.h | 1 - 6 files changed, 60 insertions(+), 513 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index b9a974d97831..3e1647cd5ad4 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -63,19 +63,6 @@ static const struct of_device_id l3_device_ids[] = { /* maximum subwindows permitted per liodn */ static u32 max_subwindow_count; -/* Pool for fspi allocation */ -static struct gen_pool *spaace_pool; - -/** - * pamu_get_max_subwin_cnt() - Return the maximum supported - * subwindow count per liodn. - * - */ -u32 pamu_get_max_subwin_cnt(void) -{ - return max_subwindow_count; -} - /** * pamu_get_ppaace() - Return the primary PACCE * @liodn: liodn PAACT index for desired PAACE @@ -155,13 +142,6 @@ static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) return fls64(addrspace_size) - 2; } -/* Derive the PAACE window count encoding for the subwindow count */ -static unsigned int map_subwindow_cnt_to_wce(u32 subwindow_cnt) -{ - /* window count is 2^(WCE+1) bytes */ - return __ffs(subwindow_cnt) - 1; -} - /* * Set the PAACE type as primary and set the coherency required domain * attribute @@ -174,89 +154,11 @@ static void pamu_init_ppaace(struct paace *ppaace) PAACE_M_COHERENCE_REQ); } -/* - * Set the PAACE type as secondary and set the coherency required domain - * attribute. - */ -static void pamu_init_spaace(struct paace *spaace) -{ - set_bf(spaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_SECONDARY); - set_bf(spaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, - PAACE_M_COHERENCE_REQ); -} - -/* - * Return the spaace (corresponding to the secondary window index) - * for a particular ppaace. - */ -static struct paace *pamu_get_spaace(struct paace *paace, u32 wnum) -{ - u32 subwin_cnt; - struct paace *spaace = NULL; - - subwin_cnt = 1UL << (get_bf(paace->impl_attr, PAACE_IA_WCE) + 1); - - if (wnum < subwin_cnt) - spaace = &spaact[paace->fspi + wnum]; - else - pr_debug("secondary paace out of bounds\n"); - - return spaace; -} - -/** - * pamu_get_fspi_and_allocate() - Allocates fspi index and reserves subwindows - * required for primary PAACE in the secondary - * PAACE table. - * @subwin_cnt: Number of subwindows to be reserved. - * - * A PPAACE entry may have a number of associated subwindows. A subwindow - * corresponds to a SPAACE entry in the SPAACT table. Each PAACE entry stores - * the index (fspi) of the first SPAACE entry in the SPAACT table. This - * function returns the index of the first SPAACE entry. The remaining - * SPAACE entries are reserved contiguously from that index. - * - * Returns a valid fspi index in the range of 0 - SPAACE_NUMBER_ENTRIES on success. - * If no SPAACE entry is available or the allocator can not reserve the required - * number of contiguous entries function returns ULONG_MAX indicating a failure. - * - */ -static unsigned long pamu_get_fspi_and_allocate(u32 subwin_cnt) -{ - unsigned long spaace_addr; - - spaace_addr = gen_pool_alloc(spaace_pool, subwin_cnt * sizeof(struct paace)); - if (!spaace_addr) - return ULONG_MAX; - - return (spaace_addr - (unsigned long)spaact) / (sizeof(struct paace)); -} - -/* Release the subwindows reserved for a particular LIODN */ -void pamu_free_subwins(int liodn) -{ - struct paace *ppaace; - u32 subwin_cnt, size; - - ppaace = pamu_get_ppaace(liodn); - if (!ppaace) { - pr_debug("Invalid liodn entry\n"); - return; - } - - if (get_bf(ppaace->addr_bitfields, PPAACE_AF_MW)) { - subwin_cnt = 1UL << (get_bf(ppaace->impl_attr, PAACE_IA_WCE) + 1); - size = (subwin_cnt - 1) * sizeof(struct paace); - gen_pool_free(spaace_pool, (unsigned long)&spaact[ppaace->fspi], size); - set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); - } -} - /* * Function used for updating stash destination for the coressponding * LIODN. */ -int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) +int pamu_update_paace_stash(int liodn, u32 value) { struct paace *paace; @@ -265,11 +167,6 @@ int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) pr_debug("Invalid liodn entry\n"); return -ENOENT; } - if (subwin) { - paace = pamu_get_spaace(paace, subwin - 1); - if (!paace) - return -ENOENT; - } set_bf(paace->impl_attr, PAACE_IA_CID, value); mb(); @@ -277,31 +174,6 @@ int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) return 0; } -/* Disable a subwindow corresponding to the LIODN */ -int pamu_disable_spaace(int liodn, u32 subwin) -{ - struct paace *paace; - - paace = pamu_get_ppaace(liodn); - if (!paace) { - pr_debug("Invalid liodn entry\n"); - return -ENOENT; - } - if (subwin) { - paace = pamu_get_spaace(paace, subwin - 1); - if (!paace) - return -ENOENT; - set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); - } else { - set_bf(paace->addr_bitfields, PAACE_AF_AP, - PAACE_AP_PERMS_DENIED); - } - - mb(); - - return 0; -} - /** * pamu_config_paace() - Sets up PPAACE entry for specified liodn * @@ -314,17 +186,15 @@ int pamu_disable_spaace(int liodn, u32 subwin) * stashid not defined * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then * snoopid not defined - * @subwin_cnt: number of sub-windows * @prot: window permissions * * Returns 0 upon success else error code < 0 returned */ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, u32 omi, unsigned long rpn, u32 snoopid, u32 stashid, - u32 subwin_cnt, int prot) + int prot) { struct paace *ppaace; - unsigned long fspi; if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { pr_debug("window size too small or not a power of two %pa\n", @@ -368,116 +238,12 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, if (~snoopid != 0) ppaace->domain_attr.to_host.snpid = snoopid; - if (subwin_cnt) { - /* The first entry is in the primary PAACE instead */ - fspi = pamu_get_fspi_and_allocate(subwin_cnt - 1); - if (fspi == ULONG_MAX) { - pr_debug("spaace indexes exhausted\n"); - return -EINVAL; - } - - /* window count is 2^(WCE+1) bytes */ - set_bf(ppaace->impl_attr, PAACE_IA_WCE, - map_subwindow_cnt_to_wce(subwin_cnt)); - set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0x1); - ppaace->fspi = fspi; - } else { - set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); - ppaace->twbah = rpn >> 20; - set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); - set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); - set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); - set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); - } - mb(); - - return 0; -} - -/** - * pamu_config_spaace() - Sets up SPAACE entry for specified subwindow - * - * @liodn: Logical IO device number - * @subwin_cnt: number of sub-windows associated with dma-window - * @subwin: subwindow index - * @subwin_size: size of subwindow - * @omi: Operation mapping index - * @rpn: real (true physical) page number - * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then - * snoopid not defined - * @stashid: cache stash id for associated cpu - * @enable: enable/disable subwindow after reconfiguration - * @prot: sub window permissions - * - * Returns 0 upon success else error code < 0 returned - */ -int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, - phys_addr_t subwin_size, u32 omi, unsigned long rpn, - u32 snoopid, u32 stashid, int enable, int prot) -{ - struct paace *paace; - - /* setup sub-windows */ - if (!subwin_cnt) { - pr_debug("Invalid subwindow count\n"); - return -EINVAL; - } - - paace = pamu_get_ppaace(liodn); - if (subwin > 0 && subwin < subwin_cnt && paace) { - paace = pamu_get_spaace(paace, subwin - 1); - - if (paace && !(paace->addr_bitfields & PAACE_V_VALID)) { - pamu_init_spaace(paace); - set_bf(paace->addr_bitfields, SPAACE_AF_LIODN, liodn); - } - } - - if (!paace) { - pr_debug("Invalid liodn entry\n"); - return -ENOENT; - } - - if ((subwin_size & (subwin_size - 1)) || subwin_size < PAMU_PAGE_SIZE) { - pr_debug("subwindow size out of range, or not a power of 2\n"); - return -EINVAL; - } - - if (rpn == ULONG_MAX) { - pr_debug("real page number out of range\n"); - return -EINVAL; - } - - /* window size is 2^(WSE+1) bytes */ - set_bf(paace->win_bitfields, PAACE_WIN_SWSE, - map_addrspace_size_to_wse(subwin_size)); - - set_bf(paace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); - paace->twbah = rpn >> 20; - set_bf(paace->win_bitfields, PAACE_WIN_TWBAL, rpn); - set_bf(paace->addr_bitfields, PAACE_AF_AP, prot); - - /* configure snoop id */ - if (~snoopid != 0) - paace->domain_attr.to_host.snpid = snoopid; - - /* set up operation mapping if it's configured */ - if (omi < OME_NUMBER_ENTRIES) { - set_bf(paace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); - paace->op_encode.index_ot.omi = omi; - } else if (~omi != 0) { - pr_debug("bad operation mapping index: %d\n", omi); - return -EINVAL; - } - - if (~stashid != 0) - set_bf(paace->impl_attr, PAACE_IA_CID, stashid); - - smp_wmb(); - - if (enable) - set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); - + set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); + ppaace->twbah = rpn >> 20; + set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); + set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); + set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); + set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); mb(); return 0; @@ -1129,17 +895,6 @@ static int fsl_pamu_probe(struct platform_device *pdev) spaact_phys = virt_to_phys(spaact); omt_phys = virt_to_phys(omt); - spaace_pool = gen_pool_create(ilog2(sizeof(struct paace)), -1); - if (!spaace_pool) { - ret = -ENOMEM; - dev_err(dev, "Failed to allocate spaace gen pool\n"); - goto error; - } - - ret = gen_pool_add(spaace_pool, (unsigned long)spaact, SPAACT_SIZE, -1); - if (ret) - goto error_genpool; - pamubypenr = in_be32(&guts_regs->pamubypenr); for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; @@ -1167,9 +922,6 @@ static int fsl_pamu_probe(struct platform_device *pdev) return 0; -error_genpool: - gen_pool_destroy(spaace_pool); - error: if (irq != NO_IRQ) free_irq(irq, data); diff --git a/drivers/iommu/fsl_pamu.h b/drivers/iommu/fsl_pamu.h index e1496ba96160..04fd843d718d 100644 --- a/drivers/iommu/fsl_pamu.h +++ b/drivers/iommu/fsl_pamu.h @@ -383,18 +383,12 @@ struct ome { int pamu_domain_init(void); int pamu_enable_liodn(int liodn); int pamu_disable_liodn(int liodn); -void pamu_free_subwins(int liodn); int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, u32 omi, unsigned long rpn, u32 snoopid, uint32_t stashid, - u32 subwin_cnt, int prot); -int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin_addr, - phys_addr_t subwin_size, u32 omi, unsigned long rpn, - uint32_t snoopid, u32 stashid, int enable, int prot); + int prot); u32 get_stash_id(u32 stash_dest_hint, u32 vcpu); void get_ome_index(u32 *omi_index, struct device *dev); -int pamu_update_paace_stash(int liodn, u32 subwin, u32 value); -int pamu_disable_spaace(int liodn, u32 subwin); -u32 pamu_get_max_subwin_cnt(void); +int pamu_update_paace_stash(int liodn, u32 value); #endif /* __FSL_PAMU_H */ diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index a4da5597755d..e6bdd38fc184 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -56,65 +56,19 @@ static int __init iommu_init_mempool(void) static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) { - u32 win_cnt = dma_domain->win_cnt; struct dma_window *win_ptr = &dma_domain->win_arr[0]; struct iommu_domain_geometry *geom; geom = &dma_domain->iommu_domain.geometry; - if (!win_cnt) { - pr_debug("Number of windows/geometry not configured for the domain\n"); - return 0; - } - - if (win_cnt > 1) { - u64 subwin_size; - dma_addr_t subwin_iova; - u32 wnd; - - subwin_size = (geom->aperture_end + 1) >> ilog2(win_cnt); - subwin_iova = iova & ~(subwin_size - 1); - wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); - win_ptr = &dma_domain->win_arr[wnd]; - } - if (win_ptr->valid) return win_ptr->paddr + (iova & (win_ptr->size - 1)); return 0; } -static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) -{ - struct dma_window *sub_win_ptr = &dma_domain->win_arr[0]; - int i, ret; - unsigned long rpn, flags; - - for (i = 0; i < dma_domain->win_cnt; i++) { - if (sub_win_ptr[i].valid) { - rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT; - spin_lock_irqsave(&iommu_lock, flags); - ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, - sub_win_ptr[i].size, - ~(u32)0, - rpn, - dma_domain->snoop_id, - dma_domain->stash_id, - (i > 0) ? 1 : 0, - sub_win_ptr[i].prot); - spin_unlock_irqrestore(&iommu_lock, flags); - if (ret) { - pr_debug("SPAACE configuration failed for liodn %d\n", - liodn); - return ret; - } - } - } - - return ret; -} - -static int map_win(int liodn, struct fsl_dma_domain *dma_domain) +/* Map the DMA window corresponding to the LIODN */ +static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) { int ret; struct dma_window *wnd = &dma_domain->win_arr[0]; @@ -127,7 +81,7 @@ static int map_win(int liodn, struct fsl_dma_domain *dma_domain) ~(u32)0, wnd->paddr >> PAMU_PAGE_SHIFT, dma_domain->snoop_id, dma_domain->stash_id, - 0, wnd->prot); + wnd->prot); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) pr_debug("PAACE configuration failed for liodn %d\n", liodn); @@ -135,50 +89,27 @@ static int map_win(int liodn, struct fsl_dma_domain *dma_domain) return ret; } -/* Map the DMA window corresponding to the LIODN */ -static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) -{ - if (dma_domain->win_cnt > 1) - return map_subwins(liodn, dma_domain); - else - return map_win(liodn, dma_domain); -} - /* Update window/subwindow mapping for the LIODN */ static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) { int ret; struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; + phys_addr_t wnd_addr; unsigned long flags; spin_lock_irqsave(&iommu_lock, flags); - if (dma_domain->win_cnt > 1) { - ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr, - wnd->size, - ~(u32)0, - wnd->paddr >> PAMU_PAGE_SHIFT, - dma_domain->snoop_id, - dma_domain->stash_id, - (wnd_nr > 0) ? 1 : 0, - wnd->prot); - if (ret) - pr_debug("Subwindow reconfiguration failed for liodn %d\n", - liodn); - } else { - phys_addr_t wnd_addr; - wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; + wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; - ret = pamu_config_ppaace(liodn, wnd_addr, - wnd->size, - ~(u32)0, - wnd->paddr >> PAMU_PAGE_SHIFT, - dma_domain->snoop_id, dma_domain->stash_id, - 0, wnd->prot); - if (ret) - pr_debug("Window reconfiguration failed for liodn %d\n", - liodn); - } + ret = pamu_config_ppaace(liodn, wnd_addr, + wnd->size, + ~(u32)0, + wnd->paddr >> PAMU_PAGE_SHIFT, + dma_domain->snoop_id, dma_domain->stash_id, + wnd->prot); + if (ret) + pr_debug("Window reconfiguration failed for liodn %d\n", + liodn); spin_unlock_irqrestore(&iommu_lock, flags); @@ -192,21 +123,12 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, unsigned long flags; spin_lock_irqsave(&iommu_lock, flags); - if (!dma_domain->win_arr) { - pr_debug("Windows not configured, stash destination update failed for liodn %d\n", - liodn); + ret = pamu_update_paace_stash(liodn, val); + if (ret) { + pr_debug("Failed to update SPAACE %d field for liodn %d\n ", + i, liodn); spin_unlock_irqrestore(&iommu_lock, flags); - return -EINVAL; - } - - for (i = 0; i < dma_domain->win_cnt; i++) { - ret = pamu_update_paace_stash(liodn, i, val); - if (ret) { - pr_debug("Failed to update SPAACE %d field for liodn %d\n ", - i, liodn); - spin_unlock_irqrestore(&iommu_lock, flags); - return ret; - } + return ret; } spin_unlock_irqrestore(&iommu_lock, flags); @@ -217,14 +139,12 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, /* Set the geometry parameters for a LIODN */ static int pamu_set_liodn(int liodn, struct device *dev, struct fsl_dma_domain *dma_domain, - struct iommu_domain_geometry *geom_attr, - u32 win_cnt) + struct iommu_domain_geometry *geom_attr) { phys_addr_t window_addr, window_size; - phys_addr_t subwin_size; - int ret = 0, i; u32 omi_index = ~(u32)0; unsigned long flags; + int ret; /* * Configure the omi_index at the geometry setup time. @@ -241,34 +161,14 @@ static int pamu_set_liodn(int liodn, struct device *dev, if (!ret) ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, 0, dma_domain->snoop_id, - dma_domain->stash_id, win_cnt, 0); + dma_domain->stash_id, 0); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) { - pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n", - liodn, win_cnt); + pr_debug("PAACE configuration failed for liodn %d\n", + liodn); return ret; } - if (win_cnt > 1) { - subwin_size = window_size >> ilog2(win_cnt); - for (i = 0; i < win_cnt; i++) { - spin_lock_irqsave(&iommu_lock, flags); - ret = pamu_disable_spaace(liodn, i); - if (!ret) - ret = pamu_config_spaace(liodn, win_cnt, i, - subwin_size, omi_index, - 0, dma_domain->snoop_id, - dma_domain->stash_id, - 0, 0); - spin_unlock_irqrestore(&iommu_lock, flags); - if (ret) { - pr_debug("SPAACE configuration failed for liodn %d\n", - liodn); - return ret; - } - } - } - return ret; } @@ -292,14 +192,12 @@ static int check_size(u64 size, dma_addr_t iova) return 0; } -static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) +static void remove_device_ref(struct device_domain_info *info) { unsigned long flags; list_del(&info->link); spin_lock_irqsave(&iommu_lock, flags); - if (win_cnt > 1) - pamu_free_subwins(info->liodn); pamu_disable_liodn(info->liodn); spin_unlock_irqrestore(&iommu_lock, flags); spin_lock_irqsave(&device_domain_lock, flags); @@ -317,7 +215,7 @@ static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) /* Remove the device from the domain device list */ list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { if (!dev || (info->dev == dev)) - remove_device_ref(info, dma_domain->win_cnt); + remove_device_ref(info); } spin_unlock_irqrestore(&dma_domain->domain_lock, flags); } @@ -399,7 +297,6 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) dma_domain->stash_id = ~(u32)0; dma_domain->snoop_id = ~(u32)0; - dma_domain->win_cnt = pamu_get_max_subwin_cnt(); INIT_LIST_HEAD(&dma_domain->devices); spin_lock_init(&dma_domain->domain_lock); @@ -411,24 +308,6 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) return &dma_domain->iommu_domain; } -/* Configure geometry settings for all LIODNs associated with domain */ -static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, - struct iommu_domain_geometry *geom_attr, - u32 win_cnt) -{ - struct device_domain_info *info; - int ret = 0; - - list_for_each_entry(info, &dma_domain->devices, link) { - ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, - geom_attr, win_cnt); - if (ret) - break; - } - - return ret; -} - /* Update stash destination for all LIODNs associated with the domain */ static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) { @@ -475,39 +354,30 @@ static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, pamu_prot |= PAACE_AP_PERMS_UPDATE; spin_lock_irqsave(&dma_domain->domain_lock, flags); - if (!dma_domain->win_arr) { - pr_debug("Number of windows not configured\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -ENODEV; - } - - if (wnd_nr >= dma_domain->win_cnt) { + if (wnd_nr > 0) { pr_debug("Invalid window index\n"); spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return -EINVAL; } - win_size = (domain->geometry.aperture_end + 1) >> - ilog2(dma_domain->win_cnt); + win_size = (domain->geometry.aperture_end + 1) >> ilog2(1); if (size > win_size) { pr_debug("Invalid window size\n"); spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return -EINVAL; } - if (dma_domain->win_cnt == 1) { - if (dma_domain->enabled) { - pr_debug("Disable the window before updating the mapping\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EBUSY; - } + if (dma_domain->enabled) { + pr_debug("Disable the window before updating the mapping\n"); + spin_unlock_irqrestore(&dma_domain->domain_lock, flags); + return -EBUSY; + } - ret = check_size(size, domain->geometry.aperture_start); - if (ret) { - pr_debug("Aperture start not aligned to the size\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } + ret = check_size(size, domain->geometry.aperture_start); + if (ret) { + pr_debug("Aperture start not aligned to the size\n"); + spin_unlock_irqrestore(&dma_domain->domain_lock, flags); + return -EINVAL; } wnd = &dma_domain->win_arr[wnd_nr]; @@ -560,22 +430,18 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, * for the domain. If yes, set the geometry for * the LIODN. */ - if (dma_domain->win_arr) { - u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; - - ret = pamu_set_liodn(liodn[i], dev, dma_domain, - &domain->geometry, win_cnt); + ret = pamu_set_liodn(liodn[i], dev, dma_domain, + &domain->geometry); + if (ret) + break; + if (dma_domain->mapped) { + /* + * Create window/subwindow mapping for + * the LIODN. + */ + ret = map_liodn(liodn[i], dma_domain); if (ret) break; - if (dma_domain->mapped) { - /* - * Create window/subwindow mapping for - * the LIODN. - */ - ret = map_liodn(liodn[i], dma_domain); - if (ret) - break; - } } } spin_unlock_irqrestore(&dma_domain->domain_lock, flags); @@ -706,48 +572,6 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en return 0; } -static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) -{ - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - unsigned long flags; - int ret; - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - /* Ensure domain is inactive i.e. DMA should be disabled for the domain */ - if (dma_domain->enabled) { - pr_debug("Can't set geometry attributes as domain is active\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EBUSY; - } - - /* - * Ensure we have valid window count i.e. it should be less than - * maximum permissible limit and should be a power of two. - */ - if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) { - pr_debug("Invalid window count\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } - - ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, - w_count > 1 ? w_count : 0); - if (!ret) { - kfree(dma_domain->win_arr); - dma_domain->win_arr = kcalloc(w_count, - sizeof(*dma_domain->win_arr), - GFP_ATOMIC); - if (!dma_domain->win_arr) { - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -ENOMEM; - } - dma_domain->win_cnt = w_count; - } - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - - return ret; -} - static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, enum iommu_attr attr_type, void *data) { @@ -761,9 +585,6 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, case DOMAIN_ATTR_FSL_PAMU_ENABLE: ret = configure_domain_dma_state(dma_domain, *(int *)data); break; - case DOMAIN_ATTR_WINDOWS: - ret = fsl_pamu_set_windows(domain, *(u32 *)data); - break; default: pr_debug("Unsupported attribute type\n"); ret = -EINVAL; diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index 53d359d66fe5..d3523ee9999d 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -17,23 +17,11 @@ struct dma_window { }; struct fsl_dma_domain { - /* - * Number of windows assocaited with this domain. - * During domain initialization, it is set to the - * the maximum number of subwindows allowed for a LIODN. - * Minimum value for this is 1 indicating a single PAMU - * window, without any sub windows. Value can be set/ - * queried by set_attr/get_attr API for DOMAIN_ATTR_WINDOWS. - * Value can only be set once the geometry has been configured. - */ - u32 win_cnt; /* * win_arr contains information of the configured - * windows for a domain. This is allocated only - * when the number of windows for the domain are - * set. + * windows for a domain. */ - struct dma_window *win_arr; + struct dma_window win_arr[1]; /* list of devices associated with the domain */ struct list_head devices; /* dma_domain states: diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index c958e6310d30..3d56ec4b373b 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -55,13 +55,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) dev_err(dev, "%s(): iommu_domain_alloc() failed", __func__); goto no_iommu; } - ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, - &window_count); - if (ret < 0) { - dev_err(dev, "%s(): iommu_domain_set_attr() = %d", __func__, - ret); - goto out_domain_free; - } stash_attr.cpu = cpu; stash_attr.cache = PAMU_ATTR_CACHE_L1; ret = iommu_domain_set_attr(pcfg->iommu_domain, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 95ce205347a9..761aa8b69936 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -109,7 +109,6 @@ enum iommu_cap { enum iommu_attr { DOMAIN_ATTR_GEOMETRY, DOMAIN_ATTR_PAGING, - DOMAIN_ATTR_WINDOWS, DOMAIN_ATTR_FSL_PAMU_STASH, DOMAIN_ATTR_FSL_PAMU_ENABLE, DOMAIN_ATTR_NESTING, /* two stages of translation */ -- cgit v1.2.3 From 376dfd2a2ff41596a6efc8ea56f8b0de172b04a6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:42 +0200 Subject: iommu/fsl_pamu: remove ->domain_window_enable The only thing that fsl_pamu_window_enable does for the current caller is to fill in the prot value in the only dma_window structure, and to propagate a few values from the iommu_domain_geometry struture into the dma_window. Remove the dma_window entirely, hardcode the prot value and otherwise use the iommu_domain_geometry structure instead. Remove the now unused ->domain_window_enable iommu method. Signed-off-by: Christoph Hellwig Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-7-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 182 +++--------------------------------- drivers/iommu/fsl_pamu_domain.h | 15 --- drivers/iommu/iommu.c | 11 --- drivers/soc/fsl/qbman/qman_portal.c | 7 -- include/linux/iommu.h | 17 ---- 5 files changed, 14 insertions(+), 218 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index e6bdd38fc184..689035e9d409 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -54,34 +54,18 @@ static int __init iommu_init_mempool(void) return 0; } -static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) -{ - struct dma_window *win_ptr = &dma_domain->win_arr[0]; - struct iommu_domain_geometry *geom; - - geom = &dma_domain->iommu_domain.geometry; - - if (win_ptr->valid) - return win_ptr->paddr + (iova & (win_ptr->size - 1)); - - return 0; -} - /* Map the DMA window corresponding to the LIODN */ static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) { int ret; - struct dma_window *wnd = &dma_domain->win_arr[0]; - phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; + struct iommu_domain_geometry *geom = &dma_domain->iommu_domain.geometry; unsigned long flags; spin_lock_irqsave(&iommu_lock, flags); - ret = pamu_config_ppaace(liodn, wnd_addr, - wnd->size, - ~(u32)0, - wnd->paddr >> PAMU_PAGE_SHIFT, - dma_domain->snoop_id, dma_domain->stash_id, - wnd->prot); + ret = pamu_config_ppaace(liodn, geom->aperture_start, + geom->aperture_end + 1, ~(u32)0, + 0, dma_domain->snoop_id, dma_domain->stash_id, + PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) pr_debug("PAACE configuration failed for liodn %d\n", liodn); @@ -89,33 +73,6 @@ static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) return ret; } -/* Update window/subwindow mapping for the LIODN */ -static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) -{ - int ret; - struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; - phys_addr_t wnd_addr; - unsigned long flags; - - spin_lock_irqsave(&iommu_lock, flags); - - wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; - - ret = pamu_config_ppaace(liodn, wnd_addr, - wnd->size, - ~(u32)0, - wnd->paddr >> PAMU_PAGE_SHIFT, - dma_domain->snoop_id, dma_domain->stash_id, - wnd->prot); - if (ret) - pr_debug("Window reconfiguration failed for liodn %d\n", - liodn); - - spin_unlock_irqrestore(&iommu_lock, flags); - - return ret; -} - static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, u32 val) { @@ -172,26 +129,6 @@ static int pamu_set_liodn(int liodn, struct device *dev, return ret; } -static int check_size(u64 size, dma_addr_t iova) -{ - /* - * Size must be a power of two and at least be equal - * to PAMU page size. - */ - if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { - pr_debug("Size too small or not a power of two\n"); - return -EINVAL; - } - - /* iova must be page size aligned */ - if (iova & (size - 1)) { - pr_debug("Address is not aligned with window size\n"); - return -EINVAL; - } - - return 0; -} - static void remove_device_ref(struct device_domain_info *info) { unsigned long flags; @@ -257,13 +194,10 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - if (iova < domain->geometry.aperture_start || iova > domain->geometry.aperture_end) return 0; - - return get_phys_addr(dma_domain, iova); + return iova; } static bool fsl_pamu_capable(enum iommu_cap cap) @@ -279,7 +213,6 @@ static void fsl_pamu_domain_free(struct iommu_domain *domain) detach_device(NULL, dma_domain); dma_domain->enabled = 0; - dma_domain->mapped = 0; kmem_cache_free(fsl_pamu_domain_cache, dma_domain); } @@ -323,84 +256,6 @@ static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) return ret; } -/* Update domain mappings for all LIODNs associated with the domain */ -static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) -{ - struct device_domain_info *info; - int ret = 0; - - list_for_each_entry(info, &dma_domain->devices, link) { - ret = update_liodn(info->liodn, dma_domain, wnd_nr); - if (ret) - break; - } - return ret; -} - - -static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, - phys_addr_t paddr, u64 size, int prot) -{ - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - struct dma_window *wnd; - int pamu_prot = 0; - int ret; - unsigned long flags; - u64 win_size; - - if (prot & IOMMU_READ) - pamu_prot |= PAACE_AP_PERMS_QUERY; - if (prot & IOMMU_WRITE) - pamu_prot |= PAACE_AP_PERMS_UPDATE; - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - if (wnd_nr > 0) { - pr_debug("Invalid window index\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } - - win_size = (domain->geometry.aperture_end + 1) >> ilog2(1); - if (size > win_size) { - pr_debug("Invalid window size\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } - - if (dma_domain->enabled) { - pr_debug("Disable the window before updating the mapping\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EBUSY; - } - - ret = check_size(size, domain->geometry.aperture_start); - if (ret) { - pr_debug("Aperture start not aligned to the size\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -EINVAL; - } - - wnd = &dma_domain->win_arr[wnd_nr]; - if (!wnd->valid) { - wnd->paddr = paddr; - wnd->size = size; - wnd->prot = pamu_prot; - - ret = update_domain_mapping(dma_domain, wnd_nr); - if (!ret) { - wnd->valid = 1; - dma_domain->mapped++; - } - } else { - pr_debug("Disable the window before updating the mapping\n"); - ret = -EBUSY; - } - - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - - return ret; -} - /* * Attach the LIODN to the DMA domain and configure the geometry * and window mappings. @@ -434,15 +289,14 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, &domain->geometry); if (ret) break; - if (dma_domain->mapped) { - /* - * Create window/subwindow mapping for - * the LIODN. - */ - ret = map_liodn(liodn[i], dma_domain); - if (ret) - break; - } + + /* + * Create window/subwindow mapping for + * the LIODN. + */ + ret = map_liodn(liodn[i], dma_domain); + if (ret) + break; } spin_unlock_irqrestore(&dma_domain->domain_lock, flags); @@ -552,13 +406,6 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en int ret; spin_lock_irqsave(&dma_domain->domain_lock, flags); - - if (enable && !dma_domain->mapped) { - pr_debug("Can't enable DMA domain without valid mapping\n"); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - return -ENODEV; - } - dma_domain->enabled = enable; list_for_each_entry(info, &dma_domain->devices, link) { ret = (enable) ? pamu_enable_liodn(info->liodn) : @@ -717,7 +564,6 @@ static const struct iommu_ops fsl_pamu_ops = { .domain_free = fsl_pamu_domain_free, .attach_dev = fsl_pamu_attach_device, .detach_dev = fsl_pamu_detach_device, - .domain_window_enable = fsl_pamu_window_enable, .iova_to_phys = fsl_pamu_iova_to_phys, .domain_set_attr = fsl_pamu_set_domain_attr, .probe_device = fsl_pamu_probe_device, diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index d3523ee9999d..13ee06e0ef01 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -9,24 +9,10 @@ #include "fsl_pamu.h" -struct dma_window { - phys_addr_t paddr; - u64 size; - int valid; - int prot; -}; - struct fsl_dma_domain { - /* - * win_arr contains information of the configured - * windows for a domain. - */ - struct dma_window win_arr[1]; /* list of devices associated with the domain */ struct list_head devices; /* dma_domain states: - * mapped - A particular mapping has been created - * within the configured geometry. * enabled - DMA has been enabled for the given * domain. This translates to setting of the * valid bit for the primary PAACE in the PAMU @@ -35,7 +21,6 @@ struct fsl_dma_domain { * enabled for it. * */ - int mapped; int enabled; /* stash_id obtained from the stash attribute details */ u32 stash_id; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index e10cfa99057c..33bc0f8f0a74 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2610,17 +2610,6 @@ size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); } -int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, - phys_addr_t paddr, u64 size, int prot) -{ - if (unlikely(domain->ops->domain_window_enable == NULL)) - return -ENODEV; - - return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size, - prot); -} -EXPORT_SYMBOL_GPL(iommu_domain_window_enable); - /** * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework * @domain: the iommu domain where the fault has happened diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 3d56ec4b373b..9ee1663f422c 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -65,13 +65,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) __func__, ret); goto out_domain_free; } - ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36, - IOMMU_READ | IOMMU_WRITE); - if (ret < 0) { - dev_err(dev, "%s(): iommu_domain_window_enable() = %d", - __func__, ret); - goto out_domain_free; - } ret = iommu_attach_device(pcfg->iommu_domain, dev); if (ret < 0) { dev_err(dev, "%s(): iommu_device_attach() = %d", __func__, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 761aa8b69936..69537e83aae4 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -217,7 +217,6 @@ struct iommu_iotlb_gather { * @get_resv_regions: Request list of reserved regions for a device * @put_resv_regions: Free list of reserved regions for a device * @apply_resv_region: Temporary helper call-back for iova reserved ranges - * @domain_window_enable: Configure and enable a particular window for a domain * @of_xlate: add OF master IDs to iommu grouping * @is_attach_deferred: Check if domain attach should be deferred from iommu * driver init to device driver init (default no) @@ -275,10 +274,6 @@ struct iommu_ops { struct iommu_domain *domain, struct iommu_resv_region *region); - /* Window handling functions */ - int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, - phys_addr_t paddr, u64 size, int prot); - int (*of_xlate)(struct device *dev, struct of_phandle_args *args); bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); @@ -521,11 +516,6 @@ extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, void *data); -/* Window handling function prototypes */ -extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, - phys_addr_t offset, u64 size, - int prot); - extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, unsigned long iova, int flags); @@ -749,13 +739,6 @@ static inline void iommu_iotlb_sync(struct iommu_domain *domain, { } -static inline int iommu_domain_window_enable(struct iommu_domain *domain, - u32 wnd_nr, phys_addr_t paddr, - u64 size, int prot) -{ - return -ENODEV; -} - static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { return 0; -- cgit v1.2.3 From 4eeb96f6efac10e66fd10e718d2adeece3879121 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:43 +0200 Subject: iommu/fsl_pamu: replace DOMAIN_ATTR_FSL_PAMU_STASH with a direct call Add a fsl_pamu_configure_l1_stash API that qman_portal can call directly instead of indirecting through the iommu attr API. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-8-hch@lst.de Signed-off-by: Joerg Roedel --- arch/powerpc/include/asm/fsl_pamu_stash.h | 12 +++--------- drivers/iommu/fsl_pamu_domain.c | 16 +++------------- drivers/iommu/fsl_pamu_domain.h | 2 -- drivers/soc/fsl/qbman/qman_portal.c | 18 +++--------------- include/linux/iommu.h | 1 - 5 files changed, 9 insertions(+), 40 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/powerpc/include/asm/fsl_pamu_stash.h b/arch/powerpc/include/asm/fsl_pamu_stash.h index 30a31ad2123d..c0fbadb70b5d 100644 --- a/arch/powerpc/include/asm/fsl_pamu_stash.h +++ b/arch/powerpc/include/asm/fsl_pamu_stash.h @@ -7,6 +7,8 @@ #ifndef __FSL_PAMU_STASH_H #define __FSL_PAMU_STASH_H +struct iommu_domain; + /* cache stash targets */ enum pamu_stash_target { PAMU_ATTR_CACHE_L1 = 1, @@ -14,14 +16,6 @@ enum pamu_stash_target { PAMU_ATTR_CACHE_L3, }; -/* - * This attribute allows configuring stashig specific parameters - * in the PAMU hardware. - */ - -struct pamu_stash_attribute { - u32 cpu; /* cpu number */ - u32 cache; /* cache to stash to: L1,L2,L3 */ -}; +int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu); #endif /* __FSL_PAMU_STASH_H */ diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 689035e9d409..557a152c1d2c 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -372,27 +372,20 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain, } /* Set the domain stash attribute */ -static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) +int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu) { - struct pamu_stash_attribute *stash_attr = data; + struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); unsigned long flags; int ret; spin_lock_irqsave(&dma_domain->domain_lock, flags); - - memcpy(&dma_domain->dma_stash, stash_attr, - sizeof(struct pamu_stash_attribute)); - - dma_domain->stash_id = get_stash_id(stash_attr->cache, - stash_attr->cpu); + dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu); if (dma_domain->stash_id == ~(u32)0) { pr_debug("Invalid stash attributes\n"); spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return -EINVAL; } - ret = update_domain_stash(dma_domain, dma_domain->stash_id); - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return ret; @@ -426,9 +419,6 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, int ret = 0; switch (attr_type) { - case DOMAIN_ATTR_FSL_PAMU_STASH: - ret = configure_domain_stash(dma_domain, data); - break; case DOMAIN_ATTR_FSL_PAMU_ENABLE: ret = configure_domain_dma_state(dma_domain, *(int *)data); break; diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index 13ee06e0ef01..cd488004acd1 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -22,9 +22,7 @@ struct fsl_dma_domain { * */ int enabled; - /* stash_id obtained from the stash attribute details */ u32 stash_id; - struct pamu_stash_attribute dma_stash; u32 snoop_id; struct iommu_domain iommu_domain; spinlock_t domain_lock; diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 9ee1663f422c..798b3a1ffd0b 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -47,7 +47,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) #ifdef CONFIG_FSL_PAMU struct device *dev = pcfg->dev; int window_count = 1; - struct pamu_stash_attribute stash_attr; int ret; pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); @@ -55,13 +54,9 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) dev_err(dev, "%s(): iommu_domain_alloc() failed", __func__); goto no_iommu; } - stash_attr.cpu = cpu; - stash_attr.cache = PAMU_ATTR_CACHE_L1; - ret = iommu_domain_set_attr(pcfg->iommu_domain, - DOMAIN_ATTR_FSL_PAMU_STASH, - &stash_attr); + ret = fsl_pamu_configure_l1_stash(pcfg->iommu_domain, cpu); if (ret < 0) { - dev_err(dev, "%s(): iommu_domain_set_attr() = %d", + dev_err(dev, "%s(): fsl_pamu_configure_l1_stash() = %d", __func__, ret); goto out_domain_free; } @@ -143,15 +138,8 @@ static void qman_portal_update_sdest(const struct qm_portal_config *pcfg, unsigned int cpu) { #ifdef CONFIG_FSL_PAMU /* TODO */ - struct pamu_stash_attribute stash_attr; - int ret; - if (pcfg->iommu_domain) { - stash_attr.cpu = cpu; - stash_attr.cache = PAMU_ATTR_CACHE_L1; - ret = iommu_domain_set_attr(pcfg->iommu_domain, - DOMAIN_ATTR_FSL_PAMU_STASH, &stash_attr); - if (ret < 0) { + if (fsl_pamu_configure_l1_stash(pcfg->iommu_domain, cpu) < 0) { dev_err(pcfg->dev, "Failed to update pamu stash setting\n"); return; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 69537e83aae4..a3968122aa69 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -109,7 +109,6 @@ enum iommu_cap { enum iommu_attr { DOMAIN_ATTR_GEOMETRY, DOMAIN_ATTR_PAGING, - DOMAIN_ATTR_FSL_PAMU_STASH, DOMAIN_ATTR_FSL_PAMU_ENABLE, DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, -- cgit v1.2.3 From dae7747ae41ccdc92ad94d56b05793a3ee983e8a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:44 +0200 Subject: iommu/fsl_pamu: merge pamu_set_liodn and map_liodn Merge the two fuctions that configure the ppaace into a single coherent function. I somehow doubt we need the two pamu_config_ppaace calls, but keep the existing behavior just to be on the safe side. Signed-off-by: Christoph Hellwig Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-9-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 65 +++++++++++------------------------------ 1 file changed, 17 insertions(+), 48 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 557a152c1d2c..198725ef2795 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -54,25 +54,6 @@ static int __init iommu_init_mempool(void) return 0; } -/* Map the DMA window corresponding to the LIODN */ -static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) -{ - int ret; - struct iommu_domain_geometry *geom = &dma_domain->iommu_domain.geometry; - unsigned long flags; - - spin_lock_irqsave(&iommu_lock, flags); - ret = pamu_config_ppaace(liodn, geom->aperture_start, - geom->aperture_end + 1, ~(u32)0, - 0, dma_domain->snoop_id, dma_domain->stash_id, - PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); - spin_unlock_irqrestore(&iommu_lock, flags); - if (ret) - pr_debug("PAACE configuration failed for liodn %d\n", liodn); - - return ret; -} - static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, u32 val) { @@ -94,11 +75,11 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, } /* Set the geometry parameters for a LIODN */ -static int pamu_set_liodn(int liodn, struct device *dev, - struct fsl_dma_domain *dma_domain, - struct iommu_domain_geometry *geom_attr) +static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, + int liodn) { - phys_addr_t window_addr, window_size; + struct iommu_domain *domain = &dma_domain->iommu_domain; + struct iommu_domain_geometry *geom = &domain->geometry; u32 omi_index = ~(u32)0; unsigned long flags; int ret; @@ -110,22 +91,25 @@ static int pamu_set_liodn(int liodn, struct device *dev, */ get_ome_index(&omi_index, dev); - window_addr = geom_attr->aperture_start; - window_size = geom_attr->aperture_end + 1; - spin_lock_irqsave(&iommu_lock, flags); ret = pamu_disable_liodn(liodn); - if (!ret) - ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, - 0, dma_domain->snoop_id, - dma_domain->stash_id, 0); + if (ret) + goto out_unlock; + ret = pamu_config_ppaace(liodn, geom->aperture_start, + geom->aperture_end + 1, omi_index, 0, + dma_domain->snoop_id, dma_domain->stash_id, 0); + if (ret) + goto out_unlock; + ret = pamu_config_ppaace(liodn, geom->aperture_start, + geom->aperture_end + 1, ~(u32)0, + 0, dma_domain->snoop_id, dma_domain->stash_id, + PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); +out_unlock: spin_unlock_irqrestore(&iommu_lock, flags); if (ret) { pr_debug("PAACE configuration failed for liodn %d\n", liodn); - return ret; } - return ret; } @@ -265,7 +249,6 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, int num) { unsigned long flags; - struct iommu_domain *domain = &dma_domain->iommu_domain; int ret = 0; int i; @@ -280,21 +263,7 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, } attach_device(dma_domain, liodn[i], dev); - /* - * Check if geometry has already been configured - * for the domain. If yes, set the geometry for - * the LIODN. - */ - ret = pamu_set_liodn(liodn[i], dev, dma_domain, - &domain->geometry); - if (ret) - break; - - /* - * Create window/subwindow mapping for - * the LIODN. - */ - ret = map_liodn(liodn[i], dma_domain); + ret = pamu_set_liodn(dma_domain, dev, liodn[i]); if (ret) break; } -- cgit v1.2.3 From 85e362ca462548092b744acbaed9472ab14706b0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:45 +0200 Subject: iommu/fsl_pamu: merge handle_attach_device into fsl_pamu_attach_device No good reason to split this functionality over two functions. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-10-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 59 ++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 39 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 198725ef2795..41927c3c4177 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -240,45 +240,13 @@ static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) return ret; } -/* - * Attach the LIODN to the DMA domain and configure the geometry - * and window mappings. - */ -static int handle_attach_device(struct fsl_dma_domain *dma_domain, - struct device *dev, const u32 *liodn, - int num) -{ - unsigned long flags; - int ret = 0; - int i; - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - for (i = 0; i < num; i++) { - /* Ensure that LIODN value is valid */ - if (liodn[i] >= PAACE_NUMBER_ENTRIES) { - pr_debug("Invalid liodn %d, attach device failed for %pOF\n", - liodn[i], dev->of_node); - ret = -EINVAL; - break; - } - - attach_device(dma_domain, liodn[i], dev); - ret = pamu_set_liodn(dma_domain, dev, liodn[i]); - if (ret) - break; - } - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - - return ret; -} - static int fsl_pamu_attach_device(struct iommu_domain *domain, struct device *dev) { struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); + unsigned long flags; + int len, ret = 0, i; const u32 *liodn; - u32 liodn_cnt; - int len, ret = 0; struct pci_dev *pdev = NULL; struct pci_controller *pci_ctl; @@ -298,14 +266,27 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, } liodn = of_get_property(dev->of_node, "fsl,liodn", &len); - if (liodn) { - liodn_cnt = len / sizeof(u32); - ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt); - } else { + if (!liodn) { pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); - ret = -EINVAL; + return -EINVAL; } + spin_lock_irqsave(&dma_domain->domain_lock, flags); + for (i = 0; i < len / sizeof(u32); i++) { + /* Ensure that LIODN value is valid */ + if (liodn[i] >= PAACE_NUMBER_ENTRIES) { + pr_debug("Invalid liodn %d, attach device failed for %pOF\n", + liodn[i], dev->of_node); + ret = -EINVAL; + break; + } + + attach_device(dma_domain, liodn[i], dev); + ret = pamu_set_liodn(dma_domain, dev, liodn[i]); + if (ret) + break; + } + spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return ret; } -- cgit v1.2.3 From 7d61cb6ff0122a017ae907aed62478a4db9c5991 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:46 +0200 Subject: iommu/fsl_pamu: enable the liodn when attaching a device Instead of a separate call to enable all devices from the list, just enable the liodn once the device is attached to the iommu domain. This also remove the DOMAIN_ATTR_FSL_PAMU_ENABLE iommu_attr. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-11-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 47 +++---------------------------------- drivers/iommu/fsl_pamu_domain.h | 10 -------- drivers/soc/fsl/qbman/qman_portal.c | 11 --------- include/linux/iommu.h | 1 - 4 files changed, 3 insertions(+), 66 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 41927c3c4177..c2e7e17570e7 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -195,9 +195,6 @@ static void fsl_pamu_domain_free(struct iommu_domain *domain) /* remove all the devices from the device list */ detach_device(NULL, dma_domain); - - dma_domain->enabled = 0; - kmem_cache_free(fsl_pamu_domain_cache, dma_domain); } @@ -285,6 +282,9 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, ret = pamu_set_liodn(dma_domain, dev, liodn[i]); if (ret) break; + ret = pamu_enable_liodn(liodn[i]); + if (ret) + break; } spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return ret; @@ -341,46 +341,6 @@ int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu) return ret; } -/* Configure domain dma state i.e. enable/disable DMA */ -static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) -{ - struct device_domain_info *info; - unsigned long flags; - int ret; - - spin_lock_irqsave(&dma_domain->domain_lock, flags); - dma_domain->enabled = enable; - list_for_each_entry(info, &dma_domain->devices, link) { - ret = (enable) ? pamu_enable_liodn(info->liodn) : - pamu_disable_liodn(info->liodn); - if (ret) - pr_debug("Unable to set dma state for liodn %d", - info->liodn); - } - spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - - return 0; -} - -static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, - enum iommu_attr attr_type, void *data) -{ - struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); - int ret = 0; - - switch (attr_type) { - case DOMAIN_ATTR_FSL_PAMU_ENABLE: - ret = configure_domain_dma_state(dma_domain, *(int *)data); - break; - default: - pr_debug("Unsupported attribute type\n"); - ret = -EINVAL; - break; - } - - return ret; -} - static struct iommu_group *get_device_iommu_group(struct device *dev) { struct iommu_group *group; @@ -505,7 +465,6 @@ static const struct iommu_ops fsl_pamu_ops = { .attach_dev = fsl_pamu_attach_device, .detach_dev = fsl_pamu_detach_device, .iova_to_phys = fsl_pamu_iova_to_phys, - .domain_set_attr = fsl_pamu_set_domain_attr, .probe_device = fsl_pamu_probe_device, .release_device = fsl_pamu_release_device, .device_group = fsl_pamu_device_group, diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index cd488004acd1..5f4ed253f61b 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -12,16 +12,6 @@ struct fsl_dma_domain { /* list of devices associated with the domain */ struct list_head devices; - /* dma_domain states: - * enabled - DMA has been enabled for the given - * domain. This translates to setting of the - * valid bit for the primary PAACE in the PAMU - * PAACT table. Domain geometry should be set and - * it must have a valid mapping before DMA can be - * enabled for it. - * - */ - int enabled; u32 stash_id; u32 snoop_id; struct iommu_domain iommu_domain; diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 798b3a1ffd0b..bf38eb0042ed 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -46,7 +46,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) { #ifdef CONFIG_FSL_PAMU struct device *dev = pcfg->dev; - int window_count = 1; int ret; pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); @@ -66,14 +65,6 @@ static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) ret); goto out_domain_free; } - ret = iommu_domain_set_attr(pcfg->iommu_domain, - DOMAIN_ATTR_FSL_PAMU_ENABLE, - &window_count); - if (ret < 0) { - dev_err(dev, "%s(): iommu_domain_set_attr() = %d", __func__, - ret); - goto out_detach_device; - } no_iommu: #endif @@ -82,8 +73,6 @@ no_iommu: return; #ifdef CONFIG_FSL_PAMU -out_detach_device: - iommu_detach_device(pcfg->iommu_domain, NULL); out_domain_free: iommu_domain_free(pcfg->iommu_domain); pcfg->iommu_domain = NULL; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index a3968122aa69..f5caaa8d39be 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -109,7 +109,6 @@ enum iommu_cap { enum iommu_attr { DOMAIN_ATTR_GEOMETRY, DOMAIN_ATTR_PAGING, - DOMAIN_ATTR_FSL_PAMU_ENABLE, DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, -- cgit v1.2.3 From fd78696ece54a83c8fc4f6ff1c959b7bcf25c410 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:47 +0200 Subject: iommu/fsl_pamu: remove the snoop_id field The snoop_id is always set to ~(u32)0. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-12-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu_domain.c | 5 ++--- drivers/iommu/fsl_pamu_domain.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index c2e7e17570e7..e9c1e0dd68f0 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -97,12 +97,12 @@ static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, goto out_unlock; ret = pamu_config_ppaace(liodn, geom->aperture_start, geom->aperture_end + 1, omi_index, 0, - dma_domain->snoop_id, dma_domain->stash_id, 0); + ~(u32)0, dma_domain->stash_id, 0); if (ret) goto out_unlock; ret = pamu_config_ppaace(liodn, geom->aperture_start, geom->aperture_end + 1, ~(u32)0, - 0, dma_domain->snoop_id, dma_domain->stash_id, + 0, ~(u32)0, dma_domain->stash_id, PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); out_unlock: spin_unlock_irqrestore(&iommu_lock, flags); @@ -210,7 +210,6 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) return NULL; dma_domain->stash_id = ~(u32)0; - dma_domain->snoop_id = ~(u32)0; INIT_LIST_HEAD(&dma_domain->devices); spin_lock_init(&dma_domain->domain_lock); diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h index 5f4ed253f61b..95ac1b3cab3b 100644 --- a/drivers/iommu/fsl_pamu_domain.h +++ b/drivers/iommu/fsl_pamu_domain.h @@ -13,7 +13,6 @@ struct fsl_dma_domain { /* list of devices associated with the domain */ struct list_head devices; u32 stash_id; - u32 snoop_id; struct iommu_domain iommu_domain; spinlock_t domain_lock; }; -- cgit v1.2.3 From 151f9414b88bf62cdc671892ca1409ddf05ae72f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:48 +0200 Subject: iommu/fsl_pamu: remove the rpn and snoop_id arguments to pamu_config_ppaac These are always wired to fixed values, so don't bother passing them as arguments. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210401155256.298656-13-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu.c | 14 +++----------- drivers/iommu/fsl_pamu.h | 3 +-- drivers/iommu/fsl_pamu_domain.c | 6 +++--- 3 files changed, 7 insertions(+), 16 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 3e1647cd5ad4..742fa0e8c45b 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -181,18 +181,14 @@ int pamu_update_paace_stash(int liodn, u32 value) * @win_addr: starting address of DSA window * @win-size: size of DSA window * @omi: Operation mapping index -- if ~omi == 0 then omi not defined - * @rpn: real (true physical) page number * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then * stashid not defined - * @snoopid: snoop id for hardware coherency -- if ~snoopid == 0 then - * snoopid not defined * @prot: window permissions * * Returns 0 upon success else error code < 0 returned */ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, - u32 omi, unsigned long rpn, u32 snoopid, u32 stashid, - int prot) + u32 omi, u32 stashid, int prot) { struct paace *ppaace; @@ -234,13 +230,9 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, if (~stashid != 0) set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); - /* configure snoop id */ - if (~snoopid != 0) - ppaace->domain_attr.to_host.snpid = snoopid; - set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); - ppaace->twbah = rpn >> 20; - set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, rpn); + ppaace->twbah = 0; + set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, 0); set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); diff --git a/drivers/iommu/fsl_pamu.h b/drivers/iommu/fsl_pamu.h index 04fd843d718d..c96b29f0c707 100644 --- a/drivers/iommu/fsl_pamu.h +++ b/drivers/iommu/fsl_pamu.h @@ -384,8 +384,7 @@ int pamu_domain_init(void); int pamu_enable_liodn(int liodn); int pamu_disable_liodn(int liodn); int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, - u32 omi, unsigned long rpn, u32 snoopid, uint32_t stashid, - int prot); + u32 omi, uint32_t stashid, int prot); u32 get_stash_id(u32 stash_dest_hint, u32 vcpu); void get_ome_index(u32 *omi_index, struct device *dev); diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index e9c1e0dd68f0..c83f1e7c2cb0 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -96,13 +96,13 @@ static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, if (ret) goto out_unlock; ret = pamu_config_ppaace(liodn, geom->aperture_start, - geom->aperture_end + 1, omi_index, 0, - ~(u32)0, dma_domain->stash_id, 0); + geom->aperture_end + 1, omi_index, + dma_domain->stash_id, 0); if (ret) goto out_unlock; ret = pamu_config_ppaace(liodn, geom->aperture_start, geom->aperture_end + 1, ~(u32)0, - 0, ~(u32)0, dma_domain->stash_id, + dma_domain->stash_id, PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); out_unlock: spin_unlock_irqrestore(&iommu_lock, flags); -- cgit v1.2.3 From 57fa44be7fdb0d8803ecf11886d41b5002eae8f2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:49 +0200 Subject: iommu/fsl_pamu: hardcode the window address and size in pamu_config_ppaace The win_addr and win_size parameters are always set to 0 and 1 << 36 respectively, so just hard code them. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210401155256.298656-14-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu.c | 23 ++++------------------- drivers/iommu/fsl_pamu.h | 3 +-- drivers/iommu/fsl_pamu_domain.c | 10 ++-------- 3 files changed, 7 insertions(+), 29 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 742fa0e8c45b..fc38b1fba7cf 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -178,8 +178,6 @@ int pamu_update_paace_stash(int liodn, u32 value) * pamu_config_paace() - Sets up PPAACE entry for specified liodn * * @liodn: Logical IO device number - * @win_addr: starting address of DSA window - * @win-size: size of DSA window * @omi: Operation mapping index -- if ~omi == 0 then omi not defined * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then * stashid not defined @@ -187,35 +185,22 @@ int pamu_update_paace_stash(int liodn, u32 value) * * Returns 0 upon success else error code < 0 returned */ -int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, - u32 omi, u32 stashid, int prot) +int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) { struct paace *ppaace; - if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { - pr_debug("window size too small or not a power of two %pa\n", - &win_size); - return -EINVAL; - } - - if (win_addr & (win_size - 1)) { - pr_debug("window address is not aligned with window size\n"); - return -EINVAL; - } - ppaace = pamu_get_ppaace(liodn); if (!ppaace) return -ENOENT; /* window size is 2^(WSE+1) bytes */ set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, - map_addrspace_size_to_wse(win_size)); + map_addrspace_size_to_wse(1ULL << 36)); pamu_init_ppaace(ppaace); - ppaace->wbah = win_addr >> (PAMU_PAGE_SHIFT + 20); - set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, - (win_addr >> PAMU_PAGE_SHIFT)); + ppaace->wbah = 0; + set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); /* set up operation mapping if it's configured */ if (omi < OME_NUMBER_ENTRIES) { diff --git a/drivers/iommu/fsl_pamu.h b/drivers/iommu/fsl_pamu.h index c96b29f0c707..36df7975ff64 100644 --- a/drivers/iommu/fsl_pamu.h +++ b/drivers/iommu/fsl_pamu.h @@ -383,8 +383,7 @@ struct ome { int pamu_domain_init(void); int pamu_enable_liodn(int liodn); int pamu_disable_liodn(int liodn); -int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, - u32 omi, uint32_t stashid, int prot); +int pamu_config_ppaace(int liodn, u32 omi, uint32_t stashid, int prot); u32 get_stash_id(u32 stash_dest_hint, u32 vcpu); void get_ome_index(u32 *omi_index, struct device *dev); diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index c83f1e7c2cb0..32944d67baa4 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -78,8 +78,6 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, int liodn) { - struct iommu_domain *domain = &dma_domain->iommu_domain; - struct iommu_domain_geometry *geom = &domain->geometry; u32 omi_index = ~(u32)0; unsigned long flags; int ret; @@ -95,14 +93,10 @@ static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, ret = pamu_disable_liodn(liodn); if (ret) goto out_unlock; - ret = pamu_config_ppaace(liodn, geom->aperture_start, - geom->aperture_end + 1, omi_index, - dma_domain->stash_id, 0); + ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0); if (ret) goto out_unlock; - ret = pamu_config_ppaace(liodn, geom->aperture_start, - geom->aperture_end + 1, ~(u32)0, - dma_domain->stash_id, + ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id, PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); out_unlock: spin_unlock_irqrestore(&iommu_lock, flags); -- cgit v1.2.3 From 9fb5fad562fa0a41c84691714d99c23f54168a9e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:50 +0200 Subject: iommu: remove DOMAIN_ATTR_PAGING DOMAIN_ATTR_PAGING is never used. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-15-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 5 ----- include/linux/iommu.h | 1 - 2 files changed, 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 33bc0f8f0a74..93b3c2048c21 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2668,7 +2668,6 @@ int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { struct iommu_domain_geometry *geometry; - bool *paging; int ret = 0; switch (attr) { @@ -2676,10 +2675,6 @@ int iommu_domain_get_attr(struct iommu_domain *domain, geometry = data; *geometry = domain->geometry; - break; - case DOMAIN_ATTR_PAGING: - paging = data; - *paging = (domain->pgsize_bitmap != 0UL); break; default: if (!domain->ops->domain_get_attr) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index f5caaa8d39be..85b51084b820 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -108,7 +108,6 @@ enum iommu_cap { enum iommu_attr { DOMAIN_ATTR_GEOMETRY, - DOMAIN_ATTR_PAGING, DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, -- cgit v1.2.3 From bc9a05eef113e75cfa792fdf24dae011bc3d5294 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:51 +0200 Subject: iommu: remove DOMAIN_ATTR_GEOMETRY The geometry information can be trivially queried from the iommu_domain struture. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-16-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 20 +++----------------- drivers/vfio/vfio_iommu_type1.c | 26 ++++++++++++-------------- drivers/vhost/vdpa.c | 10 +++------- include/linux/iommu.h | 1 - 4 files changed, 18 insertions(+), 39 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 93b3c2048c21..703e67d8cb04 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2667,23 +2667,9 @@ core_initcall(iommu_init); int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { - struct iommu_domain_geometry *geometry; - int ret = 0; - - switch (attr) { - case DOMAIN_ATTR_GEOMETRY: - geometry = data; - *geometry = domain->geometry; - - break; - default: - if (!domain->ops->domain_get_attr) - return -EINVAL; - - ret = domain->ops->domain_get_attr(domain, attr, data); - } - - return ret; + if (!domain->ops->domain_get_attr) + return -EINVAL; + return domain->ops->domain_get_attr(domain, attr, data); } EXPORT_SYMBOL_GPL(iommu_domain_get_attr); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 4bb162c1d649..c8e57f22f421 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -2252,7 +2252,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, int ret; bool resv_msi, msi_remap; phys_addr_t resv_msi_base = 0; - struct iommu_domain_geometry geo; + struct iommu_domain_geometry *geo; LIST_HEAD(iova_copy); LIST_HEAD(group_resv_regions); @@ -2333,10 +2333,9 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_domain; /* Get aperture info */ - iommu_domain_get_attr(domain->domain, DOMAIN_ATTR_GEOMETRY, &geo); - - if (vfio_iommu_aper_conflict(iommu, geo.aperture_start, - geo.aperture_end)) { + geo = &domain->domain->geometry; + if (vfio_iommu_aper_conflict(iommu, geo->aperture_start, + geo->aperture_end)) { ret = -EINVAL; goto out_detach; } @@ -2359,8 +2358,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, if (ret) goto out_detach; - ret = vfio_iommu_aper_resize(&iova_copy, geo.aperture_start, - geo.aperture_end); + ret = vfio_iommu_aper_resize(&iova_copy, geo->aperture_start, + geo->aperture_end); if (ret) goto out_detach; @@ -2493,7 +2492,6 @@ static void vfio_iommu_aper_expand(struct vfio_iommu *iommu, struct list_head *iova_copy) { struct vfio_domain *domain; - struct iommu_domain_geometry geo; struct vfio_iova *node; dma_addr_t start = 0; dma_addr_t end = (dma_addr_t)~0; @@ -2502,12 +2500,12 @@ static void vfio_iommu_aper_expand(struct vfio_iommu *iommu, return; list_for_each_entry(domain, &iommu->domain_list, next) { - iommu_domain_get_attr(domain->domain, DOMAIN_ATTR_GEOMETRY, - &geo); - if (geo.aperture_start > start) - start = geo.aperture_start; - if (geo.aperture_end < end) - end = geo.aperture_end; + struct iommu_domain_geometry *geo = &domain->domain->geometry; + + if (geo->aperture_start > start) + start = geo->aperture_start; + if (geo->aperture_end < end) + end = geo->aperture_end; } /* Modify aperture limits. The new aper is either same or bigger */ diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index ef688c8c0e0e..25824fab433d 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -826,18 +826,14 @@ static void vhost_vdpa_free_domain(struct vhost_vdpa *v) static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v) { struct vdpa_iova_range *range = &v->range; - struct iommu_domain_geometry geo; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; if (ops->get_iova_range) { *range = ops->get_iova_range(vdpa); - } else if (v->domain && - !iommu_domain_get_attr(v->domain, - DOMAIN_ATTR_GEOMETRY, &geo) && - geo.force_aperture) { - range->first = geo.aperture_start; - range->last = geo.aperture_end; + } else if (v->domain && v->domain->geometry.force_aperture) { + range->first = v->domain->geometry.aperture_start; + range->last = v->domain->geometry.aperture_end; } else { range->first = 0; range->last = ULLONG_MAX; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 85b51084b820..207446cc6d3f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -107,7 +107,6 @@ enum iommu_cap { */ enum iommu_attr { - DOMAIN_ATTR_GEOMETRY, DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, -- cgit v1.2.3 From 7e147547783a9035df816864b6a45ffbb254d700 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:52 +0200 Subject: iommu: remove DOMAIN_ATTR_NESTING Use an explicit enable_nesting method instead. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-17-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 43 +++++++++++------------------ drivers/iommu/arm/arm-smmu/arm-smmu.c | 30 ++++++++++---------- drivers/iommu/intel/iommu.c | 31 ++++++--------------- drivers/iommu/iommu.c | 10 +++++++ drivers/vfio/vfio_iommu_type1.c | 5 +--- include/linux/iommu.h | 4 ++- 6 files changed, 55 insertions(+), 68 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1c7721b89fec..7f3a122e6b5f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2555,15 +2555,6 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); switch (domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_NESTING: - *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); - return 0; - default: - return -ENODEV; - } - break; case IOMMU_DOMAIN_DMA: switch (attr) { case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: @@ -2587,23 +2578,6 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain, mutex_lock(&smmu_domain->init_mutex); switch (domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_NESTING: - if (smmu_domain->smmu) { - ret = -EPERM; - goto out_unlock; - } - - if (*(int *)data) - smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; - else - smmu_domain->stage = ARM_SMMU_DOMAIN_S1; - break; - default: - ret = -ENODEV; - } - break; case IOMMU_DOMAIN_DMA: switch(attr) { case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: @@ -2617,11 +2591,25 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain, ret = -EINVAL; } -out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; } +static int arm_smmu_enable_nesting(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) + ret = -EPERM; + else + smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; + mutex_unlock(&smmu_domain->init_mutex); + + return ret; +} + static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); @@ -2721,6 +2709,7 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, + .enable_nesting = arm_smmu_enable_nesting, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index d8c6bfde6a61..0aa6d6672749 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1489,9 +1489,6 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, switch(domain->type) { case IOMMU_DOMAIN_UNMANAGED: switch (attr) { - case DOMAIN_ATTR_NESTING: - *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); - return 0; case DOMAIN_ATTR_IO_PGTABLE_CFG: { struct io_pgtable_domain_attr *pgtbl_cfg = data; *pgtbl_cfg = smmu_domain->pgtbl_cfg; @@ -1519,6 +1516,21 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, } } +static int arm_smmu_enable_nesting(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) + ret = -EPERM; + else + smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; + mutex_unlock(&smmu_domain->init_mutex); + + return ret; +} + static int arm_smmu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { @@ -1530,17 +1542,6 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain, switch(domain->type) { case IOMMU_DOMAIN_UNMANAGED: switch (attr) { - case DOMAIN_ATTR_NESTING: - if (smmu_domain->smmu) { - ret = -EPERM; - goto out_unlock; - } - - if (*(int *)data) - smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; - else - smmu_domain->stage = ARM_SMMU_DOMAIN_S1; - break; case DOMAIN_ATTR_IO_PGTABLE_CFG: { struct io_pgtable_domain_attr *pgtbl_cfg = data; @@ -1633,6 +1634,7 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, + .enable_nesting = arm_smmu_enable_nesting, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 1ea0f0bd6973..761feac91767 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5396,32 +5396,19 @@ static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain, } static int -intel_iommu_domain_set_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) +intel_iommu_enable_nesting(struct iommu_domain *domain) { struct dmar_domain *dmar_domain = to_dmar_domain(domain); unsigned long flags; - int ret = 0; - - if (domain->type != IOMMU_DOMAIN_UNMANAGED) - return -EINVAL; + int ret = -ENODEV; - switch (attr) { - case DOMAIN_ATTR_NESTING: - spin_lock_irqsave(&device_domain_lock, flags); - if (nested_mode_support() && - list_empty(&dmar_domain->devices)) { - dmar_domain->flags |= DOMAIN_FLAG_NESTING_MODE; - dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL; - } else { - ret = -ENODEV; - } - spin_unlock_irqrestore(&device_domain_lock, flags); - break; - default: - ret = -EINVAL; - break; + spin_lock_irqsave(&device_domain_lock, flags); + if (nested_mode_support() && list_empty(&dmar_domain->devices)) { + dmar_domain->flags |= DOMAIN_FLAG_NESTING_MODE; + dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL; + ret = 0; } + spin_unlock_irqrestore(&device_domain_lock, flags); return ret; } @@ -5550,7 +5537,7 @@ const struct iommu_ops intel_iommu_ops = { .domain_alloc = intel_iommu_domain_alloc, .domain_free = intel_iommu_domain_free, .domain_get_attr = intel_iommu_domain_get_attr, - .domain_set_attr = intel_iommu_domain_set_attr, + .enable_nesting = intel_iommu_enable_nesting, .attach_dev = intel_iommu_attach_device, .detach_dev = intel_iommu_detach_device, .aux_attach_dev = intel_iommu_aux_attach_device, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 703e67d8cb04..4738bc024240 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2690,6 +2690,16 @@ int iommu_domain_set_attr(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_domain_set_attr); +int iommu_enable_nesting(struct iommu_domain *domain) +{ + if (domain->type != IOMMU_DOMAIN_UNMANAGED) + return -EINVAL; + if (!domain->ops->enable_nesting) + return -EINVAL; + return domain->ops->enable_nesting(domain); +} +EXPORT_SYMBOL_GPL(iommu_enable_nesting); + void iommu_get_resv_regions(struct device *dev, struct list_head *list) { const struct iommu_ops *ops = dev->bus->iommu_ops; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index c8e57f22f421..1201482e6eec 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -2320,10 +2320,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, } if (iommu->nesting) { - int attr = 1; - - ret = iommu_domain_set_attr(domain->domain, DOMAIN_ATTR_NESTING, - &attr); + ret = iommu_enable_nesting(domain->domain); if (ret) goto out_domain; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 207446cc6d3f..d7d76b5e1192 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -107,7 +107,6 @@ enum iommu_cap { */ enum iommu_attr { - DOMAIN_ATTR_NESTING, /* two stages of translation */ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, DOMAIN_ATTR_MAX, @@ -210,6 +209,7 @@ struct iommu_iotlb_gather { * @device_group: find iommu group for a particular device * @domain_get_attr: Query domain attributes * @domain_set_attr: Change domain attributes + * @enable_nesting: Enable nesting * @get_resv_regions: Request list of reserved regions for a device * @put_resv_regions: Free list of reserved regions for a device * @apply_resv_region: Temporary helper call-back for iova reserved ranges @@ -262,6 +262,7 @@ struct iommu_ops { enum iommu_attr attr, void *data); int (*domain_set_attr)(struct iommu_domain *domain, enum iommu_attr attr, void *data); + int (*enable_nesting)(struct iommu_domain *domain); /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); @@ -511,6 +512,7 @@ extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, void *data); extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, void *data); +int iommu_enable_nesting(struct iommu_domain *domain); extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, unsigned long iova, int flags); -- cgit v1.2.3 From 3189713a1b84ac02cce3217955ae68d0d67b15b7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:53 +0200 Subject: iommu: remove iommu_set_cmd_line_dma_api and iommu_cmd_line_dma_api Don't obsfucate the trivial bit flag check. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210401155256.298656-18-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 4738bc024240..7d1f50ac3709 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -70,16 +70,6 @@ static const char * const iommu_group_resv_type_string[] = { #define IOMMU_CMD_LINE_DMA_API BIT(0) -static void iommu_set_cmd_line_dma_api(void) -{ - iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; -} - -static bool iommu_cmd_line_dma_api(void) -{ - return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API); -} - static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -130,9 +120,7 @@ static const char *iommu_domain_type_str(unsigned int t) static int __init iommu_subsys_init(void) { - bool cmd_line = iommu_cmd_line_dma_api(); - - if (!cmd_line) { + if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) iommu_set_default_passthrough(false); else @@ -146,7 +134,8 @@ static int __init iommu_subsys_init(void) pr_info("Default domain type: %s %s\n", iommu_domain_type_str(iommu_def_domain_type), - cmd_line ? "(set via kernel command line)" : ""); + (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ? + "(set via kernel command line)" : ""); return 0; } @@ -2757,16 +2746,14 @@ EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); void iommu_set_default_passthrough(bool cmd_line) { if (cmd_line) - iommu_set_cmd_line_dma_api(); - + iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; } void iommu_set_default_translated(bool cmd_line) { if (cmd_line) - iommu_set_cmd_line_dma_api(); - + iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; iommu_def_domain_type = IOMMU_DOMAIN_DMA; } -- cgit v1.2.3 From a250c23f15c21c556becd4986f453255e545807c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 1 Apr 2021 17:52:54 +0200 Subject: iommu: remove DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE Instead make the global iommu_dma_strict paramete in iommu.c canonical by exporting helpers to get and set it and use those directly in the drivers. This make sure that the iommu.strict parameter also works for the AMD and Intel IOMMU drivers on x86. As those default to lazy flushing a new IOMMU_CMD_LINE_STRICT is used to turn the value into a tristate to represent the default if not overriden by an explicit parameter. [ported on top of the other iommu_attr changes and added a few small missing bits] Signed-off-by: Robin Murphy . Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210401155256.298656-19-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 23 +---------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 50 +--------------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 - drivers/iommu/arm/arm-smmu/arm-smmu.c | 27 ++---------- drivers/iommu/dma-iommu.c | 9 +--- drivers/iommu/intel/iommu.c | 64 ++++++----------------------- drivers/iommu/iommu.c | 30 ++++++++++---- include/linux/iommu.h | 4 +- 8 files changed, 43 insertions(+), 165 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a69a8b573e40..ce6393d2224d 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1771,26 +1771,6 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev) return acpihid_device_group(dev); } -static int amd_iommu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - switch (domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - return -ENODEV; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - *(int *)data = !amd_iommu_unmap_flush; - return 0; - default: - return -ENODEV; - } - break; - default: - return -EINVAL; - } -} - /***************************************************************************** * * The next functions belong to the dma_ops mapping/unmapping code. @@ -1855,7 +1835,7 @@ int __init amd_iommu_init_dma_ops(void) pr_info("IO/TLB flush on unmap enabled\n"); else pr_info("Lazy IO/TLB flushing enabled\n"); - + iommu_set_dma_strict(amd_iommu_unmap_flush); return 0; } @@ -2257,7 +2237,6 @@ const struct iommu_ops amd_iommu_ops = { .release_device = amd_iommu_release_device, .probe_finalize = amd_iommu_probe_finalize, .device_group = amd_iommu_device_group, - .domain_get_attr = amd_iommu_domain_get_attr, .get_resv_regions = amd_iommu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 7f3a122e6b5f..7fe45838298c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2040,7 +2040,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain, .iommu_dev = smmu->dev, }; - if (smmu_domain->non_strict) + if (!iommu_get_dma_strict(domain)) pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); @@ -2549,52 +2549,6 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) return group; } -static int arm_smmu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - - switch (domain->type) { - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - *(int *)data = smmu_domain->non_strict; - return 0; - default: - return -ENODEV; - } - break; - default: - return -EINVAL; - } -} - -static int arm_smmu_domain_set_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - int ret = 0; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - - mutex_lock(&smmu_domain->init_mutex); - - switch (domain->type) { - case IOMMU_DOMAIN_DMA: - switch(attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - smmu_domain->non_strict = *(int *)data; - break; - default: - ret = -ENODEV; - } - break; - default: - ret = -EINVAL; - } - - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - static int arm_smmu_enable_nesting(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -2707,8 +2661,6 @@ static struct iommu_ops arm_smmu_ops = { .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, - .domain_get_attr = arm_smmu_domain_get_attr, - .domain_set_attr = arm_smmu_domain_set_attr, .enable_nesting = arm_smmu_enable_nesting, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 7b15b7580c6e..05cf2bf42d33 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -677,7 +677,6 @@ struct arm_smmu_domain { struct mutex init_mutex; /* Protects smmu pointer */ struct io_pgtable_ops *pgtbl_ops; - bool non_strict; atomic_t nr_ats_masters; enum arm_smmu_domain_stage stage; diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 0aa6d6672749..f17c54a76ef6 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -761,6 +761,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .iommu_dev = smmu->dev, }; + if (!iommu_get_dma_strict(domain)) + pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; + if (smmu->impl && smmu->impl->init_context) { ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); if (ret) @@ -1499,18 +1502,6 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, return -ENODEV; } break; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: { - bool non_strict = smmu_domain->pgtbl_cfg.quirks & - IO_PGTABLE_QUIRK_NON_STRICT; - *(int *)data = non_strict; - return 0; - } - default: - return -ENODEV; - } - break; default: return -EINVAL; } @@ -1557,18 +1548,6 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain, ret = -ENODEV; } break; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - if (*(int *)data) - smmu_domain->pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; - else - smmu_domain->pgtbl_cfg.quirks &= ~IO_PGTABLE_QUIRK_NON_STRICT; - break; - default: - ret = -ENODEV; - } - break; default: ret = -EINVAL; } diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0af4048f9b3f..efaf5eab40a1 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -306,10 +306,7 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) cookie = container_of(iovad, struct iommu_dma_cookie, iovad); domain = cookie->fq_domain; - /* - * The IOMMU driver supporting DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE - * implies that ops->flush_iotlb_all must be non-NULL. - */ + domain->ops->flush_iotlb_all(domain); } @@ -336,7 +333,6 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, struct iommu_dma_cookie *cookie = domain->iova_cookie; unsigned long order, base_pfn; struct iova_domain *iovad; - int attr; if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) return -EINVAL; @@ -373,8 +369,7 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, init_iova_domain(iovad, 1UL << order, base_pfn); if (!cookie->fq_domain && (!dev || !dev_is_untrusted(dev)) && - !iommu_domain_get_attr(domain, DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && - attr) { + domain->ops->flush_iotlb_all && !iommu_get_dma_strict(domain)) { if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, iommu_dma_entry_dtor)) pr_warn("iova flush queue initialization failed\n"); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 761feac91767..1db4d8934526 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4347,6 +4347,17 @@ int __init intel_iommu_init(void) down_read(&dmar_global_lock); for_each_active_iommu(iommu, drhd) { + /* + * The flush queue implementation does not perform + * page-selective invalidations that are required for efficient + * TLB flushes in virtual environments. The benefit of batching + * is likely to be much lower than the overhead of synchronizing + * the virtual and physical IOMMU page-tables. + */ + if (!intel_iommu_strict && cap_caching_mode(iommu->cap)) { + pr_warn("IOMMU batching is disabled due to virtualization"); + intel_iommu_strict = 1; + } iommu_device_sysfs_add(&iommu->iommu, NULL, intel_iommu_groups, "%s", iommu->name); @@ -4355,6 +4366,7 @@ int __init intel_iommu_init(void) } up_read(&dmar_global_lock); + iommu_set_dma_strict(intel_iommu_strict); bus_set_iommu(&pci_bus_type, &intel_iommu_ops); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); @@ -5413,57 +5425,6 @@ intel_iommu_enable_nesting(struct iommu_domain *domain) return ret; } -static bool domain_use_flush_queue(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool r = true; - - if (intel_iommu_strict) - return false; - - /* - * The flush queue implementation does not perform page-selective - * invalidations that are required for efficient TLB flushes in virtual - * environments. The benefit of batching is likely to be much lower than - * the overhead of synchronizing the virtual and physical IOMMU - * page-tables. - */ - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (!cap_caching_mode(iommu->cap)) - continue; - - pr_warn_once("IOMMU batching is disabled due to virtualization"); - r = false; - break; - } - rcu_read_unlock(); - - return r; -} - -static int -intel_iommu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - switch (domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - return -ENODEV; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - *(int *)data = domain_use_flush_queue(); - return 0; - default: - return -ENODEV; - } - break; - default: - return -EINVAL; - } -} - /* * Check that the device does not live on an external facing PCI port that is * marked as untrusted. Such devices should not be able to apply quirks and @@ -5536,7 +5497,6 @@ const struct iommu_ops intel_iommu_ops = { .capable = intel_iommu_capable, .domain_alloc = intel_iommu_domain_alloc, .domain_free = intel_iommu_domain_free, - .domain_get_attr = intel_iommu_domain_get_attr, .enable_nesting = intel_iommu_enable_nesting, .attach_dev = intel_iommu_attach_device, .detach_dev = intel_iommu_detach_device, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 7d1f50ac3709..b09d3f07f3df 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -69,6 +69,7 @@ static const char * const iommu_group_resv_type_string[] = { }; #define IOMMU_CMD_LINE_DMA_API BIT(0) +#define IOMMU_CMD_LINE_STRICT BIT(1) static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); @@ -318,10 +319,29 @@ early_param("iommu.passthrough", iommu_set_def_domain_type); static int __init iommu_dma_setup(char *str) { - return kstrtobool(str, &iommu_dma_strict); + int ret = kstrtobool(str, &iommu_dma_strict); + + if (!ret) + iommu_cmd_line |= IOMMU_CMD_LINE_STRICT; + return ret; } early_param("iommu.strict", iommu_dma_setup); +void iommu_set_dma_strict(bool strict) +{ + if (strict || !(iommu_cmd_line & IOMMU_CMD_LINE_STRICT)) + iommu_dma_strict = strict; +} + +bool iommu_get_dma_strict(struct iommu_domain *domain) +{ + /* only allow lazy flushing for DMA domains */ + if (domain->type == IOMMU_DOMAIN_DMA) + return iommu_dma_strict; + return true; +} +EXPORT_SYMBOL_GPL(iommu_get_dma_strict); + static ssize_t iommu_group_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) { @@ -1500,14 +1520,6 @@ static int iommu_group_alloc_default_domain(struct bus_type *bus, group->default_domain = dom; if (!group->domain) group->domain = dom; - - if (!iommu_dma_strict) { - int attr = 1; - iommu_domain_set_attr(dom, - DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, - &attr); - } - return 0; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d7d76b5e1192..9349bdd62e91 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -107,7 +107,6 @@ enum iommu_cap { */ enum iommu_attr { - DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, DOMAIN_ATTR_IO_PGTABLE_CFG, DOMAIN_ATTR_MAX, }; @@ -514,6 +513,9 @@ extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, void *data); int iommu_enable_nesting(struct iommu_domain *domain); +void iommu_set_dma_strict(bool val); +bool iommu_get_dma_strict(struct iommu_domain *domain); + extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, unsigned long iova, int flags); -- cgit v1.2.3 From 4fc52b81e87be583efb834df5b58245cb9ddd3e7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:55 +0200 Subject: iommu: remove DOMAIN_ATTR_IO_PGTABLE_CFG Use an explicit set_pgtable_quirks method instead that just passes the actual quirk bitmask instead. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Acked-by: Li Yang Link: https://lore.kernel.org/r/20210401155256.298656-20-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/gpu/drm/msm/adreno/adreno_gpu.c | 5 +-- drivers/iommu/arm/arm-smmu/arm-smmu.c | 64 ++++++--------------------------- drivers/iommu/arm/arm-smmu/arm-smmu.h | 2 +- drivers/iommu/iommu.c | 11 ++++++ include/linux/io-pgtable.h | 4 --- include/linux/iommu.h | 12 ++++++- 6 files changed, 35 insertions(+), 63 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 0f184c3dd9d9..4a0b14dad93e 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -188,10 +188,7 @@ int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid) void adreno_set_llc_attributes(struct iommu_domain *iommu) { - struct io_pgtable_domain_attr pgtbl_cfg; - - pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; - iommu_domain_set_attr(iommu, DOMAIN_ATTR_IO_PGTABLE_CFG, &pgtbl_cfg); + iommu_set_pgtable_quirks(iommu, IO_PGTABLE_QUIRK_ARM_OUTER_WBWA); } struct msm_gem_address_space * diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index f17c54a76ef6..3c6adcdb201b 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -770,8 +770,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_clear_smmu; } - if (smmu_domain->pgtbl_cfg.quirks) - pgtbl_cfg.quirks |= smmu_domain->pgtbl_cfg.quirks; + if (smmu_domain->pgtbl_quirks) + pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { @@ -1484,29 +1484,6 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) return group; } -static int arm_smmu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - - switch(domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_IO_PGTABLE_CFG: { - struct io_pgtable_domain_attr *pgtbl_cfg = data; - *pgtbl_cfg = smmu_domain->pgtbl_cfg; - - return 0; - } - default: - return -ENODEV; - } - break; - default: - return -EINVAL; - } -} - static int arm_smmu_enable_nesting(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -1522,37 +1499,19 @@ static int arm_smmu_enable_nesting(struct iommu_domain *domain) return ret; } -static int arm_smmu_domain_set_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) +static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, + unsigned long quirks) { - int ret = 0; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; mutex_lock(&smmu_domain->init_mutex); - - switch(domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_IO_PGTABLE_CFG: { - struct io_pgtable_domain_attr *pgtbl_cfg = data; - - if (smmu_domain->smmu) { - ret = -EPERM; - goto out_unlock; - } - - smmu_domain->pgtbl_cfg = *pgtbl_cfg; - break; - } - default: - ret = -ENODEV; - } - break; - default: - ret = -EINVAL; - } -out_unlock: + if (smmu_domain->smmu) + ret = -EPERM; + else + smmu_domain->pgtbl_quirks = quirks; mutex_unlock(&smmu_domain->init_mutex); + return ret; } @@ -1611,9 +1570,8 @@ static struct iommu_ops arm_smmu_ops = { .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, - .domain_get_attr = arm_smmu_domain_get_attr, - .domain_set_attr = arm_smmu_domain_set_attr, .enable_nesting = arm_smmu_enable_nesting, + .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index d2a2d1bc58ba..c31a59d35c64 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -364,7 +364,7 @@ enum arm_smmu_domain_stage { struct arm_smmu_domain { struct arm_smmu_device *smmu; struct io_pgtable_ops *pgtbl_ops; - struct io_pgtable_domain_attr pgtbl_cfg; + unsigned long pgtbl_quirks; const struct iommu_flush_ops *flush_ops; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b09d3f07f3df..7ea5135b1431 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2701,6 +2701,17 @@ int iommu_enable_nesting(struct iommu_domain *domain) } EXPORT_SYMBOL_GPL(iommu_enable_nesting); +int iommu_set_pgtable_quirks(struct iommu_domain *domain, + unsigned long quirk) +{ + if (domain->type != IOMMU_DOMAIN_UNMANAGED) + return -EINVAL; + if (!domain->ops->set_pgtable_quirks) + return -EINVAL; + return domain->ops->set_pgtable_quirks(domain, quirk); +} +EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); + void iommu_get_resv_regions(struct device *dev, struct list_head *list) { const struct iommu_ops *ops = dev->bus->iommu_ops; diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index a4c9ca2c31f1..4d40dfa75b55 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -204,10 +204,6 @@ struct io_pgtable { #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) -struct io_pgtable_domain_attr { - unsigned long quirks; -}; - static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) { if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 9349bdd62e91..fbac49fe0880 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -107,7 +107,6 @@ enum iommu_cap { */ enum iommu_attr { - DOMAIN_ATTR_IO_PGTABLE_CFG, DOMAIN_ATTR_MAX, }; @@ -209,6 +208,7 @@ struct iommu_iotlb_gather { * @domain_get_attr: Query domain attributes * @domain_set_attr: Change domain attributes * @enable_nesting: Enable nesting + * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*) * @get_resv_regions: Request list of reserved regions for a device * @put_resv_regions: Free list of reserved regions for a device * @apply_resv_region: Temporary helper call-back for iova reserved ranges @@ -262,6 +262,8 @@ struct iommu_ops { int (*domain_set_attr)(struct iommu_domain *domain, enum iommu_attr attr, void *data); int (*enable_nesting)(struct iommu_domain *domain); + int (*set_pgtable_quirks)(struct iommu_domain *domain, + unsigned long quirks); /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); @@ -512,6 +514,8 @@ extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, void *data); int iommu_enable_nesting(struct iommu_domain *domain); +int iommu_set_pgtable_quirks(struct iommu_domain *domain, + unsigned long quirks); void iommu_set_dma_strict(bool val); bool iommu_get_dma_strict(struct iommu_domain *domain); @@ -891,6 +895,12 @@ static inline int iommu_domain_set_attr(struct iommu_domain *domain, return -EINVAL; } +static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain, + unsigned long quirks) +{ + return 0; +} + static inline int iommu_device_register(struct iommu_device *iommu) { return -ENODEV; -- cgit v1.2.3 From 7876a83ffe8c23c7049a63c747a7b96cafaf10a4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Apr 2021 17:52:56 +0200 Subject: iommu: remove iommu_domain_{get,set}_attr Remove the now unused iommu attr infrastructure. Signed-off-by: Christoph Hellwig Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210401155256.298656-21-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 26 -------------------------- include/linux/iommu.h | 36 ------------------------------------ 2 files changed, 62 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 7ea5135b1431..c4ad9c644802 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2665,32 +2665,6 @@ static int __init iommu_init(void) } core_initcall(iommu_init); -int iommu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - if (!domain->ops->domain_get_attr) - return -EINVAL; - return domain->ops->domain_get_attr(domain, attr, data); -} -EXPORT_SYMBOL_GPL(iommu_domain_get_attr); - -int iommu_domain_set_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - int ret = 0; - - switch (attr) { - default: - if (domain->ops->domain_set_attr == NULL) - return -EINVAL; - - ret = domain->ops->domain_set_attr(domain, attr, data); - } - - return ret; -} -EXPORT_SYMBOL_GPL(iommu_domain_set_attr); - int iommu_enable_nesting(struct iommu_domain *domain) { if (domain->type != IOMMU_DOMAIN_UNMANAGED) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index fbac49fe0880..a5b3af54fbb8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -96,20 +96,6 @@ enum iommu_cap { IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ }; -/* - * Following constraints are specifc to FSL_PAMUV1: - * -aperture must be power of 2, and naturally aligned - * -number of windows must be power of 2, and address space size - * of each window is determined by aperture size / # of windows - * -the actual size of the mapped region of a window must be power - * of 2 starting with 4KB and physical address must be naturally - * aligned. - */ - -enum iommu_attr { - DOMAIN_ATTR_MAX, -}; - /* These are the possible reserved region types */ enum iommu_resv_type { /* Memory regions which must be mapped 1:1 at all times */ @@ -205,8 +191,6 @@ struct iommu_iotlb_gather { * @probe_finalize: Do final setup work after the device is added to an IOMMU * group and attached to the groups domain * @device_group: find iommu group for a particular device - * @domain_get_attr: Query domain attributes - * @domain_set_attr: Change domain attributes * @enable_nesting: Enable nesting * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*) * @get_resv_regions: Request list of reserved regions for a device @@ -257,10 +241,6 @@ struct iommu_ops { void (*release_device)(struct device *dev); void (*probe_finalize)(struct device *dev); struct iommu_group *(*device_group)(struct device *dev); - int (*domain_get_attr)(struct iommu_domain *domain, - enum iommu_attr attr, void *data); - int (*domain_set_attr)(struct iommu_domain *domain, - enum iommu_attr attr, void *data); int (*enable_nesting)(struct iommu_domain *domain); int (*set_pgtable_quirks)(struct iommu_domain *domain, unsigned long quirks); @@ -509,10 +489,6 @@ extern int iommu_page_response(struct device *dev, extern int iommu_group_id(struct iommu_group *group); extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); -extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, - void *data); -extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, - void *data); int iommu_enable_nesting(struct iommu_domain *domain); int iommu_set_pgtable_quirks(struct iommu_domain *domain, unsigned long quirks); @@ -883,18 +859,6 @@ static inline int iommu_group_id(struct iommu_group *group) return -ENODEV; } -static inline int iommu_domain_get_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - return -EINVAL; -} - -static inline int iommu_domain_set_attr(struct iommu_domain *domain, - enum iommu_attr attr, void *data) -{ - return -EINVAL; -} - static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain, unsigned long quirks) { -- cgit v1.2.3 From d151c85c52a314c6ecb91ab35b3f696a6778b509 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 2 Apr 2021 16:33:09 +0200 Subject: iommu/amd: Remove the unused device errata code The device errata mechism is entirely unused, so remove it. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210402143312.372386-2-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 1 - drivers/iommu/amd/iommu.c | 56 ++----------------------------------- include/linux/amd-iommu.h | 18 ------------ 3 files changed, 3 insertions(+), 72 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 6937e3674a16..94c1a7a9876d 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -693,7 +693,6 @@ struct iommu_dev_data { } ats; /* ATS state */ bool pri_tlp; /* PASID TLB required for PPR completions */ - u32 errata; /* Bitmap for errata to apply */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 70814ab0b374..9b1a01602bb3 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -290,15 +290,6 @@ static bool pci_iommuv2_capable(struct pci_dev *pdev) return true; } -static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) -{ - struct iommu_dev_data *dev_data; - - dev_data = dev_iommu_priv_get(&pdev->dev); - - return dev_data->errata & (1 << erratum) ? true : false; -} - /* * This function checks if the driver got a valid device from the caller to * avoid dereferencing invalid pointers. @@ -1531,33 +1522,9 @@ static void pdev_iommuv2_disable(struct pci_dev *pdev) pci_disable_pasid(pdev); } -/* FIXME: Change generic reset-function to do the same */ -static int pri_reset_while_enabled(struct pci_dev *pdev) -{ - u16 control; - int pos; - - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); - if (!pos) - return -EINVAL; - - pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control); - control |= PCI_PRI_CTRL_RESET; - pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control); - - return 0; -} - static int pdev_iommuv2_enable(struct pci_dev *pdev) { - bool reset_enable; - int reqs, ret; - - /* FIXME: Hardcode number of outstanding requests for now */ - reqs = 32; - if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE)) - reqs = 1; - reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET); + int ret; /* Only allow access to user-accessible pages */ ret = pci_enable_pasid(pdev, 0); @@ -1570,16 +1537,11 @@ static int pdev_iommuv2_enable(struct pci_dev *pdev) goto out_err; /* Enable PRI */ - ret = pci_enable_pri(pdev, reqs); + /* FIXME: Hardcode number of outstanding requests for now */ + ret = pci_enable_pri(pdev, 32); if (ret) goto out_err; - if (reset_enable) { - ret = pri_reset_while_enabled(pdev); - if (ret) - goto out_err; - } - ret = pci_enable_ats(pdev, PAGE_SHIFT); if (ret) goto out_err; @@ -2594,18 +2556,6 @@ struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev) } EXPORT_SYMBOL(amd_iommu_get_v2_domain); -void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum) -{ - struct iommu_dev_data *dev_data; - - if (!amd_iommu_v2_supported()) - return; - - dev_data = dev_iommu_priv_get(&pdev->dev); - dev_data->errata |= (1 << erratum); -} -EXPORT_SYMBOL(amd_iommu_enable_device_erratum); - int amd_iommu_device_info(struct pci_dev *pdev, struct amd_iommu_device_info *info) { diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 450717299928..474065ed88a4 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -32,24 +32,6 @@ struct pci_dev; extern int amd_iommu_detect(void); extern int amd_iommu_init_hardware(void); -/** - * amd_iommu_enable_device_erratum() - Enable erratum workaround for device - * in the IOMMUv2 driver - * @pdev: The PCI device the workaround is necessary for - * @erratum: The erratum workaround to enable - * - * The function needs to be called before amd_iommu_init_device(). - * Possible values for the erratum number are for now: - * - AMD_PRI_DEV_ERRATUM_ENABLE_RESET - Reset PRI capability when PRI - * is enabled - * - AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE - Limit number of outstanding PRI - * requests to one - */ -#define AMD_PRI_DEV_ERRATUM_ENABLE_RESET 0 -#define AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE 1 - -extern void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum); - /** * amd_iommu_init_device() - Init device for use with IOMMUv2 driver * @pdev: The PCI device to initialize -- cgit v1.2.3 From 42fa2bda18d2d666e184dbd0406ec8340b4f4b17 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 2 Apr 2021 16:33:10 +0200 Subject: iommu/amd: Remove the unused amd_iommu_get_v2_domain function Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210402143312.372386-3-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 - drivers/iommu/amd/iommu.c | 34 ---------------------------------- 2 files changed, 35 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 026ce7f8d993..ed46dc754a2a 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -65,7 +65,6 @@ extern int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, unsigned long cr3); extern int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid); -extern struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev); #ifdef CONFIG_IRQ_REMAP extern int amd_iommu_create_irq_domain(struct amd_iommu *iommu); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 9b1a01602bb3..a519aaec50e4 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2522,40 +2522,6 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, } EXPORT_SYMBOL(amd_iommu_complete_ppr); -struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev) -{ - struct protection_domain *pdomain; - struct iommu_dev_data *dev_data; - struct device *dev = &pdev->dev; - struct iommu_domain *io_domain; - - if (!check_device(dev)) - return NULL; - - dev_data = dev_iommu_priv_get(&pdev->dev); - pdomain = dev_data->domain; - io_domain = iommu_get_domain_for_dev(dev); - - if (pdomain == NULL && dev_data->defer_attach) { - dev_data->defer_attach = false; - pdomain = to_pdomain(io_domain); - attach_device(dev, pdomain); - } - - if (pdomain == NULL) - return NULL; - - if (io_domain->type != IOMMU_DOMAIN_DMA) - return NULL; - - /* Only return IOMMUv2 domains */ - if (!(pdomain->flags & PD_IOMMUV2_MASK)) - return NULL; - - return &pdomain->domain; -} -EXPORT_SYMBOL(amd_iommu_get_v2_domain); - int amd_iommu_device_info(struct pci_dev *pdev, struct amd_iommu_device_info *info) { -- cgit v1.2.3 From b29a1fc7595a868fc49ed3083a2766c3cab06c19 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 2 Apr 2021 16:33:11 +0200 Subject: iommu/amd: Remove a few unused exports Remove exports for functions that are only used in the AMD IOMMU driver itself, or the also always builtin perf events support. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210402143312.372386-4-hch@lst.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 9126efcbaf2c..a96734feac11 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -208,7 +208,6 @@ u16 *amd_iommu_alias_table; * for a specific device. It is also indexed by the PCI device id. */ struct amd_iommu **amd_iommu_rlookup_table; -EXPORT_SYMBOL(amd_iommu_rlookup_table); /* * This table is used to find the irq remapping table for a given device id @@ -268,7 +267,6 @@ bool translation_pre_enabled(struct amd_iommu *iommu) { return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); } -EXPORT_SYMBOL(translation_pre_enabled); static void clear_translation_pre_enabled(struct amd_iommu *iommu) { @@ -3273,7 +3271,6 @@ struct amd_iommu *get_amd_iommu(unsigned int idx) return iommu; return NULL; } -EXPORT_SYMBOL(get_amd_iommu); /**************************************************************************** * @@ -3355,7 +3352,6 @@ int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); } -EXPORT_SYMBOL(amd_iommu_pc_get_reg); int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) { @@ -3364,4 +3360,3 @@ int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); } -EXPORT_SYMBOL(amd_iommu_pc_set_reg); -- cgit v1.2.3 From fc1b6620501f1a4b88f583549c63666180bea177 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 2 Apr 2021 16:33:12 +0200 Subject: iommu/amd: Move a few prototypes to include/linux/amd-iommu.h A few functions that were intentended for the perf events support are currently declared in arch/x86/events/amd/iommu.h, which mens they are not in scope for the actual function definition. Also amdkfd has started using a few of them using externs in a .c file. End that misery by moving the prototypes to the proper header. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210402143312.372386-5-hch@lst.de Signed-off-by: Joerg Roedel --- arch/x86/events/amd/iommu.c | 1 + arch/x86/events/amd/iommu.h | 19 ------------------- drivers/gpu/drm/amd/amdkfd/kfd_iommu.c | 4 ---- drivers/iommu/amd/amd_iommu.h | 1 - include/linux/amd-iommu.h | 12 ++++++++++++ 5 files changed, 13 insertions(+), 24 deletions(-) (limited to 'drivers/iommu') diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c index be50ef8572cc..a573fac1ece8 100644 --- a/arch/x86/events/amd/iommu.c +++ b/arch/x86/events/amd/iommu.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "../perf_event.h" #include "iommu.h" diff --git a/arch/x86/events/amd/iommu.h b/arch/x86/events/amd/iommu.h index 0e5c036fd7be..095505c6213b 100644 --- a/arch/x86/events/amd/iommu.h +++ b/arch/x86/events/amd/iommu.h @@ -21,23 +21,4 @@ #define PC_MAX_SPEC_BNKS 64 #define PC_MAX_SPEC_CNTRS 16 -struct amd_iommu; - -/* amd_iommu_init.c external support functions */ -extern int amd_iommu_get_num_iommus(void); - -extern bool amd_iommu_pc_supported(void); - -extern u8 amd_iommu_pc_get_max_banks(unsigned int idx); - -extern u8 amd_iommu_pc_get_max_counters(unsigned int idx); - -extern int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); - -extern int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); - -extern struct amd_iommu *get_amd_iommu(int idx); - #endif /*_PERF_EVENT_AMD_IOMMU_H_*/ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c index 66bbca61e3ef..3c56333bbb9c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c @@ -329,10 +329,6 @@ int kfd_iommu_resume(struct kfd_dev *kfd) return 0; } -extern bool amd_iommu_pc_supported(void); -extern u8 amd_iommu_pc_get_max_banks(u16 devid); -extern u8 amd_iommu_pc_get_max_counters(u16 devid); - /** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology */ int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index ed46dc754a2a..55dd38d814d9 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -11,7 +11,6 @@ #include "amd_iommu_types.h" -extern int amd_iommu_get_num_iommus(void); extern int amd_iommu_init_dma_ops(void); extern int amd_iommu_init_passthrough(void); extern irqreturn_t amd_iommu_int_thread(int irq, void *data); diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 474065ed88a4..58e6c3806c09 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -10,6 +10,8 @@ #include +struct amd_iommu; + /* * This is mainly used to communicate information back-and-forth * between SVM and IOMMU for setting up and tearing down posted @@ -194,4 +196,14 @@ static inline int amd_iommu_deactivate_guest_mode(void *data) } #endif /* defined(CONFIG_AMD_IOMMU) && defined(CONFIG_IRQ_REMAP) */ +int amd_iommu_get_num_iommus(void); +bool amd_iommu_pc_supported(void); +u8 amd_iommu_pc_get_max_banks(unsigned int idx); +u8 amd_iommu_pc_get_max_counters(unsigned int idx); +int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, + u64 *value); +int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, + u64 *value); +struct amd_iommu *get_amd_iommu(unsigned int idx); + #endif /* _ASM_X86_AMD_IOMMU_H */ -- cgit v1.2.3 From 03d205094af45bca4f8e0498c461a893aa3ec6d9 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:54:11 +0800 Subject: iommu/vt-d: Report the right page fault address The Address field of the Page Request Descriptor only keeps bit [63:12] of the offending address. Convert it to a full address before reporting it to device drivers. Fixes: eb8d93ea3c1d3 ("iommu/vt-d: Report page request faults for guest SVA") Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320025415.641201-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index f77184a9defb..5165cea90421 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -852,7 +852,7 @@ intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) /* Fill in event data for device specific processing */ memset(&event, 0, sizeof(struct iommu_fault_event)); event.fault.type = IOMMU_FAULT_PAGE_REQ; - event.fault.prm.addr = desc->addr; + event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; event.fault.prm.pasid = desc->pasid; event.fault.prm.grpid = desc->prg_index; event.fault.prm.perm = prq_to_iommu_prot(desc); -- cgit v1.2.3 From eea53c5816889ee8b64544fa2e9311a81184ff9c Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:54:12 +0800 Subject: iommu/vt-d: Remove WO permissions on second-level paging entries When the first level page table is used for IOVA translation, it only supports Read-Only and Read-Write permissions. The Write-Only permission is not supported as the PRESENT bit (implying Read permission) should always set. When using second level, we still give separate permissions that allows WriteOnly which seems inconsistent and awkward. We want to have consistent behavior. After moving to 1st level, we don't want things to work sometimes, and break if we use 2nd level for the same mappings. Hence remove this configuration. Suggested-by: Ashok Raj Fixes: b802d070a52a1 ("iommu/vt-d: Use iova over first level") Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320025415.641201-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 0ee5f1bd8af2..b0f901ee6358 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2312,8 +2312,9 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -EINVAL; attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); + attr |= DMA_FL_PTE_PRESENT; if (domain_use_first_level(domain)) { - attr |= DMA_FL_PTE_PRESENT | DMA_FL_PTE_XD | DMA_FL_PTE_US; + attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US; if (domain->domain.type == IOMMU_DOMAIN_DMA) { attr |= DMA_FL_PTE_ACCESS; -- cgit v1.2.3 From c0474a606ecb9326227b4d68059942f9db88a897 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:54:13 +0800 Subject: iommu/vt-d: Invalidate PASID cache when root/context entry changed When the Intel IOMMU is operating in the scalable mode, some information from the root and context table may be used to tag entries in the PASID cache. Software should invalidate the PASID-cache when changing root or context table entries. Suggested-by: Ashok Raj Fixes: 7373a8cc38197 ("iommu/vt-d: Setup context and enable RID2PASID support") Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320025415.641201-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 18 +++++++++--------- include/linux/intel-iommu.h | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b0f901ee6358..c981e69bc107 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1347,6 +1347,11 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) readl, (sts & DMA_GSTS_RTPS), sts); raw_spin_unlock_irqrestore(&iommu->register_lock, flag); + + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + if (sm_supported(iommu)) + qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); } void iommu_flush_write_buffer(struct intel_iommu *iommu) @@ -2430,6 +2435,10 @@ static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); + + if (sm_supported(iommu)) + qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0); + iommu->flush.flush_iotlb(iommu, did_old, 0, @@ -3278,8 +3287,6 @@ static int __init init_dmars(void) register_pasid_allocator(iommu); #endif iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); } #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA @@ -3469,12 +3476,7 @@ static int init_iommu_hw(void) } iommu_flush_write_buffer(iommu); - iommu_set_root_entry(iommu); - - iommu->flush.flush_context(iommu, 0, 0, 0, - DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); } @@ -3857,8 +3859,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) goto disable_iommu; iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index e0f8c2ade3e8..03faf20a6817 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -378,6 +378,7 @@ enum { /* PASID cache invalidation granu */ #define QI_PC_ALL_PASIDS 0 #define QI_PC_PASID_SEL 1 +#define QI_PC_GLOBAL 3 #define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) #define QI_EIOTLB_IH(ih) (((u64)ih) << 6) -- cgit v1.2.3 From 8b74b6ab253866450c131e9134642efb40439c91 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 20 Mar 2021 10:54:15 +0800 Subject: iommu/vt-d: Avoid unnecessary cache flush in pasid entry teardown When a present pasid entry is disassembled, all kinds of pasid related caches need to be flushed. But when a pasid entry is not being used (PRESENT bit not set), we don't need to do this. Check the PRESENT bit in intel_pasid_tear_down_entry() and avoid flushing caches if it's not set. Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210320025415.641201-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 8f2a702ce429..477b2e1d303c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -517,6 +517,9 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, if (WARN_ON(!pte)) return; + if (!(pte->val[0] & PASID_PTE_PRESENT)) + return; + did = pasid_get_domain_id(pte); intel_pasid_clear_entry(dev, pasid, fault_ignore); -- cgit v1.2.3 From e0bb4b73540495111ff2723e41cf5add2f031021 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Wed, 7 Apr 2021 16:44:48 +0800 Subject: iommu/arm-smmu-v3: Remove the unused fields for PREFETCH_CONFIG command Per SMMUv3 spec, there is no Size and Addr field in the PREFETCH_CONFIG command and they're not used by the driver. Remove them. We can add them back if we're going to use PREFETCH_ADDR in the future. Signed-off-by: Zenghui Yu Link: https://lore.kernel.org/r/20210407084448.1838-1-yuzenghui@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 -- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 -- 2 files changed, 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 941ba5484731..b9552120b708 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -245,8 +245,6 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) break; case CMDQ_OP_PREFETCH_CFG: cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); - cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size); - cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK; break; case CMDQ_OP_CFGI_CD: cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 230b6f6b3901..49b11582519a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -410,8 +410,6 @@ struct arm_smmu_cmdq_ent { #define CMDQ_OP_PREFETCH_CFG 0x1 struct { u32 sid; - u8 size; - u64 addr; } prefetch; #define CMDQ_OP_CFGI_STE 0x3 -- cgit v1.2.3 From 268aa4548277a1e50f326c6fbca75dd1073574d4 Mon Sep 17 00:00:00 2001 From: Nadav Amit Date: Tue, 23 Mar 2021 14:06:19 -0700 Subject: iommu/amd: Page-specific invalidations for more than one page Currently, IOMMU invalidations and device-IOTLB invalidations using AMD IOMMU fall back to full address-space invalidation if more than a single page need to be flushed. Full flushes are especially inefficient when the IOMMU is virtualized by a hypervisor, since it requires the hypervisor to synchronize the entire address-space. AMD IOMMUs allow to provide a mask to perform page-specific invalidations for multiple pages that match the address. The mask is encoded as part of the address, and the first zero bit in the address (in bits [51:12]) indicates the mask size. Use this hardware feature to perform selective IOMMU and IOTLB flushes. Combine the logic between both for better code reuse. The IOMMU invalidations passed a smoke-test. The device IOTLB invalidations are untested. Cc: Joerg Roedel Cc: Will Deacon Cc: Jiajun Cao Cc: iommu@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Nadav Amit Link: https://lore.kernel.org/r/20210323210619.513069-1-namit@vmware.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 76 ++++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 34 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a519aaec50e4..f164f31909f8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -852,33 +852,58 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); } -static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, - size_t size, u16 domid, int pde) +/* + * Builds an invalidation address which is suitable for one page or multiple + * pages. Sets the size bit (S) as needed is more than one page is flushed. + */ +static inline u64 build_inv_address(u64 address, size_t size) { - u64 pages; - bool s; + u64 pages, end, msb_diff; pages = iommu_num_pages(address, size, PAGE_SIZE); - s = false; - if (pages > 1) { + if (pages == 1) + return address & PAGE_MASK; + + end = address + size - 1; + + /* + * msb_diff would hold the index of the most significant bit that + * flipped between the start and end. + */ + msb_diff = fls64(end ^ address) - 1; + + /* + * Bits 63:52 are sign extended. If for some reason bit 51 is different + * between the start and the end, invalidate everything. + */ + if (unlikely(msb_diff > 51)) { + address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; + } else { /* - * If we have to flush more than one page, flush all - * TLB entries for this domain + * The msb-bit must be clear on the address. Just set all the + * lower bits. */ - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = true; + address |= 1ull << (msb_diff - 1); } + /* Clear bits 11:0 */ address &= PAGE_MASK; + /* Set the size bit - we flush more than one 4kb page */ + return address | CMD_INV_IOMMU_PAGES_SIZE_MASK; +} + +static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, + size_t size, u16 domid, int pde) +{ + u64 inv_address = build_inv_address(address, size); + memset(cmd, 0, sizeof(*cmd)); cmd->data[1] |= domid; - cmd->data[2] = lower_32_bits(address); - cmd->data[3] = upper_32_bits(address); + cmd->data[2] = lower_32_bits(inv_address); + cmd->data[3] = upper_32_bits(inv_address); CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); - if (s) /* size bit - we flush more than one 4kb page */ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; if (pde) /* PDE bit - we want to flush everything, not only the PTEs */ cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; } @@ -886,32 +911,15 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, u64 address, size_t size) { - u64 pages; - bool s; - - pages = iommu_num_pages(address, size, PAGE_SIZE); - s = false; - - if (pages > 1) { - /* - * If we have to flush more than one page, flush all - * TLB entries for this domain - */ - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = true; - } - - address &= PAGE_MASK; + u64 inv_address = build_inv_address(address, size); memset(cmd, 0, sizeof(*cmd)); cmd->data[0] = devid; cmd->data[0] |= (qdep & 0xff) << 24; cmd->data[1] = devid; - cmd->data[2] = lower_32_bits(address); - cmd->data[3] = upper_32_bits(address); + cmd->data[2] = lower_32_bits(inv_address); + cmd->data[3] = upper_32_bits(inv_address); CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); - if (s) - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; } static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, u32 pasid, -- cgit v1.2.3 From d0272ea1839b2fd013f11289c22166005ef774ce Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Apr 2021 22:16:22 +0200 Subject: iommu/exynos: Remove unneeded local variable initialization The initialization of 'fault_addr' local variable is not needed as it is shortly after overwritten. Addresses-Coverity: Unused value Signed-off-by: Krzysztof Kozlowski Acked-by: Marek Szyprowski Link: https://lore.kernel.org/r/20210408201622.78009-1-krzysztof.kozlowski@canonical.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index de324b4eedfe..8fa9a591fb96 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -407,7 +407,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) struct sysmmu_drvdata *data = dev_id; const struct sysmmu_fault_info *finfo; unsigned int i, n, itype; - sysmmu_iova_t fault_addr = -1; + sysmmu_iova_t fault_addr; unsigned short reg_status, reg_clear; int ret = -ENOSYS; -- cgit v1.2.3 From 45ed93374a3a66ed35412f18fa356b3550c9f622 Mon Sep 17 00:00:00 2001 From: Shaokun Zhang Date: Fri, 9 Apr 2021 11:30:40 +0800 Subject: iommu/amd: Remove duplicate check of devid 'devid' has been checked in function check_device, no need to double check and clean up this. Cc: Joerg Roedel Cc: Will Deacon Signed-off-by: Shaokun Zhang Link: https://lore.kernel.org/r/1617939040-35579-1-git-send-email-zhangshaokun@hisilicon.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f164f31909f8..4ed9bc72cf98 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1685,9 +1685,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) return ERR_PTR(-ENODEV); devid = get_device_id(dev); - if (devid < 0) - return ERR_PTR(devid); - iommu = amd_iommu_rlookup_table[devid]; if (dev_iommu_priv_get(dev)) @@ -1989,16 +1986,12 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + int devid = get_device_id(dev); struct amd_iommu *iommu; - int devid; if (!check_device(dev)) return; - devid = get_device_id(dev); - if (devid < 0) - return; - if (dev_data->domain != NULL) detach_device(dev); -- cgit v1.2.3 From 715601e4e36903a653cd4294dfd3ed0019101991 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Fri, 9 Apr 2021 03:58:47 -0500 Subject: Revert "iommu/amd: Fix performance counter initialization" This reverts commit 6778ff5b21bd8e78c8bd547fd66437cf2657fd9b. The original commit tries to address an issue, where PMC power-gating causing the IOMMU PMC pre-init test to fail on certain desktop/mobile platforms where the power-gating is normally enabled. There have been several reports that the workaround still does not guarantee to work, and can add up to 100 ms (on the worst case) to the boot process on certain platforms such as the MSI B350M MORTAR with AMD Ryzen 3 2200G. Therefore, revert this commit as a prelude to removing the pre-init test. Link: https://lore.kernel.org/linux-iommu/alpine.LNX.3.20.13.2006030935570.3181@monopod.intra.ispras.ru/ Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=201753 Cc: Tj (Elloe Linux) Cc: Shuah Khan Cc: Alexander Monakov Cc: David Coe Signed-off-by: Paul Menzel Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20210409085848.3908-2-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 45 +++++++++++---------------------------------- 1 file changed, 11 insertions(+), 34 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index a96734feac11..bd7380804df2 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -256,8 +255,6 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); static void init_device_table_dma(void); -static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value, bool is_write); static bool amd_iommu_pre_enabled = true; @@ -1715,11 +1712,13 @@ static int __init init_iommu_all(struct acpi_table_header *table) return 0; } -static void __init init_iommu_perf_ctr(struct amd_iommu *iommu) +static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, + u8 fxn, u64 *value, bool is_write); + +static void init_iommu_perf_ctr(struct amd_iommu *iommu) { - int retry; struct pci_dev *pdev = iommu->dev; - u64 val = 0xabcd, val2 = 0, save_reg, save_src; + u64 val = 0xabcd, val2 = 0, save_reg = 0; if (!iommu_feature(iommu, FEATURE_PC)) return; @@ -1727,39 +1726,17 @@ static void __init init_iommu_perf_ctr(struct amd_iommu *iommu) amd_iommu_pc_present = true; /* save the value to restore, if writable */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false) || - iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, false)) - goto pc_false; - - /* - * Disable power gating by programing the performance counter - * source to 20 (i.e. counts the reads and writes from/to IOMMU - * Reserved Register [MMIO Offset 1FF8h] that are ignored.), - * which never get incremented during this init phase. - * (Note: The event is also deprecated.) - */ - val = 20; - if (iommu_pc_get_set_reg(iommu, 0, 0, 8, &val, true)) + if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false)) goto pc_false; /* Check if the performance counters can be written to */ - val = 0xabcd; - for (retry = 5; retry; retry--) { - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true) || - iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false) || - val2) - break; - - /* Wait about 20 msec for power gating to disable and retry. */ - msleep(20); - } - - /* restore */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true) || - iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, true)) + if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || + (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || + (val != val2)) goto pc_false; - if (val != val2) + /* restore */ + if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true)) goto pc_false; pci_info(pdev, "IOMMU performance counters supported\n"); -- cgit v1.2.3 From 994d6608efe4a4c8834bdc5014c86f4bc6aceea6 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Fri, 9 Apr 2021 03:58:48 -0500 Subject: iommu/amd: Remove performance counter pre-initialization test In early AMD desktop/mobile platforms (during 2013), when the IOMMU Performance Counter (PMC) support was first introduced in commit 30861ddc9cca ("perf/x86/amd: Add IOMMU Performance Counter resource management"), there was a HW bug where the counters could not be accessed. The result was reading of the counter always return zero. At the time, the suggested workaround was to add a test logic prior to initializing the PMC feature to check if the counters can be programmed and read back the same value. This has been working fine until the more recent desktop/mobile platforms start enabling power gating for the PMC, which prevents access to the counters. This results in the PMC support being disabled unnecesarily. Unfortunatly, there is no documentation of since which generation of hardware the original PMC HW bug was fixed. Although, it was fixed soon after the first introduction of the PMC. Base on this, we assume that the buggy platforms are less likely to be in used, and it should be relatively safe to remove this legacy logic. Link: https://lore.kernel.org/linux-iommu/alpine.LNX.3.20.13.2006030935570.3181@monopod.intra.ispras.ru/ Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=201753 Cc: Tj (Elloe Linux) Cc: Shuah Khan Cc: Alexander Monakov Cc: David Coe Cc: Paul Menzel Signed-off-by: Suravee Suthikulpanit Tested-by: Shuah Khan Link: https://lore.kernel.org/r/20210409085848.3908-3-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index bd7380804df2..625d520da140 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1712,33 +1712,16 @@ static int __init init_iommu_all(struct acpi_table_header *table) return 0; } -static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value, bool is_write); - static void init_iommu_perf_ctr(struct amd_iommu *iommu) { + u64 val; struct pci_dev *pdev = iommu->dev; - u64 val = 0xabcd, val2 = 0, save_reg = 0; if (!iommu_feature(iommu, FEATURE_PC)) return; amd_iommu_pc_present = true; - /* save the value to restore, if writable */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false)) - goto pc_false; - - /* Check if the performance counters can be written to */ - if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || - (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || - (val != val2)) - goto pc_false; - - /* restore */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true)) - goto pc_false; - pci_info(pdev, "IOMMU performance counters supported\n"); val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); @@ -1746,11 +1729,6 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) iommu->max_counters = (u8) ((val >> 7) & 0xf); return; - -pc_false: - pci_err(pdev, "Unable to read/write to IOMMU perf counter.\n"); - amd_iommu_pc_present = false; - return; } static ssize_t amd_iommu_show_cap(struct device *dev, -- cgit v1.2.3 From 906f86c860304e84c36cd4c42a6a73928c4ed74c Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sun, 11 Apr 2021 14:23:12 +0800 Subject: iommu/vt-d: Fix build error of pasid_enable_wpe() with !X86 Commit f68c7f539b6e9 ("iommu/vt-d: Enable write protect for supervisor SVM") added pasid_enable_wpe() which hit below compile error with !X86. ../drivers/iommu/intel/pasid.c: In function 'pasid_enable_wpe': ../drivers/iommu/intel/pasid.c:554:22: error: implicit declaration of function 'read_cr0' [-Werror=implicit-function-declaration] 554 | unsigned long cr0 = read_cr0(); | ^~~~~~~~ In file included from ../include/linux/build_bug.h:5, from ../include/linux/bits.h:22, from ../include/linux/bitops.h:6, from ../drivers/iommu/intel/pasid.c:12: ../drivers/iommu/intel/pasid.c:557:23: error: 'X86_CR0_WP' undeclared (first use in this function) 557 | if (unlikely(!(cr0 & X86_CR0_WP))) { | ^~~~~~~~~~ ../include/linux/compiler.h:78:42: note: in definition of macro 'unlikely' 78 | # define unlikely(x) __builtin_expect(!!(x), 0) | ^ ../drivers/iommu/intel/pasid.c:557:23: note: each undeclared identifier is reported only once for each function it appears in 557 | if (unlikely(!(cr0 & X86_CR0_WP))) { | ^~~~~~~~~~ ../include/linux/compiler.h:78:42: note: in definition of macro 'unlikely' 78 | # define unlikely(x) __builtin_expect(!!(x), 0) | Add the missing dependency. Cc: Sanjay Kumar Cc: Jacob Pan Cc: Randy Dunlap Reported-by: kernel test robot Reported-by: Randy Dunlap Fixes: f68c7f539b6e9 ("iommu/vt-d: Enable write protect for supervisor SVM") Signed-off-by: Lu Baolu Acked-by: Randy Dunlap # build-tested Link: https://lore.kernel.org/r/20210411062312.3057579-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 477b2e1d303c..72646bafc52f 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -551,6 +551,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu, static inline int pasid_enable_wpe(struct pasid_entry *pte) { +#ifdef CONFIG_X86 unsigned long cr0 = read_cr0(); /* CR0.WP is normally set but just to be sure */ @@ -558,6 +559,7 @@ static inline int pasid_enable_wpe(struct pasid_entry *pte) pr_err_ratelimited("No CPU write protect!\n"); return -EINVAL; } +#endif pasid_set_wpe(pte); return 0; -- cgit v1.2.3 From 745610c4a3e3baaebf6d1f8cd5b4d82892432520 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 11 Apr 2021 09:08:17 +0200 Subject: iommu/vt-d: Fix an error handling path in 'intel_prepare_irq_remapping()' If 'intel_cap_audit()' fails, we should return directly, as already done in the surrounding error handling path. Fixes: ad3d19029979 ("iommu/vt-d: Audit IOMMU Capabilities and add helper functions") Signed-off-by: Christophe JAILLET Acked-by: Lu Baolu Link: https://lore.kernel.org/r/98d531caabe66012b4fffc7813fd4b9470afd517.1618124777.git.christophe.jaillet@wanadoo.fr Signed-off-by: Joerg Roedel --- drivers/iommu/intel/irq_remapping.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 611ef5243cb6..5c16ebe037a1 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -736,7 +736,7 @@ static int __init intel_prepare_irq_remapping(void) return -ENODEV; if (intel_cap_audit(CAP_AUDIT_STATIC_IRQR, NULL)) - goto error; + return -ENODEV; if (!dmar_ir_support()) return -ENODEV; -- cgit v1.2.3 From 304c73ba69459d4c18c2a4b843be6f5777b4b85c Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 12 Apr 2021 20:01:41 +0200 Subject: iommu/amd: Put newline after closing bracket in warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, on the Dell OptiPlex 5055 the EFR mismatch warning looks like below. [ 1.479774] smpboot: CPU0: AMD Ryzen 5 PRO 1500 Quad-Core Processor (family: 0x17, model: 0x1, stepping: 0x1) […] [ 2.507370] AMD-Vi: [Firmware Warn]: EFR mismatch. Use IVHD EFR (0xf77ef22294ada : 0x400f77ef22294ada ). Add the newline after the `).`, so it’s on one line. Fixes: a44092e326d4 ("iommu/amd: Use IVHD EFR for early initialization of IOMMU features") Cc: iommu@lists.linux-foundation.org Cc: Suravee Suthikulpanit Cc: Brijesh Singh Cc: Robert Richter Signed-off-by: Paul Menzel Link: https://lore.kernel.org/r/20210412180141.29605-1-pmenzel@molgen.mpg.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 625d520da140..6aac8720fc8a 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1790,7 +1790,7 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu) * IVHD and MMIO conflict. */ if (features != iommu->features) - pr_warn(FW_WARN "EFR mismatch. Use IVHD EFR (%#llx : %#llx\n).", + pr_warn(FW_WARN "EFR mismatch. Use IVHD EFR (%#llx : %#llx).\n", features, iommu->features); } -- cgit v1.2.3 From 38c527aeb41926c71902dd42f788a8b093b21416 Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Thu, 15 Apr 2021 08:46:28 +0800 Subject: iommu/vt-d: Force to flush iotlb before creating superpage The translation caches may preserve obsolete data when the mapping size is changed, suppose the following sequence which can reveal the problem with high probability. 1.mmap(4GB,MAP_HUGETLB) 2. while (1) { (a) DMA MAP 0,0xa0000 (b) DMA UNMAP 0,0xa0000 (c) DMA MAP 0,0xc0000000 * DMA read IOVA 0 may failure here (Not present) * if the problem occurs. (d) DMA UNMAP 0,0xc0000000 } The page table(only focus on IOVA 0) after (a) is: PML4: 0x19db5c1003 entry:0xffff899bdcd2f000 PDPE: 0x1a1cacb003 entry:0xffff89b35b5c1000 PDE: 0x1a30a72003 entry:0xffff89b39cacb000 PTE: 0x21d200803 entry:0xffff89b3b0a72000 The page table after (b) is: PML4: 0x19db5c1003 entry:0xffff899bdcd2f000 PDPE: 0x1a1cacb003 entry:0xffff89b35b5c1000 PDE: 0x1a30a72003 entry:0xffff89b39cacb000 PTE: 0x0 entry:0xffff89b3b0a72000 The page table after (c) is: PML4: 0x19db5c1003 entry:0xffff899bdcd2f000 PDPE: 0x1a1cacb003 entry:0xffff89b35b5c1000 PDE: 0x21d200883 entry:0xffff89b39cacb000 (*) Because the PDE entry after (b) is present, it won't be flushed even if the iommu driver flush cache when unmap, so the obsolete data may be preserved in cache, which would cause the wrong translation at end. However, we can see the PDE entry is finally switch to 2M-superpage mapping, but it does not transform to 0x21d200883 directly: 1. PDE: 0x1a30a72003 2. __domain_mapping dma_pte_free_pagetable Set the PDE entry to ZERO Set the PDE entry to 0x21d200883 So we must flush the cache after the entry switch to ZERO to avoid the obsolete info be preserved. Cc: David Woodhouse Cc: Lu Baolu Cc: Nadav Amit Cc: Alex Williamson Cc: Joerg Roedel Cc: Kevin Tian Cc: Gonglei (Arei) Fixes: 6491d4d02893 ("intel-iommu: Free old page tables before creating superpage") Cc: # v3.0+ Link: https://lore.kernel.org/linux-iommu/670baaf8-4ff8-4e84-4be3-030b95ab5a5e@huawei.com/ Suggested-by: Lu Baolu Signed-off-by: Longpeng(Mike) Acked-by: Lu Baolu Link: https://lore.kernel.org/r/20210415004628.1779-1-longpeng2@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 52 +++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 14 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index c981e69bc107..ab2a026dd844 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2301,6 +2301,41 @@ static inline int hardware_largepage_caps(struct dmar_domain *domain, return level; } +/* + * Ensure that old small page tables are removed to make room for superpage(s). + * We're going to add new large pages, so make sure we don't remove their parent + * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared. + */ +static void switch_to_super_page(struct dmar_domain *domain, + unsigned long start_pfn, + unsigned long end_pfn, int level) +{ + unsigned long lvl_pages = lvl_to_nr_pages(level); + struct dma_pte *pte = NULL; + int i; + + while (start_pfn <= end_pfn) { + if (!pte) + pte = pfn_to_dma_pte(domain, start_pfn, &level); + + if (dma_pte_present(pte)) { + dma_pte_free_pagetable(domain, start_pfn, + start_pfn + lvl_pages - 1, + level + 1); + + for_each_domain_iommu(i, domain) + iommu_flush_iotlb_psi(g_iommus[i], domain, + start_pfn, lvl_pages, + 0, 0); + } + + pte++; + start_pfn += lvl_pages; + if (first_pte_in_page(pte)) + pte = NULL; + } +} + static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long phys_pfn, unsigned long nr_pages, int prot) @@ -2342,22 +2377,11 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -ENOMEM; /* It is large page*/ if (largepage_lvl > 1) { - unsigned long nr_superpages, end_pfn; + unsigned long end_pfn; pteval |= DMA_PTE_LARGE_PAGE; - lvl_pages = lvl_to_nr_pages(largepage_lvl); - - nr_superpages = nr_pages / lvl_pages; - end_pfn = iov_pfn + nr_superpages * lvl_pages - 1; - - /* - * Ensure that old small page tables are - * removed to make room for superpage(s). - * We're adding new large pages, so make sure - * we don't remove their parent tables. - */ - dma_pte_free_pagetable(domain, iov_pfn, end_pfn, - largepage_lvl + 1); + end_pfn = ((iov_pfn + nr_pages) & level_mask(largepage_lvl)) - 1; + switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl); } else { pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE; } -- cgit v1.2.3 From 84b6269c3c28b920088d4d5ae3723dc740221ae9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 15 Apr 2021 16:44:42 +0200 Subject: iommu/fsl-pamu: Fix uninitialized variable warning The variable 'i' in the function update_liodn_stash() is not initialized and only used in a debug printk(). So it has no meaning at all, remove it. Reported-by: kernel test robot Signed-off-by: Joerg Roedel Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210415144442.25103-1-joro@8bytes.org --- drivers/iommu/fsl_pamu_domain.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 32944d67baa4..0ac781186dbd 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -57,14 +57,13 @@ static int __init iommu_init_mempool(void) static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, u32 val) { - int ret = 0, i; + int ret = 0; unsigned long flags; spin_lock_irqsave(&iommu_lock, flags); ret = pamu_update_paace_stash(liodn, val); if (ret) { - pr_debug("Failed to update SPAACE %d field for liodn %d\n ", - i, liodn); + pr_debug("Failed to update SPAACE for liodn %d\n ", liodn); spin_unlock_irqrestore(&iommu_lock, flags); return ret; } -- cgit v1.2.3 From b34ea31fe013569d42b7e8681ef3f717f77c5b72 Mon Sep 17 00:00:00 2001 From: Dafna Hirschfeld Date: Fri, 16 Apr 2021 12:54:49 +0200 Subject: iommu/mediatek: Always enable the clk on resume In mtk_iommu_runtime_resume always enable the clk, even if m4u_dom is null. Otherwise the 'suspend' cb might disable the clk which is already disabled causing the warning: [ 1.586104] infra_m4u already disabled [ 1.586133] WARNING: CPU: 0 PID: 121 at drivers/clk/clk.c:952 clk_core_disable+0xb0/0xb8 [ 1.594391] mtk-iommu 10205000.iommu: bound 18001000.larb (ops mtk_smi_larb_component_ops) [ 1.598108] Modules linked in: [ 1.598114] CPU: 0 PID: 121 Comm: kworker/0:2 Not tainted 5.12.0-rc5 #69 [ 1.609246] mtk-iommu 10205000.iommu: bound 14027000.larb (ops mtk_smi_larb_component_ops) [ 1.617487] Hardware name: Google Elm (DT) [ 1.617491] Workqueue: pm pm_runtime_work [ 1.620545] mtk-iommu 10205000.iommu: bound 19001000.larb (ops mtk_smi_larb_component_ops) [ 1.627229] pstate: 60000085 (nZCv daIf -PAN -UAO -TCO BTYPE=--) [ 1.659297] pc : clk_core_disable+0xb0/0xb8 [ 1.663475] lr : clk_core_disable+0xb0/0xb8 [ 1.667652] sp : ffff800011b9bbe0 [ 1.670959] x29: ffff800011b9bbe0 x28: 0000000000000000 [ 1.676267] x27: ffff800011448000 x26: ffff8000100cfd98 [ 1.681574] x25: ffff800011b9bd48 x24: 0000000000000000 [ 1.686882] x23: 0000000000000000 x22: ffff8000106fad90 [ 1.692189] x21: 000000000000000a x20: ffff0000c0048500 [ 1.697496] x19: ffff0000c0048500 x18: ffffffffffffffff [ 1.702804] x17: 0000000000000000 x16: 0000000000000000 [ 1.708112] x15: ffff800011460300 x14: fffffffffffe0000 [ 1.713420] x13: ffff8000114602d8 x12: 0720072007200720 [ 1.718727] x11: 0720072007200720 x10: 0720072007200720 [ 1.724035] x9 : ffff800011b9bbe0 x8 : ffff800011b9bbe0 [ 1.729342] x7 : 0000000000000009 x6 : ffff8000114b8328 [ 1.734649] x5 : 0000000000000000 x4 : 0000000000000000 [ 1.739956] x3 : 00000000ffffffff x2 : ffff800011460298 [ 1.745263] x1 : 1af1d7de276f4500 x0 : 0000000000000000 [ 1.750572] Call trace: [ 1.753010] clk_core_disable+0xb0/0xb8 [ 1.756840] clk_core_disable_lock+0x24/0x40 [ 1.761105] clk_disable+0x20/0x30 [ 1.764501] mtk_iommu_runtime_suspend+0x88/0xa8 [ 1.769114] pm_generic_runtime_suspend+0x2c/0x48 [ 1.773815] __rpm_callback+0xe0/0x178 [ 1.777559] rpm_callback+0x24/0x88 [ 1.781041] rpm_suspend+0xdc/0x470 [ 1.784523] rpm_idle+0x12c/0x170 [ 1.787831] pm_runtime_work+0xa8/0xc0 [ 1.791573] process_one_work+0x1e8/0x360 [ 1.795580] worker_thread+0x44/0x478 [ 1.799237] kthread+0x150/0x158 [ 1.802460] ret_from_fork+0x10/0x30 [ 1.806034] ---[ end trace 82402920ef64573b ]--- [ 1.810728] ------------[ cut here ]------------ In addition, we now don't need to enable the clock from the function mtk_iommu_hw_init since it is already enabled by the resume. Fixes: c0b57581b73b ("iommu/mediatek: Add power-domain operation") Signed-off-by: Dafna Hirschfeld Reviewed-by: Yong Wu Link: https://lore.kernel.org/r/20210416105449.4744-1-dafna.hirschfeld@collabora.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 75375935f301..cdda3a85fc08 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -690,13 +690,6 @@ static const struct iommu_ops mtk_iommu_ops = { static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) { u32 regval; - int ret; - - ret = clk_prepare_enable(data->bclk); - if (ret) { - dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); - return ret; - } if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | @@ -762,7 +755,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, dev_name(data->dev), (void *)data)) { writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); - clk_disable_unprepare(data->bclk); dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); return -ENODEV; } @@ -979,14 +971,19 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) void __iomem *base = data->base; int ret; - /* Avoid first resume to affect the default value of registers below. */ - if (!m4u_dom) - return 0; ret = clk_prepare_enable(data->bclk); if (ret) { dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); return ret; } + + /* + * Uppon first resume, only enable the clk and return, since the values of the + * registers are not yet set. + */ + if (!m4u_dom) + return 0; + writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); -- cgit v1.2.3 From f90a9a85fb946e11092f8ca43e30210a2d427bf4 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 12 Apr 2021 14:48:42 +0800 Subject: iommu/mediatek-v1: Avoid build fail when build as module When this driver build as module, It build fail like: ERROR: modpost: "of_phandle_iterator_args" [drivers/iommu/mtk_iommu_v1.ko] undefined! This patch remove this interface to avoid this build fail. Reported-by: Valdis Kletnieks Signed-off-by: Yong Wu Link: https://lore.kernel.org/r/20210412064843.11614-1-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 62 ++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 34 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index be1b20e3f20e..bed0bb9d63fd 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -424,23 +424,21 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct of_phandle_args iommu_spec; - struct of_phandle_iterator it; struct mtk_iommu_data *data; - int err; + int err, idx = 0; - of_for_each_phandle(&it, err, dev->of_node, "iommus", - "#iommu-cells", -1) { - int count = of_phandle_iterator_args(&it, iommu_spec.args, - MAX_PHANDLE_ARGS); - iommu_spec.np = of_node_get(it.node); - iommu_spec.args_count = count; + while (!of_parse_phandle_with_args(dev->of_node, "iommus", + "#iommu-cells", + idx, &iommu_spec)) { - mtk_iommu_create_mapping(dev, &iommu_spec); + err = mtk_iommu_create_mapping(dev, &iommu_spec); + of_node_put(iommu_spec.np); + if (err) + return ERR_PTR(err); /* dev->iommu_fwspec might have changed */ fwspec = dev_iommu_fwspec_get(dev); - - of_node_put(iommu_spec.np); + idx++; } if (!fwspec || fwspec->ops != &mtk_iommu_ops) @@ -549,10 +547,8 @@ static int mtk_iommu_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct resource *res; struct component_match *match = NULL; - struct of_phandle_args larb_spec; - struct of_phandle_iterator it; void *protect; - int larb_nr, ret, err; + int larb_nr, ret, i; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -580,35 +576,33 @@ static int mtk_iommu_probe(struct platform_device *pdev) if (IS_ERR(data->bclk)) return PTR_ERR(data->bclk); - larb_nr = 0; - of_for_each_phandle(&it, err, dev->of_node, - "mediatek,larbs", NULL, 0) { + larb_nr = of_count_phandle_with_args(dev->of_node, + "mediatek,larbs", NULL); + if (larb_nr < 0) + return larb_nr; + + for (i = 0; i < larb_nr; i++) { + struct device_node *larbnode; struct platform_device *plarbdev; - int count = of_phandle_iterator_args(&it, larb_spec.args, - MAX_PHANDLE_ARGS); - if (count) - continue; + larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); + if (!larbnode) + return -EINVAL; - larb_spec.np = of_node_get(it.node); - if (!of_device_is_available(larb_spec.np)) + if (!of_device_is_available(larbnode)) { + of_node_put(larbnode); continue; + } - plarbdev = of_find_device_by_node(larb_spec.np); + plarbdev = of_find_device_by_node(larbnode); if (!plarbdev) { - plarbdev = of_platform_device_create( - larb_spec.np, NULL, - platform_bus_type.dev_root); - if (!plarbdev) { - of_node_put(larb_spec.np); - return -EPROBE_DEFER; - } + of_node_put(larbnode); + return -EPROBE_DEFER; } + data->larb_imu[i].dev = &plarbdev->dev; - data->larb_imu[larb_nr].dev = &plarbdev->dev; component_match_add_release(dev, &match, release_of, - compare_of, larb_spec.np); - larb_nr++; + compare_of, larbnode); } platform_set_drvdata(pdev, data); -- cgit v1.2.3 From ac304c070c54413efabf29f9e73c54576d329774 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 12 Apr 2021 14:48:43 +0800 Subject: iommu/mediatek-v1: Add error handle for mtk_iommu_probe In the original code, we lack the error handle. This patch adds them. Signed-off-by: Yong Wu Link: https://lore.kernel.org/r/20210412064843.11614-2-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index bed0bb9d63fd..8ba9a2ec5509 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -620,12 +620,26 @@ static int mtk_iommu_probe(struct platform_device *pdev) ret = iommu_device_register(&data->iommu); if (ret) - return ret; + goto out_sysfs_remove; - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + if (!iommu_present(&platform_bus_type)) { + ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + if (ret) + goto out_dev_unreg; + } - return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + if (ret) + goto out_bus_set_null; + return ret; + +out_bus_set_null: + bus_set_iommu(&platform_bus_type, NULL); +out_dev_unreg: + iommu_device_unregister(&data->iommu); +out_sysfs_remove: + iommu_device_sysfs_remove(&data->iommu); + return ret; } static int mtk_iommu_remove(struct platform_device *pdev) -- cgit v1.2.3