summaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd_iommu.c42
-rw-r--r--drivers/iommu/arm-smmu.c331
-rw-r--r--drivers/iommu/dmar.c28
-rw-r--r--drivers/iommu/exynos-iommu.c51
-rw-r--r--drivers/iommu/fsl_pamu_domain.c15
-rw-r--r--drivers/iommu/intel-iommu.c16
-rw-r--r--drivers/iommu/intel_irq_remapping.c7
-rw-r--r--drivers/iommu/iommu.c216
-rw-r--r--drivers/iommu/irq_remapping.c11
-rw-r--r--drivers/iommu/irq_remapping.h2
-rw-r--r--drivers/iommu/msm_iommu.c7
-rw-r--r--drivers/iommu/omap-iommu.c27
-rw-r--r--drivers/iommu/omap-iommu.h1
-rw-r--r--drivers/iommu/tegra-gart.c9
-rw-r--r--drivers/iommu/tegra-smmu.c9
15 files changed, 443 insertions, 329 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 23c5be6a3a1e..505a9adac2d5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -282,17 +282,13 @@ static bool check_device(struct device *dev)
return true;
}
-static int init_iommu_group(struct device *dev)
+static void init_iommu_group(struct device *dev)
{
struct iommu_group *group;
group = iommu_group_get_for_dev(dev);
-
- if (IS_ERR(group))
- return PTR_ERR(group);
-
- iommu_group_put(group);
- return 0;
+ if (!IS_ERR(group))
+ iommu_group_put(group);
}
static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
@@ -362,7 +358,6 @@ static int iommu_init_device(struct device *dev)
struct pci_dev *pdev = to_pci_dev(dev);
struct iommu_dev_data *dev_data;
u16 alias;
- int ret;
if (dev->archdata.iommu)
return 0;
@@ -389,12 +384,6 @@ static int iommu_init_device(struct device *dev)
list_add(&dev_data->alias_list, &alias_data->alias_list);
}
- ret = init_iommu_group(dev);
- if (ret) {
- free_dev_data(dev_data);
- return ret;
- }
-
if (pci_iommuv2_capable(pdev)) {
struct amd_iommu *iommu;
@@ -480,6 +469,15 @@ int __init amd_iommu_init_devices(void)
goto out_free;
}
+ /*
+ * Initialize IOMMU groups only after iommu_init_device() has
+ * had a chance to populate any IVRS defined aliases.
+ */
+ for_each_pci_dev(pdev) {
+ if (check_device(&pdev->dev))
+ init_iommu_group(&pdev->dev);
+ }
+
return 0;
out_free:
@@ -2437,6 +2435,7 @@ static int device_change_notifier(struct notifier_block *nb,
case BUS_NOTIFY_ADD_DEVICE:
iommu_init_device(dev);
+ init_iommu_group(dev);
/*
* dev_data is still NULL and
@@ -3405,20 +3404,20 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
return paddr;
}
-static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool amd_iommu_capable(enum iommu_cap cap)
{
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
- return 1;
+ return true;
case IOMMU_CAP_INTR_REMAP:
- return irq_remapping_enabled;
+ return (irq_remapping_enabled == 1);
}
- return 0;
+ return false;
}
static const struct iommu_ops amd_iommu_ops = {
+ .capable = amd_iommu_capable,
.domain_init = amd_iommu_domain_init,
.domain_destroy = amd_iommu_domain_destroy,
.attach_dev = amd_iommu_attach_device,
@@ -3426,7 +3425,6 @@ static const struct iommu_ops amd_iommu_ops = {
.map = amd_iommu_map,
.unmap = amd_iommu_unmap,
.iova_to_phys = amd_iommu_iova_to_phys,
- .domain_has_cap = amd_iommu_domain_has_cap,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};
@@ -4256,7 +4254,7 @@ static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
return 0;
}
-static int setup_hpet_msi(unsigned int irq, unsigned int id)
+static int alloc_hpet_msi(unsigned int irq, unsigned int id)
{
struct irq_2_irte *irte_info;
struct irq_cfg *cfg;
@@ -4295,6 +4293,6 @@ struct irq_remap_ops amd_iommu_irq_ops = {
.compose_msi_msg = compose_msi_msg,
.msi_alloc_irq = msi_alloc_irq,
.msi_setup_irq = msi_setup_irq,
- .setup_hpet_msi = setup_hpet_msi,
+ .alloc_hpet_msi = alloc_hpet_msi,
};
#endif
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index ca18d6d42a9b..60558f794922 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -24,7 +24,7 @@
* - v7/v8 long-descriptor format
* - Non-secure access to the SMMU
* - 4k and 64k pages, with contiguous pte hints.
- * - Up to 42-bit addressing (dependent on VA_BITS)
+ * - Up to 48-bit addressing (dependent on VA_BITS)
* - Context fault reporting
*/
@@ -59,7 +59,7 @@
/* SMMU global address space */
#define ARM_SMMU_GR0(smmu) ((smmu)->base)
-#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize)
+#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
/*
* SMMU global address space with conditional offset to access secure
@@ -146,6 +146,8 @@
#define ID0_CTTW (1 << 14)
#define ID0_NUMIRPT_SHIFT 16
#define ID0_NUMIRPT_MASK 0xff
+#define ID0_NUMSIDB_SHIFT 9
+#define ID0_NUMSIDB_MASK 0xf
#define ID0_NUMSMRG_SHIFT 0
#define ID0_NUMSMRG_MASK 0xff
@@ -222,7 +224,7 @@
/* Translation context bank */
#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
-#define ARM_SMMU_CB(smmu, n) ((n) * (smmu)->pagesize)
+#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
#define ARM_SMMU_CB_SCTLR 0x0
#define ARM_SMMU_CB_RESUME 0x8
@@ -324,6 +326,16 @@
#define FSYNR0_WNR (1 << 4)
+static int force_stage;
+module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(force_stage,
+ "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
+
+enum arm_smmu_arch_version {
+ ARM_SMMU_V1 = 1,
+ ARM_SMMU_V2,
+};
+
struct arm_smmu_smr {
u8 idx;
u16 mask;
@@ -347,7 +359,7 @@ struct arm_smmu_device {
void __iomem *base;
unsigned long size;
- unsigned long pagesize;
+ unsigned long pgshift;
#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
@@ -358,7 +370,7 @@ struct arm_smmu_device {
#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
u32 options;
- int version;
+ enum arm_smmu_arch_version version;
u32 num_context_banks;
u32 num_s2_context_banks;
@@ -368,8 +380,9 @@ struct arm_smmu_device {
u32 num_mapping_groups;
DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
- unsigned long input_size;
+ unsigned long s1_input_size;
unsigned long s1_output_size;
+ unsigned long s2_input_size;
unsigned long s2_output_size;
u32 num_global_irqs;
@@ -424,17 +437,17 @@ static void parse_driver_options(struct arm_smmu_device *smmu)
} while (arm_smmu_options[++i].opt);
}
-static struct device *dev_get_master_dev(struct device *dev)
+static struct device_node *dev_get_dev_node(struct device *dev)
{
if (dev_is_pci(dev)) {
struct pci_bus *bus = to_pci_dev(dev)->bus;
while (!pci_is_root_bus(bus))
bus = bus->parent;
- return bus->bridge->parent;
+ return bus->bridge->parent->of_node;
}
- return dev;
+ return dev->of_node;
}
static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
@@ -459,15 +472,17 @@ static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
}
static struct arm_smmu_master_cfg *
-find_smmu_master_cfg(struct arm_smmu_device *smmu, struct device *dev)
+find_smmu_master_cfg(struct device *dev)
{
- struct arm_smmu_master *master;
+ struct arm_smmu_master_cfg *cfg = NULL;
+ struct iommu_group *group = iommu_group_get(dev);
- if (dev_is_pci(dev))
- return dev->archdata.iommu;
+ if (group) {
+ cfg = iommu_group_get_iommudata(group);
+ iommu_group_put(group);
+ }
- master = find_smmu_master(smmu, dev->of_node);
- return master ? &master->cfg : NULL;
+ return cfg;
}
static int insert_smmu_master(struct arm_smmu_device *smmu,
@@ -524,9 +539,18 @@ static int register_smmu_master(struct arm_smmu_device *smmu,
master->of_node = masterspec->np;
master->cfg.num_streamids = masterspec->args_count;
- for (i = 0; i < master->cfg.num_streamids; ++i)
- master->cfg.streamids[i] = masterspec->args[i];
+ for (i = 0; i < master->cfg.num_streamids; ++i) {
+ u16 streamid = masterspec->args[i];
+ if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
+ (streamid >= smmu->num_mapping_groups)) {
+ dev_err(dev,
+ "stream ID for master device %s greater than maximum allowed (%d)\n",
+ masterspec->np->name, smmu->num_mapping_groups);
+ return -ERANGE;
+ }
+ master->cfg.streamids[i] = streamid;
+ }
return insert_smmu_master(smmu, master);
}
@@ -534,7 +558,7 @@ static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
{
struct arm_smmu_device *smmu;
struct arm_smmu_master *master = NULL;
- struct device_node *dev_node = dev_get_master_dev(dev)->of_node;
+ struct device_node *dev_node = dev_get_dev_node(dev);
spin_lock(&arm_smmu_devices_lock);
list_for_each_entry(smmu, &arm_smmu_devices, list) {
@@ -623,7 +647,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
if (fsr & FSR_IGN)
dev_err_ratelimited(smmu->dev,
- "Unexpected context fault (fsr 0x%u)\n",
+ "Unexpected context fault (fsr 0x%x)\n",
fsr);
fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
@@ -718,7 +742,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
/* CBAR */
reg = cfg->cbar;
- if (smmu->version == 1)
+ if (smmu->version == ARM_SMMU_V1)
reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
/*
@@ -733,7 +757,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
}
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
- if (smmu->version > 1) {
+ if (smmu->version > ARM_SMMU_V1) {
/* CBA2R */
#ifdef CONFIG_64BIT
reg = CBA2R_RW64_64BIT;
@@ -744,7 +768,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
/* TTBCR2 */
- switch (smmu->input_size) {
+ switch (smmu->s1_input_size) {
case 32:
reg = (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT);
break;
@@ -752,6 +776,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT);
break;
case 39:
+ case 40:
reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT);
break;
case 42:
@@ -773,6 +798,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT);
break;
case 39:
+ case 40:
reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT);
break;
case 42:
@@ -804,14 +830,14 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
* TTBCR
* We use long descriptor, with inner-shareable WBWA tables in TTBR0.
*/
- if (smmu->version > 1) {
+ if (smmu->version > ARM_SMMU_V1) {
if (PAGE_SIZE == SZ_4K)
reg = TTBCR_TG0_4K;
else
reg = TTBCR_TG0_64K;
if (!stage1) {
- reg |= (64 - smmu->s1_output_size) << TTBCR_T0SZ_SHIFT;
+ reg |= (64 - smmu->s2_input_size) << TTBCR_T0SZ_SHIFT;
switch (smmu->s2_output_size) {
case 32:
@@ -834,7 +860,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
break;
}
} else {
- reg |= (64 - smmu->input_size) << TTBCR_T0SZ_SHIFT;
+ reg |= (64 - smmu->s1_input_size) << TTBCR_T0SZ_SHIFT;
}
} else {
reg = 0;
@@ -843,8 +869,11 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
reg |= TTBCR_EAE |
(TTBCR_SH_IS << TTBCR_SH0_SHIFT) |
(TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) |
- (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT) |
- (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT);
+ (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT);
+
+ if (!stage1)
+ reg |= (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT);
+
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
/* MAIR0 (stage-1 only) */
@@ -868,10 +897,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
static int arm_smmu_init_domain_context(struct iommu_domain *domain,
struct arm_smmu_device *smmu)
{
- int irq, ret, start;
+ int irq, start, ret = 0;
+ unsigned long flags;
struct arm_smmu_domain *smmu_domain = domain->priv;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ spin_lock_irqsave(&smmu_domain->lock, flags);
+ if (smmu_domain->smmu)
+ goto out_unlock;
+
if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) {
/*
* We will likely want to change this if/when KVM gets
@@ -890,16 +924,20 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
smmu->num_context_banks);
if (IS_ERR_VALUE(ret))
- return ret;
+ goto out_unlock;
cfg->cbndx = ret;
- if (smmu->version == 1) {
+ if (smmu->version == ARM_SMMU_V1) {
cfg->irptndx = atomic_inc_return(&smmu->irptndx);
cfg->irptndx %= smmu->num_context_irqs;
} else {
cfg->irptndx = cfg->cbndx;
}
+ ACCESS_ONCE(smmu_domain->smmu) = smmu;
+ arm_smmu_init_context_bank(smmu_domain);
+ spin_unlock_irqrestore(&smmu_domain->lock, flags);
+
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
"arm-smmu-context-fault", domain);
@@ -907,15 +945,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
cfg->irptndx, irq);
cfg->irptndx = INVALID_IRPTNDX;
- goto out_free_context;
}
- smmu_domain->smmu = smmu;
- arm_smmu_init_context_bank(smmu_domain);
return 0;
-out_free_context:
- __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
+out_unlock:
+ spin_unlock_irqrestore(&smmu_domain->lock, flags);
return ret;
}
@@ -975,7 +1010,6 @@ static void arm_smmu_free_ptes(pmd_t *pmd)
{
pgtable_t table = pmd_pgtable(*pmd);
- pgtable_page_dtor(table);
__free_page(table);
}
@@ -1108,6 +1142,9 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
struct arm_smmu_smr *smrs = cfg->smrs;
+ if (!smrs)
+ return;
+
/* Invalidate the SMRs before freeing back to the allocator */
for (i = 0; i < cfg->num_streamids; ++i) {
u8 idx = smrs[i].idx;
@@ -1120,20 +1157,6 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
kfree(smrs);
}
-static void arm_smmu_bypass_stream_mapping(struct arm_smmu_device *smmu,
- struct arm_smmu_master_cfg *cfg)
-{
- int i;
- void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
-
- for (i = 0; i < cfg->num_streamids; ++i) {
- u16 sid = cfg->streamids[i];
-
- writel_relaxed(S2CR_TYPE_BYPASS,
- gr0_base + ARM_SMMU_GR0_S2CR(sid));
- }
-}
-
static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_master_cfg *cfg)
{
@@ -1141,9 +1164,10 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_device *smmu = smmu_domain->smmu;
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+ /* Devices in an IOMMU group may already be configured */
ret = arm_smmu_master_configure_smrs(smmu, cfg);
if (ret)
- return ret;
+ return ret == -EEXIST ? 0 : ret;
for (i = 0; i < cfg->num_streamids; ++i) {
u32 idx, s2cr;
@@ -1160,58 +1184,75 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_master_cfg *cfg)
{
+ int i;
struct arm_smmu_device *smmu = smmu_domain->smmu;
+ void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+
+ /* An IOMMU group is torn down by the first device to be removed */
+ if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
+ return;
/*
* We *must* clear the S2CR first, because freeing the SMR means
* that it can be re-allocated immediately.
*/
- arm_smmu_bypass_stream_mapping(smmu, cfg);
+ for (i = 0; i < cfg->num_streamids; ++i) {
+ u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
+
+ writel_relaxed(S2CR_TYPE_BYPASS,
+ gr0_base + ARM_SMMU_GR0_S2CR(idx));
+ }
+
arm_smmu_master_free_smrs(smmu, cfg);
}
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
- int ret = -EINVAL;
+ int ret;
struct arm_smmu_domain *smmu_domain = domain->priv;
- struct arm_smmu_device *smmu;
+ struct arm_smmu_device *smmu, *dom_smmu;
struct arm_smmu_master_cfg *cfg;
- unsigned long flags;
- smmu = dev_get_master_dev(dev)->archdata.iommu;
+ smmu = find_smmu_for_device(dev);
if (!smmu) {
dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
return -ENXIO;
}
+ if (dev->archdata.iommu) {
+ dev_err(dev, "already attached to IOMMU domain\n");
+ return -EEXIST;
+ }
+
/*
* Sanity check the domain. We don't support domains across
* different SMMUs.
*/
- spin_lock_irqsave(&smmu_domain->lock, flags);
- if (!smmu_domain->smmu) {
+ dom_smmu = ACCESS_ONCE(smmu_domain->smmu);
+ if (!dom_smmu) {
/* Now that we have a master, we can finalise the domain */
ret = arm_smmu_init_domain_context(domain, smmu);
if (IS_ERR_VALUE(ret))
- goto err_unlock;
- } else if (smmu_domain->smmu != smmu) {
+ return ret;
+
+ dom_smmu = smmu_domain->smmu;
+ }
+
+ if (dom_smmu != smmu) {
dev_err(dev,
"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
- dev_name(smmu_domain->smmu->dev),
- dev_name(smmu->dev));
- goto err_unlock;
+ dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
+ return -EINVAL;
}
- spin_unlock_irqrestore(&smmu_domain->lock, flags);
/* Looks ok, so add the device to the domain */
- cfg = find_smmu_master_cfg(smmu_domain->smmu, dev);
+ cfg = find_smmu_master_cfg(dev);
if (!cfg)
return -ENODEV;
- return arm_smmu_domain_add_master(smmu_domain, cfg);
-
-err_unlock:
- spin_unlock_irqrestore(&smmu_domain->lock, flags);
+ ret = arm_smmu_domain_add_master(smmu_domain, cfg);
+ if (!ret)
+ dev->archdata.iommu = domain;
return ret;
}
@@ -1220,9 +1261,12 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
struct arm_smmu_domain *smmu_domain = domain->priv;
struct arm_smmu_master_cfg *cfg;
- cfg = find_smmu_master_cfg(smmu_domain->smmu, dev);
- if (cfg)
- arm_smmu_domain_remove_master(smmu_domain, cfg);
+ cfg = find_smmu_master_cfg(dev);
+ if (!cfg)
+ return;
+
+ dev->archdata.iommu = NULL;
+ arm_smmu_domain_remove_master(smmu_domain, cfg);
}
static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
@@ -1247,10 +1291,6 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
return -ENOMEM;
arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE);
- if (!pgtable_page_ctor(table)) {
- __free_page(table);
- return -ENOMEM;
- }
pmd_populate(NULL, pmd, table);
arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd));
}
@@ -1368,6 +1408,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, next, pfn,
prot, stage);
phys += next - addr;
+ pfn = __phys_to_pfn(phys);
} while (pmd++, addr = next, addr < end);
return ret;
@@ -1420,9 +1461,11 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
if (cfg->cbar == CBAR_TYPE_S2_TRANS) {
stage = 2;
+ input_mask = (1ULL << smmu->s2_input_size) - 1;
output_mask = (1ULL << smmu->s2_output_size) - 1;
} else {
stage = 1;
+ input_mask = (1ULL << smmu->s1_input_size) - 1;
output_mask = (1ULL << smmu->s1_output_size) - 1;
}
@@ -1432,7 +1475,6 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
if (size & ~PAGE_MASK)
return -EINVAL;
- input_mask = (1ULL << smmu->input_size) - 1;
if ((phys_addr_t)iova & ~input_mask)
return -ERANGE;
@@ -1515,20 +1557,19 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK);
}
-static int arm_smmu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool arm_smmu_capable(enum iommu_cap cap)
{
- struct arm_smmu_domain *smmu_domain = domain->priv;
- struct arm_smmu_device *smmu = smmu_domain->smmu;
- u32 features = smmu ? smmu->features : 0;
-
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
- return features & ARM_SMMU_FEAT_COHERENT_WALK;
+ /*
+ * Return true here as the SMMU can always send out coherent
+ * requests.
+ */
+ return true;
case IOMMU_CAP_INTR_REMAP:
- return 1; /* MSIs are just memory writes */
+ return true; /* MSIs are just memory writes */
default:
- return 0;
+ return false;
}
}
@@ -1538,17 +1579,19 @@ static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
return 0; /* Continue walking */
}
+static void __arm_smmu_release_pci_iommudata(void *data)
+{
+ kfree(data);
+}
+
static int arm_smmu_add_device(struct device *dev)
{
struct arm_smmu_device *smmu;
+ struct arm_smmu_master_cfg *cfg;
struct iommu_group *group;
+ void (*releasefn)(void *) = NULL;
int ret;
- if (dev->archdata.iommu) {
- dev_warn(dev, "IOMMU driver already assigned to device\n");
- return -EINVAL;
- }
-
smmu = find_smmu_for_device(dev);
if (!smmu)
return -ENODEV;
@@ -1560,7 +1603,6 @@ static int arm_smmu_add_device(struct device *dev)
}
if (dev_is_pci(dev)) {
- struct arm_smmu_master_cfg *cfg;
struct pci_dev *pdev = to_pci_dev(dev);
cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
@@ -1576,11 +1618,20 @@ static int arm_smmu_add_device(struct device *dev)
*/
pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid,
&cfg->streamids[0]);
- dev->archdata.iommu = cfg;
+ releasefn = __arm_smmu_release_pci_iommudata;
} else {
- dev->archdata.iommu = smmu;
+ struct arm_smmu_master *master;
+
+ master = find_smmu_master(smmu, dev->of_node);
+ if (!master) {
+ ret = -ENODEV;
+ goto out_put_group;
+ }
+
+ cfg = &master->cfg;
}
+ iommu_group_set_iommudata(group, cfg, releasefn);
ret = iommu_group_add_device(group, dev);
out_put_group:
@@ -1590,14 +1641,11 @@ out_put_group:
static void arm_smmu_remove_device(struct device *dev)
{
- if (dev_is_pci(dev))
- kfree(dev->archdata.iommu);
-
- dev->archdata.iommu = NULL;
iommu_group_remove_device(dev);
}
static const struct iommu_ops arm_smmu_ops = {
+ .capable = arm_smmu_capable,
.domain_init = arm_smmu_domain_init,
.domain_destroy = arm_smmu_domain_destroy,
.attach_dev = arm_smmu_attach_dev,
@@ -1605,7 +1653,6 @@ static const struct iommu_ops arm_smmu_ops = {
.map = arm_smmu_map,
.unmap = arm_smmu_unmap,
.iova_to_phys = arm_smmu_iova_to_phys,
- .domain_has_cap = arm_smmu_domain_has_cap,
.add_device = arm_smmu_add_device,
.remove_device = arm_smmu_remove_device,
.pgsize_bitmap = (SECTION_SIZE |
@@ -1626,7 +1673,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Mark all SMRn as invalid and all S2CRn as bypass */
for (i = 0; i < smmu->num_mapping_groups; ++i) {
- writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i));
+ writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
writel_relaxed(S2CR_TYPE_BYPASS,
gr0_base + ARM_SMMU_GR0_S2CR(i));
}
@@ -1691,10 +1738,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
u32 id;
dev_notice(smmu->dev, "probing hardware configuration...\n");
-
- /* Primecell ID */
- id = readl_relaxed(gr0_base + ARM_SMMU_GR0_PIDR2);
- smmu->version = ((id >> PIDR2_ARCH_SHIFT) & PIDR2_ARCH_MASK) + 1;
dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
/* ID0 */
@@ -1705,6 +1748,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
return -ENODEV;
}
#endif
+
+ /* Restrict available stages based on module parameter */
+ if (force_stage == 1)
+ id &= ~(ID0_S2TS | ID0_NTS);
+ else if (force_stage == 2)
+ id &= ~(ID0_S1TS | ID0_NTS);
+
if (id & ID0_S1TS) {
smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
dev_notice(smmu->dev, "\tstage 1 translation\n");
@@ -1721,8 +1771,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
}
if (!(smmu->features &
- (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2 |
- ARM_SMMU_FEAT_TRANS_NESTED))) {
+ (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
dev_err(smmu->dev, "\tno translation support!\n");
return -ENODEV;
}
@@ -1761,16 +1810,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
dev_notice(smmu->dev,
"\tstream matching with %u register groups, mask 0x%x",
smmu->num_mapping_groups, mask);
+ } else {
+ smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
+ ID0_NUMSIDB_MASK;
}
/* ID1 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
- smmu->pagesize = (id & ID1_PAGESIZE) ? SZ_64K : SZ_4K;
+ smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
/* Check for size mismatch of SMMU address space from mapped region */
size = 1 <<
(((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
- size *= (smmu->pagesize << 1);
+ size *= 2 << smmu->pgshift;
if (smmu->size != size)
dev_warn(smmu->dev,
"SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
@@ -1789,23 +1841,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
/* ID2 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
+ smmu->s1_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size);
- /*
- * Stage-1 output limited by stage-2 input size due to pgd
- * allocation (PTRS_PER_PGD).
- */
+ /* Stage-2 input size limited due to pgd allocation (PTRS_PER_PGD) */
#ifdef CONFIG_64BIT
- smmu->s1_output_size = min_t(unsigned long, VA_BITS, size);
+ smmu->s2_input_size = min_t(unsigned long, VA_BITS, size);
#else
- smmu->s1_output_size = min(32UL, size);
+ smmu->s2_input_size = min(32UL, size);
#endif
/* The stage-2 output mask is also applied for bypass */
size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
smmu->s2_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size);
- if (smmu->version == 1) {
- smmu->input_size = 32;
+ if (smmu->version == ARM_SMMU_V1) {
+ smmu->s1_input_size = 32;
} else {
#ifdef CONFIG_64BIT
size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
@@ -1813,7 +1863,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
#else
size = 32;
#endif
- smmu->input_size = size;
+ smmu->s1_input_size = size;
if ((PAGE_SIZE == SZ_4K && !(id & ID2_PTFS_4K)) ||
(PAGE_SIZE == SZ_64K && !(id & ID2_PTFS_64K)) ||
@@ -1824,15 +1874,30 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
}
}
- dev_notice(smmu->dev,
- "\t%lu-bit VA, %lu-bit IPA, %lu-bit PA\n",
- smmu->input_size, smmu->s1_output_size,
- smmu->s2_output_size);
+ if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
+ dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
+ smmu->s1_input_size, smmu->s1_output_size);
+
+ if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
+ dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
+ smmu->s2_input_size, smmu->s2_output_size);
+
return 0;
}
+static const struct of_device_id arm_smmu_of_match[] = {
+ { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 },
+ { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 },
+ { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 },
+ { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 },
+ { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 },
+ { },
+};
+MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
+
static int arm_smmu_device_dt_probe(struct platform_device *pdev)
{
+ const struct of_device_id *of_id;
struct resource *res;
struct arm_smmu_device *smmu;
struct device *dev = &pdev->dev;
@@ -1847,6 +1912,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
}
smmu->dev = dev;
+ of_id = of_match_node(arm_smmu_of_match, dev->of_node);
+ smmu->version = (enum arm_smmu_arch_version)of_id->data;
+
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
@@ -1889,6 +1957,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
smmu->irqs[i] = irq;
}
+ err = arm_smmu_device_cfg_probe(smmu);
+ if (err)
+ return err;
+
i = 0;
smmu->masters = RB_ROOT;
while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
@@ -1905,13 +1977,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
}
dev_notice(dev, "registered %d master devices\n", i);
- err = arm_smmu_device_cfg_probe(smmu);
- if (err)
- goto out_put_masters;
-
parse_driver_options(smmu);
- if (smmu->version > 1 &&
+ if (smmu->version > ARM_SMMU_V1 &&
smmu->num_context_banks != smmu->num_context_irqs) {
dev_err(dev,
"found only %d context interrupt(s) but %d required\n",
@@ -1992,17 +2060,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
return 0;
}
-#ifdef CONFIG_OF
-static struct of_device_id arm_smmu_of_match[] = {
- { .compatible = "arm,smmu-v1", },
- { .compatible = "arm,smmu-v2", },
- { .compatible = "arm,mmu-400", },
- { .compatible = "arm,mmu-500", },
- { },
-};
-MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
-#endif
-
static struct platform_driver arm_smmu_driver = {
.driver = {
.owner = THIS_MODULE,
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 60ab474bfff3..c5c61cabd6e3 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -155,6 +155,7 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
if (event == BUS_NOTIFY_ADD_DEVICE) {
for (tmp = dev; tmp; tmp = tmp->bus->self) {
level--;
+ info->path[level].bus = tmp->bus->number;
info->path[level].device = PCI_SLOT(tmp->devfn);
info->path[level].function = PCI_FUNC(tmp->devfn);
if (pci_is_root_bus(tmp->bus))
@@ -177,17 +178,33 @@ static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
int i;
if (info->bus != bus)
- return false;
+ goto fallback;
if (info->level != count)
- return false;
+ goto fallback;
for (i = 0; i < count; i++) {
if (path[i].device != info->path[i].device ||
path[i].function != info->path[i].function)
- return false;
+ goto fallback;
}
return true;
+
+fallback:
+
+ if (count != 1)
+ return false;
+
+ i = info->level - 1;
+ if (bus == info->path[i].bus &&
+ path[0].device == info->path[i].device &&
+ path[0].function == info->path[i].function) {
+ pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
+ bus, path[0].device, path[0].function);
+ return true;
+ }
+
+ return false;
}
/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
@@ -247,7 +264,7 @@ int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
for_each_active_dev_scope(devices, count, index, tmp)
if (tmp == &info->dev->dev) {
- rcu_assign_pointer(devices[index].dev, NULL);
+ RCU_INIT_POINTER(devices[index].dev, NULL);
synchronize_rcu();
put_device(tmp);
return 1;
@@ -678,8 +695,7 @@ static int __init dmar_acpi_dev_scope_init(void)
andd->device_name);
continue;
}
- acpi_bus_get_device(h, &adev);
- if (!adev) {
+ if (acpi_bus_get_device(h, &adev)) {
pr_err("Failed to get device for ACPI object %s\n",
andd->device_name);
continue;
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index d037e87a1fe5..74233186f6f7 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -32,7 +32,7 @@
typedef u32 sysmmu_iova_t;
typedef u32 sysmmu_pte_t;
-/* We does not consider super section mapping (16MB) */
+/* We do not consider super section mapping (16MB) */
#define SECT_ORDER 20
#define LPAGE_ORDER 16
#define SPAGE_ORDER 12
@@ -307,7 +307,7 @@ static void show_fault_information(const char *name,
static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
{
- /* SYSMMU is in blocked when interrupt occurred. */
+ /* SYSMMU is in blocked state when interrupt occurred. */
struct sysmmu_drvdata *data = dev_id;
enum exynos_sysmmu_inttype itype;
sysmmu_iova_t addr = -1;
@@ -567,8 +567,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
/*
* L2TLB invalidation required
* 4KB page: 1 invalidation
- * 64KB page: 16 invalidation
- * 1MB page: 64 invalidation
+ * 64KB page: 16 invalidations
+ * 1MB page: 64 invalidations
* because it is set-associative TLB
* with 8-way and 64 sets.
* 1MB page can be cached in one of all sets.
@@ -714,7 +714,7 @@ static int exynos_iommu_domain_init(struct iommu_domain *domain)
if (!priv->lv2entcnt)
goto err_counter;
- /* w/a of System MMU v3.3 to prevent caching 1MiB mapping */
+ /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
priv->pgtable[i + 0] = ZERO_LV2LINK;
priv->pgtable[i + 1] = ZERO_LV2LINK;
@@ -861,14 +861,14 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
pgtable_flush(sent, sent + 1);
/*
- * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache
- * may caches the address of zero_l2_table. This function
- * replaces the zero_l2_table with new L2 page table to write
- * valid mappings.
+ * If pre-fetched SLPD is a faulty SLPD in zero_l2_table,
+ * FLPD cache may cache the address of zero_l2_table. This
+ * function replaces the zero_l2_table with new L2 page table
+ * to write valid mappings.
* Accessing the valid area may cause page fault since FLPD
- * cache may still caches zero_l2_table for the valid area
- * instead of new L2 page table that have the mapping
- * information of the valid area
+ * cache may still cache zero_l2_table for the valid area
+ * instead of new L2 page table that has the mapping
+ * information of the valid area.
* Thus any replacement of zero_l2_table with other valid L2
* page table must involve FLPD cache invalidation for System
* MMU v3.3.
@@ -963,27 +963,27 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
/*
* *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
*
- * System MMU v3.x have an advanced logic to improve address translation
+ * System MMU v3.x has advanced logic to improve address translation
* performance with caching more page table entries by a page table walk.
- * However, the logic has a bug that caching fault page table entries and System
- * MMU reports page fault if the cached fault entry is hit even though the fault
- * entry is updated to a valid entry after the entry is cached.
- * To prevent caching fault page table entries which may be updated to valid
- * entries later, the virtual memory manager should care about the w/a about the
- * problem. The followings describe w/a.
+ * However, the logic has a bug that while caching faulty page table entries,
+ * System MMU reports page fault if the cached fault entry is hit even though
+ * the fault entry is updated to a valid entry after the entry is cached.
+ * To prevent caching faulty page table entries which may be updated to valid
+ * entries later, the virtual memory manager should care about the workaround
+ * for the problem. The following describes the workaround.
*
* Any two consecutive I/O virtual address regions must have a hole of 128KiB
- * in maximum to prevent misbehavior of System MMU 3.x. (w/a of h/w bug)
+ * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug).
*
- * Precisely, any start address of I/O virtual region must be aligned by
+ * Precisely, any start address of I/O virtual region must be aligned with
* the following sizes for System MMU v3.1 and v3.2.
* System MMU v3.1: 128KiB
* System MMU v3.2: 256KiB
*
* Because System MMU v3.3 caches page table entries more aggressively, it needs
- * more w/a.
- * - Any two consecutive I/O virtual regions must be have a hole of larger size
- * than or equal size to 128KiB.
+ * more workarounds.
+ * - Any two consecutive I/O virtual regions must have a hole of size larger
+ * than or equal to 128KiB.
* - Start address of an I/O virtual region must be aligned by 128KiB.
*/
static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
@@ -1061,7 +1061,8 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
goto err;
}
- *ent = ZERO_LV2LINK; /* w/a for h/w bug in Sysmem MMU v3.3 */
+ /* workaround for h/w bug in System MMU v3.3 */
+ *ent = ZERO_LV2LINK;
pgtable_flush(ent, ent + 1);
size = SECT_SIZE;
goto done;
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index 61d1dafa242d..c828f80d48b0 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -411,8 +411,7 @@ static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
return get_phys_addr(dma_domain, iova);
}
-static int fsl_pamu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool fsl_pamu_capable(enum iommu_cap cap)
{
return cap == IOMMU_CAP_CACHE_COHERENCY;
}
@@ -984,7 +983,7 @@ static int fsl_pamu_add_device(struct device *dev)
struct iommu_group *group = ERR_PTR(-ENODEV);
struct pci_dev *pdev;
const u32 *prop;
- int ret, len;
+ int ret = 0, len;
/*
* For platform devices we allocate a separate group for
@@ -1007,7 +1006,13 @@ static int fsl_pamu_add_device(struct device *dev)
if (IS_ERR(group))
return PTR_ERR(group);
- ret = iommu_group_add_device(group, dev);
+ /*
+ * Check if device has already been added to an iommu group.
+ * Group could have already been created for a PCI device in
+ * the iommu_group_get_for_dev path.
+ */
+ if (!dev->iommu_group)
+ ret = iommu_group_add_device(group, dev);
iommu_group_put(group);
return ret;
@@ -1074,6 +1079,7 @@ static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
}
static const struct iommu_ops fsl_pamu_ops = {
+ .capable = fsl_pamu_capable,
.domain_init = fsl_pamu_domain_init,
.domain_destroy = fsl_pamu_domain_destroy,
.attach_dev = fsl_pamu_attach_device,
@@ -1083,7 +1089,6 @@ static const struct iommu_ops fsl_pamu_ops = {
.domain_get_windows = fsl_pamu_get_windows,
.domain_set_windows = fsl_pamu_set_windows,
.iova_to_phys = fsl_pamu_iova_to_phys,
- .domain_has_cap = fsl_pamu_domain_has_cap,
.domain_set_attr = fsl_pamu_set_domain_attr,
.domain_get_attr = fsl_pamu_get_domain_attr,
.add_device = fsl_pamu_add_device,
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 5619f264862d..a27d6cb1a793 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -3865,8 +3865,7 @@ static int device_notifier(struct notifier_block *nb,
if (iommu_dummy(dev))
return 0;
- if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
- action != BUS_NOTIFY_DEL_DEVICE)
+ if (action != BUS_NOTIFY_REMOVED_DEVICE)
return 0;
/*
@@ -4415,17 +4414,14 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
return phys;
}
-static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool intel_iommu_capable(enum iommu_cap cap)
{
- struct dmar_domain *dmar_domain = domain->priv;
-
if (cap == IOMMU_CAP_CACHE_COHERENCY)
- return dmar_domain->iommu_snooping;
+ return domain_update_iommu_snooping(NULL) == 1;
if (cap == IOMMU_CAP_INTR_REMAP)
- return irq_remapping_enabled;
+ return irq_remapping_enabled == 1;
- return 0;
+ return false;
}
static int intel_iommu_add_device(struct device *dev)
@@ -4464,6 +4460,7 @@ static void intel_iommu_remove_device(struct device *dev)
}
static const struct iommu_ops intel_iommu_ops = {
+ .capable = intel_iommu_capable,
.domain_init = intel_iommu_domain_init,
.domain_destroy = intel_iommu_domain_destroy,
.attach_dev = intel_iommu_attach_device,
@@ -4471,7 +4468,6 @@ static const struct iommu_ops intel_iommu_ops = {
.map = intel_iommu_map,
.unmap = intel_iommu_unmap,
.iova_to_phys = intel_iommu_iova_to_phys,
- .domain_has_cap = intel_iommu_domain_has_cap,
.add_device = intel_iommu_add_device,
.remove_device = intel_iommu_remove_device,
.pgsize_bitmap = INTEL_IOMMU_PGSIZES,
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 0df41f6264f5..7c80661b35c1 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -438,8 +438,7 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
(addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
/* Set interrupt-remapping table pointer */
- iommu->gcmd |= DMA_GCMD_SIRTP;
- writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
+ writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
readl, (sts & DMA_GSTS_IRTPS), sts);
@@ -1139,7 +1138,7 @@ static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
return ret;
}
-static int intel_setup_hpet_msi(unsigned int irq, unsigned int id)
+static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id)
{
int ret = -1;
struct intel_iommu *iommu;
@@ -1170,5 +1169,5 @@ struct irq_remap_ops intel_irq_remap_ops = {
.compose_msi_msg = intel_compose_msi_msg,
.msi_alloc_irq = intel_msi_alloc_irq,
.msi_setup_irq = intel_msi_setup_irq,
- .setup_hpet_msi = intel_setup_hpet_msi,
+ .alloc_hpet_msi = intel_alloc_hpet_msi,
};
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index ac4adb337038..ed8b04867b1f 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -30,6 +30,7 @@
#include <linux/notifier.h>
#include <linux/err.h>
#include <linux/pci.h>
+#include <linux/bitops.h>
#include <trace/events/iommu.h>
static struct kset *iommu_group_kset;
@@ -519,6 +520,9 @@ int iommu_group_id(struct iommu_group *group)
}
EXPORT_SYMBOL_GPL(iommu_group_id);
+static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
+ unsigned long *devfns);
+
/*
* To consider a PCI device isolated, we require ACS to support Source
* Validation, Request Redirection, Completer Redirection, and Upstream
@@ -529,6 +533,86 @@ EXPORT_SYMBOL_GPL(iommu_group_id);
*/
#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
+/*
+ * For multifunction devices which are not isolated from each other, find
+ * all the other non-isolated functions and look for existing groups. For
+ * each function, we also need to look for aliases to or from other devices
+ * that may already have a group.
+ */
+static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
+ unsigned long *devfns)
+{
+ struct pci_dev *tmp = NULL;
+ struct iommu_group *group;
+
+ if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
+ return NULL;
+
+ for_each_pci_dev(tmp) {
+ if (tmp == pdev || tmp->bus != pdev->bus ||
+ PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
+ pci_acs_enabled(tmp, REQ_ACS_FLAGS))
+ continue;
+
+ group = get_pci_alias_group(tmp, devfns);
+ if (group) {
+ pci_dev_put(tmp);
+ return group;
+ }
+ }
+
+ return NULL;
+}
+
+/*
+ * Look for aliases to or from the given device for exisiting groups. The
+ * dma_alias_devfn only supports aliases on the same bus, therefore the search
+ * space is quite small (especially since we're really only looking at pcie
+ * device, and therefore only expect multiple slots on the root complex or
+ * downstream switch ports). It's conceivable though that a pair of
+ * multifunction devices could have aliases between them that would cause a
+ * loop. To prevent this, we use a bitmap to track where we've been.
+ */
+static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
+ unsigned long *devfns)
+{
+ struct pci_dev *tmp = NULL;
+ struct iommu_group *group;
+
+ if (test_and_set_bit(pdev->devfn & 0xff, devfns))
+ return NULL;
+
+ group = iommu_group_get(&pdev->dev);
+ if (group)
+ return group;
+
+ for_each_pci_dev(tmp) {
+ if (tmp == pdev || tmp->bus != pdev->bus)
+ continue;
+
+ /* We alias them or they alias us */
+ if (((pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) &&
+ pdev->dma_alias_devfn == tmp->devfn) ||
+ ((tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) &&
+ tmp->dma_alias_devfn == pdev->devfn)) {
+
+ group = get_pci_alias_group(tmp, devfns);
+ if (group) {
+ pci_dev_put(tmp);
+ return group;
+ }
+
+ group = get_pci_function_alias_group(tmp, devfns);
+ if (group) {
+ pci_dev_put(tmp);
+ return group;
+ }
+ }
+ }
+
+ return NULL;
+}
+
struct group_for_pci_data {
struct pci_dev *pdev;
struct iommu_group *group;
@@ -557,7 +641,7 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
struct group_for_pci_data data;
struct pci_bus *bus;
struct iommu_group *group = NULL;
- struct pci_dev *tmp;
+ u64 devfns[4] = { 0 };
/*
* Find the upstream DMA alias for the device. A device must not
@@ -591,76 +675,21 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
}
/*
- * Next we need to consider DMA alias quirks. If one device aliases
- * to another, they should be grouped together. It's theoretically
- * possible that aliases could create chains of devices where each
- * device aliases another device. If we then factor in multifunction
- * ACS grouping requirements, each alias could incorporate a new slot
- * with multiple functions, each with aliases. This is all extremely
- * unlikely as DMA alias quirks are typically only used for PCIe
- * devices where we usually have a single slot per bus. Furthermore,
- * the alias quirk is usually to another function within the slot
- * (and ACS multifunction is not supported) or to a different slot
- * that doesn't physically exist. The likely scenario is therefore
- * that everything on the bus gets grouped together. To reduce the
- * problem space, share the IOMMU group for all devices on the bus
- * if a DMA alias quirk is present on the bus.
+ * Look for existing groups on device aliases. If we alias another
+ * device or another device aliases us, use the same group.
*/
- tmp = NULL;
- for_each_pci_dev(tmp) {
- if (tmp->bus != pdev->bus ||
- !(tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN))
- continue;
-
- pci_dev_put(tmp);
- tmp = NULL;
-
- /* We have an alias quirk, search for an existing group */
- for_each_pci_dev(tmp) {
- struct iommu_group *group_tmp;
-
- if (tmp->bus != pdev->bus)
- continue;
-
- group_tmp = iommu_group_get(&tmp->dev);
- if (!group) {
- group = group_tmp;
- continue;
- }
-
- if (group_tmp) {
- WARN_ON(group != group_tmp);
- iommu_group_put(group_tmp);
- }
- }
-
- return group ? group : iommu_group_alloc();
- }
-
- /*
- * Non-multifunction devices or multifunction devices supporting
- * ACS get their own group.
- */
- if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
- return iommu_group_alloc();
+ group = get_pci_alias_group(pdev, (unsigned long *)devfns);
+ if (group)
+ return group;
/*
- * Multifunction devices not supporting ACS share a group with other
- * similar devices in the same slot.
+ * Look for existing groups on non-isolated functions on the same
+ * slot and aliases of those funcions, if any. No need to clear
+ * the search bitmap, the tested devfns are still valid.
*/
- tmp = NULL;
- for_each_pci_dev(tmp) {
- if (tmp == pdev || tmp->bus != pdev->bus ||
- PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
- pci_acs_enabled(tmp, REQ_ACS_FLAGS))
- continue;
-
- group = iommu_group_get(&tmp->dev);
- if (group) {
- pci_dev_put(tmp);
- return group;
- }
- }
+ group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
+ if (group)
+ return group;
/* No shared group found, allocate new */
return iommu_group_alloc();
@@ -678,15 +707,17 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
*/
struct iommu_group *iommu_group_get_for_dev(struct device *dev)
{
- struct iommu_group *group = ERR_PTR(-EIO);
+ struct iommu_group *group;
int ret;
group = iommu_group_get(dev);
if (group)
return group;
- if (dev_is_pci(dev))
- group = iommu_group_get_for_pci_dev(to_pci_dev(dev));
+ if (!dev_is_pci(dev))
+ return ERR_PTR(-EINVAL);
+
+ group = iommu_group_get_for_pci_dev(to_pci_dev(dev));
if (IS_ERR(group))
return group;
@@ -768,18 +799,26 @@ static int iommu_bus_notifier(struct notifier_block *nb,
return 0;
}
-static struct notifier_block iommu_bus_nb = {
- .notifier_call = iommu_bus_notifier,
-};
-
-static void iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
+static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
{
+ int err;
+ struct notifier_block *nb;
struct iommu_callback_data cb = {
.ops = ops,
};
- bus_register_notifier(bus, &iommu_bus_nb);
- bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
+ nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
+ if (!nb)
+ return -ENOMEM;
+
+ nb->notifier_call = iommu_bus_notifier;
+
+ err = bus_register_notifier(bus, nb);
+ if (err) {
+ kfree(nb);
+ return err;
+ }
+ return bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
}
/**
@@ -803,9 +842,7 @@ int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
bus->iommu_ops = ops;
/* Do IOMMU specific setup for this bus-type */
- iommu_bus_init(bus, ops);
-
- return 0;
+ return iommu_bus_init(bus, ops);
}
EXPORT_SYMBOL_GPL(bus_set_iommu);
@@ -815,6 +852,15 @@ bool iommu_present(struct bus_type *bus)
}
EXPORT_SYMBOL_GPL(iommu_present);
+bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
+{
+ if (!bus->iommu_ops || !bus->iommu_ops->capable)
+ return false;
+
+ return bus->iommu_ops->capable(cap);
+}
+EXPORT_SYMBOL_GPL(iommu_capable);
+
/**
* iommu_set_fault_handler() - set a fault handler for an iommu domain
* @domain: iommu domain
@@ -945,16 +991,6 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
-int iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
-{
- if (unlikely(domain->ops->domain_has_cap == NULL))
- return 0;
-
- return domain->ops->domain_has_cap(domain, cap);
-}
-EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
-
static size_t iommu_pgsize(struct iommu_domain *domain,
unsigned long addr_merge, size_t size)
{
diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c
index 33c439524080..74a1767c89b5 100644
--- a/drivers/iommu/irq_remapping.c
+++ b/drivers/iommu/irq_remapping.c
@@ -12,6 +12,7 @@
#include <asm/processor.h>
#include <asm/x86_init.h>
#include <asm/apic.h>
+#include <asm/hpet.h>
#include "irq_remapping.h"
@@ -345,10 +346,16 @@ static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
{
- if (!remap_ops || !remap_ops->setup_hpet_msi)
+ int ret;
+
+ if (!remap_ops || !remap_ops->alloc_hpet_msi)
return -ENODEV;
- return remap_ops->setup_hpet_msi(irq, id);
+ ret = remap_ops->alloc_hpet_msi(irq, id);
+ if (ret)
+ return -EINVAL;
+
+ return default_setup_hpet_msi(irq, id);
}
void panic_if_irq_remap(const char *msg)
diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h
index 90c4dae5a46b..fde250f86e60 100644
--- a/drivers/iommu/irq_remapping.h
+++ b/drivers/iommu/irq_remapping.h
@@ -80,7 +80,7 @@ struct irq_remap_ops {
int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int);
/* Setup interrupt remapping for an HPET MSI */
- int (*setup_hpet_msi)(unsigned int, unsigned int);
+ int (*alloc_hpet_msi)(unsigned int, unsigned int);
};
extern struct irq_remap_ops intel_irq_remap_ops;
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 49f41d6e02f1..6e3dcc289d59 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -603,10 +603,9 @@ fail:
return ret;
}
-static int msm_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool msm_iommu_capable(enum iommu_cap cap)
{
- return 0;
+ return false;
}
static void print_ctx_regs(void __iomem *base, int ctx)
@@ -675,6 +674,7 @@ fail:
}
static const struct iommu_ops msm_iommu_ops = {
+ .capable = msm_iommu_capable,
.domain_init = msm_iommu_domain_init,
.domain_destroy = msm_iommu_domain_destroy,
.attach_dev = msm_iommu_attach_dev,
@@ -682,7 +682,6 @@ static const struct iommu_ops msm_iommu_ops = {
.map = msm_iommu_map,
.unmap = msm_iommu_unmap,
.iova_to_phys = msm_iommu_iova_to_phys,
- .domain_has_cap = msm_iommu_domain_has_cap,
.pgsize_bitmap = MSM_IOMMU_PGSIZES,
};
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index e202b0c24120..36278870e84a 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -26,6 +26,7 @@
#include <linux/of.h>
#include <linux/of_iommu.h>
#include <linux/of_irq.h>
+#include <linux/of_platform.h>
#include <asm/cacheflush.h>
@@ -892,19 +893,11 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
goto err_enable;
flush_iotlb_all(obj);
- if (!try_module_get(obj->owner)) {
- err = -ENODEV;
- goto err_module;
- }
-
spin_unlock(&obj->iommu_lock);
dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
return obj;
-err_module:
- if (obj->refcount == 1)
- iommu_disable(obj);
err_enable:
obj->refcount--;
spin_unlock(&obj->iommu_lock);
@@ -925,8 +918,6 @@ static void omap_iommu_detach(struct omap_iommu *obj)
if (--obj->refcount == 0)
iommu_disable(obj);
- module_put(obj->owner);
-
obj->iopgd = NULL;
spin_unlock(&obj->iommu_lock);
@@ -1006,7 +997,7 @@ static int omap_iommu_remove(struct platform_device *pdev)
return 0;
}
-static struct of_device_id omap_iommu_of_match[] = {
+static const struct of_device_id omap_iommu_of_match[] = {
{ .compatible = "ti,omap2-iommu" },
{ .compatible = "ti,omap4-iommu" },
{ .compatible = "ti,dra7-iommu" },
@@ -1091,6 +1082,11 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
int ret = 0;
+ if (!arch_data || !arch_data->name) {
+ dev_err(dev, "device doesn't have an associated iommu\n");
+ return -EINVAL;
+ }
+
spin_lock(&omap_domain->lock);
/* only a single device is supported per domain for now */
@@ -1239,6 +1235,7 @@ static int omap_iommu_add_device(struct device *dev)
{
struct omap_iommu_arch_data *arch_data;
struct device_node *np;
+ struct platform_device *pdev;
/*
* Allocate the archdata iommu structure for DT-based devices.
@@ -1253,13 +1250,19 @@ static int omap_iommu_add_device(struct device *dev)
if (!np)
return 0;
+ pdev = of_find_device_by_node(np);
+ if (WARN_ON(!pdev)) {
+ of_node_put(np);
+ return -EINVAL;
+ }
+
arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
if (!arch_data) {
of_node_put(np);
return -ENOMEM;
}
- arch_data->name = kstrdup(dev_name(dev), GFP_KERNEL);
+ arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL);
dev->archdata.iommu = arch_data;
of_node_put(np);
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index 1275a822934b..4f1b68c08c15 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -28,7 +28,6 @@ struct iotlb_entry {
struct omap_iommu {
const char *name;
- struct module *owner;
void __iomem *regbase;
struct device *dev;
void *isr_priv;
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index b10a8ecede8e..a6d76abf2c06 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -303,13 +303,13 @@ static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
return pa;
}
-static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool gart_iommu_capable(enum iommu_cap cap)
{
- return 0;
+ return false;
}
static const struct iommu_ops gart_iommu_ops = {
+ .capable = gart_iommu_capable,
.domain_init = gart_iommu_domain_init,
.domain_destroy = gart_iommu_domain_destroy,
.attach_dev = gart_iommu_attach_dev,
@@ -317,7 +317,6 @@ static const struct iommu_ops gart_iommu_ops = {
.map = gart_iommu_map,
.unmap = gart_iommu_unmap,
.iova_to_phys = gart_iommu_iova_to_phys,
- .domain_has_cap = gart_iommu_domain_has_cap,
.pgsize_bitmap = GART_IOMMU_PGSIZES,
};
@@ -416,7 +415,7 @@ static const struct dev_pm_ops tegra_gart_pm_ops = {
.resume = tegra_gart_resume,
};
-static struct of_device_id tegra_gart_of_match[] = {
+static const struct of_device_id tegra_gart_of_match[] = {
{ .compatible = "nvidia,tegra20-gart", },
{ },
};
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 3ded3894623c..3afdf43f732a 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -780,10 +780,9 @@ static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain,
return PFN_PHYS(pfn);
}
-static int smmu_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static bool smmu_iommu_capable(enum iommu_cap cap)
{
- return 0;
+ return false;
}
static int smmu_iommu_attach_dev(struct iommu_domain *domain,
@@ -949,6 +948,7 @@ static void smmu_iommu_domain_destroy(struct iommu_domain *domain)
}
static const struct iommu_ops smmu_iommu_ops = {
+ .capable = smmu_iommu_capable,
.domain_init = smmu_iommu_domain_init,
.domain_destroy = smmu_iommu_domain_destroy,
.attach_dev = smmu_iommu_attach_dev,
@@ -956,7 +956,6 @@ static const struct iommu_ops smmu_iommu_ops = {
.map = smmu_iommu_map,
.unmap = smmu_iommu_unmap,
.iova_to_phys = smmu_iommu_iova_to_phys,
- .domain_has_cap = smmu_iommu_domain_has_cap,
.pgsize_bitmap = SMMU_IOMMU_PGSIZES,
};
@@ -1260,7 +1259,7 @@ static const struct dev_pm_ops tegra_smmu_pm_ops = {
.resume = tegra_smmu_resume,
};
-static struct of_device_id tegra_smmu_of_match[] = {
+static const struct of_device_id tegra_smmu_of_match[] = {
{ .compatible = "nvidia,tegra30-smmu", },
{ },
};
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-24 18:07:43 +0100