diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/Kconfig | 44 | ||||
| -rw-r--r-- | drivers/iommu/Makefile | 8 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 246 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 280 | ||||
| -rw-r--r-- | drivers/iommu/intr_remapping.c | 93 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 114 | ||||
| -rw-r--r-- | drivers/iommu/msm_iommu.c | 9 | ||||
| -rw-r--r-- | drivers/iommu/omap-iommu-debug.c | 418 | ||||
| -rw-r--r-- | drivers/iommu/omap-iommu.c | 1245 | ||||
| -rw-r--r-- | drivers/iommu/omap-iovmm.c | 742 |
11 files changed, 2878 insertions, 325 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index b57b3fa492f3..5414253b185a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -59,10 +59,14 @@ config AMD_IOMMU_STATS If unsure, say N. # Intel IOMMU support -config DMAR - bool "Support for DMA Remapping Devices" +config DMAR_TABLE + bool + +config INTEL_IOMMU + bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) select IOMMU_API + select DMAR_TABLE help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. @@ -70,18 +74,18 @@ config DMAR and include PCI device scope covered by these DMA remapping devices. -config DMAR_DEFAULT_ON +config INTEL_IOMMU_DEFAULT_ON def_bool y - prompt "Enable DMA Remapping Devices by default" - depends on DMAR + prompt "Enable Intel DMA Remapping Devices by default" + depends on INTEL_IOMMU help Selecting this option will enable a DMAR device at boot time if one is found. If this option is not selected, DMAR support can be enabled by passing intel_iommu=on to the kernel. -config DMAR_BROKEN_GFX_WA +config INTEL_IOMMU_BROKEN_GFX_WA bool "Workaround broken graphics drivers (going away soon)" - depends on DMAR && BROKEN && X86 + depends on INTEL_IOMMU && BROKEN && X86 ---help--- Current Graphics drivers tend to use physical address for DMA and avoid using DMA APIs. Setting this config @@ -90,21 +94,41 @@ config DMAR_BROKEN_GFX_WA to use physical addresses for DMA, at least until this option is removed in the 2.6.32 kernel. -config DMAR_FLOPPY_WA +config INTEL_IOMMU_FLOPPY_WA def_bool y - depends on DMAR && X86 + depends on INTEL_IOMMU && X86 ---help--- Floppy disk drivers are known to bypass DMA API calls thereby failing to work when IOMMU is enabled. This workaround will setup a 1:1 mapping for the first 16MiB to make floppy (an ISA device) work. -config INTR_REMAP +config IRQ_REMAP bool "Support for Interrupt Remapping (EXPERIMENTAL)" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL + select DMAR_TABLE ---help--- Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or to support platforms with CPU's having > 8 bit APIC ID, say Y. +# OMAP IOMMU support +config OMAP_IOMMU + bool "OMAP IOMMU Support" + depends on ARCH_OMAP + select IOMMU_API + +config OMAP_IOVMM + tristate "OMAP IO Virtual Memory Manager Support" + depends on OMAP_IOMMU + +config OMAP_IOMMU_DEBUG + tristate "Export OMAP IOMMU/IOVMM internals in DebugFS" + depends on OMAP_IOVMM && DEBUG_FS + help + Select this to see extensive information about + the internal state of OMAP IOMMU/IOVMM in debugfs. + + Say N unless you know you need this. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 4d4d77df7cac..2f4448794bc7 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,9 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o -obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o -obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o +obj-$(CONFIG_DMAR_TABLE) += dmar.o +obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o +obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o +obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o +obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o +obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0e4227f457af..4ee277a8521a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1283,7 +1283,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, if (!pte || !IOMMU_PTE_PRESENT(*pte)) continue; - dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1); + dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1); } update_domain(&dma_dom->domain); @@ -2495,7 +2495,7 @@ static unsigned device_dma_ops_init(void) void __init amd_iommu_init_api(void) { - register_iommu(&amd_iommu_ops); + bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6dcc7e2d54de..35c1e17fce1d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -46,7 +46,7 @@ */ LIST_HEAD(dmar_drhd_units); -static struct acpi_table_header * __initdata dmar_tbl; +struct acpi_table_header * __initdata dmar_tbl; static acpi_size dmar_tbl_size; static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd) @@ -118,8 +118,8 @@ static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope, return 0; } -static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, - struct pci_dev ***devices, u16 segment) +int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, + struct pci_dev ***devices, u16 segment) { struct acpi_dmar_device_scope *scope; void * tmp = start; @@ -217,133 +217,6 @@ static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru) return ret; } -#ifdef CONFIG_DMAR -LIST_HEAD(dmar_rmrr_units); - -static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) -{ - list_add(&rmrr->list, &dmar_rmrr_units); -} - - -static int __init -dmar_parse_one_rmrr(struct acpi_dmar_header *header) -{ - struct acpi_dmar_reserved_memory *rmrr; - struct dmar_rmrr_unit *rmrru; - - rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); - if (!rmrru) - return -ENOMEM; - - rmrru->hdr = header; - rmrr = (struct acpi_dmar_reserved_memory *)header; - rmrru->base_address = rmrr->base_address; - rmrru->end_address = rmrr->end_address; - - dmar_register_rmrr_unit(rmrru); - return 0; -} - -static int __init -rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) -{ - struct acpi_dmar_reserved_memory *rmrr; - int ret; - - rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; - ret = dmar_parse_dev_scope((void *)(rmrr + 1), - ((void *)rmrr) + rmrr->header.length, - &rmrru->devices_cnt, &rmrru->devices, rmrr->segment); - - if (ret || (rmrru->devices_cnt == 0)) { - list_del(&rmrru->list); - kfree(rmrru); - } - return ret; -} - -static LIST_HEAD(dmar_atsr_units); - -static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) -{ - struct acpi_dmar_atsr *atsr; - struct dmar_atsr_unit *atsru; - - atsr = container_of(hdr, struct acpi_dmar_atsr, header); - atsru = kzalloc(sizeof(*atsru), GFP_KERNEL); - if (!atsru) - return -ENOMEM; - - atsru->hdr = hdr; - atsru->include_all = atsr->flags & 0x1; - - list_add(&atsru->list, &dmar_atsr_units); - - return 0; -} - -static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) -{ - int rc; - struct acpi_dmar_atsr *atsr; - - if (atsru->include_all) - return 0; - - atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); - rc = dmar_parse_dev_scope((void *)(atsr + 1), - (void *)atsr + atsr->header.length, - &atsru->devices_cnt, &atsru->devices, - atsr->segment); - if (rc || !atsru->devices_cnt) { - list_del(&atsru->list); - kfree(atsru); - } - - return rc; -} - -int dmar_find_matched_atsr_unit(struct pci_dev *dev) -{ - int i; - struct pci_bus *bus; - struct acpi_dmar_atsr *atsr; - struct dmar_atsr_unit *atsru; - - dev = pci_physfn(dev); - - list_for_each_entry(atsru, &dmar_atsr_units, list) { - atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); - if (atsr->segment == pci_domain_nr(dev->bus)) - goto found; - } - - return 0; - -found: - for (bus = dev->bus; bus; bus = bus->parent) { - struct pci_dev *bridge = bus->self; - - if (!bridge || !pci_is_pcie(bridge) || - bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) - return 0; - - if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) { - for (i = 0; i < atsru->devices_cnt; i++) - if (atsru->devices[i] == bridge) - return 1; - break; - } - } - - if (atsru->include_all) - return 1; - - return 0; -} -#endif - #ifdef CONFIG_ACPI_NUMA static int __init dmar_parse_one_rhsa(struct acpi_dmar_header *header) @@ -484,14 +357,10 @@ parse_dmar_table(void) ret = dmar_parse_one_drhd(entry_header); break; case ACPI_DMAR_TYPE_RESERVED_MEMORY: -#ifdef CONFIG_DMAR ret = dmar_parse_one_rmrr(entry_header); -#endif break; case ACPI_DMAR_TYPE_ATSR: -#ifdef CONFIG_DMAR ret = dmar_parse_one_atsr(entry_header); -#endif break; case ACPI_DMAR_HARDWARE_AFFINITY: #ifdef CONFIG_ACPI_NUMA @@ -557,34 +426,31 @@ dmar_find_matched_drhd_unit(struct pci_dev *dev) int __init dmar_dev_scope_init(void) { + static int dmar_dev_scope_initialized; struct dmar_drhd_unit *drhd, *drhd_n; int ret = -ENODEV; + if (dmar_dev_scope_initialized) + return dmar_dev_scope_initialized; + + if (list_empty(&dmar_drhd_units)) + goto fail; + list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) { ret = dmar_parse_dev(drhd); if (ret) - return ret; + goto fail; } -#ifdef CONFIG_DMAR - { - struct dmar_rmrr_unit *rmrr, *rmrr_n; - struct dmar_atsr_unit *atsr, *atsr_n; - - list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) { - ret = rmrr_parse_dev(rmrr); - if (ret) - return ret; - } + ret = dmar_parse_rmrr_atsr_dev(); + if (ret) + goto fail; - list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) { - ret = atsr_parse_dev(atsr); - if (ret) - return ret; - } - } -#endif + dmar_dev_scope_initialized = 1; + return 0; +fail: + dmar_dev_scope_initialized = ret; return ret; } @@ -611,14 +477,6 @@ int __init dmar_table_init(void) return -ENODEV; } -#ifdef CONFIG_DMAR - if (list_empty(&dmar_rmrr_units)) - printk(KERN_INFO PREFIX "No RMRR found\n"); - - if (list_empty(&dmar_atsr_units)) - printk(KERN_INFO PREFIX "No ATSR found\n"); -#endif - return 0; } @@ -682,9 +540,6 @@ int __init check_zero_address(void) return 1; failed: -#ifdef CONFIG_DMAR - dmar_disabled = 1; -#endif return 0; } @@ -696,22 +551,21 @@ int __init detect_intel_iommu(void) if (ret) ret = check_zero_address(); { -#ifdef CONFIG_INTR_REMAP struct acpi_table_dmar *dmar; dmar = (struct acpi_table_dmar *) dmar_tbl; - if (ret && cpu_has_x2apic && dmar->flags & 0x1) + + if (ret && intr_remapping_enabled && cpu_has_x2apic && + dmar->flags & 0x1) printk(KERN_INFO - "Queued invalidation will be enabled to support " - "x2apic and Intr-remapping.\n"); -#endif -#ifdef CONFIG_DMAR + "Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); + if (ret && !no_iommu && !iommu_detected && !dmar_disabled) { iommu_detected = 1; /* Make sure ACS will be enabled */ pci_request_acs(); } -#endif + #ifdef CONFIG_X86 if (ret) x86_init.iommu.iommu_init = intel_iommu_init; @@ -758,7 +612,6 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) goto err_unmap; } -#ifdef CONFIG_DMAR agaw = iommu_calculate_agaw(iommu); if (agaw < 0) { printk(KERN_ERR @@ -773,7 +626,6 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->seq_id); goto err_unmap; } -#endif iommu->agaw = agaw; iommu->msagaw = msagaw; @@ -800,7 +652,7 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) (unsigned long long)iommu->cap, (unsigned long long)iommu->ecap); - spin_lock_init(&iommu->register_lock); + raw_spin_lock_init(&iommu->register_lock); drhd->iommu = iommu; return 0; @@ -817,9 +669,7 @@ void free_iommu(struct intel_iommu *iommu) if (!iommu) return; -#ifdef CONFIG_DMAR free_dmar_iommu(iommu); -#endif if (iommu->reg) iounmap(iommu->reg); @@ -921,11 +771,11 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu) restart: rc = 0; - spin_lock_irqsave(&qi->q_lock, flags); + raw_spin_lock_irqsave(&qi->q_lock, flags); while (qi->free_cnt < 3) { - spin_unlock_irqrestore(&qi->q_lock, flags); + raw_spin_unlock_irqrestore(&qi->q_lock, flags); cpu_relax(); - spin_lock_irqsave(&qi->q_lock, flags); + raw_spin_lock_irqsave(&qi->q_lock, flags); } index = qi->free_head; @@ -965,15 +815,15 @@ restart: if (rc) break; - spin_unlock(&qi->q_lock); + raw_spin_unlock(&qi->q_lock); cpu_relax(); - spin_lock(&qi->q_lock); + raw_spin_lock(&qi->q_lock); } qi->desc_status[index] = QI_DONE; reclaim_free_desc(qi); - spin_unlock_irqrestore(&qi->q_lock, flags); + raw_spin_unlock_irqrestore(&qi->q_lock, flags); if (rc == -EAGAIN) goto restart; @@ -1062,7 +912,7 @@ void dmar_disable_qi(struct intel_iommu *iommu) if (!ecap_qis(iommu->ecap)) return; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_QIES)) @@ -1082,7 +932,7 @@ void dmar_disable_qi(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, !(sts & DMA_GSTS_QIES), sts); end: - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } /* @@ -1097,7 +947,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) qi->free_head = qi->free_tail = 0; qi->free_cnt = QI_LENGTH; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); /* write zero to the tail reg */ writel(0, iommu->reg + DMAR_IQT_REG); @@ -1110,7 +960,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } /* @@ -1159,7 +1009,7 @@ int dmar_enable_qi(struct intel_iommu *iommu) qi->free_head = qi->free_tail = 0; qi->free_cnt = QI_LENGTH; - spin_lock_init(&qi->q_lock); + raw_spin_lock_init(&qi->q_lock); __dmar_enable_qi(iommu); @@ -1225,11 +1075,11 @@ void dmar_msi_unmask(struct irq_data *data) unsigned long flag; /* unmask it */ - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(0, iommu->reg + DMAR_FECTL_REG); /* Read a reg to force flush the post write */ readl(iommu->reg + DMAR_FECTL_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_mask(struct irq_data *data) @@ -1238,11 +1088,11 @@ void dmar_msi_mask(struct irq_data *data) struct intel_iommu *iommu = irq_data_get_irq_handler_data(data); /* mask it */ - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG); /* Read a reg to force flush the post write */ readl(iommu->reg + DMAR_FECTL_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_write(int irq, struct msi_msg *msg) @@ -1250,11 +1100,11 @@ void dmar_msi_write(int irq, struct msi_msg *msg) struct intel_iommu *iommu = irq_get_handler_data(irq); unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(msg->data, iommu->reg + DMAR_FEDATA_REG); writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG); writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_read(int irq, struct msi_msg *msg) @@ -1262,11 +1112,11 @@ void dmar_msi_read(int irq, struct msi_msg *msg) struct intel_iommu *iommu = irq_get_handler_data(irq); unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); msg->data = readl(iommu->reg + DMAR_FEDATA_REG); msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG); msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } static int dmar_fault_do_one(struct intel_iommu *iommu, int type, @@ -1303,7 +1153,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) u32 fault_status; unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); fault_status = readl(iommu->reg + DMAR_FSTS_REG); if (fault_status) printk(KERN_ERR "DRHD: handling fault status reg %x\n", @@ -1342,7 +1192,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) writel(DMA_FRCD_F, iommu->reg + reg + fault_index * PRIMARY_FAULT_REG_LEN + 12); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); dmar_fault_do_one(iommu, type, fault_reason, source_id, guest_addr); @@ -1350,14 +1200,14 @@ irqreturn_t dmar_fault(int irq, void *dev_id) fault_index++; if (fault_index >= cap_num_fault_regs(iommu->cap)) fault_index = 0; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); } clear_rest: /* clear all the other faults */ fault_status = readl(iommu->reg + DMAR_FSTS_REG); writel(fault_status, iommu->reg + DMAR_FSTS_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return IRQ_HANDLED; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c621c98c99da..bb161d2fa03c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -306,6 +306,11 @@ static inline bool dma_pte_present(struct dma_pte *pte) return (pte->val & 3) != 0; } +static inline bool dma_pte_superpage(struct dma_pte *pte) +{ + return (pte->val & (1 << 7)); +} + static inline int first_pte_in_page(struct dma_pte *pte) { return !((unsigned long)pte & ~VTD_PAGE_MASK); @@ -393,17 +398,20 @@ static long list_size; static void domain_remove_dev_info(struct dmar_domain *domain); -#ifdef CONFIG_DMAR_DEFAULT_ON +#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; #else int dmar_disabled = 1; -#endif /*CONFIG_DMAR_DEFAULT_ON*/ +#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; +int intel_iommu_gfx_mapped; +EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); + #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) static DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); @@ -577,17 +585,18 @@ static void domain_update_iommu_snooping(struct dmar_domain *domain) static void domain_update_iommu_superpage(struct dmar_domain *domain) { - int i, mask = 0xf; + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + int mask = 0xf; if (!intel_iommu_superpage) { domain->iommu_superpage = 0; return; } - domain->iommu_superpage = 4; /* 1TiB */ - - for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) { - mask |= cap_super_page_val(g_iommus[i]->cap); + /* set iommu_superpage to the smallest common denominator */ + for_each_active_iommu(iommu, drhd) { + mask &= cap_super_page_val(iommu->cap); if (!mask) { break; } @@ -730,29 +739,23 @@ out: } static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, - unsigned long pfn, int large_level) + unsigned long pfn, int target_level) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; struct dma_pte *parent, *pte = NULL; int level = agaw_to_level(domain->agaw); - int offset, target_level; + int offset; BUG_ON(!domain->pgd); BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width); parent = domain->pgd; - /* Search pte */ - if (!large_level) - target_level = 1; - else - target_level = large_level; - while (level > 0) { void *tmp_page; offset = pfn_level_offset(pfn, level); pte = &parent[offset]; - if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE)) + if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte))) break; if (level == target_level) break; @@ -816,13 +819,14 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain, } /* clear last level pte, a tlb flush should be followed */ -static void dma_pte_clear_range(struct dmar_domain *domain, +static int dma_pte_clear_range(struct dmar_domain *domain, unsigned long start_pfn, unsigned long last_pfn) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; unsigned int large_page = 1; struct dma_pte *first_pte, *pte; + int order; BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); @@ -846,6 +850,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain, (void *)pte - (void *)first_pte); } while (start_pfn && start_pfn <= last_pfn); + + order = (large_page - 1) * 9; + return order; } /* free page table pages. last level pte should already be cleared */ @@ -932,7 +939,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) addr = iommu->root_entry; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr)); writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG); @@ -941,7 +948,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_RTPS), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } static void iommu_flush_write_buffer(struct intel_iommu *iommu) @@ -952,14 +959,14 @@ static void iommu_flush_write_buffer(struct intel_iommu *iommu) if (!rwbf_quirk && !cap_rwbf(iommu->cap)) return; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(val & DMA_GSTS_WBFS)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -986,14 +993,14 @@ static void __iommu_flush_context(struct intel_iommu *iommu, } val |= DMA_CCMD_ICC; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_CCMD_REG, val); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, dmar_readq, (!(val & DMA_CCMD_ICC)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -1032,7 +1039,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, if (cap_write_drain(iommu->cap)) val |= DMA_TLB_WRITE_DRAIN; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); /* Note: Only uses first TLB reg currently */ if (val_iva) dmar_writeq(iommu->reg + tlb_offset, val_iva); @@ -1042,7 +1049,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, IOMMU_WAIT_OP(iommu, tlb_offset + 8, dmar_readq, (!(val & DMA_TLB_IVT)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); /* check IOTLB invalidation granularity */ if (DMA_TLB_IAIG(val) == 0) @@ -1158,7 +1165,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) u32 pmen; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); pmen = readl(iommu->reg + DMAR_PMEN_REG); pmen &= ~DMA_PMEN_EPM; writel(pmen, iommu->reg + DMAR_PMEN_REG); @@ -1167,7 +1174,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG, readl, !(pmen & DMA_PMEN_PRS), pmen); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } static int iommu_enable_translation(struct intel_iommu *iommu) @@ -1175,7 +1182,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); iommu->gcmd |= DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1183,7 +1190,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_TES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); return 0; } @@ -1192,7 +1199,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->gcmd &= ~DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1200,7 +1207,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(sts & DMA_GSTS_TES)), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return 0; } @@ -2150,7 +2157,7 @@ static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, rmrr->end_address); } -#ifdef CONFIG_DMAR_FLOPPY_WA +#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA static inline void iommu_prepare_isa(void) { struct pci_dev *pdev; @@ -2173,7 +2180,7 @@ static inline void iommu_prepare_isa(void) { return; } -#endif /* !CONFIG_DMAR_FLPY_WA */ +#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */ static int md_domain_init(struct dmar_domain *domain, int guest_width); @@ -2484,7 +2491,7 @@ static int __init init_dmars(void) if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; -#ifdef CONFIG_DMAR_BROKEN_GFX_WA +#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA iommu_identity_mapping |= IDENTMAP_GFX; #endif @@ -3226,9 +3233,6 @@ static void __init init_no_remapping_devices(void) } } - if (dmar_map_gfx) - return; - for_each_drhd_unit(drhd) { int i; if (drhd->ignored || drhd->include_all) @@ -3236,18 +3240,23 @@ static void __init init_no_remapping_devices(void) for (i = 0; i < drhd->devices_cnt; i++) if (drhd->devices[i] && - !IS_GFX_DEVICE(drhd->devices[i])) + !IS_GFX_DEVICE(drhd->devices[i])) break; if (i < drhd->devices_cnt) continue; - /* bypass IOMMU if it is just for gfx devices */ - drhd->ignored = 1; - for (i = 0; i < drhd->devices_cnt; i++) { - if (!drhd->devices[i]) - continue; - drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + /* This IOMMU has *only* gfx devices. Either bypass it or + set the gfx_mapped flag, as appropriate */ + if (dmar_map_gfx) { + intel_iommu_gfx_mapped = 1; + } else { + drhd->ignored = 1; + for (i = 0; i < drhd->devices_cnt; i++) { + if (!drhd->devices[i]) + continue; + drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + } } } } @@ -3320,7 +3329,7 @@ static int iommu_suspend(void) for_each_active_iommu(iommu, drhd) { iommu_disable_translation(iommu); - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->iommu_state[SR_DMAR_FECTL_REG] = readl(iommu->reg + DMAR_FECTL_REG); @@ -3331,7 +3340,7 @@ static int iommu_suspend(void) iommu->iommu_state[SR_DMAR_FEUADDR_REG] = readl(iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } return 0; @@ -3358,7 +3367,7 @@ static void iommu_resume(void) for_each_active_iommu(iommu, drhd) { - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->iommu_state[SR_DMAR_FECTL_REG], iommu->reg + DMAR_FECTL_REG); @@ -3369,7 +3378,7 @@ static void iommu_resume(void) writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG], iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } for_each_active_iommu(iommu, drhd) @@ -3390,6 +3399,151 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ +LIST_HEAD(dmar_rmrr_units); + +static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) +{ + list_add(&rmrr->list, &dmar_rmrr_units); +} + + +int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header) +{ + struct acpi_dmar_reserved_memory *rmrr; + struct dmar_rmrr_unit *rmrru; + + rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); + if (!rmrru) + return -ENOMEM; + + rmrru->hdr = header; + rmrr = (struct acpi_dmar_reserved_memory *)header; + rmrru->base_address = rmrr->base_address; + rmrru->end_address = rmrr->end_address; + + dmar_register_rmrr_unit(rmrru); + return 0; +} + +static int __init +rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) +{ + struct acpi_dmar_reserved_memory *rmrr; + int ret; + + rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; + ret = dmar_parse_dev_scope((void *)(rmrr + 1), + ((void *)rmrr) + rmrr->header.length, + &rmrru->devices_cnt, &rmrru->devices, rmrr->segment); + + if (ret || (rmrru->devices_cnt == 0)) { + list_del(&rmrru->list); + kfree(rmrru); + } + return ret; +} + +static LIST_HEAD(dmar_atsr_units); + +int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) +{ + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + atsr = container_of(hdr, struct acpi_dmar_atsr, header); + atsru = kzalloc(sizeof(*atsru), GFP_KERNEL); + if (!atsru) + return -ENOMEM; + + atsru->hdr = hdr; + atsru->include_all = atsr->flags & 0x1; + + list_add(&atsru->list, &dmar_atsr_units); + + return 0; +} + +static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) +{ + int rc; + struct acpi_dmar_atsr *atsr; + + if (atsru->include_all) + return 0; + + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + rc = dmar_parse_dev_scope((void *)(atsr + 1), + (void *)atsr + atsr->header.length, + &atsru->devices_cnt, &atsru->devices, + atsr->segment); + if (rc || !atsru->devices_cnt) { + list_del(&atsru->list); + kfree(atsru); + } + + return rc; +} + +int dmar_find_matched_atsr_unit(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus; + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + dev = pci_physfn(dev); + + list_for_each_entry(atsru, &dmar_atsr_units, list) { + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + if (atsr->segment == pci_domain_nr(dev->bus)) + goto found; + } + + return 0; + +found: + for (bus = dev->bus; bus; bus = bus->parent) { + struct pci_dev *bridge = bus->self; + + if (!bridge || !pci_is_pcie(bridge) || + bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) + return 0; + + if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) { + for (i = 0; i < atsru->devices_cnt; i++) + if (atsru->devices[i] == bridge) + return 1; + break; + } + } + + if (atsru->include_all) + return 1; + + return 0; +} + +int dmar_parse_rmrr_atsr_dev(void) +{ + struct dmar_rmrr_unit *rmrr, *rmrr_n; + struct dmar_atsr_unit *atsr, *atsr_n; + int ret = 0; + + list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) { + ret = rmrr_parse_dev(rmrr); + if (ret) + return ret; + } + + list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) { + ret = atsr_parse_dev(atsr); + if (ret) + return ret; + } + + return ret; +} + /* * Here we only respond to action of unbound device from driver. * @@ -3439,16 +3593,12 @@ int __init intel_iommu_init(void) return -ENODEV; } - if (dmar_dev_scope_init()) { + if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); return -ENODEV; } - /* - * Check the need for DMA-remapping initialization now. - * Above initialization will also be used by Interrupt-remapping. - */ if (no_iommu || dmar_disabled) return -ENODEV; @@ -3458,6 +3608,12 @@ int __init intel_iommu_init(void) return -ENODEV; } + if (list_empty(&dmar_rmrr_units)) + printk(KERN_INFO "DMAR: No RMRR found\n"); + + if (list_empty(&dmar_atsr_units)) + printk(KERN_INFO "DMAR: No ATSR found\n"); + if (dmar_init_reserved_ranges()) { if (force_on) panic("tboot: Failed to reserve iommu ranges\n"); @@ -3486,7 +3642,7 @@ int __init intel_iommu_init(void) init_iommu_pm_ops(); - register_iommu(&intel_iommu_ops); + bus_set_iommu(&pci_bus_type, &intel_iommu_ops); bus_register_notifier(&pci_bus_type, &device_nb); @@ -3568,6 +3724,8 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, found = 1; } + spin_unlock_irqrestore(&device_domain_lock, flags); + if (found == 0) { unsigned long tmp_flags; spin_lock_irqsave(&domain->iommu_lock, tmp_flags); @@ -3584,8 +3742,6 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&iommu->lock, tmp_flags); } } - - spin_unlock_irqrestore(&device_domain_lock, flags); } static void vm_domain_remove_all_dev_info(struct dmar_domain *domain) @@ -3739,6 +3895,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain) vm_domain_exit(dmar_domain); return -ENOMEM; } + domain_update_iommu_cap(dmar_domain); domain->priv = dmar_domain; return 0; @@ -3864,14 +4021,15 @@ static int intel_iommu_unmap(struct iommu_domain *domain, { struct dmar_domain *dmar_domain = domain->priv; size_t size = PAGE_SIZE << gfp_order; + int order; - dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, + order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, (iova + size - 1) >> VTD_PAGE_SHIFT); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; - return gfp_order; + return order; } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, @@ -3950,7 +4108,11 @@ static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev) if (!(ggc & GGC_MEMORY_VT_ENABLED)) { printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); dmar_map_gfx = 0; - } + } else if (dmar_map_gfx) { + /* we have to ensure the gfx device is idle before we flush */ + printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); + intel_iommu_strict = 1; + } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt); diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 1a89d4a2cadf..07c9f189f314 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -21,6 +21,7 @@ int intr_remapping_enabled; static int disable_intremap; static int disable_sourceid_checking; +static int no_x2apic_optout; static __init int setup_nointremap(char *str) { @@ -34,18 +35,26 @@ static __init int setup_intremap(char *str) if (!str) return -EINVAL; - if (!strncmp(str, "on", 2)) - disable_intremap = 0; - else if (!strncmp(str, "off", 3)) - disable_intremap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; + while (*str) { + if (!strncmp(str, "on", 2)) + disable_intremap = 0; + else if (!strncmp(str, "off", 3)) + disable_intremap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } return 0; } early_param("intremap", setup_intremap); -static DEFINE_SPINLOCK(irq_2_ir_lock); +static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); static struct irq_2_iommu *irq_2_iommu(unsigned int irq) { @@ -62,12 +71,12 @@ int get_irte(int irq, struct irte *entry) if (!entry || !irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); index = irq_iommu->irte_index + irq_iommu->sub_handle; *entry = *(irq_iommu->iommu->ir_table->base + index); - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return 0; } @@ -101,7 +110,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) return -1; } - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); do { for (i = index; i < index + count; i++) if (table->base[i].present) @@ -113,7 +122,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) index = (index + count) % INTR_REMAP_TABLE_ENTRIES; if (index == start_index) { - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); printk(KERN_ERR "can't allocate an IRTE\n"); return -1; } @@ -127,7 +136,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) irq_iommu->sub_handle = 0; irq_iommu->irte_mask = mask; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return index; } @@ -152,10 +161,10 @@ int map_irq_to_irte_handle(int irq, u16 *sub_handle) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); *sub_handle = irq_iommu->sub_handle; index = irq_iommu->irte_index; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return index; } @@ -167,14 +176,14 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); irq_iommu->iommu = iommu; irq_iommu->irte_index = index; irq_iommu->sub_handle = subhandle; irq_iommu->irte_mask = 0; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return 0; } @@ -190,7 +199,7 @@ int modify_irte(int irq, struct irte *irte_modified) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); iommu = irq_iommu->iommu; @@ -202,7 +211,7 @@ int modify_irte(int irq, struct irte *irte_modified) __iommu_flush_cache(iommu, irte, sizeof(*irte)); rc = qi_flush_iec(iommu, index, 0); - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return rc; } @@ -270,7 +279,7 @@ int free_irte(int irq) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); rc = clear_entries(irq_iommu); @@ -279,7 +288,7 @@ int free_irte(int irq) irq_iommu->sub_handle = 0; irq_iommu->irte_mask = 0; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return rc; } @@ -409,7 +418,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) addr = virt_to_phys((void *)iommu->ir_table->base); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); dmar_writeq(iommu->reg + DMAR_IRTA_REG, (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); @@ -420,7 +429,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRTPS), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); /* * global invalidation of interrupt entry cache before enabling @@ -428,7 +437,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) */ qi_global_iec(iommu); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); /* Enable interrupt-remapping */ iommu->gcmd |= DMA_GCMD_IRE; @@ -437,7 +446,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } @@ -485,7 +494,7 @@ static void iommu_disable_intr_remapping(struct intel_iommu *iommu) */ qi_global_iec(iommu); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_IRES)) @@ -498,7 +507,16 @@ static void iommu_disable_intr_remapping(struct intel_iommu *iommu) readl, !(sts & DMA_GSTS_IRES), sts); end: - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +static int __init dmar_x2apic_optout(void) +{ + struct acpi_table_dmar *dmar; + dmar = (struct acpi_table_dmar *)dmar_tbl; + if (!dmar || no_x2apic_optout) + return 0; + return dmar->flags & DMAR_X2APIC_OPT_OUT; } int __init intr_remapping_supported(void) @@ -521,16 +539,25 @@ int __init intr_remapping_supported(void) return 1; } -int __init enable_intr_remapping(int eim) +int __init enable_intr_remapping(void) { struct dmar_drhd_unit *drhd; int setup = 0; + int eim = 0; if (parse_ioapics_under_ir() != 1) { printk(KERN_INFO "Not enable interrupt remapping\n"); return -1; } + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + WARN(!eim, KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled\n" + "This will leave your machine vulnerable to irq-injection attacks\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + } + for_each_drhd_unit(drhd) { struct intel_iommu *iommu = drhd->iommu; @@ -606,8 +633,9 @@ int __init enable_intr_remapping(int eim) goto error; intr_remapping_enabled = 1; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); - return 0; + return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; error: /* @@ -745,6 +773,15 @@ int __init parse_ioapics_under_ir(void) return ir_supported; } +int ir_dev_scope_init(void) +{ + if (!intr_remapping_enabled) + return 0; + + return dmar_dev_scope_init(); +} +rootfs_initcall(ir_dev_scope_init); + void disable_intr_remapping(void) { struct dmar_drhd_unit *drhd; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 6e6b6a11b3ce..2fb2963df553 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -16,6 +16,8 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include <linux/device.h> +#include <linux/kernel.h> #include <linux/bug.h> #include <linux/types.h> #include <linux/module.h> @@ -23,32 +25,78 @@ #include <linux/errno.h> #include <linux/iommu.h> -static struct iommu_ops *iommu_ops; +static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops) +{ +} -void register_iommu(struct iommu_ops *ops) +/** + * bus_set_iommu - set iommu-callbacks for the bus + * @bus: bus. + * @ops: the callbacks provided by the iommu-driver + * + * This function is called by an iommu driver to set the iommu methods + * used for a particular bus. Drivers for devices on that bus can use + * the iommu-api after these ops are registered. + * This special function is needed because IOMMUs are usually devices on + * the bus itself, so the iommu drivers are not initialized when the bus + * is set up. With this function the iommu-driver can set the iommu-ops + * afterwards. + */ +int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops) { - if (iommu_ops) - BUG(); + if (bus->iommu_ops != NULL) + return -EBUSY; + + bus->iommu_ops = ops; + + /* Do IOMMU specific setup for this bus-type */ + iommu_bus_init(bus, ops); - iommu_ops = ops; + return 0; } +EXPORT_SYMBOL_GPL(bus_set_iommu); -bool iommu_found(void) +bool iommu_present(struct bus_type *bus) { - return iommu_ops != NULL; + return bus->iommu_ops != NULL; } -EXPORT_SYMBOL_GPL(iommu_found); +EXPORT_SYMBOL_GPL(iommu_present); -struct iommu_domain *iommu_domain_alloc(void) +/** + * iommu_set_fault_handler() - set a fault handler for an iommu domain + * @domain: iommu domain + * @handler: fault handler + * + * This function should be used by IOMMU users which want to be notified + * whenever an IOMMU fault happens. + * + * The fault handler itself should return 0 on success, and an appropriate + * error code otherwise. + */ +void iommu_set_fault_handler(struct iommu_domain *domain, + iommu_fault_handler_t handler) +{ + BUG_ON(!domain); + + domain->handler = handler; +} +EXPORT_SYMBOL_GPL(iommu_set_fault_handler); + +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) { struct iommu_domain *domain; int ret; + if (bus == NULL || bus->iommu_ops == NULL) + return NULL; + domain = kmalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; - ret = iommu_ops->domain_init(domain); + domain->ops = bus->iommu_ops; + + ret = domain->ops->domain_init(domain); if (ret) goto out_free; @@ -63,62 +111,78 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) { - iommu_ops->domain_destroy(domain); + if (likely(domain->ops->domain_destroy != NULL)) + domain->ops->domain_destroy(domain); + kfree(domain); } EXPORT_SYMBOL_GPL(iommu_domain_free); int iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - return iommu_ops->attach_dev(domain, dev); + if (unlikely(domain->ops->attach_dev == NULL)) + return -ENODEV; + + return domain->ops->attach_dev(domain, dev); } EXPORT_SYMBOL_GPL(iommu_attach_device); void iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - iommu_ops->detach_dev(domain, dev); + if (unlikely(domain->ops->detach_dev == NULL)) + return; + + domain->ops->detach_dev(domain, dev); } EXPORT_SYMBOL_GPL(iommu_detach_device); phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, unsigned long iova) { - return iommu_ops->iova_to_phys(domain, iova); + if (unlikely(domain->ops->iova_to_phys == NULL)) + return 0; + + return domain->ops->iova_to_phys(domain, iova); } EXPORT_SYMBOL_GPL(iommu_iova_to_phys); int iommu_domain_has_cap(struct iommu_domain *domain, unsigned long cap) { - return iommu_ops->domain_has_cap(domain, cap); + if (unlikely(domain->ops->domain_has_cap == NULL)) + return 0; + + return domain->ops->domain_has_cap(domain, cap); } EXPORT_SYMBOL_GPL(iommu_domain_has_cap); int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, int gfp_order, int prot) { - unsigned long invalid_mask; size_t size; - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; + if (unlikely(domain->ops->map == NULL)) + return -ENODEV; - BUG_ON((iova | paddr) & invalid_mask); + size = PAGE_SIZE << gfp_order; - return iommu_ops->map(domain, iova, paddr, gfp_order, prot); + BUG_ON(!IS_ALIGNED(iova | paddr, size)); + + return domain->ops->map(domain, iova, paddr, gfp_order, prot); } EXPORT_SYMBOL_GPL(iommu_map); int iommu_unmap(struct iommu_domain *domain, unsigned long iova, int gfp_order) { - unsigned long invalid_mask; size_t size; - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; + if (unlikely(domain->ops->unmap == NULL)) + return -ENODEV; + + size = PAGE_SIZE << gfp_order; - BUG_ON(iova & invalid_mask); + BUG_ON(!IS_ALIGNED(iova, size)); - return iommu_ops->unmap(domain, iova, gfp_order); + return domain->ops->unmap(domain, iova, gfp_order); } EXPORT_SYMBOL_GPL(iommu_unmap); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 1a584e077c61..5865dd2e28f9 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -543,6 +543,13 @@ static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, } ret = __flush_iotlb(domain); + + /* + * the IOMMU API requires us to return the order of the unmapped + * page (on success). + */ + if (!ret) + ret = order; fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; @@ -721,7 +728,7 @@ static void __init setup_iommu_tex_classes(void) static int __init msm_iommu_init(void) { setup_iommu_tex_classes(); - register_iommu(&msm_iommu_ops); + bus_set_iommu(&platform_bus_type, &msm_iommu_ops); return 0; } diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c new file mode 100644 index 000000000000..9c192e79f806 --- /dev/null +++ b/drivers/iommu/omap-iommu-debug.c @@ -0,0 +1,418 @@ +/* + * omap iommu: debugfs interface + * + * Copyright (C) 2008-2009 Nokia Corporation + * + * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/err.h> +#include <linux/clk.h> +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/platform_device.h> +#include <linux/debugfs.h> + +#include <plat/iommu.h> +#include <plat/iovmm.h> + +#include <plat/iopgtable.h> + +#define MAXCOLUMN 100 /* for short messages */ + +static DEFINE_MUTEX(iommu_debug_lock); + +static struct dentry *iommu_debug_root; + +static ssize_t debug_read_ver(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + u32 ver = omap_iommu_arch_version(); + char buf[MAXCOLUMN], *p = buf; + + p += sprintf(p, "H/W version: %d.%d\n", (ver >> 4) & 0xf , ver & 0xf); + + return simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); +} + +static ssize_t debug_read_regs(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + char *p, *buf; + ssize_t bytes; + + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + mutex_lock(&iommu_debug_lock); + + bytes = omap_iommu_dump_ctx(obj, p, count); + bytes = simple_read_from_buffer(userbuf, count, ppos, buf, bytes); + + mutex_unlock(&iommu_debug_lock); + kfree(buf); + + return bytes; +} + +static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + char *p, *buf; + ssize_t bytes, rest; + + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + mutex_lock(&iommu_debug_lock); + + p += sprintf(p, "%8s %8s\n", "cam:", "ram:"); + p += sprintf(p, "-----------------------------------------\n"); + rest = count - (p - buf); + p += omap_dump_tlb_entries(obj, p, rest); + + bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); + + mutex_unlock(&iommu_debug_lock); + kfree(buf); + + return bytes; +} + +static ssize_t debug_write_pagetable(struct file *file, + const char __user *userbuf, size_t count, loff_t *ppos) +{ + struct iotlb_entry e; + struct cr_regs cr; + int err; + struct omap_iommu *obj = file->private_data; + char buf[MAXCOLUMN], *p = buf; + + count = min(count, sizeof(buf)); + + mutex_lock(&iommu_debug_lock); + if (copy_from_user(p, userbuf, count)) { + mutex_unlock(&iommu_debug_lock); + return -EFAULT; + } + + sscanf(p, "%x %x", &cr.cam, &cr.ram); + if (!cr.cam || !cr.ram) { + mutex_unlock(&iommu_debug_lock); + return -EINVAL; + } + + omap_iotlb_cr_to_e(&cr, &e); + err = omap_iopgtable_store_entry(obj, &e); + if (err) + dev_err(obj->dev, "%s: fail to store cr\n", __func__); + + mutex_unlock(&iommu_debug_lock); + return count; +} + +#define dump_ioptable_entry_one(lv, da, val) \ + ({ \ + int __err = 0; \ + ssize_t bytes; \ + const int maxcol = 22; \ + const char *str = "%d: %08x %08x\n"; \ + bytes = snprintf(p, maxcol, str, lv, da, val); \ + p += bytes; \ + len -= bytes; \ + if (len < maxcol) \ + __err = -ENOMEM; \ + __err; \ + }) + +static ssize_t dump_ioptable(struct omap_iommu *obj, char *buf, ssize_t len) +{ + int i; + u32 *iopgd; + char *p = buf; + + spin_lock(&obj->page_table_lock); + + iopgd = iopgd_offset(obj, 0); + for (i = 0; i < PTRS_PER_IOPGD; i++, iopgd++) { + int j, err; + u32 *iopte; + u32 da; + + if (!*iopgd) + continue; + + if (!(*iopgd & IOPGD_TABLE)) { + da = i << IOPGD_SHIFT; + + err = dump_ioptable_entry_one(1, da, *iopgd); + if (err) + goto out; + continue; + } + + iopte = iopte_offset(iopgd, 0); + + for (j = 0; j < PTRS_PER_IOPTE; j++, iopte++) { + if (!*iopte) + continue; + + da = (i << IOPGD_SHIFT) + (j << IOPTE_SHIFT); + err = dump_ioptable_entry_one(2, da, *iopgd); + if (err) + goto out; + } + } +out: + spin_unlock(&obj->page_table_lock); + + return p - buf; +} + +static ssize_t debug_read_pagetable(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + char *p, *buf; + size_t bytes; + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + p += sprintf(p, "L: %8s %8s\n", "da:", "pa:"); + p += sprintf(p, "-----------------------------------------\n"); + + mutex_lock(&iommu_debug_lock); + + bytes = PAGE_SIZE - (p - buf); + p += dump_ioptable(obj, p, bytes); + + bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); + + mutex_unlock(&iommu_debug_lock); + free_page((unsigned long)buf); + + return bytes; +} + +static ssize_t debug_read_mmap(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + char *p, *buf; + struct iovm_struct *tmp; + int uninitialized_var(i); + ssize_t bytes; + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + p += sprintf(p, "%-3s %-8s %-8s %6s %8s\n", + "No", "start", "end", "size", "flags"); + p += sprintf(p, "-------------------------------------------------\n"); + + mutex_lock(&iommu_debug_lock); + + list_for_each_entry(tmp, &obj->mmap, list) { + size_t len; + const char *str = "%3d %08x-%08x %6x %8x\n"; + const int maxcol = 39; + + len = tmp->da_end - tmp->da_start; + p += snprintf(p, maxcol, str, + i, tmp->da_start, tmp->da_end, len, tmp->flags); + + if (PAGE_SIZE - (p - buf) < maxcol) + break; + i++; + } + + bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); + + mutex_unlock(&iommu_debug_lock); + free_page((unsigned long)buf); + + return bytes; +} + +static ssize_t debug_read_mem(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + char *p, *buf; + struct iovm_struct *area; + ssize_t bytes; + + count = min_t(ssize_t, count, PAGE_SIZE); + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + mutex_lock(&iommu_debug_lock); + + area = omap_find_iovm_area(obj, (u32)ppos); + if (IS_ERR(area)) { + bytes = -EINVAL; + goto err_out; + } + memcpy(p, area->va, count); + p += count; + + bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); +err_out: + mutex_unlock(&iommu_debug_lock); + free_page((unsigned long)buf); + + return bytes; +} + +static ssize_t debug_write_mem(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct omap_iommu *obj = file->private_data; + struct iovm_struct *area; + char *p, *buf; + + count = min_t(size_t, count, PAGE_SIZE); + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + p = buf; + + mutex_lock(&iommu_debug_lock); + + if (copy_from_user(p, userbuf, count)) { + count = -EFAULT; + goto err_out; + } + + area = omap_find_iovm_area(obj, (u32)ppos); + if (IS_ERR(area)) { + count = -EINVAL; + goto err_out; + } + memcpy(area->va, p, count); +err_out: + mutex_unlock(&iommu_debug_lock); + free_page((unsigned long)buf); + + return count; +} + +static int debug_open_generic(struct inode *inode, struct file *file) +{ + file->private_data = inode->i_private; + return 0; +} + +#define DEBUG_FOPS(name) \ + static const struct file_operations debug_##name##_fops = { \ + .open = debug_open_generic, \ + .read = debug_read_##name, \ + .write = debug_write_##name, \ + .llseek = generic_file_llseek, \ + }; + +#define DEBUG_FOPS_RO(name) \ + static const struct file_operations debug_##name##_fops = { \ + .open = debug_open_generic, \ + .read = debug_read_##name, \ + .llseek = generic_file_llseek, \ + }; + +DEBUG_FOPS_RO(ver); +DEBUG_FOPS_RO(regs); +DEBUG_FOPS_RO(tlb); +DEBUG_FOPS(pagetable); +DEBUG_FOPS_RO(mmap); +DEBUG_FOPS(mem); + +#define __DEBUG_ADD_FILE(attr, mode) \ + { \ + struct dentry *dent; \ + dent = debugfs_create_file(#attr, mode, parent, \ + obj, &debug_##attr##_fops); \ + if (!dent) \ + return -ENOMEM; \ + } + +#define DEBUG_ADD_FILE(name) __DEBUG_ADD_FILE(name, 600) +#define DEBUG_ADD_FILE_RO(name) __DEBUG_ADD_FILE(name, 400) + +static int iommu_debug_register(struct device *dev, void *data) +{ + struct platform_device *pdev = to_platform_device(dev); + struct omap_iommu *obj = platform_get_drvdata(pdev); + struct dentry *d, *parent; + + if (!obj || !obj->dev) + return -EINVAL; + + d = debugfs_create_dir(obj->name, iommu_debug_root); + if (!d) + return -ENOMEM; + parent = d; + + d = debugfs_create_u8("nr_tlb_entries", 400, parent, + (u8 *)&obj->nr_tlb_entries); + if (!d) + return -ENOMEM; + + DEBUG_ADD_FILE_RO(ver); + DEBUG_ADD_FILE_RO(regs); + DEBUG_ADD_FILE_RO(tlb); + DEBUG_ADD_FILE(pagetable); + DEBUG_ADD_FILE_RO(mmap); + DEBUG_ADD_FILE(mem); + + return 0; +} + +static int __init iommu_debug_init(void) +{ + struct dentry *d; + int err; + + d = debugfs_create_dir("iommu", NULL); + if (!d) + return -ENOMEM; + iommu_debug_root = d; + + err = omap_foreach_iommu_device(d, iommu_debug_register); + if (err) + goto err_out; + return 0; + +err_out: + debugfs_remove_recursive(iommu_debug_root); + return err; +} +module_init(iommu_debug_init) + +static void __exit iommu_debugfs_exit(void) +{ + debugfs_remove_recursive(iommu_debug_root); +} +module_exit(iommu_debugfs_exit) + +MODULE_DESCRIPTION("omap iommu: debugfs interface"); +MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c new file mode 100644 index 000000000000..8f32b2bf7587 --- /dev/null +++ b/drivers/iommu/omap-iommu.c @@ -0,0 +1,1245 @@ +/* + * omap iommu: tlb and pagetable primitives + * + * Copyright (C) 2008-2010 Nokia Corporation + * + * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>, + * Paul Mundt and Toshihiro Kobayashi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/err.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/interrupt.h> +#include <linux/ioport.h> +#include <linux/clk.h> +#include <linux/platform_device.h> +#include <linux/iommu.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> + +#include <asm/cacheflush.h> + +#include <plat/iommu.h> + +#include <plat/iopgtable.h> + +#define for_each_iotlb_cr(obj, n, __i, cr) \ + for (__i = 0; \ + (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ + __i++) + +/** + * struct omap_iommu_domain - omap iommu domain + * @pgtable: the page table + * @iommu_dev: an omap iommu device attached to this domain. only a single + * iommu device can be attached for now. + * @lock: domain lock, should be taken when attaching/detaching + */ +struct omap_iommu_domain { + u32 *pgtable; + struct omap_iommu *iommu_dev; + spinlock_t lock; +}; + +/* accommodate the difference between omap1 and omap2/3 */ +static const struct iommu_functions *arch_iommu; + +static struct platform_driver omap_iommu_driver; +static struct kmem_cache *iopte_cachep; + +/** + * omap_install_iommu_arch - Install archtecure specific iommu functions + * @ops: a pointer to architecture specific iommu functions + * + * There are several kind of iommu algorithm(tlb, pagetable) among + * omap series. This interface installs such an iommu algorighm. + **/ +int omap_install_iommu_arch(const struct iommu_functions *ops) +{ + if (arch_iommu) + return -EBUSY; + + arch_iommu = ops; + return 0; +} +EXPORT_SYMBOL_GPL(omap_install_iommu_arch); + +/** + * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions + * @ops: a pointer to architecture specific iommu functions + * + * This interface uninstalls the iommu algorighm installed previously. + **/ +void omap_uninstall_iommu_arch(const struct iommu_functions *ops) +{ + if (arch_iommu != ops) + pr_err("%s: not your arch\n", __func__); + + arch_iommu = NULL; +} +EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch); + +/** + * omap_iommu_save_ctx - Save registers for pm off-mode support + * @obj: target iommu + **/ +void omap_iommu_save_ctx(struct omap_iommu *obj) +{ + arch_iommu->save_ctx(obj); +} +EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); + +/** + * omap_iommu_restore_ctx - Restore registers for pm off-mode support + * @obj: target iommu + **/ +void omap_iommu_restore_ctx(struct omap_iommu *obj) +{ + arch_iommu->restore_ctx(obj); +} +EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); + +/** + * omap_iommu_arch_version - Return running iommu arch version + **/ +u32 omap_iommu_arch_version(void) +{ + return arch_iommu->version; +} +EXPORT_SYMBOL_GPL(omap_iommu_arch_version); + +static int iommu_enable(struct omap_iommu *obj) +{ + int err; + + if (!obj) + return -EINVAL; + + if (!arch_iommu) + return -ENODEV; + + clk_enable(obj->clk); + + err = arch_iommu->enable(obj); + + clk_disable(obj->clk); + return err; +} + +static void iommu_disable(struct omap_iommu *obj) +{ + if (!obj) + return; + + clk_enable(obj->clk); + + arch_iommu->disable(obj); + + clk_disable(obj->clk); +} + +/* + * TLB operations + */ +void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e) +{ + BUG_ON(!cr || !e); + + arch_iommu->cr_to_e(cr, e); +} +EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e); + +static inline int iotlb_cr_valid(struct cr_regs *cr) +{ + if (!cr) + return -EINVAL; + + return arch_iommu->cr_valid(cr); +} + +static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj, + struct iotlb_entry *e) +{ + if (!e) + return NULL; + + return arch_iommu->alloc_cr(obj, e); +} + +static u32 iotlb_cr_to_virt(struct cr_regs *cr) +{ + return arch_iommu->cr_to_virt(cr); +} + +static u32 get_iopte_attr(struct iotlb_entry *e) +{ + return arch_iommu->get_pte_attr(e); +} + +static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) +{ + return arch_iommu->fault_isr(obj, da); +} + +static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) +{ + u32 val; + + val = iommu_read_reg(obj, MMU_LOCK); + + l->base = MMU_LOCK_BASE(val); + l->vict = MMU_LOCK_VICT(val); + +} + +static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) +{ + u32 val; + + val = (l->base << MMU_LOCK_BASE_SHIFT); + val |= (l->vict << MMU_LOCK_VICT_SHIFT); + + iommu_write_reg(obj, val, MMU_LOCK); +} + +static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr) +{ + arch_iommu->tlb_read_cr(obj, cr); +} + +static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) +{ + arch_iommu->tlb_load_cr(obj, cr); + + iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); + iommu_write_reg(obj, 1, MMU_LD_TLB); +} + +/** + * iotlb_dump_cr - Dump an iommu tlb entry into buf + * @obj: target iommu + * @cr: contents of cam and ram register + * @buf: output buffer + **/ +static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, + char *buf) +{ + BUG_ON(!cr || !buf); + + return arch_iommu->dump_cr(obj, cr, buf); +} + +/* only used in iotlb iteration for-loop */ +static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) +{ + struct cr_regs cr; + struct iotlb_lock l; + + iotlb_lock_get(obj, &l); + l.vict = n; + iotlb_lock_set(obj, &l); + iotlb_read_cr(obj, &cr); + + return cr; +} + +/** + * load_iotlb_entry - Set an iommu tlb entry + * @obj: target iommu + * @e: an iommu tlb entry info + **/ +#ifdef PREFETCH_IOTLB +static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + int err = 0; + struct iotlb_lock l; + struct cr_regs *cr; + + if (!obj || !obj->nr_tlb_entries || !e) + return -EINVAL; + + clk_enable(obj->clk); + + iotlb_lock_get(obj, &l); + if (l.base == obj->nr_tlb_entries) { + dev_warn(obj->dev, "%s: preserve entries full\n", __func__); + err = -EBUSY; + goto out; + } + if (!e->prsvd) { + int i; + struct cr_regs tmp; + + for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp) + if (!iotlb_cr_valid(&tmp)) + break; + + if (i == obj->nr_tlb_entries) { + dev_dbg(obj->dev, "%s: full: no entry\n", __func__); + err = -EBUSY; + goto out; + } + + iotlb_lock_get(obj, &l); + } else { + l.vict = l.base; + iotlb_lock_set(obj, &l); + } + + cr = iotlb_alloc_cr(obj, e); + if (IS_ERR(cr)) { + clk_disable(obj->clk); + return PTR_ERR(cr); + } + + iotlb_load_cr(obj, cr); + kfree(cr); + + if (e->prsvd) + l.base++; + /* increment victim for next tlb load */ + if (++l.vict == obj->nr_tlb_entries) + l.vict = l.base; + iotlb_lock_set(obj, &l); +out: + clk_disable(obj->clk); + return err; +} + +#else /* !PREFETCH_IOTLB */ + +static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + return 0; +} + +#endif /* !PREFETCH_IOTLB */ + +static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + return load_iotlb_entry(obj, e); +} + +/** + * flush_iotlb_page - Clear an iommu tlb entry + * @obj: target iommu + * @da: iommu device virtual address + * + * Clear an iommu tlb entry which includes 'da' address. + **/ +static void flush_iotlb_page(struct omap_iommu *obj, u32 da) +{ + int i; + struct cr_regs cr; + + clk_enable(obj->clk); + + for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) { + u32 start; + size_t bytes; + + if (!iotlb_cr_valid(&cr)) + continue; + + start = iotlb_cr_to_virt(&cr); + bytes = iopgsz_to_bytes(cr.cam & 3); + + if ((start <= da) && (da < start + bytes)) { + dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n", + __func__, start, da, bytes); + iotlb_load_cr(obj, &cr); + iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); + } + } + clk_disable(obj->clk); + + if (i == obj->nr_tlb_entries) + dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da); +} + +/** + * flush_iotlb_all - Clear all iommu tlb entries + * @obj: target iommu + **/ +static void flush_iotlb_all(struct omap_iommu *obj) +{ + struct iotlb_lock l; + + clk_enable(obj->clk); + + l.base = 0; + l.vict = 0; + iotlb_lock_set(obj, &l); + + iommu_write_reg(obj, 1, MMU_GFLUSH); + + clk_disable(obj->clk); +} + +#if defined(CONFIG_OMAP_IOMMU_DEBUG) || defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE) + +ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes) +{ + if (!obj || !buf) + return -EINVAL; + + clk_enable(obj->clk); + + bytes = arch_iommu->dump_ctx(obj, buf, bytes); + + clk_disable(obj->clk); + + return bytes; +} +EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx); + +static int +__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) +{ + int i; + struct iotlb_lock saved; + struct cr_regs tmp; + struct cr_regs *p = crs; + + clk_enable(obj->clk); + iotlb_lock_get(obj, &saved); + + for_each_iotlb_cr(obj, num, i, tmp) { + if (!iotlb_cr_valid(&tmp)) + continue; + *p++ = tmp; + } + + iotlb_lock_set(obj, &saved); + clk_disable(obj->clk); + + return p - crs; +} + +/** + * omap_dump_tlb_entries - dump cr arrays to given buffer + * @obj: target iommu + * @buf: output buffer + **/ +size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes) +{ + int i, num; + struct cr_regs *cr; + char *p = buf; + + num = bytes / sizeof(*cr); + num = min(obj->nr_tlb_entries, num); + + cr = kcalloc(num, sizeof(*cr), GFP_KERNEL); + if (!cr) + return 0; + + num = __dump_tlb_entries(obj, cr, num); + for (i = 0; i < num; i++) + p += iotlb_dump_cr(obj, cr + i, p); + kfree(cr); + + return p - buf; +} +EXPORT_SYMBOL_GPL(omap_dump_tlb_entries); + +int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *)) +{ + return driver_for_each_device(&omap_iommu_driver.driver, + NULL, data, fn); +} +EXPORT_SYMBOL_GPL(omap_foreach_iommu_device); + +#endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */ + +/* + * H/W pagetable operations + */ +static void flush_iopgd_range(u32 *first, u32 *last) +{ + /* FIXME: L2 cache should be taken care of if it exists */ + do { + asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd" + : : "r" (first)); + first += L1_CACHE_BYTES / sizeof(*first); + } while (first <= last); +} + +static void flush_iopte_range(u32 *first, u32 *last) +{ + /* FIXME: L2 cache should be taken care of if it exists */ + do { + asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte" + : : "r" (first)); + first += L1_CACHE_BYTES / sizeof(*first); + } while (first <= last); +} + +static void iopte_free(u32 *iopte) +{ + /* Note: freed iopte's must be clean ready for re-use */ + kmem_cache_free(iopte_cachep, iopte); +} + +static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da) +{ + u32 *iopte; + + /* a table has already existed */ + if (*iopgd) + goto pte_ready; + + /* + * do the allocation outside the page table lock + */ + spin_unlock(&obj->page_table_lock); + iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL); + spin_lock(&obj->page_table_lock); + + if (!*iopgd) { + if (!iopte) + return ERR_PTR(-ENOMEM); + + *iopgd = virt_to_phys(iopte) | IOPGD_TABLE; + flush_iopgd_range(iopgd, iopgd); + + dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte); + } else { + /* We raced, free the reduniovant table */ + iopte_free(iopte); + } + +pte_ready: + iopte = iopte_offset(iopgd, da); + + dev_vdbg(obj->dev, + "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", + __func__, da, iopgd, *iopgd, iopte, *iopte); + + return iopte; +} + +static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) +{ + u32 *iopgd = iopgd_offset(obj, da); + + if ((da | pa) & ~IOSECTION_MASK) { + dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", + __func__, da, pa, IOSECTION_SIZE); + return -EINVAL; + } + + *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION; + flush_iopgd_range(iopgd, iopgd); + return 0; +} + +static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) +{ + u32 *iopgd = iopgd_offset(obj, da); + int i; + + if ((da | pa) & ~IOSUPER_MASK) { + dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", + __func__, da, pa, IOSUPER_SIZE); + return -EINVAL; + } + + for (i = 0; i < 16; i++) + *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER; + flush_iopgd_range(iopgd, iopgd + 15); + return 0; +} + +static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) +{ + u32 *iopgd = iopgd_offset(obj, da); + u32 *iopte = iopte_alloc(obj, iopgd, da); + + if (IS_ERR(iopte)) + return PTR_ERR(iopte); + + *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL; + flush_iopte_range(iopte, iopte); + + dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n", + __func__, da, pa, iopte, *iopte); + + return 0; +} + +static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot) +{ + u32 *iopgd = iopgd_offset(obj, da); + u32 *iopte = iopte_alloc(obj, iopgd, da); + int i; + + if ((da | pa) & ~IOLARGE_MASK) { + dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n", + __func__, da, pa, IOLARGE_SIZE); + return -EINVAL; + } + + if (IS_ERR(iopte)) + return PTR_ERR(iopte); + + for (i = 0; i < 16; i++) + *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE; + flush_iopte_range(iopte, iopte + 15); + return 0; +} + +static int +iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e) +{ + int (*fn)(struct omap_iommu *, u32, u32, u32); + u32 prot; + int err; + + if (!obj || !e) + return -EINVAL; + + switch (e->pgsz) { + case MMU_CAM_PGSZ_16M: + fn = iopgd_alloc_super; + break; + case MMU_CAM_PGSZ_1M: + fn = iopgd_alloc_section; + break; + case MMU_CAM_PGSZ_64K: + fn = iopte_alloc_large; + break; + case MMU_CAM_PGSZ_4K: + fn = iopte_alloc_page; + break; + default: + fn = NULL; + BUG(); + break; + } + + prot = get_iopte_attr(e); + + spin_lock(&obj->page_table_lock); + err = fn(obj, e->da, e->pa, prot); + spin_unlock(&obj->page_table_lock); + + return err; +} + +/** + * omap_iopgtable_store_entry - Make an iommu pte entry + * @obj: target iommu + * @e: an iommu tlb entry info + **/ +int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e) +{ + int err; + + flush_iotlb_page(obj, e->da); + err = iopgtable_store_entry_core(obj, e); + if (!err) + prefetch_iotlb_entry(obj, e); + return err; +} +EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry); + +/** + * iopgtable_lookup_entry - Lookup an iommu pte entry + * @obj: target iommu + * @da: iommu device virtual address + * @ppgd: iommu pgd entry pointer to be returned + * @ppte: iommu pte entry pointer to be returned + **/ +static void +iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte) +{ + u32 *iopgd, *iopte = NULL; + + iopgd = iopgd_offset(obj, da); + if (!*iopgd) + goto out; + + if (iopgd_is_table(*iopgd)) + iopte = iopte_offset(iopgd, da); +out: + *ppgd = iopgd; + *ppte = iopte; +} + +static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da) +{ + size_t bytes; + u32 *iopgd = iopgd_offset(obj, da); + int nent = 1; + + if (!*iopgd) + return 0; + + if (iopgd_is_table(*iopgd)) { + int i; + u32 *iopte = iopte_offset(iopgd, da); + + bytes = IOPTE_SIZE; + if (*iopte & IOPTE_LARGE) { + nent *= 16; + /* rewind to the 1st entry */ + iopte = iopte_offset(iopgd, (da & IOLARGE_MASK)); + } + bytes *= nent; + memset(iopte, 0, nent * sizeof(*iopte)); + flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte)); + + /* + * do table walk to check if this table is necessary or not + */ + iopte = iopte_offset(iopgd, 0); + for (i = 0; i < PTRS_PER_IOPTE; i++) + if (iopte[i]) + goto out; + + iopte_free(iopte); + nent = 1; /* for the next L1 entry */ + } else { + bytes = IOPGD_SIZE; + if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) { + nent *= 16; + /* rewind to the 1st entry */ + iopgd = iopgd_offset(obj, (da & IOSUPER_MASK)); + } + bytes *= nent; + } + memset(iopgd, 0, nent * sizeof(*iopgd)); + flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd)); +out: + return bytes; +} + +/** + * iopgtable_clear_entry - Remove an iommu pte entry + * @obj: target iommu + * @da: iommu device virtual address + **/ +static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da) +{ + size_t bytes; + + spin_lock(&obj->page_table_lock); + + bytes = iopgtable_clear_entry_core(obj, da); + flush_iotlb_page(obj, da); + + spin_unlock(&obj->page_table_lock); + + return bytes; +} + +static void iopgtable_clear_entry_all(struct omap_iommu *obj) +{ + int i; + + spin_lock(&obj->page_table_lock); + + for (i = 0; i < PTRS_PER_IOPGD; i++) { + u32 da; + u32 *iopgd; + + da = i << IOPGD_SHIFT; + iopgd = iopgd_offset(obj, da); + + if (!*iopgd) + continue; + + if (iopgd_is_table(*iopgd)) + iopte_free(iopte_offset(iopgd, 0)); + + *iopgd = 0; + flush_iopgd_range(iopgd, iopgd); + } + + flush_iotlb_all(obj); + + spin_unlock(&obj->page_table_lock); +} + +/* + * Device IOMMU generic operations + */ +static irqreturn_t iommu_fault_handler(int irq, void *data) +{ + u32 da, errs; + u32 *iopgd, *iopte; + struct omap_iommu *obj = data; + struct iommu_domain *domain = obj->domain; + + if (!obj->refcount) + return IRQ_NONE; + + clk_enable(obj->clk); + errs = iommu_report_fault(obj, &da); + clk_disable(obj->clk); + if (errs == 0) + return IRQ_HANDLED; + + /* Fault callback or TLB/PTE Dynamic loading */ + if (!report_iommu_fault(domain, obj->dev, da, 0)) + return IRQ_HANDLED; + + iommu_disable(obj); + + iopgd = iopgd_offset(obj, da); + + if (!iopgd_is_table(*iopgd)) { + dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p " + "*pgd:px%08x\n", obj->name, errs, da, iopgd, *iopgd); + return IRQ_NONE; + } + + iopte = iopte_offset(iopgd, da); + + dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x " + "pte:0x%p *pte:0x%08x\n", obj->name, errs, da, iopgd, *iopgd, + iopte, *iopte); + + return IRQ_NONE; +} + +static int device_match_by_alias(struct device *dev, void *data) +{ + struct omap_iommu *obj = to_iommu(dev); + const char *name = data; + + pr_debug("%s: %s %s\n", __func__, obj->name, name); + + return strcmp(obj->name, name) == 0; +} + +/** + * omap_find_iommu_device() - find an omap iommu device by name + * @name: name of the iommu device + * + * The generic iommu API requires the caller to provide the device + * he wishes to attach to a certain iommu domain. + * + * Drivers generally should not bother with this as it should just + * be taken care of by the DMA-API using dev_archdata. + * + * This function is provided as an interim solution until the latter + * materializes, and omap3isp is fully migrated to the DMA-API. + */ +struct device *omap_find_iommu_device(const char *name) +{ + return driver_find_device(&omap_iommu_driver.driver, NULL, + (void *)name, + device_match_by_alias); +} +EXPORT_SYMBOL_GPL(omap_find_iommu_device); + +/** + * omap_iommu_attach() - attach iommu device to an iommu domain + * @dev: target omap iommu device + * @iopgd: page table + **/ +static struct omap_iommu *omap_iommu_attach(struct device *dev, u32 *iopgd) +{ + int err = -ENOMEM; + struct omap_iommu *obj = to_iommu(dev); + + spin_lock(&obj->iommu_lock); + + /* an iommu device can only be attached once */ + if (++obj->refcount > 1) { + dev_err(dev, "%s: already attached!\n", obj->name); + err = -EBUSY; + goto err_enable; + } + + obj->iopgd = iopgd; + err = iommu_enable(obj); + if (err) + goto err_enable; + flush_iotlb_all(obj); + + if (!try_module_get(obj->owner)) + goto err_module; + + spin_unlock(&obj->iommu_lock); + + dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); + return obj; + +err_module: + if (obj->refcount == 1) + iommu_disable(obj); +err_enable: + obj->refcount--; + spin_unlock(&obj->iommu_lock); + return ERR_PTR(err); +} + +/** + * omap_iommu_detach - release iommu device + * @obj: target iommu + **/ +static void omap_iommu_detach(struct omap_iommu *obj) +{ + if (!obj || IS_ERR(obj)) + return; + + spin_lock(&obj->iommu_lock); + + if (--obj->refcount == 0) + iommu_disable(obj); + + module_put(obj->owner); + + obj->iopgd = NULL; + + spin_unlock(&obj->iommu_lock); + + dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); +} + +/* + * OMAP Device MMU(IOMMU) detection + */ +static int __devinit omap_iommu_probe(struct platform_device *pdev) +{ + int err = -ENODEV; + int irq; + struct omap_iommu *obj; + struct resource *res; + struct iommu_platform_data *pdata = pdev->dev.platform_data; + + if (pdev->num_resources != 2) + return -EINVAL; + + obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL); + if (!obj) + return -ENOMEM; + + obj->clk = clk_get(&pdev->dev, pdata->clk_name); + if (IS_ERR(obj->clk)) + goto err_clk; + + obj->nr_tlb_entries = pdata->nr_tlb_entries; + obj->name = pdata->name; + obj->dev = &pdev->dev; + obj->ctx = (void *)obj + sizeof(*obj); + obj->da_start = pdata->da_start; + obj->da_end = pdata->da_end; + + spin_lock_init(&obj->iommu_lock); + mutex_init(&obj->mmap_lock); + spin_lock_init(&obj->page_table_lock); + INIT_LIST_HEAD(&obj->mmap); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + err = -ENODEV; + goto err_mem; + } + + res = request_mem_region(res->start, resource_size(res), + dev_name(&pdev->dev)); + if (!res) { + err = -EIO; + goto err_mem; + } + + obj->regbase = ioremap(res->start, resource_size(res)); + if (!obj->regbase) { + err = -ENOMEM; + goto err_ioremap; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + err = -ENODEV; + goto err_irq; + } + err = request_irq(irq, iommu_fault_handler, IRQF_SHARED, + dev_name(&pdev->dev), obj); + if (err < 0) + goto err_irq; + platform_set_drvdata(pdev, obj); + + dev_info(&pdev->dev, "%s registered\n", obj->name); + return 0; + +err_irq: + iounmap(obj->regbase); +err_ioremap: + release_mem_region(res->start, resource_size(res)); +err_mem: + clk_put(obj->clk); +err_clk: + kfree(obj); + return err; +} + +static int __devexit omap_iommu_remove(struct platform_device *pdev) +{ + int irq; + struct resource *res; + struct omap_iommu *obj = platform_get_drvdata(pdev); + + platform_set_drvdata(pdev, NULL); + + iopgtable_clear_entry_all(obj); + + irq = platform_get_irq(pdev, 0); + free_irq(irq, obj); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + release_mem_region(res->start, resource_size(res)); + iounmap(obj->regbase); + + clk_put(obj->clk); + dev_info(&pdev->dev, "%s removed\n", obj->name); + kfree(obj); + return 0; +} + +static struct platform_driver omap_iommu_driver = { + .probe = omap_iommu_probe, + .remove = __devexit_p(omap_iommu_remove), + .driver = { + .name = "omap-iommu", + }, +}; + +static void iopte_cachep_ctor(void *iopte) +{ + clean_dcache_area(iopte, IOPTE_TABLE_SIZE); +} + +static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, + phys_addr_t pa, int order, int prot) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + size_t bytes = PAGE_SIZE << order; + struct iotlb_entry e; + int omap_pgsz; + u32 ret, flags; + + /* we only support mapping a single iommu page for now */ + omap_pgsz = bytes_to_iopgsz(bytes); + if (omap_pgsz < 0) { + dev_err(dev, "invalid size to map: %d\n", bytes); + return -EINVAL; + } + + dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); + + flags = omap_pgsz | prot; + + iotlb_init_entry(&e, da, pa, flags); + + ret = omap_iopgtable_store_entry(oiommu, &e); + if (ret) + dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret); + + return ret; +} + +static int omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, + int order) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + size_t unmap_size; + + dev_dbg(dev, "unmapping da 0x%lx order %d\n", da, order); + + unmap_size = iopgtable_clear_entry(oiommu, da); + + return unmap_size ? get_order(unmap_size) : -EINVAL; +} + +static int +omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu; + int ret = 0; + + spin_lock(&omap_domain->lock); + + /* only a single device is supported per domain for now */ + if (omap_domain->iommu_dev) { + dev_err(dev, "iommu domain is already attached\n"); + ret = -EBUSY; + goto out; + } + + /* get a handle to and enable the omap iommu */ + oiommu = omap_iommu_attach(dev, omap_domain->pgtable); + if (IS_ERR(oiommu)) { + ret = PTR_ERR(oiommu); + dev_err(dev, "can't get omap iommu: %d\n", ret); + goto out; + } + + omap_domain->iommu_dev = oiommu; + oiommu->domain = domain; + +out: + spin_unlock(&omap_domain->lock); + return ret; +} + +static void omap_iommu_detach_dev(struct iommu_domain *domain, + struct device *dev) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = to_iommu(dev); + + spin_lock(&omap_domain->lock); + + /* only a single device is supported per domain for now */ + if (omap_domain->iommu_dev != oiommu) { + dev_err(dev, "invalid iommu device\n"); + goto out; + } + + iopgtable_clear_entry_all(oiommu); + + omap_iommu_detach(oiommu); + + omap_domain->iommu_dev = NULL; + +out: + spin_unlock(&omap_domain->lock); +} + +static int omap_iommu_domain_init(struct iommu_domain *domain) +{ + struct omap_iommu_domain *omap_domain; + + omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); + if (!omap_domain) { + pr_err("kzalloc failed\n"); + goto out; + } + + omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); + if (!omap_domain->pgtable) { + pr_err("kzalloc failed\n"); + goto fail_nomem; + } + + /* + * should never fail, but please keep this around to ensure + * we keep the hardware happy + */ + BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)); + + clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE); + spin_lock_init(&omap_domain->lock); + + domain->priv = omap_domain; + + return 0; + +fail_nomem: + kfree(omap_domain); +out: + return -ENOMEM; +} + +/* assume device was already detached */ +static void omap_iommu_domain_destroy(struct iommu_domain *domain) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + + domain->priv = NULL; + + kfree(omap_domain->pgtable); + kfree(omap_domain); +} + +static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, + unsigned long da) +{ + struct omap_iommu_domain *omap_domain = domain->priv; + struct omap_iommu *oiommu = omap_domain->iommu_dev; + struct device *dev = oiommu->dev; + u32 *pgd, *pte; + phys_addr_t ret = 0; + + iopgtable_lookup_entry(oiommu, da, &pgd, &pte); + + if (pte) { + if (iopte_is_small(*pte)) + ret = omap_iommu_translate(*pte, da, IOPTE_MASK); + else if (iopte_is_large(*pte)) + ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); + else + dev_err(dev, "bogus pte 0x%x", *pte); + } else { + if (iopgd_is_section(*pgd)) + ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); + else if (iopgd_is_super(*pgd)) + ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); + else + dev_err(dev, "bogus pgd 0x%x", *pgd); + } + + return ret; +} + +static int omap_iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + return 0; +} + +static struct iommu_ops omap_iommu_ops = { + .domain_init = omap_iommu_domain_init, + .domain_destroy = omap_iommu_domain_destroy, + .attach_dev = omap_iommu_attach_dev, + .detach_dev = omap_iommu_detach_dev, + .map = omap_iommu_map, + .unmap = omap_iommu_unmap, + .iova_to_phys = omap_iommu_iova_to_phys, + .domain_has_cap = omap_iommu_domain_has_cap, +}; + +static int __init omap_iommu_init(void) +{ + struct kmem_cache *p; + const unsigned long flags = SLAB_HWCACHE_ALIGN; + size_t align = 1 << 10; /* L2 pagetable alignement */ + + p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags, + iopte_cachep_ctor); + if (!p) + return -ENOMEM; + iopte_cachep = p; + + bus_set_iommu(&platform_bus_type, &omap_iommu_ops); + + return platform_driver_register(&omap_iommu_driver); +} +module_init(omap_iommu_init); + +static void __exit omap_iommu_exit(void) +{ + kmem_cache_destroy(iopte_cachep); + + platform_driver_unregister(&omap_iommu_driver); +} +module_exit(omap_iommu_exit); + +MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives"); +MODULE_ALIAS("platform:omap-iommu"); +MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/omap-iovmm.c b/drivers/iommu/omap-iovmm.c new file mode 100644 index 000000000000..e8fdb8830f69 --- /dev/null +++ b/drivers/iommu/omap-iovmm.c @@ -0,0 +1,742 @@ +/* + * omap iommu: simple virtual address space management + * + * Copyright (C) 2008-2009 Nokia Corporation + * + * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/device.h> +#include <linux/scatterlist.h> +#include <linux/iommu.h> + +#include <asm/cacheflush.h> +#include <asm/mach/map.h> + +#include <plat/iommu.h> +#include <plat/iovmm.h> + +#include <plat/iopgtable.h> + +static struct kmem_cache *iovm_area_cachep; + +/* return the offset of the first scatterlist entry in a sg table */ +static unsigned int sgtable_offset(const struct sg_table *sgt) +{ + if (!sgt || !sgt->nents) + return 0; + + return sgt->sgl->offset; +} + +/* return total bytes of sg buffers */ +static size_t sgtable_len(const struct sg_table *sgt) +{ + unsigned int i, total = 0; + struct scatterlist *sg; + + if (!sgt) + return 0; + + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + size_t bytes; + + bytes = sg->length + sg->offset; + + if (!iopgsz_ok(bytes)) { + pr_err("%s: sg[%d] not iommu pagesize(%u %u)\n", + __func__, i, bytes, sg->offset); + return 0; + } + + if (i && sg->offset) { + pr_err("%s: sg[%d] offset not allowed in internal " + "entries\n", __func__, i); + return 0; + } + + total += bytes; + } + + return total; +} +#define sgtable_ok(x) (!!sgtable_len(x)) + +static unsigned max_alignment(u32 addr) +{ + int i; + unsigned pagesize[] = { SZ_16M, SZ_1M, SZ_64K, SZ_4K, }; + for (i = 0; i < ARRAY_SIZE(pagesize) && addr & (pagesize[i] - 1); i++) + ; + return (i < ARRAY_SIZE(pagesize)) ? pagesize[i] : 0; +} + +/* + * calculate the optimal number sg elements from total bytes based on + * iommu superpages + */ +static unsigned sgtable_nents(size_t bytes, u32 da, u32 pa) +{ + unsigned nr_entries = 0, ent_sz; + + if (!IS_ALIGNED(bytes, PAGE_SIZE)) { + pr_err("%s: wrong size %08x\n", __func__, bytes); + return 0; + } + + while (bytes) { + ent_sz = max_alignment(da | pa); + ent_sz = min_t(unsigned, ent_sz, iopgsz_max(bytes)); + nr_entries++; + da += ent_sz; + pa += ent_sz; + bytes -= ent_sz; + } + + return nr_entries; +} + +/* allocate and initialize sg_table header(a kind of 'superblock') */ +static struct sg_table *sgtable_alloc(const size_t bytes, u32 flags, + u32 da, u32 pa) +{ + unsigned int nr_entries; + int err; + struct sg_table *sgt; + + if (!bytes) + return ERR_PTR(-EINVAL); + + if (!IS_ALIGNED(bytes, PAGE_SIZE)) + return ERR_PTR(-EINVAL); + + if (flags & IOVMF_LINEAR) { + nr_entries = sgtable_nents(bytes, da, pa); + if (!nr_entries) + return ERR_PTR(-EINVAL); + } else + nr_entries = bytes / PAGE_SIZE; + + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) + return ERR_PTR(-ENOMEM); + + err = sg_alloc_table(sgt, nr_entries, GFP_KERNEL); + if (err) { + kfree(sgt); + return ERR_PTR(err); + } + + pr_debug("%s: sgt:%p(%d entries)\n", __func__, sgt, nr_entries); + + return sgt; +} + +/* free sg_table header(a kind of superblock) */ +static void sgtable_free(struct sg_table *sgt) +{ + if (!sgt) + return; + + sg_free_table(sgt); + kfree(sgt); + + pr_debug("%s: sgt:%p\n", __func__, sgt); +} + +/* map 'sglist' to a contiguous mpu virtual area and return 'va' */ +static void *vmap_sg(const struct sg_table *sgt) +{ + u32 va; + size_t total; + unsigned int i; + struct scatterlist *sg; + struct vm_struct *new; + const struct mem_type *mtype; + + mtype = get_mem_type(MT_DEVICE); + if (!mtype) + return ERR_PTR(-EINVAL); + + total = sgtable_len(sgt); + if (!total) + return ERR_PTR(-EINVAL); + + new = __get_vm_area(total, VM_IOREMAP, VMALLOC_START, VMALLOC_END); + if (!new) + return ERR_PTR(-ENOMEM); + va = (u32)new->addr; + + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + size_t bytes; + u32 pa; + int err; + + pa = sg_phys(sg) - sg->offset; + bytes = sg->length + sg->offset; + + BUG_ON(bytes != PAGE_SIZE); + + err = ioremap_page(va, pa, mtype); + if (err) + goto err_out; + + va += bytes; + } + + flush_cache_vmap((unsigned long)new->addr, + (unsigned long)(new->addr + total)); + return new->addr; + +err_out: + WARN_ON(1); /* FIXME: cleanup some mpu mappings */ + vunmap(new->addr); + return ERR_PTR(-EAGAIN); +} + +static inline void vunmap_sg(const void *va) +{ + vunmap(va); +} + +static struct iovm_struct *__find_iovm_area(struct omap_iommu *obj, + const u32 da) +{ + struct iovm_struct *tmp; + + list_for_each_entry(tmp, &obj->mmap, list) { + if ((da >= tmp->da_start) && (da < tmp->da_end)) { + size_t len; + + len = tmp->da_end - tmp->da_start; + + dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n", + __func__, tmp->da_start, da, tmp->da_end, len, + tmp->flags); + + return tmp; + } + } + + return NULL; +} + +/** + * omap_find_iovm_area - find iovma which includes @da + * @da: iommu device virtual address + * + * Find the existing iovma starting at @da + */ +struct iovm_struct *omap_find_iovm_area(struct omap_iommu *obj, u32 da) +{ + struct iovm_struct *area; + + mutex_lock(&obj->mmap_lock); + area = __find_iovm_area(obj, da); + mutex_unlock(&obj->mmap_lock); + + return area; +} +EXPORT_SYMBOL_GPL(omap_find_iovm_area); + +/* + * This finds the hole(area) which fits the requested address and len + * in iovmas mmap, and returns the new allocated iovma. + */ +static struct iovm_struct *alloc_iovm_area(struct omap_iommu *obj, u32 da, + size_t bytes, u32 flags) +{ + struct iovm_struct *new, *tmp; + u32 start, prev_end, alignment; + + if (!obj || !bytes) + return ERR_PTR(-EINVAL); + + start = da; + alignment = PAGE_SIZE; + + if (~flags & IOVMF_DA_FIXED) { + /* Don't map address 0 */ + start = obj->da_start ? obj->da_start : alignment; + + if (flags & IOVMF_LINEAR) + alignment = iopgsz_max(bytes); + start = roundup(start, alignment); + } else if (start < obj->da_start || start > obj->da_end || + obj->da_end - start < bytes) { + return ERR_PTR(-EINVAL); + } + + tmp = NULL; + if (list_empty(&obj->mmap)) + goto found; + + prev_end = 0; + list_for_each_entry(tmp, &obj->mmap, list) { + + if (prev_end > start) + break; + + if (tmp->da_start > start && (tmp->da_start - start) >= bytes) + goto found; + + if (tmp->da_end >= start && ~flags & IOVMF_DA_FIXED) + start = roundup(tmp->da_end + 1, alignment); + + prev_end = tmp->da_end; + } + + if ((start >= prev_end) && (obj->da_end - start >= bytes)) + goto found; + + dev_dbg(obj->dev, "%s: no space to fit %08x(%x) flags: %08x\n", + __func__, da, bytes, flags); + + return ERR_PTR(-EINVAL); + +found: + new = kmem_cache_zalloc(iovm_area_cachep, GFP_KERNEL); + if (!new) + return ERR_PTR(-ENOMEM); + + new->iommu = obj; + new->da_start = start; + new->da_end = start + bytes; + new->flags = flags; + + /* + * keep ascending order of iovmas + */ + if (tmp) + list_add_tail(&new->list, &tmp->list); + else + list_add(&new->list, &obj->mmap); + + dev_dbg(obj->dev, "%s: found %08x-%08x-%08x(%x) %08x\n", + __func__, new->da_start, start, new->da_end, bytes, flags); + + return new; +} + +static void free_iovm_area(struct omap_iommu *obj, struct iovm_struct *area) +{ + size_t bytes; + + BUG_ON(!obj || !area); + + bytes = area->da_end - area->da_start; + + dev_dbg(obj->dev, "%s: %08x-%08x(%x) %08x\n", + __func__, area->da_start, area->da_end, bytes, area->flags); + + list_del(&area->list); + kmem_cache_free(iovm_area_cachep, area); +} + +/** + * omap_da_to_va - convert (d) to (v) + * @obj: objective iommu + * @da: iommu device virtual address + * @va: mpu virtual address + * + * Returns mpu virtual addr which corresponds to a given device virtual addr + */ +void *omap_da_to_va(struct omap_iommu *obj, u32 da) +{ + void *va = NULL; + struct iovm_struct *area; + + mutex_lock(&obj->mmap_lock); + + area = __find_iovm_area(obj, da); + if (!area) { + dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da); + goto out; + } + va = area->va; +out: + mutex_unlock(&obj->mmap_lock); + + return va; +} +EXPORT_SYMBOL_GPL(omap_da_to_va); + +static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va) +{ + unsigned int i; + struct scatterlist *sg; + void *va = _va; + void *va_end; + + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + struct page *pg; + const size_t bytes = PAGE_SIZE; + + /* + * iommu 'superpage' isn't supported with 'omap_iommu_vmalloc()' + */ + pg = vmalloc_to_page(va); + BUG_ON(!pg); + sg_set_page(sg, pg, bytes, 0); + + va += bytes; + } + + va_end = _va + PAGE_SIZE * i; +} + +static inline void sgtable_drain_vmalloc(struct sg_table *sgt) +{ + /* + * Actually this is not necessary at all, just exists for + * consistency of the code readability. + */ + BUG_ON(!sgt); +} + +/* create 'da' <-> 'pa' mapping from 'sgt' */ +static int map_iovm_area(struct iommu_domain *domain, struct iovm_struct *new, + const struct sg_table *sgt, u32 flags) +{ + int err; + unsigned int i, j; + struct scatterlist *sg; + u32 da = new->da_start; + int order; + + if (!domain || !sgt) + return -EINVAL; + + BUG_ON(!sgtable_ok(sgt)); + + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + u32 pa; + size_t bytes; + + pa = sg_phys(sg) - sg->offset; + bytes = sg->length + sg->offset; + + flags &= ~IOVMF_PGSZ_MASK; + + if (bytes_to_iopgsz(bytes) < 0) + goto err_out; + + order = get_order(bytes); + + pr_debug("%s: [%d] %08x %08x(%x)\n", __func__, + i, da, pa, bytes); + + err = iommu_map(domain, da, pa, order, flags); + if (err) + goto err_out; + + da += bytes; + } + return 0; + +err_out: + da = new->da_start; + + for_each_sg(sgt->sgl, sg, i, j) { + size_t bytes; + + bytes = sg->length + sg->offset; + order = get_order(bytes); + + /* ignore failures.. we're already handling one */ + iommu_unmap(domain, da, order); + + da += bytes; + } + return err; +} + +/* release 'da' <-> 'pa' mapping */ +static void unmap_iovm_area(struct iommu_domain *domain, struct omap_iommu *obj, + struct iovm_struct *area) +{ + u32 start; + size_t total = area->da_end - area->da_start; + const struct sg_table *sgt = area->sgt; + struct scatterlist *sg; + int i, err; + + BUG_ON(!sgtable_ok(sgt)); + BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE)); + + start = area->da_start; + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + size_t bytes; + int order; + + bytes = sg->length + sg->offset; + order = get_order(bytes); + + err = iommu_unmap(domain, start, order); + if (err < 0) + break; + + dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n", + __func__, start, bytes, area->flags); + + BUG_ON(!IS_ALIGNED(bytes, PAGE_SIZE)); + + total -= bytes; + start += bytes; + } + BUG_ON(total); +} + +/* template function for all unmapping */ +static struct sg_table *unmap_vm_area(struct iommu_domain *domain, + struct omap_iommu *obj, const u32 da, + void (*fn)(const void *), u32 flags) +{ + struct sg_table *sgt = NULL; + struct iovm_struct *area; + + if (!IS_ALIGNED(da, PAGE_SIZE)) { + dev_err(obj->dev, "%s: alignment err(%08x)\n", __func__, da); + return NULL; + } + + mutex_lock(&obj->mmap_lock); + + area = __find_iovm_area(obj, da); + if (!area) { + dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da); + goto out; + } + + if ((area->flags & flags) != flags) { + dev_err(obj->dev, "%s: wrong flags(%08x)\n", __func__, + area->flags); + goto out; + } + sgt = (struct sg_table *)area->sgt; + + unmap_iovm_area(domain, obj, area); + + fn(area->va); + + dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n", __func__, + area->da_start, da, area->da_end, + area->da_end - area->da_start, area->flags); + + free_iovm_area(obj, area); +out: + mutex_unlock(&obj->mmap_lock); + + return sgt; +} + +static u32 map_iommu_region(struct iommu_domain *domain, struct omap_iommu *obj, + u32 da, const struct sg_table *sgt, void *va, + size_t bytes, u32 flags) +{ + int err = -ENOMEM; + struct iovm_struct *new; + + mutex_lock(&obj->mmap_lock); + + new = alloc_iovm_area(obj, da, bytes, flags); + if (IS_ERR(new)) { + err = PTR_ERR(new); + goto err_alloc_iovma; + } + new->va = va; + new->sgt = sgt; + + if (map_iovm_area(domain, new, sgt, new->flags)) + goto err_map; + + mutex_unlock(&obj->mmap_lock); + + dev_dbg(obj->dev, "%s: da:%08x(%x) flags:%08x va:%p\n", + __func__, new->da_start, bytes, new->flags, va); + + return new->da_start; + +err_map: + free_iovm_area(obj, new); +err_alloc_iovma: + mutex_unlock(&obj->mmap_lock); + return err; +} + +static inline u32 +__iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, + u32 da, const struct sg_table *sgt, + void *va, size_t bytes, u32 flags) +{ + return map_iommu_region(domain, obj, da, sgt, va, bytes, flags); +} + +/** + * omap_iommu_vmap - (d)-(p)-(v) address mapper + * @obj: objective iommu + * @sgt: address of scatter gather table + * @flags: iovma and page property + * + * Creates 1-n-1 mapping with given @sgt and returns @da. + * All @sgt element must be io page size aligned. + */ +u32 omap_iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, + const struct sg_table *sgt, u32 flags) +{ + size_t bytes; + void *va = NULL; + + if (!obj || !obj->dev || !sgt) + return -EINVAL; + + bytes = sgtable_len(sgt); + if (!bytes) + return -EINVAL; + bytes = PAGE_ALIGN(bytes); + + if (flags & IOVMF_MMIO) { + va = vmap_sg(sgt); + if (IS_ERR(va)) + return PTR_ERR(va); + } + + flags |= IOVMF_DISCONT; + flags |= IOVMF_MMIO; + + da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags); + if (IS_ERR_VALUE(da)) + vunmap_sg(va); + + return da + sgtable_offset(sgt); +} +EXPORT_SYMBOL_GPL(omap_iommu_vmap); + +/** + * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()' + * @obj: objective iommu + * @da: iommu device virtual address + * + * Free the iommu virtually contiguous memory area starting at + * @da, which was returned by 'omap_iommu_vmap()'. + */ +struct sg_table * +omap_iommu_vunmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da) +{ + struct sg_table *sgt; + /* + * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called. + * Just returns 'sgt' to the caller to free + */ + da &= PAGE_MASK; + sgt = unmap_vm_area(domain, obj, da, vunmap_sg, + IOVMF_DISCONT | IOVMF_MMIO); + if (!sgt) + dev_dbg(obj->dev, "%s: No sgt\n", __func__); + return sgt; +} +EXPORT_SYMBOL_GPL(omap_iommu_vunmap); + +/** + * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper + * @obj: objective iommu + * @da: contiguous iommu virtual memory + * @bytes: allocation size + * @flags: iovma and page property + * + * Allocate @bytes linearly and creates 1-n-1 mapping and returns + * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set. + */ +u32 +omap_iommu_vmalloc(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, + size_t bytes, u32 flags) +{ + void *va; + struct sg_table *sgt; + + if (!obj || !obj->dev || !bytes) + return -EINVAL; + + bytes = PAGE_ALIGN(bytes); + + va = vmalloc(bytes); + if (!va) + return -ENOMEM; + + flags |= IOVMF_DISCONT; + flags |= IOVMF_ALLOC; + + sgt = sgtable_alloc(bytes, flags, da, 0); + if (IS_ERR(sgt)) { + da = PTR_ERR(sgt); + goto err_sgt_alloc; + } + sgtable_fill_vmalloc(sgt, va); + + da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags); + if (IS_ERR_VALUE(da)) + goto err_iommu_vmap; + + return da; + +err_iommu_vmap: + sgtable_drain_vmalloc(sgt); + sgtable_free(sgt); +err_sgt_alloc: + vfree(va); + return da; +} +EXPORT_SYMBOL_GPL(omap_iommu_vmalloc); + +/** + * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()' + * @obj: objective iommu + * @da: iommu device virtual address + * + * Frees the iommu virtually continuous memory area starting at + * @da, as obtained from 'omap_iommu_vmalloc()'. + */ +void omap_iommu_vfree(struct iommu_domain *domain, struct omap_iommu *obj, + const u32 da) +{ + struct sg_table *sgt; + + sgt = unmap_vm_area(domain, obj, da, vfree, + IOVMF_DISCONT | IOVMF_ALLOC); + if (!sgt) + dev_dbg(obj->dev, "%s: No sgt\n", __func__); + sgtable_free(sgt); +} +EXPORT_SYMBOL_GPL(omap_iommu_vfree); + +static int __init iovmm_init(void) +{ + const unsigned long flags = SLAB_HWCACHE_ALIGN; + struct kmem_cache *p; + + p = kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct), 0, + flags, NULL); + if (!p) + return -ENOMEM; + iovm_area_cachep = p; + + return 0; +} +module_init(iovmm_init); + +static void __exit iovmm_exit(void) +{ + kmem_cache_destroy(iovm_area_cachep); +} +module_exit(iovmm_exit); + +MODULE_DESCRIPTION("omap iommu: simple virtual address space management"); +MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>"); +MODULE_LICENSE("GPL v2"); |