diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/amd_iommu.c | 590 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_init.c | 34 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_proto.h | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_types.h | 11 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 28 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 527 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 526 | ||||
| -rw-r--r-- | drivers/iommu/intel_irq_remapping.c | 255 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 373 | ||||
| -rw-r--r-- | drivers/iommu/iova.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/rockchip-iommu.c | 27 |
11 files changed, 1426 insertions, 951 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index e43d48956dea..c5677ed2cd89 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -64,10 +64,6 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - /* List of all available dev_data structures */ static LIST_HEAD(dev_data_list); static DEFINE_SPINLOCK(dev_data_list_lock); @@ -119,7 +115,7 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); -static int __init alloc_passthrough_domain(void); +static int alloc_passthrough_domain(void); /**************************************************************************** * @@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) } /* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device + * This function actually applies the mapping to the page table of the + * dma_ops domain. */ -static struct dma_ops_domain *find_protection_domain(u16 devid) +static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) { - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; - - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); + u64 addr; - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); } +} + +/* + * Inits the unity mappings required for a specific device + */ +static void init_unity_mappings_for_device(struct device *dev, + struct dma_ops_domain *dma_dom) +{ + struct unity_map_entry *e; + u16 devid; - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + devid = get_device_id(dev); - return ret; + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + alloc_unity_mapping(dma_dom, e); + } } /* @@ -290,11 +293,23 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { + struct dma_ops_domain *dma_domain; + struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) - iommu_group_put(group); + if (IS_ERR(group)) + return; + + domain = iommu_group_default_domain(group); + if (!domain) + goto out; + + dma_domain = to_pdomain(domain)->priv; + + init_unity_mappings_for_device(dev, dma_domain); +out: + iommu_group_put(group); } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) @@ -434,64 +449,15 @@ static void iommu_uninit_device(struct device *dev) /* Unlink from alias, it may change if another device is re-plugged */ dev_data->alias_data = NULL; + /* Remove dma-ops */ + dev->archdata.dma_ops = NULL; + /* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races. */ } -void __init amd_iommu_uninit_devices(void) -{ - struct iommu_dev_data *dev_data, *n; - struct pci_dev *pdev = NULL; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - iommu_uninit_device(&pdev->dev); - } - - /* Free all of our dev_data structures */ - list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) - free_dev_data(dev_data); -} - -int __init amd_iommu_init_devices(void) -{ - struct pci_dev *pdev = NULL; - int ret = 0; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - ret = iommu_init_device(&pdev->dev); - if (ret == -ENOTSUPP) - iommu_ignore_device(&pdev->dev); - else if (ret) - goto out_free; - } - - /* - * Initialize IOMMU groups only after iommu_init_device() has - * had a chance to populate any IVRS defined aliases. - */ - for_each_pci_dev(pdev) { - if (check_device(&pdev->dev)) - init_iommu_group(&pdev->dev); - } - - return 0; - -out_free: - - amd_iommu_uninit_devices(); - - return ret; -} #ifdef CONFIG_AMD_IOMMU_STATS /* @@ -1463,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops @@ -1704,14 +1582,16 @@ static unsigned long dma_ops_area_alloc(struct device *dev, unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; int i = start >> APERTURE_RANGE_SHIFT; - unsigned long boundary_size; + unsigned long boundary_size, mask; unsigned long address = -1; unsigned long limit; next_bit >>= PAGE_SHIFT; - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - PAGE_SIZE) >> PAGE_SHIFT; + mask = dma_get_seg_boundary(dev); + + boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : + 1UL << (BITS_PER_LONG - PAGE_SHIFT); for (;i < max_index; ++i) { unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; @@ -1869,9 +1749,15 @@ static void free_pt_##LVL (unsigned long __pt) \ pt = (u64 *)__pt; \ \ for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ if (!IOMMU_PTE_PRESENT(pt[i])) \ continue; \ \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ FN(p); \ } \ @@ -2008,7 +1894,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) goto free_dma_dom; dma_dom->need_flush = false; - dma_dom->target_dev = 0xffff; add_domain_to_list(&dma_dom->domain); @@ -2373,110 +2258,67 @@ static void detach_device(struct device *dev) dev_data->ats.enabled = false; } -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct protection_domain *dom = NULL; - unsigned long flags; - - dev_data = get_dev_data(dev); - - if (dev_data->domain) - return dev_data->domain; - - if (dev_data->alias_data != NULL) { - struct iommu_dev_data *alias_data = dev_data->alias_data; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - } - - return dom; -} - -static int device_change_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static int amd_iommu_add_device(struct device *dev) { - struct dma_ops_domain *dma_domain; - struct protection_domain *domain; struct iommu_dev_data *dev_data; - struct device *dev = data; + struct iommu_domain *domain; struct amd_iommu *iommu; - unsigned long flags; u16 devid; + int ret; - if (!check_device(dev)) + if (!check_device(dev) || get_dev_data(dev)) return 0; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; - iommu_init_device(dev); - init_iommu_group(dev); + ret = iommu_init_device(dev); + if (ret) { + if (ret != -ENOTSUPP) + pr_err("Failed to initialize device %s - trying to proceed anyway\n", + dev_name(dev)); - /* - * dev_data is still NULL and - * got initialized in iommu_init_device - */ - dev_data = get_dev_data(dev); + iommu_ignore_device(dev); + dev->archdata.dma_ops = &nommu_dma_ops; + goto out; + } + init_iommu_group(dev); - if (iommu_pass_through || dev_data->iommu_v2) { - dev_data->passthrough = true; - attach_device(dev, pt_domain); - break; - } + dev_data = get_dev_data(dev); - domain = domain_for_device(dev); + BUG_ON(!dev_data); - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } + if (dev_data->iommu_v2) + iommu_request_dm_for_dev(dev); + /* Domains are initialized for this device - have a look what we ended up with */ + domain = iommu_get_domain_for_dev(dev); + if (domain->type == IOMMU_DOMAIN_IDENTITY) { + dev_data->passthrough = true; + dev->archdata.dma_ops = &nommu_dma_ops; + } else { dev->archdata.dma_ops = &amd_iommu_dma_ops; - - break; - case BUS_NOTIFY_REMOVED_DEVICE: - - iommu_uninit_device(dev); - - default: - goto out; } +out: iommu_completion_wait(iommu); -out: return 0; } -static struct notifier_block device_nb = { - .notifier_call = device_change_notifier, -}; - -void amd_iommu_init_notifier(void) +static void amd_iommu_remove_device(struct device *dev) { - bus_register_notifier(&pci_bus_type, &device_nb); + struct amd_iommu *iommu; + u16 devid; + + if (!check_device(dev)) + return; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + iommu_uninit_device(dev); + iommu_completion_wait(iommu); } /***************************************************************************** @@ -2495,28 +2337,20 @@ void amd_iommu_init_notifier(void) static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); + struct iommu_domain *io_domain; if (!check_device(dev)) return ERR_PTR(-EINVAL); - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) - return ERR_PTR(-EBUSY); - - if (domain != NULL) - return domain; + io_domain = iommu_get_domain_for_dev(dev); + if (!io_domain) + return NULL; - /* Device not bound yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); + domain = to_pdomain(io_domain); + if (!dma_ops_domain(domain)) + return ERR_PTR(-EBUSY); - return &dma_dom->domain; + return domain; } static void update_device_table(struct protection_domain *domain) @@ -2930,6 +2764,7 @@ static void *alloc_coherent(struct device *dev, size_t size, size = PAGE_ALIGN(size); dma_mask = dev->coherent_dma_mask; flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); + flag |= __GFP_ZERO; page = alloc_pages(flag | __GFP_NOWARN, get_order(size)); if (!page) { @@ -3011,54 +2846,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } -/* - * The function for pre-allocating protection domains. - * - * If the driver core informs the DMA layer if a driver grabs a device - * we don't need to preallocate the protection domains anymore. - * For now we have to. - */ -static void __init prealloc_protection_domains(void) -{ - struct iommu_dev_data *dev_data; - struct dma_ops_domain *dma_dom; - struct pci_dev *dev = NULL; - u16 devid; - - for_each_pci_dev(dev) { - - /* Do we handle this device? */ - if (!check_device(&dev->dev)) - continue; - - dev_data = get_dev_data(&dev->dev); - if (!amd_iommu_force_isolation && dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(&dev->dev, pt_domain); - pr_info("AMD-Vi: Using passthrough domain for device %s\n", - dev_name(&dev->dev)); - } - - /* Is there already any domain for it? */ - if (domain_for_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - dma_dom = dma_ops_domain_alloc(); - if (!dma_dom) - continue; - init_unity_mappings_for_device(dma_dom, devid); - dma_dom->target_dev = devid; - - attach_device(&dev->dev, &dma_dom->domain); - - list_add_tail(&dma_dom->list, &iommu_pd_list); - } -} - static struct dma_map_ops amd_iommu_dma_ops = { .alloc = alloc_coherent, .free = free_coherent, @@ -3069,76 +2856,16 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -static unsigned device_dma_ops_init(void) -{ - struct iommu_dev_data *dev_data; - struct pci_dev *pdev = NULL; - unsigned unhandled = 0; - - for_each_pci_dev(pdev) { - if (!check_device(&pdev->dev)) { - - iommu_ignore_device(&pdev->dev); - - unhandled += 1; - continue; - } - - dev_data = get_dev_data(&pdev->dev); - - if (!dev_data->passthrough) - pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; - else - pdev->dev.archdata.dma_ops = &nommu_dma_ops; - } - - return unhandled; -} - -/* - * The function which clues the AMD IOMMU driver into dma_ops. - */ - -void __init amd_iommu_init_api(void) +int __init amd_iommu_init_api(void) { - bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) { - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } - - /* - * Pre-allocate the protection domains for each device. - */ - prealloc_protection_domains(); - iommu_detected = 1; swiotlb = 0; - /* Make the driver finally visible to the drivers */ - unhandled = device_dma_ops_init(); - if (unhandled && max_pfn > MAX_DMA32_PFN) { - /* There are unhandled devices - initialize swiotlb for them */ - swiotlb = 1; - } - amd_iommu_stats_init(); if (amd_iommu_unmap_flush) @@ -3147,14 +2874,6 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; - -free_domains: - - for_each_iommu(iommu) { - dma_ops_domain_free(iommu->default_dom); - } - - return ret; } /***************************************************************************** @@ -3221,7 +2940,7 @@ out_err: return NULL; } -static int __init alloc_passthrough_domain(void) +static int alloc_passthrough_domain(void) { if (pt_domain != NULL) return 0; @@ -3239,30 +2958,46 @@ static int __init alloc_passthrough_domain(void) static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { struct protection_domain *pdomain; + struct dma_ops_domain *dma_domain; - /* We only support unmanaged domains for now */ - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; - - pdomain = protection_domain_alloc(); - if (!pdomain) - goto out_free; + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; - pdomain->mode = PAGE_MODE_3_LEVEL; - pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pdomain->pt_root) - goto out_free; + pdomain->mode = PAGE_MODE_3_LEVEL; + pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pdomain->pt_root) { + protection_domain_free(pdomain); + return NULL; + } - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; + pdomain->domain.geometry.aperture_start = 0; + pdomain->domain.geometry.aperture_end = ~0ULL; + pdomain->domain.geometry.force_aperture = true; - return &pdomain->domain; + break; + case IOMMU_DOMAIN_DMA: + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) { + pr_err("AMD-Vi: Failed to allocate\n"); + return NULL; + } + pdomain = &dma_domain->domain; + break; + case IOMMU_DOMAIN_IDENTITY: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; -out_free: - protection_domain_free(pdomain); + pdomain->mode = PAGE_MODE_NONE; + break; + default: + return NULL; + } - return NULL; + return &pdomain->domain; } static void amd_iommu_domain_free(struct iommu_domain *dom) @@ -3412,6 +3147,47 @@ static bool amd_iommu_capable(enum iommu_cap cap) return false; } +static void amd_iommu_get_dm_regions(struct device *dev, + struct list_head *head) +{ + struct unity_map_entry *entry; + u16 devid; + + devid = get_device_id(dev); + + list_for_each_entry(entry, &amd_iommu_unity_map, list) { + struct iommu_dm_region *region; + + if (devid < entry->devid_start || devid > entry->devid_end) + continue; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) { + pr_err("Out of memory allocating dm-regions for %s\n", + dev_name(dev)); + return; + } + + region->start = entry->address_start; + region->length = entry->address_end - entry->address_start; + if (entry->prot & IOMMU_PROT_IR) + region->prot |= IOMMU_READ; + if (entry->prot & IOMMU_PROT_IW) + region->prot |= IOMMU_WRITE; + + list_add_tail(®ion->list, head); + } +} + +static void amd_iommu_put_dm_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_dm_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); +} + static const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -3422,6 +3198,10 @@ static const struct iommu_ops amd_iommu_ops = { .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .add_device = amd_iommu_add_device, + .remove_device = amd_iommu_remove_device, + .get_dm_regions = amd_iommu_get_dm_regions, + .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, }; diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 450ef5001a65..dbac49cea7a1 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -226,6 +226,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); +static void init_device_table_dma(void); static inline void update_last_devid(u16 devid) { @@ -1385,9 +1386,15 @@ static int __init amd_iommu_init_pci(void) break; } - ret = amd_iommu_init_devices(); + init_device_table_dma(); + + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); + + ret = amd_iommu_init_api(); - print_iommu_info(); + if (!ret) + print_iommu_info(); return ret; } @@ -1825,8 +1832,6 @@ static bool __init check_ioapic_information(void) static void __init free_dma_resources(void) { - amd_iommu_uninit_devices(); - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, get_order(MAX_DOMAIN_ID/8)); @@ -2019,27 +2024,10 @@ static bool detect_ivrs(void) static int amd_iommu_init_dma(void) { - struct amd_iommu *iommu; - int ret; - if (iommu_pass_through) - ret = amd_iommu_init_passthrough(); + return amd_iommu_init_passthrough(); else - ret = amd_iommu_init_dma_ops(); - - if (ret) - return ret; - - init_device_table_dma(); - - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); - - amd_iommu_init_api(); - - amd_iommu_init_notifier(); - - return 0; + return amd_iommu_init_dma_ops(); } /**************************************************************************** diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 72b0fd455e24..9ed1c4330551 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -30,7 +30,7 @@ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); -extern void amd_iommu_init_api(void); +extern int amd_iommu_init_api(void); /* Needed for interrupt remapping */ extern int amd_iommu_prepare(void); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 05030e523771..bb56560ac6ca 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -446,8 +446,6 @@ struct aperture_range { * Data container for a dma_ops specific protection domain */ struct dma_ops_domain { - struct list_head list; - /* generic protection domain information */ struct protection_domain domain; @@ -462,12 +460,6 @@ struct dma_ops_domain { /* This will be set to true when TLB needs to be flushed */ bool need_flush; - - /* - * if this is a preallocated domain, keep the device for which it was - * preallocated in this variable - */ - u16 target_dev; }; /* @@ -552,9 +544,6 @@ struct amd_iommu { /* if one, we need to send a completion wait command */ bool need_sync; - /* default dma_ops domain for that IOMMU */ - struct dma_ops_domain *default_dom; - /* IOMMU sysfs device */ struct device *iommu_dev; diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 9847613085e1..c5886582b64f 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -26,7 +26,7 @@ * These routines are used by both DMA-remapping and Interrupt-remapping */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */ +#define pr_fmt(fmt) "DMAR: " fmt #include <linux/pci.h> #include <linux/dmar.h> @@ -555,7 +555,7 @@ static int dmar_walk_remapping_entries(struct acpi_dmar_header *start, break; } else if (next > end) { /* Avoid passing table end */ - pr_warn(FW_BUG "record passes table end\n"); + pr_warn(FW_BUG "Record passes table end\n"); ret = -EINVAL; break; } @@ -802,7 +802,7 @@ int __init dmar_table_init(void) ret = parse_dmar_table(); if (ret < 0) { if (ret != -ENODEV) - pr_info("parse DMAR table failure.\n"); + pr_info("Parse DMAR table failure.\n"); } else if (list_empty(&dmar_drhd_units)) { pr_info("No DMAR devices found\n"); ret = -ENODEV; @@ -847,7 +847,7 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg) else addr = early_ioremap(drhd->address, VTD_PAGE_SIZE); if (!addr) { - pr_warn("IOMMU: can't validate: %llx\n", drhd->address); + pr_warn("Can't validate DRHD address: %llx\n", drhd->address); return -EINVAL; } @@ -921,14 +921,14 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) iommu->reg_size = VTD_PAGE_SIZE; if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { - pr_err("IOMMU: can't reserve memory\n"); + pr_err("Can't reserve memory\n"); err = -EBUSY; goto out; } iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); if (!iommu->reg) { - pr_err("IOMMU: can't map the region\n"); + pr_err("Can't map the region\n"); err = -ENOMEM; goto release; } @@ -952,13 +952,13 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) iommu->reg_size = map_size; if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { - pr_err("IOMMU: can't reserve memory\n"); + pr_err("Can't reserve memory\n"); err = -EBUSY; goto out; } iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); if (!iommu->reg) { - pr_err("IOMMU: can't map the region\n"); + pr_err("Can't map the region\n"); err = -ENOMEM; goto release; } @@ -1014,14 +1014,14 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) return -ENOMEM; if (dmar_alloc_seq_id(iommu) < 0) { - pr_err("IOMMU: failed to allocate seq_id\n"); + pr_err("Failed to allocate seq_id\n"); err = -ENOSPC; goto error; } err = map_iommu(iommu, drhd->reg_base_addr); if (err) { - pr_err("IOMMU: failed to map %s\n", iommu->name); + pr_err("Failed to map %s\n", iommu->name); goto error_free_seq_id; } @@ -1045,8 +1045,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->node = -1; ver = readl(iommu->reg + DMAR_VER_REG); - pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", - iommu->seq_id, + pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", + iommu->name, (unsigned long long)drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver), (unsigned long long)iommu->cap, @@ -1644,7 +1644,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) irq = dmar_alloc_hwirq(); if (irq <= 0) { - pr_err("IOMMU: no free vectors\n"); + pr_err("No free IRQ vectors\n"); return -EINVAL; } @@ -1661,7 +1661,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu); if (ret) - pr_err("IOMMU: can't request irq\n"); + pr_err("Can't request irq\n"); return ret; } diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 3e898504a7c4..97c41b8ab5d9 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -12,21 +12,22 @@ #define DEBUG #endif -#include <linux/io.h> -#include <linux/interrupt.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/pm_runtime.h> #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/err.h> -#include <linux/mm.h> +#include <linux/io.h> #include <linux/iommu.h> -#include <linux/errno.h> +#include <linux/interrupt.h> #include <linux/list.h> -#include <linux/memblock.h> -#include <linux/export.h> +#include <linux/of.h> +#include <linux/of_iommu.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> #include <asm/cacheflush.h> +#include <asm/dma-iommu.h> #include <asm/pgtable.h> typedef u32 sysmmu_iova_t; @@ -184,35 +185,50 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { "UNKNOWN FAULT" }; -/* attached to dev.archdata.iommu of the master device */ +/* + * This structure is attached to dev.archdata.iommu of the master device + * on device add, contains a list of SYSMMU controllers defined by device tree, + * which are bound to given master device. It is usually referenced by 'owner' + * pointer. +*/ struct exynos_iommu_owner { - struct list_head client; /* entry of exynos_iommu_domain.clients */ - struct device *dev; - struct device *sysmmu; - struct iommu_domain *domain; - void *vmm_data; /* IO virtual memory manager's data */ - spinlock_t lock; /* Lock to preserve consistency of System MMU */ + struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ }; +/* + * This structure exynos specific generalization of struct iommu_domain. + * It contains list of SYSMMU controllers from all master devices, which has + * been attached to this domain and page tables of IO address space defined by + * it. It is usually referenced by 'domain' pointer. + */ struct exynos_iommu_domain { - struct list_head clients; /* list of sysmmu_drvdata.node */ - sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ - short *lv2entcnt; /* free lv2 entry counter for each section */ - spinlock_t lock; /* lock for this structure */ - spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ + struct list_head clients; /* list of sysmmu_drvdata.domain_node */ + sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ + short *lv2entcnt; /* free lv2 entry counter for each section */ + spinlock_t lock; /* lock for modyfying list of clients */ + spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ struct iommu_domain domain; /* generic domain data structure */ }; +/* + * This structure hold all data of a single SYSMMU controller, this includes + * hw resources like registers and clocks, pointers and list nodes to connect + * it to all other structures, internal state and parameters read from device + * tree. It is usually referenced by 'data' pointer. + */ struct sysmmu_drvdata { - struct device *sysmmu; /* System MMU's device descriptor */ - struct device *master; /* Owner of system MMU */ - void __iomem *sfrbase; - struct clk *clk; - struct clk *clk_master; - int activations; - spinlock_t lock; - struct iommu_domain *domain; - phys_addr_t pgtable; + struct device *sysmmu; /* SYSMMU controller device */ + struct device *master; /* master device (owner) */ + void __iomem *sfrbase; /* our registers */ + struct clk *clk; /* SYSMMU's clock */ + struct clk *clk_master; /* master's device clock */ + int activations; /* number of calls to sysmmu_enable */ + spinlock_t lock; /* lock for modyfying state */ + struct exynos_iommu_domain *domain; /* domain we belong to */ + struct list_head domain_node; /* node for domain clients list */ + struct list_head owner_node; /* node for owner controllers list */ + phys_addr_t pgtable; /* assigned page table structure */ + unsigned int version; /* our version */ }; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) @@ -244,11 +260,6 @@ static void sysmmu_unblock(void __iomem *sfrbase) __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); } -static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data) -{ - return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); -} - static bool sysmmu_block(void __iomem *sfrbase) { int i = 120; @@ -345,7 +356,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) show_fault_information(dev_name(data->sysmmu), itype, base, addr); if (data->domain) - ret = report_iommu_fault(data->domain, + ret = report_iommu_fault(&data->domain->domain, data->master, addr, itype); } @@ -408,7 +419,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) unsigned int cfg = CFG_LRU | CFG_QOS(15); unsigned int ver; - ver = __raw_sysmmu_version(data); + ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); if (MMU_MAJ_VER(ver) == 3) { if (MMU_MIN_VER(ver) >= 2) { cfg |= CFG_FLPDCACHE; @@ -422,6 +433,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) } __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); + data->version = ver; } static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) @@ -442,8 +454,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) clk_disable(data->clk_master); } -static int __sysmmu_enable(struct sysmmu_drvdata *data, - phys_addr_t pgtable, struct iommu_domain *domain) +static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, + struct exynos_iommu_domain *domain) { int ret = 0; unsigned long flags; @@ -470,77 +482,17 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, return ret; } -/* __exynos_sysmmu_enable: Enables System MMU - * - * returns -error if an error occurred and System MMU is not enabled, - * 0 if the System MMU has been just enabled and 1 if System MMU was already - * enabled before. - */ -static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable, - struct iommu_domain *domain) -{ - int ret = 0; - unsigned long flags; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data; - - BUG_ON(!has_sysmmu(dev)); - - spin_lock_irqsave(&owner->lock, flags); - - data = dev_get_drvdata(owner->sysmmu); - - ret = __sysmmu_enable(data, pgtable, domain); - if (ret >= 0) - data->master = dev; - - spin_unlock_irqrestore(&owner->lock, flags); - - return ret; -} - -int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable) -{ - BUG_ON(!memblock_is_memory(pgtable)); - - return __exynos_sysmmu_enable(dev, pgtable, NULL); -} - -static bool exynos_sysmmu_disable(struct device *dev) -{ - unsigned long flags; - bool disabled = true; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data; - - BUG_ON(!has_sysmmu(dev)); - - spin_lock_irqsave(&owner->lock, flags); - - data = dev_get_drvdata(owner->sysmmu); - - disabled = __sysmmu_disable(data); - if (disabled) - data->master = NULL; - - spin_unlock_irqrestore(&owner->lock, flags); - - return disabled; -} - static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { - if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3)) + if (data->version == MAKE_MMU_VER(3, 3)) __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); } -static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, +static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { unsigned long flags; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu); if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); @@ -554,14 +506,10 @@ static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, clk_disable(data->clk_master); } -static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, - size_t size) +static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, + sysmmu_iova_t iova, size_t size) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; unsigned long flags; - struct sysmmu_drvdata *data; - - data = dev_get_drvdata(owner->sysmmu); spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { @@ -580,7 +528,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, * 1MB page can be cached in one of all sets. * 64KB page can be one of 16 consecutive sets. */ - if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2) + if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); if (sysmmu_block(data->sfrbase)) { @@ -591,32 +539,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); } else { - dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n", - iova); - } - spin_unlock_irqrestore(&data->lock, flags); -} - -void exynos_sysmmu_tlb_invalidate(struct device *dev) -{ - struct exynos_iommu_owner *owner = dev->archdata.iommu; - unsigned long flags; - struct sysmmu_drvdata *data; - - data = dev_get_drvdata(owner->sysmmu); - - spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); - if (sysmmu_block(data->sfrbase)) { - __sysmmu_tlb_invalidate(data->sfrbase); - sysmmu_unblock(data->sfrbase); - } - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); - } else { - dev_dbg(dev, "disabled. Skipping TLB invalidation\n"); + dev_dbg(data->master, + "disabled. Skipping TLB invalidation @ %#x\n", iova); } spin_unlock_irqrestore(&data->lock, flags); } @@ -682,6 +606,36 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int exynos_sysmmu_suspend(struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev); + + dev_dbg(dev, "suspend\n"); + if (is_sysmmu_active(data)) { + __sysmmu_disable_nocount(data); + pm_runtime_put(dev); + } + return 0; +} + +static int exynos_sysmmu_resume(struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev); + + dev_dbg(dev, "resume\n"); + if (is_sysmmu_active(data)) { + pm_runtime_get_sync(dev); + __sysmmu_enable_nocount(data); + } + return 0; +} +#endif + +static const struct dev_pm_ops sysmmu_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume) +}; + static const struct of_device_id sysmmu_of_match[] __initconst = { { .compatible = "samsung,exynos-sysmmu", }, { }, @@ -692,6 +646,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { .driver = { .name = "exynos-sysmmu", .of_match_table = sysmmu_of_match, + .pm = &sysmmu_pm_ops, } }; @@ -704,104 +659,108 @@ static inline void pgtable_flush(void *vastart, void *vaend) static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) { - struct exynos_iommu_domain *exynos_domain; + struct exynos_iommu_domain *domain; int i; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; - exynos_domain = kzalloc(sizeof(*exynos_domain), GFP_KERNEL); - if (!exynos_domain) + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) return NULL; - exynos_domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); - if (!exynos_domain->pgtable) + domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); + if (!domain->pgtable) goto err_pgtable; - exynos_domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); - if (!exynos_domain->lv2entcnt) + domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + if (!domain->lv2entcnt) goto err_counter; /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ for (i = 0; i < NUM_LV1ENTRIES; i += 8) { - exynos_domain->pgtable[i + 0] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 1] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 2] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 3] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 4] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 5] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 6] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 7] = ZERO_LV2LINK; + domain->pgtable[i + 0] = ZERO_LV2LINK; + domain->pgtable[i + 1] = ZERO_LV2LINK; + domain->pgtable[i + 2] = ZERO_LV2LINK; + domain->pgtable[i + 3] = ZERO_LV2LINK; + domain->pgtable[i + 4] = ZERO_LV2LINK; + domain->pgtable[i + 5] = ZERO_LV2LINK; + domain->pgtable[i + 6] = ZERO_LV2LINK; + domain->pgtable[i + 7] = ZERO_LV2LINK; } - pgtable_flush(exynos_domain->pgtable, exynos_domain->pgtable + NUM_LV1ENTRIES); + pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES); - spin_lock_init(&exynos_domain->lock); - spin_lock_init(&exynos_domain->pgtablelock); - INIT_LIST_HEAD(&exynos_domain->clients); + spin_lock_init(&domain->lock); + spin_lock_init(&domain->pgtablelock); + INIT_LIST_HEAD(&domain->clients); - exynos_domain->domain.geometry.aperture_start = 0; - exynos_domain->domain.geometry.aperture_end = ~0UL; - exynos_domain->domain.geometry.force_aperture = true; + domain->domain.geometry.aperture_start = 0; + domain->domain.geometry.aperture_end = ~0UL; + domain->domain.geometry.force_aperture = true; - return &exynos_domain->domain; + return &domain->domain; err_counter: - free_pages((unsigned long)exynos_domain->pgtable, 2); + free_pages((unsigned long)domain->pgtable, 2); err_pgtable: - kfree(exynos_domain); + kfree(domain); return NULL; } -static void exynos_iommu_domain_free(struct iommu_domain *domain) +static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); - struct exynos_iommu_owner *owner; + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + struct sysmmu_drvdata *data, *next; unsigned long flags; int i; - WARN_ON(!list_empty(&priv->clients)); + WARN_ON(!list_empty(&domain->clients)); - spin_lock_irqsave(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(owner, &priv->clients, client) { - while (!exynos_sysmmu_disable(owner->dev)) - ; /* until System MMU is actually disabled */ + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { + if (__sysmmu_disable(data)) + data->master = NULL; + list_del_init(&data->domain_node); } - while (!list_empty(&priv->clients)) - list_del_init(priv->clients.next); - - spin_unlock_irqrestore(&priv->lock, flags); + spin_unlock_irqrestore(&domain->lock, flags); for (i = 0; i < NUM_LV1ENTRIES; i++) - if (lv1ent_page(priv->pgtable + i)) + if (lv1ent_page(domain->pgtable + i)) kmem_cache_free(lv2table_kmem_cache, - phys_to_virt(lv2table_base(priv->pgtable + i))); + phys_to_virt(lv2table_base(domain->pgtable + i))); - free_pages((unsigned long)priv->pgtable, 2); - free_pages((unsigned long)priv->lv2entcnt, 1); - kfree(priv); + free_pages((unsigned long)domain->pgtable, 2); + free_pages((unsigned long)domain->lv2entcnt, 1); + kfree(domain); } -static int exynos_iommu_attach_device(struct iommu_domain *domain, +static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct exynos_iommu_domain *priv = to_exynos_domain(domain); - phys_addr_t pagetable = virt_to_phys(priv->pgtable); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + struct sysmmu_drvdata *data; + phys_addr_t pagetable = virt_to_phys(domain->pgtable); unsigned long flags; - int ret; + int ret = -ENODEV; - spin_lock_irqsave(&priv->lock, flags); + if (!has_sysmmu(dev)) + return -ENODEV; - ret = __exynos_sysmmu_enable(dev, pagetable, domain); - if (ret == 0) { - list_add_tail(&owner->client, &priv->clients); - owner->domain = domain; - } + list_for_each_entry(data, &owner->controllers, owner_node) { + pm_runtime_get_sync(data->sysmmu); + ret = __sysmmu_enable(data, pagetable, domain); + if (ret >= 0) { + data->master = dev; - spin_unlock_irqrestore(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); + list_add_tail(&data->domain_node, &domain->clients); + spin_unlock_irqrestore(&domain->lock, flags); + } + } if (ret < 0) { dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", @@ -815,36 +774,39 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, return ret; } -static void exynos_iommu_detach_device(struct iommu_domain *domain, +static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, struct device *dev) { - struct exynos_iommu_owner *owner; - struct exynos_iommu_domain *priv = to_exynos_domain(domain); - phys_addr_t pagetable = virt_to_phys(priv->pgtable); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + phys_addr_t pagetable = virt_to_phys(domain->pgtable); + struct sysmmu_drvdata *data, *next; unsigned long flags; + bool found = false; - spin_lock_irqsave(&priv->lock, flags); + if (!has_sysmmu(dev)) + return; - list_for_each_entry(owner, &priv->clients, client) { - if (owner == dev->archdata.iommu) { - if (exynos_sysmmu_disable(dev)) { - list_del_init(&owner->client); - owner->domain = NULL; + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { + if (data->master == dev) { + if (__sysmmu_disable(data)) { + data->master = NULL; + list_del_init(&data->domain_node); } - break; + pm_runtime_put(data->sysmmu); + found = true; } } + spin_unlock_irqrestore(&domain->lock, flags); - spin_unlock_irqrestore(&priv->lock, flags); - - if (owner == dev->archdata.iommu) + if (found) dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, &pagetable); else dev_err(dev, "%s: No IOMMU is attached\n", __func__); } -static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, +static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { if (lv1ent_section(sent)) { @@ -862,6 +824,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, return ERR_PTR(-ENOMEM); *sent = mk_lv1ent_page(virt_to_phys(pent)); + kmemleak_ignore(pent); *pgcounter = NUM_LV2ENTRIES; pgtable_flush(pent, pent + NUM_LV2ENTRIES); pgtable_flush(sent, sent + 1); @@ -884,20 +847,19 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, * not currently mapped. */ if (need_flush_flpd_cache) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; - spin_lock(&priv->lock); - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_flpdcache( - owner->dev, iova); - spin_unlock(&priv->lock); + spin_lock(&domain->lock); + list_for_each_entry(data, &domain->clients, domain_node) + sysmmu_tlb_invalidate_flpdcache(data, iova); + spin_unlock(&domain->lock); } } return page_entry(sent, iova); } -static int lv1set_section(struct exynos_iommu_domain *priv, +static int lv1set_section(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, phys_addr_t paddr, short *pgcnt) { @@ -922,17 +884,17 @@ static int lv1set_section(struct exynos_iommu_domain *priv, pgtable_flush(sent, sent + 1); - spin_lock(&priv->lock); + spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; /* * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD * entry by speculative prefetch of SLPD which has no mapping. */ - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_flpdcache(owner->dev, iova); + list_for_each_entry(data, &domain->clients, domain_node) + sysmmu_tlb_invalidate_flpdcache(data, iova); } - spin_unlock(&priv->lock); + spin_unlock(&domain->lock); return 0; } @@ -992,74 +954,75 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, * than or equal to 128KiB. * - Start address of an I/O virtual region must be aligned by 128KiB. */ -static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, - phys_addr_t paddr, size_t size, int prot) +static int exynos_iommu_map(struct iommu_domain *iommu_domain, + unsigned long l_iova, phys_addr_t paddr, size_t size, + int prot) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_pte_t *entry; sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; unsigned long flags; int ret = -ENOMEM; - BUG_ON(priv->pgtable == NULL); + BUG_ON(domain->pgtable == NULL); - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - entry = section_entry(priv->pgtable, iova); + entry = section_entry(domain->pgtable, iova); if (size == SECT_SIZE) { - ret = lv1set_section(priv, entry, iova, paddr, - &priv->lv2entcnt[lv1ent_offset(iova)]); + ret = lv1set_section(domain, entry, iova, paddr, + &domain->lv2entcnt[lv1ent_offset(iova)]); } else { sysmmu_pte_t *pent; - pent = alloc_lv2entry(priv, entry, iova, - &priv->lv2entcnt[lv1ent_offset(iova)]); + pent = alloc_lv2entry(domain, entry, iova, + &domain->lv2entcnt[lv1ent_offset(iova)]); if (IS_ERR(pent)) ret = PTR_ERR(pent); else ret = lv2set_page(pent, paddr, size, - &priv->lv2entcnt[lv1ent_offset(iova)]); + &domain->lv2entcnt[lv1ent_offset(iova)]); } if (ret) pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", __func__, ret, size, iova); - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); return ret; } -static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv, - sysmmu_iova_t iova, size_t size) +static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, + sysmmu_iova_t iova, size_t size) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; unsigned long flags; - spin_lock_irqsave(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_entry(owner->dev, iova, size); + list_for_each_entry(data, &domain->clients, domain_node) + sysmmu_tlb_invalidate_entry(data, iova, size); - spin_unlock_irqrestore(&priv->lock, flags); + spin_unlock_irqrestore(&domain->lock, flags); } -static size_t exynos_iommu_unmap(struct iommu_domain *domain, - unsigned long l_iova, size_t size) +static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, + unsigned long l_iova, size_t size) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; sysmmu_pte_t *ent; size_t err_pgsize; unsigned long flags; - BUG_ON(priv->pgtable == NULL); + BUG_ON(domain->pgtable == NULL); - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - ent = section_entry(priv->pgtable, iova); + ent = section_entry(domain->pgtable, iova); if (lv1ent_section(ent)) { if (WARN_ON(size < SECT_SIZE)) { @@ -1093,7 +1056,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, *ent = 0; size = SPAGE_SIZE; pgtable_flush(ent, ent + 1); - priv->lv2entcnt[lv1ent_offset(iova)] += 1; + domain->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; } @@ -1107,15 +1070,15 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, pgtable_flush(ent, ent + SPAGES_PER_LPAGE); size = LPAGE_SIZE; - priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; + domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; done: - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); - exynos_iommu_tlb_invalidate_entry(priv, iova, size); + exynos_iommu_tlb_invalidate_entry(domain, iova, size); return size; err: - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", __func__, size, iova, err_pgsize); @@ -1123,17 +1086,17 @@ err: return 0; } -static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, +static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, dma_addr_t iova) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_pte_t *entry; unsigned long flags; phys_addr_t phys = 0; - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - entry = section_entry(priv->pgtable, iova); + entry = section_entry(domain->pgtable, iova); if (lv1ent_section(entry)) { phys = section_phys(entry) + section_offs(iova); @@ -1146,7 +1109,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, phys = spage_phys(entry) + spage_offs(iova); } - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); return phys; } @@ -1156,6 +1119,9 @@ static int exynos_iommu_add_device(struct device *dev) struct iommu_group *group; int ret; + if (!has_sysmmu(dev)) + return -ENODEV; + group = iommu_group_get(dev); if (!group) { @@ -1174,10 +1140,40 @@ static int exynos_iommu_add_device(struct device *dev) static void exynos_iommu_remove_device(struct device *dev) { + if (!has_sysmmu(dev)) + return; + iommu_group_remove_device(dev); } -static const struct iommu_ops exynos_iommu_ops = { +static int exynos_iommu_of_xlate(struct device *dev, + struct of_phandle_args *spec) +{ + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct platform_device *sysmmu = of_find_device_by_node(spec->np); + struct sysmmu_drvdata *data; + + if (!sysmmu) + return -ENODEV; + + data = platform_get_drvdata(sysmmu); + if (!data) + return -ENODEV; + + if (!owner) { + owner = kzalloc(sizeof(*owner), GFP_KERNEL); + if (!owner) + return -ENOMEM; + + INIT_LIST_HEAD(&owner->controllers); + dev->archdata.iommu = owner; + } + + list_add_tail(&data->owner_node, &owner->controllers); + return 0; +} + +static struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .domain_free = exynos_iommu_domain_free, .attach_dev = exynos_iommu_attach_device, @@ -1189,19 +1185,15 @@ static const struct iommu_ops exynos_iommu_ops = { .add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, + .of_xlate = exynos_iommu_of_xlate, }; +static bool init_done; + static int __init exynos_iommu_init(void) { - struct device_node *np; int ret; - np = of_find_matching_node(NULL, sysmmu_of_match); - if (!np) - return 0; - - of_node_put(np); - lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); if (!lv2table_kmem_cache) { @@ -1230,6 +1222,8 @@ static int __init exynos_iommu_init(void) goto err_set_iommu; } + init_done = true; + return 0; err_set_iommu: kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); @@ -1239,4 +1233,21 @@ err_reg_driver: kmem_cache_destroy(lv2table_kmem_cache); return ret; } -subsys_initcall(exynos_iommu_init); + +static int __init exynos_iommu_of_setup(struct device_node *np) +{ + struct platform_device *pdev; + + if (!init_done) + exynos_iommu_init(); + + pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + of_iommu_set_ops(np, &exynos_iommu_ops); + return 0; +} + +IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", + exynos_iommu_of_setup); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 68d43beccb7e..a98a7b27aca1 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -15,8 +15,11 @@ * Shaohua Li <shaohua.li@intel.com>, * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>, * Fenghua Yu <fenghua.yu@intel.com> + * Joerg Roedel <jroedel@suse.de> */ +#define pr_fmt(fmt) "DMAR: " fmt + #include <linux/init.h> #include <linux/bitmap.h> #include <linux/debugfs.h> @@ -40,6 +43,7 @@ #include <linux/pci-ats.h> #include <linux/memblock.h> #include <linux/dma-contiguous.h> +#include <linux/crash_dump.h> #include <asm/irq_remapping.h> #include <asm/cacheflush.h> #include <asm/iommu.h> @@ -190,7 +194,29 @@ struct root_entry { }; #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) +/* + * Take a root_entry and return the Lower Context Table Pointer (LCTP) + * if marked present. + */ +static phys_addr_t root_entry_lctp(struct root_entry *re) +{ + if (!(re->lo & 1)) + return 0; + + return re->lo & VTD_PAGE_MASK; +} + +/* + * Take a root_entry and return the Upper Context Table Pointer (UCTP) + * if marked present. + */ +static phys_addr_t root_entry_uctp(struct root_entry *re) +{ + if (!(re->hi & 1)) + return 0; + return re->hi & VTD_PAGE_MASK; +} /* * low 64 bits: * 0: present @@ -207,10 +233,38 @@ struct context_entry { u64 hi; }; -static inline bool context_present(struct context_entry *context) +static inline void context_clear_pasid_enable(struct context_entry *context) +{ + context->lo &= ~(1ULL << 11); +} + +static inline bool context_pasid_enabled(struct context_entry *context) +{ + return !!(context->lo & (1ULL << 11)); +} + +static inline void context_set_copied(struct context_entry *context) +{ + context->hi |= (1ull << 3); +} + +static inline bool context_copied(struct context_entry *context) +{ + return !!(context->hi & (1ULL << 3)); +} + +static inline bool __context_present(struct context_entry *context) { return (context->lo & 1); } + +static inline bool context_present(struct context_entry *context) +{ + return context_pasid_enabled(context) ? + __context_present(context) : + __context_present(context) && !context_copied(context); +} + static inline void context_set_present(struct context_entry *context) { context->lo |= 1; @@ -247,6 +301,11 @@ static inline void context_set_domain_id(struct context_entry *context, context->hi |= (value & ((1 << 16) - 1)) << 8; } +static inline int context_domain_id(struct context_entry *c) +{ + return((c->hi >> 8) & 0xffff); +} + static inline void context_clear_entry(struct context_entry *context) { context->lo = 0; @@ -422,6 +481,14 @@ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; +static int intel_iommu_ecs = 1; + +/* We only actually use ECS when PASID support (on the new bit 40) + * is also advertised. Some early implementations — the ones with + * PASID support on bit 28 — have issues even when we *only* use + * extended root/context tables. */ +#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \ + ecap_pasid(iommu->ecap)) int intel_iommu_gfx_mapped; EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); @@ -432,6 +499,25 @@ static LIST_HEAD(device_domain_list); static const struct iommu_ops intel_iommu_ops; +static bool translation_pre_enabled(struct intel_iommu *iommu) +{ + return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED); +} + +static void clear_translation_pre_enabled(struct intel_iommu *iommu) +{ + iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED; +} + +static void init_translation_status(struct intel_iommu *iommu) +{ + u32 gsts; + + gsts = readl(iommu->reg + DMAR_GSTS_REG); + if (gsts & DMA_GSTS_TES) + iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED; +} + /* Convert generic 'struct iommu_domain to private struct dmar_domain */ static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom) { @@ -445,26 +531,26 @@ static int __init intel_iommu_setup(char *str) while (*str) { if (!strncmp(str, "on", 2)) { dmar_disabled = 0; - printk(KERN_INFO "Intel-IOMMU: enabled\n"); + pr_info("IOMMU enabled\n"); } else if (!strncmp(str, "off", 3)) { dmar_disabled = 1; - printk(KERN_INFO "Intel-IOMMU: disabled\n"); + pr_info("IOMMU disabled\n"); } else if (!strncmp(str, "igfx_off", 8)) { dmar_map_gfx = 0; - printk(KERN_INFO - "Intel-IOMMU: disable GFX device mapping\n"); + pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { - printk(KERN_INFO - "Intel-IOMMU: Forcing DAC for PCI devices\n"); + pr_info("Forcing DAC for PCI devices\n"); dmar_forcedac = 1; } else if (!strncmp(str, "strict", 6)) { - printk(KERN_INFO - "Intel-IOMMU: disable batched IOTLB flush\n"); + pr_info("Disable batched IOTLB flush\n"); intel_iommu_strict = 1; } else if (!strncmp(str, "sp_off", 6)) { - printk(KERN_INFO - "Intel-IOMMU: disable supported super page\n"); + pr_info("Disable supported super page\n"); intel_iommu_superpage = 0; + } else if (!strncmp(str, "ecs_off", 7)) { + printk(KERN_INFO + "Intel-IOMMU: disable extended context table support\n"); + intel_iommu_ecs = 0; } str += strcspn(str, ","); @@ -669,7 +755,7 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu struct context_entry *context; u64 *entry; - if (ecap_ecs(iommu->ecap)) { + if (ecs_enabled(iommu)) { if (devfn >= 0x80) { devfn -= 0x80; entry = &root->hi; @@ -696,6 +782,11 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu return &context[devfn]; } +static int iommu_dummy(struct device *dev) +{ + return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO; +} + static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) { struct dmar_drhd_unit *drhd = NULL; @@ -705,6 +796,9 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf u16 segment = 0; int i; + if (iommu_dummy(dev)) + return NULL; + if (dev_is_pci(dev)) { pdev = to_pci_dev(dev); segment = pci_domain_nr(pdev->bus); @@ -798,7 +892,7 @@ static void free_context_table(struct intel_iommu *iommu) if (context) free_pgtable_page(context); - if (!ecap_ecs(iommu->ecap)) + if (!ecs_enabled(iommu)) continue; context = iommu_context_addr(iommu, i, 0x80, 0); @@ -1112,7 +1206,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu) root = (struct root_entry *)alloc_pgtable_page(iommu->node); if (!root) { - pr_err("IOMMU: allocating root entry for %s failed\n", + pr_err("Allocating root entry for %s failed\n", iommu->name); return -ENOMEM; } @@ -1133,7 +1227,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) unsigned long flag; addr = virt_to_phys(iommu->root_entry); - if (ecap_ecs(iommu->ecap)) + if (ecs_enabled(iommu)) addr |= DMA_RTADDR_RTT; raw_spin_lock_irqsave(&iommu->register_lock, flag); @@ -1250,9 +1344,9 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, /* check IOTLB invalidation granularity */ if (DMA_TLB_IAIG(val) == 0) - printk(KERN_ERR"IOMMU: flush IOTLB failed\n"); + pr_err("Flush IOTLB failed\n"); if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type)) - pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n", + pr_debug("TLB flush request %Lx, actual %Lx\n", (unsigned long long)DMA_TLB_IIRG(type), (unsigned long long)DMA_TLB_IAIG(val)); } @@ -1423,8 +1517,8 @@ static int iommu_init_domains(struct intel_iommu *iommu) unsigned long nlongs; ndomains = cap_ndoms(iommu->cap); - pr_debug("IOMMU%d: Number of Domains supported <%ld>\n", - iommu->seq_id, ndomains); + pr_debug("%s: Number of Domains supported <%ld>\n", + iommu->name, ndomains); nlongs = BITS_TO_LONGS(ndomains); spin_lock_init(&iommu->lock); @@ -1434,15 +1528,15 @@ static int iommu_init_domains(struct intel_iommu *iommu) */ iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); if (!iommu->domain_ids) { - pr_err("IOMMU%d: allocating domain id array failed\n", - iommu->seq_id); + pr_err("%s: Allocating domain id array failed\n", + iommu->name); return -ENOMEM; } iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), GFP_KERNEL); if (!iommu->domains) { - pr_err("IOMMU%d: allocating domain array failed\n", - iommu->seq_id); + pr_err("%s: Allocating domain array failed\n", + iommu->name); kfree(iommu->domain_ids); iommu->domain_ids = NULL; return -ENOMEM; @@ -1547,7 +1641,7 @@ static int iommu_attach_domain(struct dmar_domain *domain, num = __iommu_attach_domain(domain, iommu); spin_unlock_irqrestore(&iommu->lock, flags); if (num < 0) - pr_err("IOMMU: no free domain ids\n"); + pr_err("%s: No free domain ids\n", iommu->name); return num; } @@ -1639,7 +1733,7 @@ static int dmar_init_reserved_ranges(void) iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START), IOVA_PFN(IOAPIC_RANGE_END)); if (!iova) { - printk(KERN_ERR "Reserve IOAPIC range failed\n"); + pr_err("Reserve IOAPIC range failed\n"); return -ENODEV; } @@ -1655,7 +1749,7 @@ static int dmar_init_reserved_ranges(void) IOVA_PFN(r->start), IOVA_PFN(r->end)); if (!iova) { - printk(KERN_ERR "Reserve iova failed\n"); + pr_err("Reserve iova failed\n"); return -ENODEV; } } @@ -1702,7 +1796,7 @@ static int domain_init(struct dmar_domain *domain, int guest_width) sagaw = cap_sagaw(iommu->cap); if (!test_bit(agaw, &sagaw)) { /* hardware doesn't support it, choose a bigger one */ - pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw); + pr_debug("Hardware doesn't support agaw %d\n", agaw); agaw = find_next_bit(&sagaw, 5, agaw); if (agaw >= 5) return -ENODEV; @@ -1795,6 +1889,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain, return 0; } + context_clear_entry(context); + id = domain->id; pgd = domain->pgd; @@ -1803,7 +1899,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, id = iommu_attach_vm_domain(domain, iommu); if (id < 0) { spin_unlock_irqrestore(&iommu->lock, flags); - pr_err("IOMMU: no free domain ids\n"); + pr_err("%s: No free domain ids\n", iommu->name); return -EFAULT; } } @@ -2030,8 +2126,8 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); if (tmp) { static int dumps = 5; - printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", - iov_pfn, tmp, (unsigned long long)pteval); + pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", + iov_pfn, tmp, (unsigned long long)pteval); if (dumps) { dumps--; debug_dma_dump_mappings(NULL); @@ -2303,7 +2399,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain, if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn), dma_to_mm_pfn(last_vpfn))) { - printk(KERN_ERR "IOMMU: reserve iova failed\n"); + pr_err("Reserving iova failed\n"); return -ENOMEM; } @@ -2336,15 +2432,14 @@ static int iommu_prepare_identity_map(struct device *dev, range which is reserved in E820, so which didn't get set up to start with in si_domain */ if (domain == si_domain && hw_pass_through) { - printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n", - dev_name(dev), start, end); + pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n", + dev_name(dev), start, end); return 0; } - printk(KERN_INFO - "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n", - dev_name(dev), start, end); - + pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n", + dev_name(dev), start, end); + if (end < start) { WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n" "BIOS vendor: %s; Ver: %s; Product Version: %s\n", @@ -2401,12 +2496,11 @@ static inline void iommu_prepare_isa(void) if (!pdev) return; - printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n"); + pr_info("Prepare 0-16MiB unity mapping for LPC\n"); ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1); if (ret) - printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; " - "floppy might not work\n"); + pr_err("Failed to create 0-16MiB identity map - floppy might not work\n"); pci_dev_put(pdev); } @@ -2450,7 +2544,7 @@ static int __init si_domain_init(int hw) return -EFAULT; } - pr_debug("IOMMU: identity mapping domain is domain %d\n", + pr_debug("Identity mapping domain is domain %d\n", si_domain->id); if (hw) @@ -2650,8 +2744,8 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw hw ? CONTEXT_TT_PASS_THROUGH : CONTEXT_TT_MULTI_LEVEL); if (!ret) - pr_info("IOMMU: %s identity mapping for device %s\n", - hw ? "hardware" : "software", dev_name(dev)); + pr_info("%s identity mapping for device %s\n", + hw ? "Hardware" : "Software", dev_name(dev)); else if (ret == -ENODEV) /* device not associated with an iommu */ ret = 0; @@ -2669,10 +2763,6 @@ static int __init iommu_prepare_static_identity_mapping(int hw) int i; int ret = 0; - ret = si_domain_init(hw); - if (ret) - return -EFAULT; - for_each_pci_dev(pdev) { ret = dev_prepare_static_identity_mapping(&pdev->dev, hw); if (ret) @@ -2686,7 +2776,7 @@ static int __init iommu_prepare_static_identity_mapping(int hw) if (dev->bus != &acpi_bus_type) continue; - + adev= to_acpi_device(dev); mutex_lock(&adev->physical_node_lock); list_for_each_entry(pn, &adev->physical_node_list, node) { @@ -2728,19 +2818,200 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) */ iommu->flush.flush_context = __iommu_flush_context; iommu->flush.flush_iotlb = __iommu_flush_iotlb; - pr_info("IOMMU: %s using Register based invalidation\n", + pr_info("%s: Using Register based invalidation\n", iommu->name); } else { iommu->flush.flush_context = qi_flush_context; iommu->flush.flush_iotlb = qi_flush_iotlb; - pr_info("IOMMU: %s using Queued invalidation\n", iommu->name); + pr_info("%s: Using Queued invalidation\n", iommu->name); } } +static int copy_context_table(struct intel_iommu *iommu, + struct root_entry *old_re, + struct context_entry **tbl, + int bus, bool ext) +{ + struct context_entry *old_ce = NULL, *new_ce = NULL, ce; + int tbl_idx, pos = 0, idx, devfn, ret = 0, did; + phys_addr_t old_ce_phys; + + tbl_idx = ext ? bus * 2 : bus; + + for (devfn = 0; devfn < 256; devfn++) { + /* First calculate the correct index */ + idx = (ext ? devfn * 2 : devfn) % 256; + + if (idx == 0) { + /* First save what we may have and clean up */ + if (new_ce) { + tbl[tbl_idx] = new_ce; + __iommu_flush_cache(iommu, new_ce, + VTD_PAGE_SIZE); + pos = 1; + } + + if (old_ce) + iounmap(old_ce); + + ret = 0; + if (devfn < 0x80) + old_ce_phys = root_entry_lctp(old_re); + else + old_ce_phys = root_entry_uctp(old_re); + + if (!old_ce_phys) { + if (ext && devfn == 0) { + /* No LCTP, try UCTP */ + devfn = 0x7f; + continue; + } else { + goto out; + } + } + + ret = -ENOMEM; + old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE); + if (!old_ce) + goto out; + + new_ce = alloc_pgtable_page(iommu->node); + if (!new_ce) + goto out_unmap; + + ret = 0; + } + + /* Now copy the context entry */ + ce = old_ce[idx]; + + if (!__context_present(&ce)) + continue; + + did = context_domain_id(&ce); + if (did >= 0 && did < cap_ndoms(iommu->cap)) + set_bit(did, iommu->domain_ids); + + /* + * We need a marker for copied context entries. This + * marker needs to work for the old format as well as + * for extended context entries. + * + * Bit 67 of the context entry is used. In the old + * format this bit is available to software, in the + * extended format it is the PGE bit, but PGE is ignored + * by HW if PASIDs are disabled (and thus still + * available). + * + * So disable PASIDs first and then mark the entry + * copied. This means that we don't copy PASID + * translations from the old kernel, but this is fine as + * faults there are not fatal. + */ + context_clear_pasid_enable(&ce); + context_set_copied(&ce); + + new_ce[idx] = ce; + } + + tbl[tbl_idx + pos] = new_ce; + + __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); + +out_unmap: + iounmap(old_ce); + +out: + return ret; +} + +static int copy_translation_tables(struct intel_iommu *iommu) +{ + struct context_entry **ctxt_tbls; + struct root_entry *old_rt; + phys_addr_t old_rt_phys; + int ctxt_table_entries; + unsigned long flags; + u64 rtaddr_reg; + int bus, ret; + bool new_ext, ext; + + rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); + ext = !!(rtaddr_reg & DMA_RTADDR_RTT); + new_ext = !!ecap_ecs(iommu->ecap); + + /* + * The RTT bit can only be changed when translation is disabled, + * but disabling translation means to open a window for data + * corruption. So bail out and don't copy anything if we would + * have to change the bit. + */ + if (new_ext != ext) + return -EINVAL; + + old_rt_phys = rtaddr_reg & VTD_PAGE_MASK; + if (!old_rt_phys) + return -EINVAL; + + old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE); + if (!old_rt) + return -ENOMEM; + + /* This is too big for the stack - allocate it from slab */ + ctxt_table_entries = ext ? 512 : 256; + ret = -ENOMEM; + ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL); + if (!ctxt_tbls) + goto out_unmap; + + for (bus = 0; bus < 256; bus++) { + ret = copy_context_table(iommu, &old_rt[bus], + ctxt_tbls, bus, ext); + if (ret) { + pr_err("%s: Failed to copy context table for bus %d\n", + iommu->name, bus); + continue; + } + } + + spin_lock_irqsave(&iommu->lock, flags); + + /* Context tables are copied, now write them to the root_entry table */ + for (bus = 0; bus < 256; bus++) { + int idx = ext ? bus * 2 : bus; + u64 val; + + if (ctxt_tbls[idx]) { + val = virt_to_phys(ctxt_tbls[idx]) | 1; + iommu->root_entry[bus].lo = val; + } + + if (!ext || !ctxt_tbls[idx + 1]) + continue; + + val = virt_to_phys(ctxt_tbls[idx + 1]) | 1; + iommu->root_entry[bus].hi = val; + } + + spin_unlock_irqrestore(&iommu->lock, flags); + + kfree(ctxt_tbls); + + __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE); + + ret = 0; + +out_unmap: + iounmap(old_rt); + + return ret; +} + static int __init init_dmars(void) { struct dmar_drhd_unit *drhd; struct dmar_rmrr_unit *rmrr; + bool copied_tables = false; struct device *dev; struct intel_iommu *iommu; int i, ret; @@ -2761,8 +3032,7 @@ static int __init init_dmars(void) g_num_of_iommus++; continue; } - printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n", - DMAR_UNITS_SUPPORTED); + pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED); } /* Preallocate enough resources for IOMMU hot-addition */ @@ -2772,7 +3042,7 @@ static int __init init_dmars(void) g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *), GFP_KERNEL); if (!g_iommus) { - printk(KERN_ERR "Allocating global iommu array failed\n"); + pr_err("Allocating global iommu array failed\n"); ret = -ENOMEM; goto error; } @@ -2787,10 +3057,21 @@ static int __init init_dmars(void) for_each_active_iommu(iommu, drhd) { g_iommus[iommu->seq_id] = iommu; + intel_iommu_init_qi(iommu); + ret = iommu_init_domains(iommu); if (ret) goto free_iommu; + init_translation_status(iommu); + + if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { + iommu_disable_translation(iommu); + clear_translation_pre_enabled(iommu); + pr_warn("Translation was enabled for %s but we are not in kdump mode\n", + iommu->name); + } + /* * TBD: * we could share the same root & context tables @@ -2799,13 +3080,41 @@ static int __init init_dmars(void) ret = iommu_alloc_root_entry(iommu); if (ret) goto free_iommu; + + if (translation_pre_enabled(iommu)) { + pr_info("Translation already enabled - trying to copy translation structures\n"); + + ret = copy_translation_tables(iommu); + if (ret) { + /* + * We found the IOMMU with translation + * enabled - but failed to copy over the + * old root-entry table. Try to proceed + * by disabling translation now and + * allocating a clean root-entry table. + * This might cause DMAR faults, but + * probably the dump will still succeed. + */ + pr_err("Failed to copy translation tables from previous kernel for %s\n", + iommu->name); + iommu_disable_translation(iommu); + clear_translation_pre_enabled(iommu); + } else { + pr_info("Copied translation tables from previous kernel for %s\n", + iommu->name); + copied_tables = true; + } + } + + iommu_flush_write_buffer(iommu); + iommu_set_root_entry(iommu); + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; } - for_each_active_iommu(iommu, drhd) - intel_iommu_init_qi(iommu); - if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; @@ -2813,9 +3122,24 @@ static int __init init_dmars(void) iommu_identity_mapping |= IDENTMAP_GFX; #endif + if (iommu_identity_mapping) { + ret = si_domain_init(hw_pass_through); + if (ret) + goto free_iommu; + } + check_tylersburg_isoch(); /* + * If we copied translations from a previous kernel in the kdump + * case, we can not assign the devices to domains now, as that + * would eliminate the old mappings. So skip this part and defer + * the assignment to device driver initialization time. + */ + if (copied_tables) + goto domains_done; + + /* * If pass through is not set or not enabled, setup context entries for * identity mappings for rmrr, gfx, and isa and may fall back to static * identity mapping if iommu_identity_mapping is set. @@ -2823,7 +3147,7 @@ static int __init init_dmars(void) if (iommu_identity_mapping) { ret = iommu_prepare_static_identity_mapping(hw_pass_through); if (ret) { - printk(KERN_CRIT "Failed to setup IOMMU pass-through\n"); + pr_crit("Failed to setup IOMMU pass-through\n"); goto free_iommu; } } @@ -2841,20 +3165,21 @@ static int __init init_dmars(void) * endfor * endfor */ - printk(KERN_INFO "IOMMU: Setting RMRR:\n"); + pr_info("Setting RMRR:\n"); for_each_rmrr_units(rmrr) { /* some BIOS lists non-exist devices in DMAR table. */ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, dev) { ret = iommu_prepare_rmrr_dev(rmrr, dev); if (ret) - printk(KERN_ERR - "IOMMU: mapping reserved region failed\n"); + pr_err("Mapping reserved region failed\n"); } } iommu_prepare_isa(); +domains_done: + /* * for each drhd * enable fault log @@ -2879,11 +3204,9 @@ static int __init init_dmars(void) if (ret) goto free_iommu; - iommu_set_root_entry(iommu); + if (!translation_pre_enabled(iommu)) + iommu_enable_translation(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); - iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); } @@ -2924,7 +3247,7 @@ static struct iova *intel_alloc_iova(struct device *dev, } iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); if (unlikely(!iova)) { - printk(KERN_ERR "Allocating %ld-page iova for %s failed", + pr_err("Allocating %ld-page iova for %s failed", nrpages, dev_name(dev)); return NULL; } @@ -2939,7 +3262,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); if (!domain) { - printk(KERN_ERR "Allocating domain for %s failed", + pr_err("Allocating domain for %s failed\n", dev_name(dev)); return NULL; } @@ -2948,7 +3271,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) if (unlikely(!domain_context_mapped(dev))) { ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL); if (ret) { - printk(KERN_ERR "Domain context map for %s failed", + pr_err("Domain context map for %s failed\n", dev_name(dev)); return NULL; } @@ -2969,11 +3292,6 @@ static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev) return __get_valid_domain_for_dev(dev); } -static int iommu_dummy(struct device *dev) -{ - return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO; -} - /* Check if the dev needs to go through non-identity map and unmap process.*/ static int iommu_no_mapping(struct device *dev) { @@ -2995,8 +3313,8 @@ static int iommu_no_mapping(struct device *dev) * to non-identity mapping. */ domain_remove_one_dev_info(si_domain, dev); - printk(KERN_INFO "32bit %s uses non-identity mapping\n", - dev_name(dev)); + pr_info("32bit %s uses non-identity mapping\n", + dev_name(dev)); return 0; } } else { @@ -3011,8 +3329,8 @@ static int iommu_no_mapping(struct device *dev) CONTEXT_TT_PASS_THROUGH : CONTEXT_TT_MULTI_LEVEL); if (!ret) { - printk(KERN_INFO "64bit %s uses identity mapping\n", - dev_name(dev)); + pr_info("64bit %s uses identity mapping\n", + dev_name(dev)); return 1; } } @@ -3081,7 +3399,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, error: if (iova) __free_iova(&domain->iovad, iova); - printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n", + pr_err("Device %s request: %zx@%llx dir %d --- failed\n", dev_name(dev), size, (unsigned long long)paddr, dir); return 0; } @@ -3396,7 +3714,7 @@ static inline int iommu_domain_cache_init(void) NULL); if (!iommu_domain_cache) { - printk(KERN_ERR "Couldn't create iommu_domain cache\n"); + pr_err("Couldn't create iommu_domain cache\n"); ret = -ENOMEM; } @@ -3413,7 +3731,7 @@ static inline int iommu_devinfo_cache_init(void) SLAB_HWCACHE_ALIGN, NULL); if (!iommu_devinfo_cache) { - printk(KERN_ERR "Couldn't create devinfo cache\n"); + pr_err("Couldn't create devinfo cache\n"); ret = -ENOMEM; } @@ -3790,19 +4108,19 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) return 0; if (hw_pass_through && !ecap_pass_through(iommu->ecap)) { - pr_warn("IOMMU: %s doesn't support hardware pass through.\n", + pr_warn("%s: Doesn't support hardware pass through.\n", iommu->name); return -ENXIO; } if (!ecap_sc_support(iommu->ecap) && domain_update_iommu_snooping(iommu)) { - pr_warn("IOMMU: %s doesn't support snooping.\n", + pr_warn("%s: Doesn't support snooping.\n", iommu->name); return -ENXIO; } sp = domain_update_iommu_superpage(iommu) - 1; if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { - pr_warn("IOMMU: %s doesn't support large page.\n", + pr_warn("%s: Doesn't support large page.\n", iommu->name); return -ENXIO; } @@ -4033,7 +4351,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, start = mhp->start_pfn << PAGE_SHIFT; end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1; if (iommu_domain_identity_map(si_domain, start, end)) { - pr_warn("dmar: failed to build identity map for [%llx-%llx]\n", + pr_warn("Failed to build identity map for [%llx-%llx]\n", start, end); return NOTIFY_BAD; } @@ -4051,7 +4369,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, iova = find_iova(&si_domain->iovad, start_vpfn); if (iova == NULL) { - pr_debug("dmar: failed get IOVA for PFN %lx\n", + pr_debug("Failed get IOVA for PFN %lx\n", start_vpfn); break; } @@ -4059,7 +4377,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, iova = split_and_remove_iova(&si_domain->iovad, iova, start_vpfn, last_vpfn); if (iova == NULL) { - pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n", + pr_warn("Failed to split IOVA PFN [%lx-%lx]\n", start_vpfn, last_vpfn); return NOTIFY_BAD; } @@ -4168,13 +4486,6 @@ int __init intel_iommu_init(void) goto out_free_dmar; } - /* - * Disable translation if already enabled prior to OS handover. - */ - for_each_active_iommu(iommu, drhd) - if (iommu->gcmd & DMA_GCMD_TE) - iommu_disable_translation(iommu); - if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); @@ -4185,10 +4496,10 @@ int __init intel_iommu_init(void) goto out_free_dmar; if (list_empty(&dmar_rmrr_units)) - printk(KERN_INFO "DMAR: No RMRR found\n"); + pr_info("No RMRR found\n"); if (list_empty(&dmar_atsr_units)) - printk(KERN_INFO "DMAR: No ATSR found\n"); + pr_info("No ATSR found\n"); if (dmar_init_reserved_ranges()) { if (force_on) @@ -4202,12 +4513,11 @@ int __init intel_iommu_init(void) if (ret) { if (force_on) panic("tboot: Failed to initialize DMARs\n"); - printk(KERN_ERR "IOMMU: dmar init failed\n"); + pr_err("Initialization failed\n"); goto out_free_reserved_range; } up_write(&dmar_global_lock); - printk(KERN_INFO - "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n"); + pr_info("Intel(R) Virtualization Technology for Directed I/O\n"); init_timer(&unmap_timer); #ifdef CONFIG_SWIOTLB @@ -4349,13 +4659,11 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); if (!dmar_domain) { - printk(KERN_ERR - "intel_iommu_domain_init: dmar_domain == NULL\n"); + pr_err("Can't allocate dmar_domain\n"); return NULL; } if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { - printk(KERN_ERR - "intel_iommu_domain_init() failed\n"); + pr_err("Domain initialization failed\n"); domain_exit(dmar_domain); return NULL; } @@ -4414,7 +4722,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, addr_width = cap_mgaw(iommu->cap); if (dmar_domain->max_addr > (1LL << addr_width)) { - printk(KERN_ERR "%s: iommu width (%d) is not " + pr_err("%s: iommu width (%d) is not " "sufficient for the mapped address (%llx)\n", __func__, addr_width, dmar_domain->max_addr); return -EFAULT; @@ -4468,7 +4776,7 @@ static int intel_iommu_map(struct iommu_domain *domain, /* check if minimum agaw is sufficient for mapped address */ end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1; if (end < max_addr) { - printk(KERN_ERR "%s: iommu width (%d) is not " + pr_err("%s: iommu width (%d) is not " "sufficient for the mapped address (%llx)\n", __func__, dmar_domain->gaw, max_addr); return -EFAULT; @@ -4609,7 +4917,7 @@ static const struct iommu_ops intel_iommu_ops = { static void quirk_iommu_g4x_gfx(struct pci_dev *dev) { /* G4x/GM45 integrated gfx dmar support is totally busted. */ - printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n"); + pr_info("Disabling IOMMU for graphics on this chipset\n"); dmar_map_gfx = 0; } @@ -4627,7 +4935,7 @@ static void quirk_iommu_rwbf(struct pci_dev *dev) * Mobile 4 Series Chipset neglects to set RWBF capability, * but needs it. Same seems to hold for the desktop versions. */ - printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n"); + pr_info("Forcing write-buffer flush capability\n"); rwbf_quirk = 1; } @@ -4657,11 +4965,11 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev) return; if (!(ggc & GGC_MEMORY_VT_ENABLED)) { - printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); + pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); dmar_map_gfx = 0; } else if (dmar_map_gfx) { /* we have to ensure the gfx device is idle before we flush */ - printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); + pr_info("Disabling batched IOTLB flush on Ironlake\n"); intel_iommu_strict = 1; } } @@ -4723,7 +5031,7 @@ static void __init check_tylersburg_isoch(void) iommu_identity_mapping |= IDENTMAP_AZALIA; return; } - - printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", + + pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", vtisochctrl); } diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 5709ae9c3e77..47fcebf39e9e 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -1,3 +1,6 @@ + +#define pr_fmt(fmt) "DMAR-IR: " fmt + #include <linux/interrupt.h> #include <linux/dmar.h> #include <linux/spinlock.h> @@ -8,6 +11,7 @@ #include <linux/irq.h> #include <linux/intel-iommu.h> #include <linux/acpi.h> +#include <linux/crash_dump.h> #include <asm/io_apic.h> #include <asm/smp.h> #include <asm/cpu.h> @@ -51,8 +55,28 @@ static struct hpet_scope ir_hpet[MAX_HPET_TBS]; */ static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); +static void iommu_disable_irq_remapping(struct intel_iommu *iommu); static int __init parse_ioapics_under_ir(void); +static bool ir_pre_enabled(struct intel_iommu *iommu) +{ + return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); +} + +static void clear_ir_pre_enabled(struct intel_iommu *iommu) +{ + iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; +} + +static void init_ir_status(struct intel_iommu *iommu) +{ + u32 gsts; + + gsts = readl(iommu->reg + DMAR_GSTS_REG); + if (gsts & DMA_GSTS_IRES) + iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; +} + static struct irq_2_iommu *irq_2_iommu(unsigned int irq) { struct irq_cfg *cfg = irq_cfg(irq); @@ -100,8 +124,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) } if (mask > ecap_max_handle_mask(iommu->ecap)) { - printk(KERN_ERR - "Requested mask %x exceeds the max invalidation handle" + pr_err("Requested mask %x exceeds the max invalidation handle" " mask value %Lx\n", mask, ecap_max_handle_mask(iommu->ecap)); return -1; @@ -333,7 +356,7 @@ static int set_ioapic_sid(struct irte *irte, int apic) up_read(&dmar_global_lock); if (sid == 0) { - pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); + pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); return -1; } @@ -360,7 +383,7 @@ static int set_hpet_sid(struct irte *irte, u8 id) up_read(&dmar_global_lock); if (sid == 0) { - pr_warning("Failed to set source-id of HPET block (%d)\n", id); + pr_warn("Failed to set source-id of HPET block (%d)\n", id); return -1; } @@ -424,11 +447,59 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev) return 0; } +static int iommu_load_old_irte(struct intel_iommu *iommu) +{ + struct irte *old_ir_table; + phys_addr_t irt_phys; + unsigned int i; + size_t size; + u64 irta; + + if (!is_kdump_kernel()) { + pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", + iommu->name); + clear_ir_pre_enabled(iommu); + iommu_disable_irq_remapping(iommu); + return -EINVAL; + } + + /* Check whether the old ir-table has the same size as ours */ + irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); + if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) + != INTR_REMAP_TABLE_REG_SIZE) + return -EINVAL; + + irt_phys = irta & VTD_PAGE_MASK; + size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte); + + /* Map the old IR table */ + old_ir_table = ioremap_cache(irt_phys, size); + if (!old_ir_table) + return -ENOMEM; + + /* Copy data over */ + memcpy(iommu->ir_table->base, old_ir_table, size); + + __iommu_flush_cache(iommu, iommu->ir_table->base, size); + + /* + * Now check the table for used entries and mark those as + * allocated in the bitmap + */ + for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) { + if (iommu->ir_table->base[i].present) + bitmap_set(iommu->ir_table->bitmap, i, 1); + } + + return 0; +} + + static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) { + unsigned long flags; u64 addr; u32 sts; - unsigned long flags; addr = virt_to_phys((void *)iommu->ir_table->base); @@ -445,10 +516,16 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) raw_spin_unlock_irqrestore(&iommu->register_lock, flags); /* - * global invalidation of interrupt entry cache before enabling - * interrupt-remapping. + * Global invalidation of interrupt entry cache to make sure the + * hardware uses the new irq remapping table. */ qi_global_iec(iommu); +} + +static void iommu_enable_irq_remapping(struct intel_iommu *iommu) +{ + unsigned long flags; + u32 sts; raw_spin_lock_irqsave(&iommu->register_lock, flags); @@ -505,12 +582,48 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) ir_table->base = page_address(pages); ir_table->bitmap = bitmap; iommu->ir_table = ir_table; + + /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (!iommu->qi) { + /* + * Clear previous faults. + */ + dmar_fault(-1, iommu); + dmar_disable_qi(iommu); + + if (dmar_enable_qi(iommu)) { + pr_err("Failed to enable queued invalidation\n"); + goto out_free_bitmap; + } + } + + init_ir_status(iommu); + + if (ir_pre_enabled(iommu)) { + if (iommu_load_old_irte(iommu)) + pr_err("Failed to copy IR table for %s from previous kernel\n", + iommu->name); + else + pr_info("Copied IR table for %s from previous kernel\n", + iommu->name); + } + + iommu_set_irq_remapping(iommu, eim_mode); + return 0; +out_free_bitmap: + kfree(bitmap); out_free_pages: __free_pages(pages, INTR_REMAP_PAGE_ORDER); out_free_table: kfree(ir_table); + + iommu->ir_table = NULL; + return -ENOMEM; } @@ -580,17 +693,17 @@ static void __init intel_cleanup_irq_remapping(void) } if (x2apic_supported()) - pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); + pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); } static int __init intel_prepare_irq_remapping(void) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; + int eim = 0; if (irq_remap_broken) { - printk(KERN_WARNING - "This system BIOS has enabled interrupt remapping\n" + pr_warn("This system BIOS has enabled interrupt remapping\n" "on a chipset that contains an erratum making that\n" "feature unstable. To maintain system stability\n" "interrupt remapping is being disabled. Please\n" @@ -606,7 +719,7 @@ static int __init intel_prepare_irq_remapping(void) return -ENODEV; if (parse_ioapics_under_ir() != 1) { - printk(KERN_INFO "Not enabling interrupt remapping\n"); + pr_info("Not enabling interrupt remapping\n"); goto error; } @@ -615,85 +728,54 @@ static int __init intel_prepare_irq_remapping(void) if (!ecap_ir_support(iommu->ecap)) goto error; - /* Do the allocations early */ - for_each_iommu(iommu, drhd) - if (intel_setup_irq_remapping(iommu)) - goto error; - - return 0; - -error: - intel_cleanup_irq_remapping(); - return -ENODEV; -} - -static int __init intel_enable_irq_remapping(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool setup = false; - int eim = 0; - + /* Detect remapping mode: lapic or x2apic */ if (x2apic_supported()) { eim = !dmar_x2apic_optout(); - if (!eim) - pr_info("x2apic is disabled because BIOS sets x2apic opt out bit. You can use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); + if (!eim) { + pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); + pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); + } } for_each_iommu(iommu, drhd) { - /* - * If the queued invalidation is already initialized, - * shouldn't disable it. - */ - if (iommu->qi) - continue; - - /* - * Clear previous faults. - */ - dmar_fault(-1, iommu); - - /* - * Disable intr remapping and queued invalidation, if already - * enabled prior to OS handover. - */ - iommu_disable_irq_remapping(iommu); - - dmar_disable_qi(iommu); - } - - /* - * check for the Interrupt-remapping support - */ - for_each_iommu(iommu, drhd) if (eim && !ecap_eim_support(iommu->ecap)) { - printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); + pr_info("%s does not support EIM\n", iommu->name); eim = 0; } + } + eim_mode = eim; if (eim) pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); - /* - * Enable queued invalidation for all the DRHD's. - */ + /* Do the initializations early */ for_each_iommu(iommu, drhd) { - int ret = dmar_enable_qi(iommu); - - if (ret) { - printk(KERN_ERR "DRHD %Lx: failed to enable queued, " - " invalidation, ecap %Lx, ret %d\n", - drhd->reg_base_addr, iommu->ecap, ret); + if (intel_setup_irq_remapping(iommu)) { + pr_err("Failed to setup irq remapping for %s\n", + iommu->name); goto error; } } + return 0; + +error: + intel_cleanup_irq_remapping(); + return -ENODEV; +} + +static int __init intel_enable_irq_remapping(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + bool setup = false; + /* * Setup Interrupt-remapping for all the DRHD's now. */ for_each_iommu(iommu, drhd) { - iommu_set_irq_remapping(iommu, eim); + if (!ir_pre_enabled(iommu)) + iommu_enable_irq_remapping(iommu); setup = true; } @@ -709,9 +791,9 @@ static int __init intel_enable_irq_remapping(void) */ x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries; - pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); + pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); - return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; + return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; error: intel_cleanup_irq_remapping(); @@ -930,6 +1012,7 @@ static int reenable_irq_remapping(int eim) /* Set up interrupt remapping for iommu.*/ iommu_set_irq_remapping(iommu, eim); + iommu_enable_irq_remapping(iommu); setup = true; } @@ -1145,14 +1228,12 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) down_read(&dmar_global_lock); iommu = map_dev_to_ir(dev); if (!iommu) { - printk(KERN_ERR - "Unable to map PCI %s to iommu\n", pci_name(dev)); + pr_err("Unable to map PCI %s to iommu\n", pci_name(dev)); index = -ENOENT; } else { index = alloc_irte(iommu, irq, nvec); if (index < 0) { - printk(KERN_ERR - "Unable to allocate %d IRTE for PCI %s\n", + pr_err("Unable to allocate %d IRTE for PCI %s\n", nvec, pci_name(dev)); index = -ENOSPC; } @@ -1242,28 +1323,12 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) /* Setup Interrupt-remapping now. */ ret = intel_setup_irq_remapping(iommu); if (ret) { - pr_err("DRHD %Lx: failed to allocate resource\n", - iommu->reg_phys); - ir_remove_ioapic_hpet_scope(iommu); - return ret; - } - - if (!iommu->qi) { - /* Clear previous faults. */ - dmar_fault(-1, iommu); - iommu_disable_irq_remapping(iommu); - dmar_disable_qi(iommu); - } - - /* Enable queued invalidation */ - ret = dmar_enable_qi(iommu); - if (!ret) { - iommu_set_irq_remapping(iommu, eim); - } else { - pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n", - iommu->reg_phys, iommu->ecap, ret); + pr_err("Failed to setup irq remapping for %s\n", + iommu->name); intel_teardown_irq_remapping(iommu); ir_remove_ioapic_hpet_scope(iommu); + } else { + iommu_enable_irq_remapping(iommu); } return ret; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d4f527e56679..49e7542510d1 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -16,7 +16,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#define pr_fmt(fmt) "%s: " fmt, __func__ +#define pr_fmt(fmt) "iommu: " fmt #include <linux/device.h> #include <linux/kernel.h> @@ -51,6 +51,8 @@ struct iommu_group { void (*iommu_data_release)(void *iommu_data); char *name; int id; + struct iommu_domain *default_domain; + struct iommu_domain *domain; }; struct iommu_device { @@ -75,6 +77,15 @@ struct iommu_group_attribute iommu_group_attr_##_name = \ #define to_iommu_group(_kobj) \ container_of(_kobj, struct iommu_group, kobj) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type); +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev); +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group); +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group); + static ssize_t iommu_group_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) { @@ -128,6 +139,8 @@ static void iommu_group_release(struct kobject *kobj) { struct iommu_group *group = to_iommu_group(kobj); + pr_debug("Releasing group %d\n", group->id); + if (group->iommu_data_release) group->iommu_data_release(group->iommu_data); @@ -135,6 +148,9 @@ static void iommu_group_release(struct kobject *kobj) ida_remove(&iommu_group_ida, group->id); mutex_unlock(&iommu_group_mutex); + if (group->default_domain) + iommu_domain_free(group->default_domain); + kfree(group->name); kfree(group); } @@ -207,6 +223,8 @@ again: */ kobject_put(&group->kobj); + pr_debug("Allocated group %d\n", group->id); + return group; } EXPORT_SYMBOL_GPL(iommu_group_alloc); @@ -307,6 +325,52 @@ int iommu_group_set_name(struct iommu_group *group, const char *name) } EXPORT_SYMBOL_GPL(iommu_group_set_name); +static int iommu_group_create_direct_mappings(struct iommu_group *group, + struct device *dev) +{ + struct iommu_domain *domain = group->default_domain; + struct iommu_dm_region *entry; + struct list_head mappings; + unsigned long pg_size; + int ret = 0; + + if (!domain || domain->type != IOMMU_DOMAIN_DMA) + return 0; + + BUG_ON(!domain->ops->pgsize_bitmap); + + pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap); + INIT_LIST_HEAD(&mappings); + + iommu_get_dm_regions(dev, &mappings); + + /* We need to consider overlapping regions for different devices */ + list_for_each_entry(entry, &mappings, list) { + dma_addr_t start, end, addr; + + start = ALIGN(entry->start, pg_size); + end = ALIGN(entry->start + entry->length, pg_size); + + for (addr = start; addr < end; addr += pg_size) { + phys_addr_t phys_addr; + + phys_addr = iommu_iova_to_phys(domain, addr); + if (phys_addr) + continue; + + ret = iommu_map(domain, addr, addr, pg_size, entry->prot); + if (ret) + goto out; + } + + } + +out: + iommu_put_dm_regions(dev, &mappings); + + return ret; +} + /** * iommu_group_add_device - add a device to an iommu group * @group: the group into which to add the device (reference should be held) @@ -363,8 +427,12 @@ rename: dev->iommu_group = group; + iommu_group_create_direct_mappings(group, dev); + mutex_lock(&group->mutex); list_add_tail(&device->list, &group->devices); + if (group->domain) + __iommu_attach_device(group->domain, dev); mutex_unlock(&group->mutex); /* Notify any listeners about change to group. */ @@ -372,6 +440,9 @@ rename: IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); trace_add_device_to_group(group->id, dev); + + pr_info("Adding device %s to group %d\n", dev_name(dev), group->id); + return 0; } EXPORT_SYMBOL_GPL(iommu_group_add_device); @@ -388,6 +459,8 @@ void iommu_group_remove_device(struct device *dev) struct iommu_group *group = dev->iommu_group; struct iommu_device *tmp_device, *device = NULL; + pr_info("Removing device %s from group %d\n", dev_name(dev), group->id); + /* Pre-notify listeners that a device is being removed. */ blocking_notifier_call_chain(&group->notifier, IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); @@ -417,6 +490,17 @@ void iommu_group_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +static int iommu_group_device_count(struct iommu_group *group) +{ + struct iommu_device *entry; + int ret = 0; + + list_for_each_entry(entry, &group->devices, list) + ret++; + + return ret; +} + /** * iommu_group_for_each_dev - iterate over each device in the group * @group: the group @@ -428,19 +512,30 @@ EXPORT_SYMBOL_GPL(iommu_group_remove_device); * The group->mutex is held across callbacks, which will block calls to * iommu_group_add/remove_device. */ -int iommu_group_for_each_dev(struct iommu_group *group, void *data, - int (*fn)(struct device *, void *)) +static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) { struct iommu_device *device; int ret = 0; - mutex_lock(&group->mutex); list_for_each_entry(device, &group->devices, list) { ret = fn(device->dev, data); if (ret) break; } + return ret; +} + + +int iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) +{ + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_group_for_each_dev(group, data, fn); mutex_unlock(&group->mutex); + return ret; } EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); @@ -692,7 +787,19 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev) return group; /* No shared group found, allocate new */ - return iommu_group_alloc(); + group = iommu_group_alloc(); + if (IS_ERR(group)) + return NULL; + + /* + * Try to allocate a default domain - needs support from the + * IOMMU driver. + */ + group->default_domain = __iommu_domain_alloc(pdev->dev.bus, + IOMMU_DOMAIN_DMA); + group->domain = group->default_domain; + + return group; } /** @@ -731,6 +838,11 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev) return group; } +struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) +{ + return group->default_domain; +} + static int add_iommu_group(struct device *dev, void *data) { struct iommu_callback_data *cb = data; @@ -741,7 +853,16 @@ static int add_iommu_group(struct device *dev, void *data) WARN_ON(dev->iommu_group); - ops->add_device(dev); + return ops->add_device(dev); +} + +static int remove_iommu_group(struct device *dev, void *data) +{ + struct iommu_callback_data *cb = data; + const struct iommu_ops *ops = cb->ops; + + if (ops->remove_device && dev->iommu_group) + ops->remove_device(dev); return 0; } @@ -761,7 +882,7 @@ static int iommu_bus_notifier(struct notifier_block *nb, if (action == BUS_NOTIFY_ADD_DEVICE) { if (ops->add_device) return ops->add_device(dev); - } else if (action == BUS_NOTIFY_DEL_DEVICE) { + } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { if (ops->remove_device && dev->iommu_group) { ops->remove_device(dev); return 0; @@ -814,19 +935,25 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) nb->notifier_call = iommu_bus_notifier; err = bus_register_notifier(bus, nb); - if (err) { - kfree(nb); - return err; - } + if (err) + goto out_free; err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); - if (err) { - bus_unregister_notifier(bus, nb); - kfree(nb); - return err; - } + if (err) + goto out_err; + return 0; + +out_err: + /* Clean up */ + bus_for_each_dev(bus, NULL, &cb, remove_iommu_group); + bus_unregister_notifier(bus, nb); + +out_free: + kfree(nb); + + return err; } /** @@ -898,22 +1025,28 @@ void iommu_set_fault_handler(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_set_fault_handler); -struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type) { struct iommu_domain *domain; if (bus == NULL || bus->iommu_ops == NULL) return NULL; - domain = bus->iommu_ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED); + domain = bus->iommu_ops->domain_alloc(type); if (!domain) return NULL; domain->ops = bus->iommu_ops; - domain->type = IOMMU_DOMAIN_UNMANAGED; + domain->type = type; return domain; } + +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +{ + return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); +} EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) @@ -922,7 +1055,8 @@ void iommu_domain_free(struct iommu_domain *domain) } EXPORT_SYMBOL_GPL(iommu_domain_free); -int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev) { int ret; if (unlikely(domain->ops->attach_dev == NULL)) @@ -933,9 +1067,38 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev) trace_attach_device_to_domain(dev); return ret; } + +int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_attach_device(domain, dev); + + /* + * We have a group - lock it to make sure the device-count doesn't + * change while we are attaching + */ + mutex_lock(&group->mutex); + ret = -EINVAL; + if (iommu_group_device_count(group) != 1) + goto out_unlock; + + ret = __iommu_attach_group(domain, group); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} EXPORT_SYMBOL_GPL(iommu_attach_device); -void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +static void __iommu_detach_device(struct iommu_domain *domain, + struct device *dev) { if (unlikely(domain->ops->detach_dev == NULL)) return; @@ -943,8 +1106,48 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev) domain->ops->detach_dev(domain, dev); trace_detach_device_from_domain(dev); } + +void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_detach_device(domain, dev); + + mutex_lock(&group->mutex); + if (iommu_group_device_count(group) != 1) { + WARN_ON(1); + goto out_unlock; + } + + __iommu_detach_group(domain, group); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); +} EXPORT_SYMBOL_GPL(iommu_detach_device); +struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) +{ + struct iommu_domain *domain; + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return NULL; + + domain = group->domain; + + iommu_group_put(group); + + return domain; +} +EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); + /* * IOMMU groups are really the natrual working unit of the IOMMU, but * the IOMMU API works on domains and devices. Bridge that gap by @@ -959,13 +1162,34 @@ static int iommu_group_do_attach_device(struct device *dev, void *data) { struct iommu_domain *domain = data; - return iommu_attach_device(domain, dev); + return __iommu_attach_device(domain, dev); +} + +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (group->default_domain && group->domain != group->default_domain) + return -EBUSY; + + ret = __iommu_group_for_each_dev(group, domain, + iommu_group_do_attach_device); + if (ret == 0) + group->domain = domain; + + return ret; } int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) { - return iommu_group_for_each_dev(group, domain, - iommu_group_do_attach_device); + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_attach_group(domain, group); + mutex_unlock(&group->mutex); + + return ret; } EXPORT_SYMBOL_GPL(iommu_attach_group); @@ -973,14 +1197,40 @@ static int iommu_group_do_detach_device(struct device *dev, void *data) { struct iommu_domain *domain = data; - iommu_detach_device(domain, dev); + __iommu_detach_device(domain, dev); return 0; } +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (!group->default_domain) { + __iommu_group_for_each_dev(group, domain, + iommu_group_do_detach_device); + group->domain = NULL; + return; + } + + if (group->domain == group->default_domain) + return; + + /* Detach by re-attaching to the default domain */ + ret = __iommu_group_for_each_dev(group, group->default_domain, + iommu_group_do_attach_device); + if (ret != 0) + WARN_ON(1); + else + group->domain = group->default_domain; +} + void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) { - iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); + mutex_lock(&group->mutex); + __iommu_detach_group(domain, group); + mutex_unlock(&group->mutex); } EXPORT_SYMBOL_GPL(iommu_detach_group); @@ -1207,7 +1457,7 @@ static int __init iommu_init(void) return 0; } -arch_initcall(iommu_init); +core_initcall(iommu_init); int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) @@ -1273,3 +1523,72 @@ int iommu_domain_set_attr(struct iommu_domain *domain, return ret; } EXPORT_SYMBOL_GPL(iommu_domain_set_attr); + +void iommu_get_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->get_dm_regions) + ops->get_dm_regions(dev, list); +} + +void iommu_put_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->put_dm_regions) + ops->put_dm_regions(dev, list); +} + +/* Request that a device is direct mapped by the IOMMU */ +int iommu_request_dm_for_dev(struct device *dev) +{ + struct iommu_domain *dm_domain; + struct iommu_group *group; + int ret; + + /* Device must already be in a group before calling this function */ + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + mutex_lock(&group->mutex); + + /* Check if the default domain is already direct mapped */ + ret = 0; + if (group->default_domain && + group->default_domain->type == IOMMU_DOMAIN_IDENTITY) + goto out; + + /* Don't change mappings of existing devices */ + ret = -EBUSY; + if (iommu_group_device_count(group) != 1) + goto out; + + /* Allocate a direct mapped domain */ + ret = -ENOMEM; + dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY); + if (!dm_domain) + goto out; + + /* Attach the device to the domain */ + ret = __iommu_attach_group(dm_domain, group); + if (ret) { + iommu_domain_free(dm_domain); + goto out; + } + + /* Make the direct mapped domain the default for this group */ + if (group->default_domain) + iommu_domain_free(group->default_domain); + group->default_domain = dm_domain; + + pr_info("Using direct mapping for device %s\n", dev_name(dev)); + + ret = 0; +out: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 9dd8208312c2..b7c3d923f3e1 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -227,6 +227,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova) /* Figure out where to put new node */ while (*new) { struct iova *this = container_of(*new, struct iova, node); + parent = *new; if (iova->pfn_lo < this->pfn_lo) @@ -350,6 +351,7 @@ void free_iova(struct iova_domain *iovad, unsigned long pfn) { struct iova *iova = find_iova(iovad, pfn); + if (iova) __free_iova(iovad, iova); @@ -369,6 +371,7 @@ void put_iova_domain(struct iova_domain *iovad) node = rb_first(&iovad->rbroot); while (node) { struct iova *iova = container_of(node, struct iova, node); + rb_erase(node, &iovad->rbroot); free_iova_mem(iova); node = rb_first(&iovad->rbroot); @@ -482,6 +485,7 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) for (node = rb_first(&from->rbroot); node; node = rb_next(node)) { struct iova *iova = container_of(node, struct iova, node); struct iova *new_iova; + new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); if (!new_iova) printk(KERN_ERR "Reserve iova range %lx@%lx failed\n", diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index cab214544237..ebf0adb8e7ea 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -551,6 +551,15 @@ static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); } +static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, + dma_addr_t iova, size_t size) +{ + rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); + if (size > SPAGE_SIZE) + rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE, + SPAGE_SIZE); +} + static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { @@ -575,12 +584,6 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, rk_table_flush(page_table, NUM_PT_ENTRIES); rk_table_flush(dte_addr, 1); - /* - * Zap the first iova of newly allocated page table so iommu evicts - * old cached value of new dte from the iotlb. - */ - rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); - done: pt_phys = rk_dte_pt_address(dte); return (u32 *)phys_to_virt(pt_phys); @@ -630,6 +633,14 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, rk_table_flush(pte_addr, pte_count); + /* + * Zap the first and last iova to evict from iotlb any previously + * mapped cachelines holding stale values for its dte and pte. + * We only zap the first and last iova, since only they could have + * dte or pte shared with an existing mapping. + */ + rk_iommu_zap_iova_first_last(rk_domain, iova, size); + return 0; unwind: /* Unmap the range of iovas that we just mapped */ @@ -774,7 +785,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, list_add_tail(&iommu->node, &rk_domain->iommus); spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); - dev_info(dev, "Attached to iommu domain\n"); + dev_dbg(dev, "Attached to iommu domain\n"); rk_iommu_disable_stall(iommu); @@ -808,7 +819,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, iommu->domain = NULL; - dev_info(dev, "Detached from iommu domain\n"); + dev_dbg(dev, "Detached from iommu domain\n"); } static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) |