diff options
author | Robin Murphy <robin.murphy@arm.com> | 2019-10-25 20:08:34 +0200 |
---|---|---|
committer | Will Deacon <will@kernel.org> | 2019-11-04 20:34:31 +0100 |
commit | c79278c185c8848fefc25cf304eecec9c4623a40 (patch) | |
tree | 382c1fd931d727b51385daeb34abd0f55aa022c1 /drivers/iommu/io-pgtable-arm.c | |
parent | iommu/io-pgtable-arm: Simplify start level lookup (diff) | |
download | linux-c79278c185c8848fefc25cf304eecec9c4623a40.tar.xz linux-c79278c185c8848fefc25cf304eecec9c4623a40.zip |
iommu/io-pgtable-arm: Simplify PGD size handling
We use data->pgd_size directly for the one-off allocation and freeing of
the top-level table, but otherwise it serves for ARM_LPAE_PGD_IDX() to
repeatedly re-calculate the effective number of top-level address bits
it represents. Flip this around so we store the form we most commonly
need, and derive the lesser-used one instead. This cuts a whole bunch of
code out of the map/unmap/iova_to_phys fast-paths.
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
Diffstat (limited to 'drivers/iommu/io-pgtable-arm.c')
-rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 33 |
1 files changed, 17 insertions, 16 deletions
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index f4c2fae11256..e67112107cef 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -40,16 +40,15 @@ (d)->pg_shift) #define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift) - -#define ARM_LPAE_PAGES_PER_PGD(d) \ - DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d)) +#define ARM_LPAE_PGD_SIZE(d) \ + (sizeof(arm_lpae_iopte) << (d)->pgd_bits) /* * Calculate the index at level l used to map virtual address a using the * pagetable in d. */ #define ARM_LPAE_PGD_IDX(l,d) \ - ((l) == (d)->start_level ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) + ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0) #define ARM_LPAE_LVL_IDX(a,l,d) \ (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ @@ -174,8 +173,8 @@ struct arm_lpae_io_pgtable { struct io_pgtable iop; + int pgd_bits; int start_level; - size_t pgd_size; unsigned long pg_shift; unsigned long bits_per_level; @@ -506,7 +505,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, unsigned long table_size; if (lvl == data->start_level) - table_size = data->pgd_size; + table_size = ARM_LPAE_PGD_SIZE(data); else table_size = ARM_LPAE_GRANULE(data); @@ -743,7 +742,7 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) static struct arm_lpae_io_pgtable * arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) { - unsigned long va_bits, pgd_bits; + unsigned long va_bits; struct arm_lpae_io_pgtable *data; int levels; @@ -775,8 +774,7 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) data->start_level = ARM_LPAE_MAX_LEVELS - levels; /* Calculate the actual size of our pgd (without concatenation) */ - pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); - data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); + data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); data->iop.ops = (struct io_pgtable_ops) { .map = arm_lpae_map, @@ -870,7 +868,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s1_cfg.mair[1] = 0; /* Looking good; allocate a pgd */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), + GFP_KERNEL, cfg); if (!data->pgd) goto out_free_data; @@ -908,9 +907,9 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) if (data->start_level == 0) { unsigned long pgd_pages; - pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); + pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { - data->pgd_size = pgd_pages << data->pg_shift; + data->pgd_bits += data->bits_per_level; data->start_level++; } } @@ -967,7 +966,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), + GFP_KERNEL, cfg); if (!data->pgd) goto out_free_data; @@ -1038,7 +1038,7 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) /* Mali seems to need a full 4-level table regardless of IAS */ if (data->start_level > 0) { data->start_level = 0; - data->pgd_size = sizeof(arm_lpae_iopte); + data->pgd_bits = 0; } /* * MEMATTR: Mali has no actual notion of a non-cacheable type, so the @@ -1055,7 +1055,8 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) (ARM_MALI_LPAE_MEMATTR_IMP_DEF << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, + cfg); if (!data->pgd) goto out_free_data; @@ -1135,7 +1136,7 @@ static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", cfg->pgsize_bitmap, cfg->ias); pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n", - ARM_LPAE_MAX_LEVELS - data->start_level, data->pgd_size, + ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data), data->pg_shift, data->bits_per_level, data->pgd); } |